From b0b6a1b902f984a0b543199b26b7f942f655b978 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 1 May 2020 00:00:44 -0400 Subject: [PATCH 1/4] Upgrade to PG 11 + Postgis 3.0.1 * Use proper postgis-maintained docker image rather than build our own * use UTF8PROC 2.5.0 * include postgres gzip support --- docker/postgis-preloaded/preload-database.sh | 2 - docker/postgis/Dockerfile | 145 ++++++++----------- docker/postgis/README.md | 4 +- docker/postgis/initdb-postgis.sh | 30 ++-- 4 files changed, 76 insertions(+), 105 deletions(-) diff --git a/docker/postgis-preloaded/preload-database.sh b/docker/postgis-preloaded/preload-database.sh index 7623c584..1f5f8e13 100755 --- a/docker/postgis-preloaded/preload-database.sh +++ b/docker/postgis-preloaded/preload-database.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -Eeo pipefail -mkdir -p "${PGDATA}" - # This code was adapted from a postgres issue # https://github.com/docker-library/postgres/issues/661#issuecomment-573192715 diff --git a/docker/postgis/Dockerfile b/docker/postgis/Dockerfile index 44c932fc..4d26651f 100644 --- a/docker/postgis/Dockerfile +++ b/docker/postgis/Dockerfile @@ -1,93 +1,60 @@ -FROM postgres:9.6 -MAINTAINER "Lukas Martinelli " -ENV POSTGIS_MAJOR=2.4 \ - POSTGIS_VERSION=2.4.8 \ - GEOS_VERSION=3.6.4 +# There is an issue updating to postgis 12: +# the postgis base image has outdated ogr2ogr (gdal-bin package) +# does not support PG12 yet, which means postgis-preloaded cannot be built easily. +# Once it is updated, or there is a different way to build it, we can upgrade to 12. +FROM postgis/postgis:11-3.0 + +LABEL maintainer="Yuri Astrakhan " + +ENV UTF8PROC_TAG=v2.5.0 \ + MAPNIK_GERMAN_L10N_TAG=v2.5.8 \ + PGSQL_GZIP_HASH=33b640b72836f5ad05840a0ee55be5e34839eb39 RUN apt-get -qq -y update \ - && apt-get -qq -y --no-install-recommends install \ - autoconf \ - automake \ - autotools-dev \ + ## + ## Install build dependencies + && apt-get -qq -y --no-install-recommends install \ build-essential \ ca-certificates \ - bison \ - cmake \ + # Required by Nominatim to download data files curl \ - dblatex \ - docbook-mathml \ - docbook-xsl \ git \ - gdal-bin \ - libcunit1-dev \ - libkakasi2-dev \ - libtool \ pandoc \ - unzip \ - xsltproc \ - # PostGIS build dependencies - libgdal-dev \ - libjson-c-dev \ - libproj-dev \ - libxml2-dev \ - postgresql-server-dev-$PG_MAJOR \ -## GEOS - && cd /opt/ \ - && curl -o /opt/geos.tar.bz2 http://download.osgeo.org/geos/geos-$GEOS_VERSION.tar.bz2 \ - && mkdir /opt/geos \ - && tar xf /opt/geos.tar.bz2 -C /opt/geos --strip-components=1 \ - && cd /opt/geos/ \ - && ./configure \ - && make -j \ - && make install \ - && rm -rf /opt/geos* \ -## Protobuf - && cd /opt/ \ - && curl -L https://github.com/google/protobuf/archive/v3.0.2.tar.gz | tar xvz && cd protobuf-3.0.2 \ - && ./autogen.sh \ - && ./configure \ - && make \ - && make install \ - && ldconfig \ - && rm -rf /opt/protobuf-3.0.2 \ -## Protobuf-c - && cd /opt/ \ - && curl -L https://github.com/protobuf-c/protobuf-c/releases/download/v1.2.1/protobuf-c-1.2.1.tar.gz | tar xvz && cd protobuf-c-1.2.1 \ - && ./configure \ - && make \ - && make install \ - && ldconfig \ - && rm -rf /opt/protobuf-c.1.2.1 \ -## Postgis - && cd /opt/ \ - && git clone https://github.com/postgis/postgis.git \ - && cd postgis \ - && git reset --hard $POSTGIS_VERSION \ - && ./autogen.sh \ - && ./configure CFLAGS="-Wall" \ - && make \ - && make install \ - && ldconfig \ - && rm -rf /opt/postgis \ -## UTF8Proc - && cd /opt/ \ - && git clone https://github.com/JuliaLang/utf8proc.git \ - && cd utf8proc \ - && git checkout -q v2.0.2 \ - && make \ - && make install \ - && ldconfig \ - && rm -rf /opt/utf8proc \ -## Mapnik German - && cd /opt/ \ - && git clone https://github.com/openmaptiles/mapnik-german-l10n.git \ - && cd mapnik-german-l10n \ - && make \ - && make install \ - && rm -rf /opt/mapnik-german-l10n \ -## Cleanup - && cd / \ - && apt-get -qq -y --auto-remove purge \ + # $PG_MAJOR is declared in postgres docker + postgresql-server-dev-$PG_MAJOR \ + libkakasi2-dev \ + libgdal-dev \ + ## + ## gzip extension + && cd /opt/ \ + && git clone https://github.com/pramsey/pgsql-gzip.git \ + && cd pgsql-gzip \ + && git checkout -q $PGSQL_GZIP_HASH \ + && make \ + && make install \ + && rm -rf /opt/pgsql-gzip \ + ## + ## UTF8Proc + && cd /opt/ \ + && git clone https://github.com/JuliaLang/utf8proc.git \ + && cd utf8proc \ + && git checkout -q $UTF8PROC_TAG \ + && make \ + && make install \ + && ldconfig \ + && rm -rf /opt/utf8proc \ + ## + ## osml10n extension (originally Mapnik German) + && cd /opt/ \ + && git clone https://github.com/giggls/mapnik-german-l10n.git \ + && cd mapnik-german-l10n \ + && git checkout -q $MAPNIK_GERMAN_L10N_TAG \ + && make \ + && make install \ + && rm -rf /opt/mapnik-german-l10n \ + ## + ## Cleanup + && apt-get -qq -y --auto-remove purge \ autoconf \ automake \ autotools-dev \ @@ -108,6 +75,14 @@ RUN apt-get -qq -y update \ pandoc \ unzip \ xsltproc \ -&& rm -rf /var/lib/apt/lists/* + libpq-dev \ + postgresql-server-dev-$PG_MAJOR \ + libxml2-dev \ + libjson-c-dev \ + libgdal-dev \ + && rm -rf /usr/local/lib/*.a \ + && rm -rf /var/lib/apt/lists/* -COPY ./initdb-postgis.sh /docker-entrypoint-initdb.d/10_postgis.sh +# The script should run after the parent's 10_postgis.sh runs +# so it must have the name that's listed after that. +COPY ./initdb-postgis.sh /docker-entrypoint-initdb.d/20_omt_postgis.sh diff --git a/docker/postgis/README.md b/docker/postgis/README.md index 0aece8ae..d20db6be 100644 --- a/docker/postgis/README.md +++ b/docker/postgis/README.md @@ -1,7 +1,7 @@ -# PostgreSQL with GEOS 3.6.4 and PostGIS 2.4.8 +# PostGIS + OSM-specific extensions Docker image [![](https://images.microbadger.com/badges/image/openmaptiles/postgis.svg)](https://microbadger.com/images/openmaptiles/postgis "Get your own image badge on microbadger.com") [![Docker Automated buil](https://img.shields.io/docker/automated/openmaptiles/postgis.svg)]() -A custom PostgreSQL Docker image based off GEOS 3.6.4 and PostGIS 2.4.8. +This images is based on PostgreSQL 11 and PostGIS 2.5 [Docker image](https://hub.docker.com/r/postgis/postgis/) and includes [osml10n extension](https://github.com/giggls/mapnik-german-l10n.git) - OSM-specific label manipulation support. ## Usage diff --git a/docker/postgis/initdb-postgis.sh b/docker/postgis/initdb-postgis.sh index 75e52acc..b7d30886 100755 --- a/docker/postgis/initdb-postgis.sh +++ b/docker/postgis/initdb-postgis.sh @@ -3,27 +3,25 @@ set -o errexit set -o pipefail set -o nounset -echo "+++++++++++++++++++++++++++" -echo " Create template_postgis" -echo "+++++++++++++++++++++++++++" - -# TODO: explain why this is needed -PGUSER="$POSTGRES_USER" psql --dbname="$POSTGRES_DB" <<-'EOSQL' - CREATE DATABASE template_postgis; - UPDATE pg_database SET datistemplate = TRUE WHERE datname = 'template_postgis'; -EOSQL - echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" -echo " Create extensions in template_postgis and $POSTGRES_DB DBs" +echo " Loading OMT postgis extensions" echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" for db in template_postgis "$POSTGRES_DB"; do echo "Loading extensions into $db" PGUSER="$POSTGRES_USER" psql --dbname="$db" <<-'EOSQL' - CREATE EXTENSION postgis; - CREATE EXTENSION hstore; - CREATE EXTENSION unaccent; - CREATE EXTENSION fuzzystrmatch; - CREATE EXTENSION osml10n; + -- Cleanup. Ideally parent container shouldn't pre-install those. + DROP EXTENSION IF EXISTS postgis_tiger_geocoder; + DROP EXTENSION IF EXISTS postgis_topology; + + -- These extensions are already loaded by the parent docker + CREATE EXTENSION IF NOT EXISTS postgis; + CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; + + -- Extensions needed for OpenMapTiles + CREATE EXTENSION IF NOT EXISTS hstore; + CREATE EXTENSION IF NOT EXISTS unaccent; + CREATE EXTENSION IF NOT EXISTS osml10n; + CREATE EXTENSION IF NOT EXISTS gzip; EOSQL done From 42ff545a10653b161d6443f14b4397f0af1adbd4 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 13 May 2020 23:25:56 -0400 Subject: [PATCH 2/4] Upgrade to Postgis12, latest ogr2ogr in preload * Use PostgreSQL 12 * skip any features with errors when importing NaturalEarth data * Just for the preload image, use the latest ogr2ogr from `sid` (unstable) --- docker/import-data/Dockerfile | 1 + docker/import-data/import_data.sh | 6 ++++++ docker/postgis-preloaded/Dockerfile | 16 ++++++++++++---- docker/postgis/Dockerfile | 2 +- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/docker/import-data/Dockerfile b/docker/import-data/Dockerfile index 629ffcb3..a2d2d072 100644 --- a/docker/import-data/Dockerfile +++ b/docker/import-data/Dockerfile @@ -10,6 +10,7 @@ RUN DIR=/downloads/natural_earth \ && wget --quiet http://naciscdn.org/naturalearth/packages/natural_earth_vector.sqlite.zip \ && unzip -oj natural_earth_vector.sqlite.zip \ && ../clean-natural-earth.sh natural_earth_vector.sqlite \ + && rm ../clean-natural-earth.sh \ && rm natural_earth_vector.sqlite.zip # Download water polygons data from http://osmdata.openstreetmap.de diff --git a/docker/import-data/import_data.sh b/docker/import-data/import_data.sh index 45090605..aa3f225e 100755 --- a/docker/import-data/import_data.sh +++ b/docker/import-data/import_data.sh @@ -32,12 +32,18 @@ fi if [ -z ${1+x} ] || [ "$1" = "natural-earth" ]; then echo "Importing Natual Earth $INFO..." + # since ogr2ogr v3 NaturalEarth data shows ring self-intersecting failures. + # See https://github.com/OSGeo/gdal/issues/2301 + # Adding -skipfailures + # TODO: add -makevalid once it becomes available, or possibly remove + # the -clipsrc, and run an update query afterwards to clip and make geometries valid. PGCLIENTENCODING=UTF8 ogr2ogr \ -progress \ -f Postgresql \ -s_srs EPSG:4326 \ -t_srs EPSG:3857 \ -clipsrc -180.1 -85.0511 180.1 85.0511 \ + -skipfailures \ "PG:$PGCONN" \ -lco GEOMETRY_NAME=geometry \ -lco OVERWRITE=YES \ diff --git a/docker/postgis-preloaded/Dockerfile b/docker/postgis-preloaded/Dockerfile index 37e27281..5041e06c 100644 --- a/docker/postgis-preloaded/Dockerfile +++ b/docker/postgis-preloaded/Dockerfile @@ -24,11 +24,19 @@ ENV DATA_DIR=/import \ POSTGRES_USER=openmaptiles \ POSTGRES_PASSWORD=openmaptiles +USER root + # NOTE: The ogr2ogr in Debian stable does not support PostgreSQL 12 -# Thus parent image must be PG11 or before -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - gdal-bin +# Use the sid source instead (Debian unstable). Once gdal 3.0+ is in stable, use this code instead: +#RUN DEBIAN_FRONTEND=noninteractive apt-get update \ +# && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ +# gdal-bin +RUN set -eux ;\ + echo "APT::Default-Release "stable";" > /etc/apt/apt.conf.d/default-release ;\ + echo "deb http://ftp.debian.org/debian sid main" > /etc/apt/sources.list.d/sid.list ;\ + DEBIAN_FRONTEND=noninteractive apt-get update ;\ + DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends -t sid install \ + locales-all gdal-bin python3-gdal libgdal26 libnetcdf18 libogdi4.1 libhdf5-hl-100 # switch to postgres user for ownership and execution USER postgres diff --git a/docker/postgis/Dockerfile b/docker/postgis/Dockerfile index 4d26651f..5656f6ca 100644 --- a/docker/postgis/Dockerfile +++ b/docker/postgis/Dockerfile @@ -2,7 +2,7 @@ # the postgis base image has outdated ogr2ogr (gdal-bin package) # does not support PG12 yet, which means postgis-preloaded cannot be built easily. # Once it is updated, or there is a different way to build it, we can upgrade to 12. -FROM postgis/postgis:11-3.0 +FROM postgis/postgis:12-3.0 LABEL maintainer="Yuri Astrakhan " From 2eee5d207cbaed825852f1fa9e379d0be78f1922 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 20 May 2020 14:49:00 -0400 Subject: [PATCH 3/4] simplify change --- docker/import-data/import_data.sh | 6 ------ docker/postgis-preloaded/Dockerfile | 14 ++++---------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/docker/import-data/import_data.sh b/docker/import-data/import_data.sh index aa3f225e..45090605 100755 --- a/docker/import-data/import_data.sh +++ b/docker/import-data/import_data.sh @@ -32,18 +32,12 @@ fi if [ -z ${1+x} ] || [ "$1" = "natural-earth" ]; then echo "Importing Natual Earth $INFO..." - # since ogr2ogr v3 NaturalEarth data shows ring self-intersecting failures. - # See https://github.com/OSGeo/gdal/issues/2301 - # Adding -skipfailures - # TODO: add -makevalid once it becomes available, or possibly remove - # the -clipsrc, and run an update query afterwards to clip and make geometries valid. PGCLIENTENCODING=UTF8 ogr2ogr \ -progress \ -f Postgresql \ -s_srs EPSG:4326 \ -t_srs EPSG:3857 \ -clipsrc -180.1 -85.0511 180.1 85.0511 \ - -skipfailures \ "PG:$PGCONN" \ -lco GEOMETRY_NAME=geometry \ -lco OVERWRITE=YES \ diff --git a/docker/postgis-preloaded/Dockerfile b/docker/postgis-preloaded/Dockerfile index 5041e06c..ffbbf029 100644 --- a/docker/postgis-preloaded/Dockerfile +++ b/docker/postgis-preloaded/Dockerfile @@ -27,16 +27,10 @@ ENV DATA_DIR=/import \ USER root # NOTE: The ogr2ogr in Debian stable does not support PostgreSQL 12 -# Use the sid source instead (Debian unstable). Once gdal 3.0+ is in stable, use this code instead: -#RUN DEBIAN_FRONTEND=noninteractive apt-get update \ -# && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ -# gdal-bin -RUN set -eux ;\ - echo "APT::Default-Release "stable";" > /etc/apt/apt.conf.d/default-release ;\ - echo "deb http://ftp.debian.org/debian sid main" > /etc/apt/sources.list.d/sid.list ;\ - DEBIAN_FRONTEND=noninteractive apt-get update ;\ - DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends -t sid install \ - locales-all gdal-bin python3-gdal libgdal26 libnetcdf18 libogdi4.1 libhdf5-hl-100 +# Thus parent image must be PG11 or before +RUN DEBIAN_FRONTEND=noninteractive apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + gdal-bin # switch to postgres user for ownership and execution USER postgres From 806876461a8a92b5d4d3162ca56a7050795e634c Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 20 May 2020 14:52:06 -0400 Subject: [PATCH 4/4] set default postgres settings during init --- docker/postgis/initdb-postgis.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/postgis/initdb-postgis.sh b/docker/postgis/initdb-postgis.sh index b7d30886..7641365c 100755 --- a/docker/postgis/initdb-postgis.sh +++ b/docker/postgis/initdb-postgis.sh @@ -3,6 +3,13 @@ set -o errexit set -o pipefail set -o nounset +echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" +echo " Pre-configuring PostgreSQL" +echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" +PGUSER="$POSTGRES_USER" psql --dbname="postgres" <<-'EOSQL' + ALTER SYSTEM SET jit = 'off'; +EOSQL + echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" echo " Loading OMT postgis extensions" echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"