Move different import scripts into one container

docker-compose.yml

@@ -26,6 +26,21 @@ import:
    - cache
   links:
    - postgis:db
+import-labels:
+  build: ./src/import-external
+  command: ./import-labels.sh
+  links:
+  - postgis:db
+import-natural-earth:
+  build: ./src/import-external
+  command: ./import-natural-earth.sh
+  links:
+  - postgis:db
+import-water:
+  build: ./src/import-external
+  command: ./import-water.sh
+  links:
+  - postgis:db
 import-external:
   build: ./src/import-external
   links:

src/import-external/Dockerfile

@@ -6,7 +6,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
       gdal-bin \
     && rm -rf /var/lib/apt/lists/*
 
-RUN mkdir -p /data/import && wget --quiet http://data.openstreetmapdata.com/water-polygons-split-3857.zip && unzip -oj water-polygons-split-3857.zip -d /data/import && rm water-polygons-split-3857.zip
+RUN mkdir -p /data/import \
+    && wget --quiet http://data.openstreetmapdata.com/water-polygons-split-3857.zip \
+    && unzip -oj water-polygons-split-3857.zip -d /data/import \
+    && rm water-polygons-split-3857.zip
 
 RUN mkdir -p /data/import \
     && wget http://naciscdn.org/naturalearth/packages/natural_earth_vector.sqlite.zip \
@@ -15,7 +18,6 @@ RUN mkdir -p /data/import \
 
 ENV IMPORT_DATA_DIR=/data/import
 
-RUN mkdir -p /usr/src/app
 COPY . /usr/src/app
 WORKDIR /usr/src/app
 

src/import-external/import-labels.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+source sql.sh
+
+readonly COUNTRIES_GEOJSON="countries.geojson"
+readonly STATES_GEOJSON="states.geojson"
+readonly SEAS_GEOJSON="seas.geojson"
+
+function import_geojson() {
+    local geojson_file=$1
+    local table_name=$2
+
+    drop_table "$table_name"
+    echo "$geojson_file"
+
+    PGCLIENTENCODING=UTF8 ogr2ogr \
+    -f Postgresql \
+    -s_srs EPSG:4326 \
+    -t_srs EPSG:3857 \
+    PG:"dbname=$OSM_DB user=$OSM_USER host=$DB_HOST port=$DB_PORT" \
+    "$geojson_file" \
+    -nln "$table_name"
+}
+
+function import_labels() {
+    echo "Inserting labels into $OSM_DB"
+
+    import_geojson "$SEAS_GEOJSON" "seas"
+    import_geojson "$STATES_GEOJSON" "states"
+    import_geojson "$COUNTRIES_GEOJSON" "countries"
+}
+
+import_labels

src/import-external/import-natural-earth.sh

@@ -3,14 +3,13 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-readonly DB_HOST=$DB_PORT_5432_TCP_ADDR
-readonly DB_PORT=$DB_PORT_5432_TCP_PORT
-readonly OSM_DB=${OSM_DB:-osm}
-readonly OSM_USER=${OSM_USER:-osm}
-readonly OSM_PASSWORD=${OSM_PASSWORD:-osm}
+source sql.sh
 
-function import_sqlite() {
-    echo "Importing Natural Earth to PostGIS..."
+readonly IMPORT_DATA_DIR=${IMPORT_DATA_DIR:-/data/import}
+readonly NATURAL_EARTH_SQLITE_FILE="$IMPORT_DATA_DIR/natural_earth_vector.sqlite"
+
+function import_natural_earth() {
+    echo "Importing Natural Earth to PostGIS"
     PGCLIENTENCODING=LATIN1 ogr2ogr \
     -progress \
     -f Postgresql \
@@ -25,4 +24,4 @@ function import_sqlite() {
     "$NATURAL_EARTH_SQLITE_FILE"
 }
 
-import_sqlite
+import_natural_earth

src/import-external/import-water.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+readonly IMPORT_DATA_DIR=${IMPORT_DATA_DIR:-/data/import}
+readonly WATER_POLYGONS_FILE="$IMPORT_DATA_DIR/water_polygons.shp"
+
+source sql.sh
+
+function import_shp() {
+	local shp_file=$1
+	shp2pgsql -g way "$shp_file" | exec_psql | hide_inserts
+}
+
+function import_water() {
+    local water_table="water_polygons"
+    echo "Removing existing table $water_table"
+    drop_table $water_table
+
+    echo "Importing $WATER_POLYGONS_FILE into table $water_table"
+    import_shp "$WATER_POLYGONS_FILE"
+
+    echo "Create index on table $water_table"
+    create_index $water_table
+}
+
+import_water

src/import-external/import.sh

@@ -3,88 +3,14 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-readonly DB_HOST=$DB_PORT_5432_TCP_ADDR
-readonly DB_PORT=$DB_PORT_5432_TCP_PORT
-readonly OSM_DB=${OSM_DB:-osm}
-readonly OSM_USER=${OSM_USER:-osm}
-readonly OSM_PASSWORD=${OSM_PASSWORD:-osm}
+source import-natural-earth.sh
+source import-labels.sh
+source import-water.sh
 
-readonly IMPORT_DATA_DIR=${IMPORT_DATA_DIR:-/data/import}
-readonly WATER_POLYGONS_FILE="$IMPORT_DATA_DIR/water_polygons.shp"
-readonly NATURAL_EARTH_SQLITE_FILE="$IMPORT_DATA_DIR/natural_earth_vector.sqlite"
-
-function run_psql() {
-    PSQL_COMMAND= PG_PASSWORD=$OSM_PASSWORD psql --host="$DB_HOST" --port=5432 --dbname="$OSM_DB" --username="$OSM_USER"
-}
-
-function exec_sql_file() {
-	local sql_file=$1
-    cat "$sql_file" | run_psql
-}
-
-function import_shp() {
-	local shp_file=$1
-	shp2pgsql -g way "$shp_file" | run_psql | grep -v "INSERT 0 1"
-}
-
-function create_index() {
-    local table=$1
-    local index_name="$table"_index
-	local index_command="CREATE INDEX $index_name ON $table USING gist (way) WITH (FILLFACTOR=100);"
-	echo $index_command | run_psql
-}
-
-function drop_table() {
-    local table=$1
-	local drop_command="DROP TABLE IF EXISTS $table;"
-	echo $drop_command | run_psql
-}
-
-function import_sqlite() {
-    echo "Importing Natural Earth to PostGIS..."
-    PGCLIENTENCODING=UTF8 ogr2ogr \
-    -progress \
-    -f Postgresql \
-    -s_srs EPSG:4326 \
-    -t_srs EPSG:3857 \
-    -clipsrc -180.1 -85.0511 180.1 85.0511 \
-    PG:"dbname=$OSM_DB user=$OSM_USER host=$DB_HOST port=$DB_PORT" \
-    -lco GEOMETRY_NAME=geom \
-    -lco DIM=2 \
-    -nlt GEOMETRY \
-    -overwrite \
-    "$NATURAL_EARTH_SQLITE_FILE"
-}
-
-function import_water() {
-    local water_table="water_polygons"
-    echo "Removing existing table $water_table"
-    drop_table $water_table
-
-    echo "Importing $WATER_POLYGONS_FILE into table $water_table"
-    import_shp "$WATER_POLYGONS_FILE"
-
-    echo "Create index on table $water_table"
-    create_index $water_table
-}
-
-function hide_inserts() {
-    grep -v "INSERT 0 1"
-}
-
-function update_scaleranks() {
-    echo "Updating scalerank in $OSM_DB"
-    exec_sql_file "update.sql"
-    echo "Inserting labels in $OSM_DB"
-    exec_sql_file "marine.sql" | hide_inserts
-    exec_sql_file "states.sql" | hide_inserts
-    exec_sql_file "countries.sql" | hide_inserts
-}
-
-function main() {
-    update_scaleranks
+function import_all_external() {
+    import_natural_earth
+    import_labels
     import_water
-    import_sqlite
 }
 
-main
+import_all_external

src/import-external/sql.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+readonly DB_HOST=$DB_PORT_5432_TCP_ADDR
+readonly DB_PORT=$DB_PORT_5432_TCP_PORT
+
+readonly OSM_DB=${OSM_DB:-osm}
+readonly OSM_USER=${OSM_USER:-osm}
+readonly OSM_PASSWORD=${OSM_PASSWORD:-osm}
+
+function exec_psql() {
+    PG_PASSWORD=$OSM_PASSWORD psql --host="$DB_HOST" --port=5432 --dbname="$OSM_DB" --username="$OSM_USER"
+}
+
+function exec_sql_file() {
+	local sql_file=$1
+    cat "$sql_file" | exec_psql
+}
+
+function create_index() {
+    local table=$1
+    local index_name="$table"_index
+	local index_command="CREATE INDEX $index_name ON $table USING gist (way);"
+	echo $index_command | exec_psql
+}
+
+function drop_table() {
+    local table=$1
+	local drop_command="DROP TABLE IF EXISTS $table;"
+	echo $drop_command | exec_psql
+}
+
+function hide_inserts() {
+    grep -v "INSERT 0 1"
+}

src/import-natural-earth/Dockerfile

@@ -1,20 +0,0 @@
-FROM mdillon/postgis:9.4
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-      wget \
-      unzip \
-      gdal-bin \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /data/import \
-    && wget http://naciscdn.org/naturalearth/packages/natural_earth_vector.sqlite.zip \
-    && unzip -oj natural_earth_vector.sqlite.zip -d /data/import \
-    && rm natural_earth_vector.sqlite.zip
-
-ENV NATURAL_EARTH_SQLITE_FILE=/data/import/natural_earth_vector.sqlite
-
-RUN mkdir -p /usr/src/app
-COPY import.sh /usr/src/app/
-WORKDIR /usr/src/app
-
-CMD ["./import.sh"]

src/import-water/Dockerfile

@@ -1,16 +0,0 @@
-FROM mdillon/postgis:9.4
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-      wget \
-      unzip \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /data/import && wget --quiet http://data.openstreetmapdata.com/water-polygons-split-3857.zip && unzip -oj water-polygons-split-3857.zip -d /data/import && rm water-polygons-split-3857.zip
-
-ENV IMPORT_DATA_DIR=/data/import
-
-RUN mkdir -p /usr/src/app
-COPY import.sh /usr/src/app/
-WORKDIR /usr/src/app
-
-CMD ["./import.sh"]

src/import-water/README.md

@@ -1,4 +0,0 @@
-# Water Polygon Import Container
-
-The import container reads `SHP` files from the local import directory containing OpenStreetMap water polygons
-and imports them into the linked PostGIS container.

src/import-water/import.sh

@@ -1,46 +0,0 @@
-#!/bin/bash
-set -o errexit
-set -o pipefail
-set -o nounset
-
-readonly IMPORT_DATA_DIR=${IMPORT_DATA_DIR:-/data/import}
-readonly WATER_SHP_DOWNLOAD_URL=${WATER_SHP_DOWNLOAD_URL:-false}
-
-readonly DB_HOST=$DB_PORT_5432_TCP_ADDR
-readonly OSM_DB=${OSM_DB:-osm}
-readonly OSM_USER=${OSM_USER:-osm}
-readonly OSM_PASSWORD=${OSM_PASSWORD:-osm}
-
-function import_shp() {
-	local shp_file=$1
-	shp2pgsql -g way "$shp_file" | PG_PASSWORD=$OSM_PASSWORD psql --host="$DB_HOST" --port=5432 --dbname="$OSM_DB" --username="$OSM_USER" | grep -v "INSERT 0 1"
-}
-
-function create_index() {
-    local table=$1
-    local index_name="$table"_index
-	local index_command="CREATE INDEX $index_name ON $table USING gist (way) WITH (FILLFACTOR=100);"
-	echo $index_command | PG_PASSWORD=$OSM_PASSWORD psql --host="$DB_HOST" --port=5432 --dbname="$OSM_DB" --username="$OSM_USER"
-}
-
-function drop_table() {
-    local table=$1
-	local drop_command="DROP TABLE IF EXISTS $table;"
-	echo $drop_command | PG_PASSWORD=$OSM_PASSWORD psql --host="$DB_HOST" --port=5432 --dbname="$OSM_DB" --username="$OSM_USER"
-}
-
-function main() {
-    local water_polygons="$IMPORT_DATA_DIR/water_polygons.shp"
-
-    local water_table="water_polygons"
-    echo "Removing existing table $water_table"
-    drop_table $water_table
-
-    echo "Importing $water_polygons into table $water_table"
-    import_shp $water_polygons
-
-    echo "Create index on table $water_table"
-    create_index $water_table
-}
-
-main