From 01ad11ce7239e1f95bfdc1fa620ea3b5e441974b Mon Sep 17 00:00:00 2001
From: John-Scott Atlakson <24574+jsma@users.noreply.github.com>
Date: Sun, 12 Nov 2023 17:33:37 -0800
Subject: [PATCH] Updates and optimizations

Renames docker-compose.yml => compose.yaml which is the new preferred name going forward
Moves Dockerfiles to subdirectory, ensuring the filenames end with Dockerfile (for syntax highlighting in IDEs)
Renamed the `web` service to `app`, since this is typical for Docker examples and is also what bakerydemo uses
Various tricks and tips to optimize the build cache to speed up subsequent builds

The new Dockerfile was partially based on the output of `docker init`, which includes some best practices:
* create non-root users to run the applications
* use bind-mounts for pip & npm installs (this means that when adding a new requirement, it will only download the new package and install everything else from the cache)

NOTE: the wagtail and libs directories are no longer COPY'd into the container for doing a `pip install`, but installing wagtail and django-modelcluster creates .egg-info directories, so the bind mounts need write access.

Added custom ignore files for each Dockerfile to restrict the build context to the absolute minimum

Added a custom requirements file to avoid relying on bakerydemo's requirements which results in duplicate installs.
The previous setup was installing bakerydemo requirements, which installed wagtail, django-modelcluster, and Willow, and then a few steps later, reinstalling these packages from the local sources. Now they are only installed once.

The CMDs for both `app` and `frontend` are now wrapped with `dumb-init` (node is not designed to run as PID 1) which handles killing the processes. Previously attempting the Ctrl-C the running containers would hang waiting for `npm` to give up the ghost, now it exits immediately.
---
 .devcontainer/devcontainer.json               |  2 +-
 .devcontainer/docker-compose.yml              |  2 +-
 .env.example                                  |  3 +-
 Dockerfile                                    | 30 ----------
 Dockerfile.frontend                           |  7 ---
 Makefile                                      | 22 ++++----
 README.md                                     | 43 +++++++--------
 ...er-compose.linux.yml => compose.linux.yaml |  0
 compose.yaml                                  | 42 ++++++++++++++
 docker-compose.yml                            | 49 -----------------
 docker/app.Dockerfile                         | 55 +++++++++++++++++++
 docker/app.Dockerfile.dockerignore            | 12 ++++
 docker/frontend.Dockerfile                    | 27 +++++++++
 docker/frontend.Dockerfile.dockerignore       |  3 +
 requirements/development.txt                  | 12 ++++
 setup-db.sh                                   |  6 +-
 16 files changed, 189 insertions(+), 126 deletions(-)
 delete mode 100644 Dockerfile
 delete mode 100644 Dockerfile.frontend
 rename docker-compose.linux.yml => compose.linux.yaml (100%)
 create mode 100644 compose.yaml
 delete mode 100644 docker-compose.yml
 create mode 100644 docker/app.Dockerfile
 create mode 100644 docker/app.Dockerfile.dockerignore
 create mode 100644 docker/frontend.Dockerfile
 create mode 100644 docker/frontend.Dockerfile.dockerignore
 create mode 100644 requirements/development.txt

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 8c92314..2daf779 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -4,7 +4,7 @@
 {
     "name": "Existing Docker Compose (Extend)",
     "dockerComposeFile": [
-        "../docker-compose.yml",
+        "../compose.yml",
         "docker-compose.yml"
     ],
     "service": "web",
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
index 40dbef6..42ec5b4 100644
--- a/.devcontainer/docker-compose.yml
+++ b/.devcontainer/docker-compose.yml
@@ -1,7 +1,7 @@
 services:
   web:
     volumes:
-      - ./.vscode:/code/.vscode:delegated,rw
+      - ./.vscode:/code/.vscode
 
     # Overrides default command so things don't shut down after the process ends.
     command: /bin/sh -c "while sleep 1000; do :; done"
diff --git a/.env.example b/.env.example
index 900cd04..c914fa7 100644
--- a/.env.example
+++ b/.env.example
@@ -2,4 +2,5 @@ DATABASE_HOST=db
 DATABASE_PORT=5432
 DATABASE_NAME=app_db
 DATABASE_USER=app_user
-DATABASE_PASSWORD=changeme
\ No newline at end of file
+DATABASE_PASSWORD=changeme
+DATABASE_URL=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST/$DATABASE_NAME
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index 08950ad..0000000
--- a/Dockerfile
+++ /dev/null
@@ -1,30 +0,0 @@
-# Use an official Python runtime as a parent image
-FROM python:3.9-bullseye
-LABEL maintainer="hello@wagtail.org"
-
-# Set environment variables
-ENV PYTHONUNBUFFERED 1
-
-# Install libenchant and create the requirements folder.
-RUN apt-get update -y \
-    && apt-get install -y libenchant-2-dev postgresql-client \
-    && mkdir -p /code/requirements
-
-# Install the bakerydemo project's dependencies into the image.
-COPY ./bakerydemo/requirements/* /code/requirements/
-RUN pip install --upgrade pip \
-    && pip install -r /code/requirements/production.txt
-
-# Install wagtail from the host. This folder will be overwritten by a volume mount during run time (so that code
-# changes show up immediately), but it also needs to be copied into the image now so that wagtail can be pip install'd.
-COPY ./wagtail /code/wagtail/
-RUN cd /code/wagtail/ \
-    && pip install -e .[testing,docs]
-
-# Install Willow from the host. This folder will be overwritten by a volume mount during run time (so that code
-# changes show up immediately), but it also needs to be copied into the image now so that Willow can be pip install'd.
-COPY ./libs/Willow /code/willow/
-RUN cd /code/willow/ \
-    && pip install -e .[testing]
-
-WORKDIR /code/bakerydemo
diff --git a/Dockerfile.frontend b/Dockerfile.frontend
deleted file mode 100644
index 2b701c8..0000000
--- a/Dockerfile.frontend
+++ /dev/null
@@ -1,7 +0,0 @@
-FROM node:20-slim
-LABEL maintainer="hello@wagtail.org"
-
-RUN apt-get update && apt-get install rsync make -y
-COPY ./wagtail/package.json ./wagtail/package-lock.json ./
-
-RUN npm --prefix / install --loglevel info
diff --git a/Makefile b/Makefile
index 9180948..715c615 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ help: ## ⁉️ - Display help comments for each make command
 		| sort
 
 build: ## Build the backend Docker image
-	docker compose build web
+	docker compose build app
 
 start: ## Bring the backend Docker container up
 	docker compose up
@@ -17,37 +17,37 @@ stop: ## Stop the backend Docker container
 	docker compose stop
 
 ssh: ## Enter the running backend Docker container for the wagtail bakery site
-	docker compose exec web bash
+	docker compose exec app bash
 
 ssh-shell: ## Enter the running Docker container shell
-	docker compose exec web python manage.py shell
+	docker compose exec app python manage.py shell
 
 ssh-fe: ## Open a shell to work with the frontend code (Node/NPM)
 	docker compose exec frontend bash
 
 ssh-wagtail: ## Enter the running Docker container for the wagtail development environment
-	docker compose exec -w /code/wagtail web bash
+	docker compose exec -w /code/wagtail app bash
 
 ssh-db: ## Open a PostgreSQL shell session
-	docker compose exec web python manage.py dbshell
+	docker compose exec app python manage.py dbshell
 
 down: ## Stop and remove all Docker containers
 	docker compose down
 
 migrations: ## Make migrations to the wagtail bakery site
-	docker compose exec web python manage.py makemigrations
+	docker compose exec app python manage.py makemigrations
 
 migrate: ## Migrate the wagtail bakery site migrations
-	docker compose exec web python manage.py migrate
+	docker compose exec app python manage.py migrate
 
 test: ## Run all wagtail tests or pass in a file with `make test file=wagtail.admin.tests.test_name`
-	docker compose exec -w /code/wagtail web python runtests.py $(file) $(FILE)
+	docker compose exec -w /code/wagtail app python runtests.py $(file) $(FILE)
 
 format-wagtail: ## Format Wagtail repo
-	docker compose exec -w /code/wagtail web make format-server
+	docker compose exec -w /code/wagtail app make format-server
 	docker compose exec frontend make format-client
 
 lint-wagtail: ## Lint the Wagtail repo (server, client, docs)
-	docker compose exec -w /code/wagtail web make lint-server
-	docker compose exec -w /code/wagtail web make lint-docs
+	docker compose exec -w /code/wagtail app make lint-server
+	docker compose exec -w /code/wagtail app make lint-docs
 	docker compose exec frontend make lint-client
diff --git a/README.md b/README.md
index 567abbc..d43a22f 100644
--- a/README.md
+++ b/README.md
@@ -64,7 +64,7 @@ WARNINGS:
 If you're running this on Linux you might get into some privilege issues that can be solved using this command (tested on Ubuntu):
 
 ```sh
-CURRENT_UID=$(id -u):$(id -g) docker compose -f docker-compose.yml -f docker-compose.linux.yml up
+CURRENT_UID=$(id -u):$(id -g) docker compose -f compose.yaml -f compose.linux.yaml up
 ```
 
 Alternatively, if you're using VSCode and have the "Remote - Containers" extension, you can open the command palette and select "Remote Containers - Reopen in Container" to attach VSCode to the container. This allows for much deeper debugging.
@@ -78,11 +78,10 @@ Alternatively, if you're using VSCode and have the "Remote - Containers" extensi
 
 ```sh
 $ docker compose ps
-  Name                Command               State           Ports
---------------------------------------------------------------------------
-db         docker-entrypoint.sh postgres    Up          5432/tcp
-frontend   docker-entrypoint.sh /bin/ ...   Up
-web        ./manage.py runserver 0.0. ...   Up          0.0.0.0:8000->8000/tcp
+NAME                     IMAGE                    COMMAND                  SERVICE    CREATED          STATUS                    PORTS
+wagtail-dev-app-1        wagtail-dev-app          "/bin/sh -c 'python …"   app        10 minutes ago   Up 10 minutes             0.0.0.0:8000->8000/tcp
+wagtail-dev-db-1         postgres:16.0-bookworm   "docker-entrypoint.s…"   db         10 minutes ago   Up 10 minutes (healthy)   5432/tcp
+wagtail-dev-frontend-1   wagtail-dev-frontend     "docker-entrypoint.s…"   frontend   10 minutes ago   Up 10 minutes
 ```
 
 ### Build the backend Docker image
@@ -94,7 +93,7 @@ make build
 or
 
 ```sh
-docker compose build web
+docker compose build app
 ```
 
 ### Bring the backend Docker container up
@@ -142,7 +141,7 @@ make test
 or
 
 ```sh
-docker compose exec -w /code/wagtail web python runtests.py
+docker compose exec -w /code/wagtail app python runtests.py
 ```
 
 ### Run tests for a specific file
@@ -154,7 +153,7 @@ make test file=wagtail.admin.tests.test_name.py
 or
 
 ```sh
-docker compose exec -w /code/wagtail web python runtests.py wagtail.admin.tests.{test_file_name_here}
+docker compose exec -w /code/wagtail app python runtests.py wagtail.admin.tests.{test_file_name_here}
 ```
 
 ### Format Wagtail codebase
@@ -164,7 +163,7 @@ make format-wagtail
 ```
 or
 ```sh
-docker compose exec -w /code/wagtail web make format-server
+docker compose exec -w /code/wagtail app make format-server
 docker compose exec frontend make format-client
 ```
 
@@ -175,8 +174,8 @@ make lint-wagtail
 ```
 or
 ```sh
-docker compose exec -w /code/wagtail web make lint-server
-docker compose exec -w /code/wagtail web make lint-docs
+docker compose exec -w /code/wagtail app make lint-server
+docker compose exec -w /code/wagtail app make lint-docs
 docker compose exec frontend make lint-client
 ```
 
@@ -189,7 +188,7 @@ make ssh-shell
 or
 
 ```sh
-docker compose exec web python manage.py shell
+docker compose exec app python manage.py shell
 ```
 
 ### Open a PostgreSQL shell session
@@ -201,7 +200,7 @@ make ssh-db
 or
 
 ```sh
-docker compose exec web python manage.py dbshell
+docker compose exec app python manage.py dbshell
 ```
 
 ### Open a shell on the web server
@@ -213,7 +212,7 @@ make ssh
 or
 
 ```sh
-docker compose exec web bash
+docker compose exec app bash
 ```
 
 ### Open a shell to work with the frontend code (Node/NPM)
@@ -237,7 +236,7 @@ make ssh-fe
 or
 
 ```sh
-docker compose exec -w /code/wagtail web bash
+docker compose exec -w /code/wagtail app bash
 ```
 
 ### Make migrations to the wagtail bakery site
@@ -249,7 +248,7 @@ make migrations
 or
 
 ```sh
-docker compose exec web python manage.py makemigrations
+docker compose exec app python manage.py makemigrations
 ```
 
 ### Migrate the wagtail bakery site
@@ -261,14 +260,14 @@ make migrate
 or
 
 ```sh
-docker compose exec web python manage.py migrate
+docker compose exec app python manage.py migrate
 ```
 
 ## Getting ready to contribute
 
 Here are other actions you will likely need to do to make your first contribution to Wagtail.
 
-Set up git remotes to Wagtail forks (run these lines outside of the Docker instances, on your machine):
+Set up git remotes to Wagtail forks (run these commands on your machine, not within Docker):
 
 ```sh
 cd ~/Development/wagtail-dev/wagtail
@@ -280,10 +279,8 @@ git remote add upstream git@github.com:wagtail/wagtail.git
 git pull --all
 ```
 
-## Contributing to Willow
-
-You can use the same setup to contribute to Willow.
-You simply do the same operations to fork the Willow project and point your local copy of Willow to your fork.
+You can use the same steps to contribute to any of the dependencies installed in `./libs`.
+By default, `django-modelcluster` and `Willow` are checked out from `git` into this folder.
 
 ## See also
 
diff --git a/docker-compose.linux.yml b/compose.linux.yaml
similarity index 100%
rename from docker-compose.linux.yml
rename to compose.linux.yaml
diff --git a/compose.yaml b/compose.yaml
new file mode 100644
index 0000000..fbf9b3a
--- /dev/null
+++ b/compose.yaml
@@ -0,0 +1,42 @@
+volumes:
+  postgres-data:
+  node_modules:
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: docker/app.Dockerfile
+    volumes:
+      - ./wagtail:/code/wagtail
+      - ./libs:/code/libs
+      - ./bakerydemo:/code/bakerydemo
+    ports:
+      - "8000:8000"
+    depends_on:
+      db:
+        condition: service_healthy
+      frontend:
+        condition: service_started
+
+  db:
+    environment:
+      - "POSTGRES_DB=${DATABASE_NAME}"
+      - "POSTGRES_USER=${DATABASE_USER}"
+      - "POSTGRES_PASSWORD=${DATABASE_PASSWORD}"
+    restart: always
+    image: postgres:16.0-bookworm
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    healthcheck:
+      test: "pg_isready --quiet --dbname=${DATABASE_URL}"
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  frontend:
+    build:
+      context: .
+      dockerfile: docker/frontend.Dockerfile
+    volumes:
+      - ./wagtail:/code/wagtail
diff --git a/docker-compose.yml b/docker-compose.yml
deleted file mode 100644
index 96d34a0..0000000
--- a/docker-compose.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-volumes:
-  postgres-data:
-  node_modules:
-  
-services:
-  web:
-    build:
-      context: .
-      dockerfile: ./Dockerfile
-    command: python manage.py runserver 0.0.0.0:8000
-    volumes:
-      - ./wagtail:/code/wagtail
-      - ./bakerydemo:/code/bakerydemo
-    ports:
-      - "8000:8000"
-    environment:
-      DATABASE_URL: "postgres://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}"
-    depends_on:
-      db:
-        condition: service_healthy
-      frontend:
-        condition: service_started
-
-  db:
-    environment:
-      - "POSTGRES_DB=${DATABASE_NAME}"
-      - "POSTGRES_USER=${DATABASE_USER}"
-      - "POSTGRES_PASSWORD=${DATABASE_PASSWORD}"
-    restart: unless-stopped
-    image: postgres:14.1
-    volumes:
-      - postgres-data:/var/lib/postgresql/data
-    healthcheck:
-      test: "pg_isready --quiet --dbname=postgres://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}"
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
-  frontend:
-    build:
-      context: .
-      dockerfile: Dockerfile.frontend
-    working_dir: /code/wagtail
-    volumes:
-      - ./wagtail:/code/wagtail
-      - node_modules:/code/wagtail/node_modules
-    command: bash -c "echo 'Copying node_modules, this may take a few minutes...' && rsync -rah --info=progress2 /node_modules /code/wagtail/ && npm run start"
-    restart: "no"
-    tty: true
diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile
new file mode 100644
index 0000000..3e0c600
--- /dev/null
+++ b/docker/app.Dockerfile
@@ -0,0 +1,55 @@
+# syntax=docker/dockerfile:1
+ARG PYTHON_VERSION=3.12.0
+FROM python:${PYTHON_VERSION}-slim-bookworm as base
+
+LABEL maintainer="hello@wagtail.org"
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+RUN apt-get update -y && apt-get install -y \
+    dumb-init \
+    libenchant-2-dev \
+    postgresql-client
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/go/dockerfile-user-best-practices/
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+
+WORKDIR /code/
+
+# Download dependencies as a separate step to take advantage of Docker's caching.
+# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
+# Leverage a bind mount to requirements.txt to avoid having to copy them into
+# into this layer.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements,target=/code/requirements \
+    --mount=type=bind,source=libs,target=/code/libs,rw \
+    --mount=type=bind,source=wagtail,target=/code/wagtail,rw \
+    python -m pip install --cache-dir=/root/.cache/pip -U pip \
+    && python -m pip install --cache-dir=/root/.cache/pip -r /code/requirements/development.txt
+
+# Switch to the non-privileged user to run the application.
+USER appuser
+
+WORKDIR /code/bakerydemo
+
+# Expose the port that the application listens on.
+EXPOSE 8000
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+# Run the application.
+CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]
diff --git a/docker/app.Dockerfile.dockerignore b/docker/app.Dockerfile.dockerignore
new file mode 100644
index 0000000..879c07c
--- /dev/null
+++ b/docker/app.Dockerfile.dockerignore
@@ -0,0 +1,12 @@
+*
+!**/.env
+!bakerydemo/requirements/*.txt
+!libs/Willow/pyproject.toml
+!libs/Willow/README.md
+!libs/Willow/willow
+!libs/django-modelcluster/setup.py
+!libs/django-modelcluster/README.rst
+!libs/django-modelcluster/modelcluster
+!requirements
+!wagtail/setup.py
+!wagtail/wagtail
\ No newline at end of file
diff --git a/docker/frontend.Dockerfile b/docker/frontend.Dockerfile
new file mode 100644
index 0000000..1b887b5
--- /dev/null
+++ b/docker/frontend.Dockerfile
@@ -0,0 +1,27 @@
+# syntax=docker/dockerfile:1
+ARG NODE_VERSION=20
+FROM node:${NODE_VERSION}-slim
+
+LABEL maintainer="hello@wagtail.org"
+
+RUN apt update -y \
+    && apt install -y \
+    dumb-init \
+    make
+
+# Download dependencies as a separate step to take advantage of Docker's caching.
+# Leverage a cache mount to /root/.npm to speed up subsequent builds.
+# Leverage a bind mounts to package.json and package-lock.json to avoid having to copy them into
+# into this layer.
+RUN --mount=type=bind,source=wagtail/package.json,target=package.json \
+    --mount=type=bind,source=wagtail/package-lock.json,target=package-lock.json,rw \
+    --mount=type=cache,target=/root/.npm \
+    npm install
+
+USER node
+
+WORKDIR /code/wagtail
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+CMD ["npm", "run", "start"]
diff --git a/docker/frontend.Dockerfile.dockerignore b/docker/frontend.Dockerfile.dockerignore
new file mode 100644
index 0000000..a24b6f3
--- /dev/null
+++ b/docker/frontend.Dockerfile.dockerignore
@@ -0,0 +1,3 @@
+*
+!wagtail/package.json
+!wagtail/package-lock.json
\ No newline at end of file
diff --git a/requirements/development.txt b/requirements/development.txt
new file mode 100644
index 0000000..5ff6f10
--- /dev/null
+++ b/requirements/development.txt
@@ -0,0 +1,12 @@
+psycopg[binary]>=3.1,<3.2
+-e ./libs/django-modelcluster
+-e ./libs/Willow[testing]
+-e ./wagtail[testing,docs]
+# Bakerydemo requirements not already satisfied by the above:
+wagtail-font-awesome-svg>=1,<2
+django-csp==3.7
+django-debug-toolbar>=4.2,<5
+django-dotenv==1.4.2
+django-extensions==3.2.3
+dj-database-url==2.1.0
+ipython
diff --git a/setup-db.sh b/setup-db.sh
index 6258d4e..50fbf3a 100755
--- a/setup-db.sh
+++ b/setup-db.sh
@@ -3,6 +3,6 @@
 # Fail if any command fails.
 set -e
 
-docker compose exec web python manage.py migrate --noinput
-docker compose exec web python manage.py load_initial_data
-docker compose exec web python manage.py update_index
+docker compose exec app python manage.py migrate --noinput
+docker compose exec app python manage.py load_initial_data
+docker compose exec app python manage.py update_index