Updates and optimizations

Renames docker-compose.yml => compose.yaml which is the new preferred name going forward
Moves Dockerfiles to subdirectory, ensuring the filenames end with Dockerfile (for syntax highlighting in IDEs)
Renamed the `web` service to `app`, since this is typical for Docker examples and is also what bakerydemo uses
Various tricks and tips to optimize the build cache to speed up subsequent builds

The new Dockerfile was partially based on the output of `docker init`, which includes some best practices:
* create non-root users to run the applications
* use bind-mounts for pip & npm installs (this means that when adding a new requirement, it will only download the new package and install everything else from the cache)

NOTE: the wagtail and libs directories are no longer COPY'd into the container for doing a `pip install`, but installing wagtail and django-modelcluster creates .egg-info directories, so the bind mounts need write access.

Added custom ignore files for each Dockerfile to restrict the build context to the absolute minimum

Added a custom requirements file to avoid relying on bakerydemo's requirements which results in duplicate installs.
The previous setup was installing bakerydemo requirements, which installed wagtail, django-modelcluster, and Willow, and then a few steps later, reinstalling these packages from the local sources. Now they are only installed once.

The CMDs for both `app` and `frontend` are now wrapped with `dumb-init` (node is not designed to run as PID 1) which handles killing the processes. Previously attempting the Ctrl-C the running containers would hang waiting for `npm` to give up the ghost, now it exits immediately.

.devcontainer/devcontainer.json

@@ -4,7 +4,7 @@
 {
     "name": "Existing Docker Compose (Extend)",
     "dockerComposeFile": [
-        "../docker-compose.yml",
+        "../compose.yml",
         "docker-compose.yml"
     ],
     "service": "web",

.devcontainer/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   web:
     volumes:
-      - ./.vscode:/code/.vscode:delegated,rw
+      - ./.vscode:/code/.vscode
 
     # Overrides default command so things don't shut down after the process ends.
     command: /bin/sh -c "while sleep 1000; do :; done"

.env.example

@@ -2,4 +2,5 @@ DATABASE_HOST=db
 DATABASE_PORT=5432
 DATABASE_NAME=app_db
 DATABASE_USER=app_user
-DATABASE_PASSWORD=changeme
+DATABASE_PASSWORD=changeme
+DATABASE_URL=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST/$DATABASE_NAME

Dockerfile

@@ -1,30 +0,0 @@
-# Use an official Python runtime as a parent image
-FROM python:3.9-bullseye
-LABEL maintainer="hello@wagtail.org"
-
-# Set environment variables
-ENV PYTHONUNBUFFERED 1
-
-# Install libenchant and create the requirements folder.
-RUN apt-get update -y \
-    && apt-get install -y libenchant-2-dev postgresql-client \
-    && mkdir -p /code/requirements
-
-# Install the bakerydemo project's dependencies into the image.
-COPY ./bakerydemo/requirements/* /code/requirements/
-RUN pip install --upgrade pip \
-    && pip install -r /code/requirements/production.txt
-
-# Install wagtail from the host. This folder will be overwritten by a volume mount during run time (so that code
-# changes show up immediately), but it also needs to be copied into the image now so that wagtail can be pip install'd.
-COPY ./wagtail /code/wagtail/
-RUN cd /code/wagtail/ \
-    && pip install -e .[testing,docs]
-
-# Install Willow from the host. This folder will be overwritten by a volume mount during run time (so that code
-# changes show up immediately), but it also needs to be copied into the image now so that Willow can be pip install'd.
-COPY ./libs/Willow /code/willow/
-RUN cd /code/willow/ \
-    && pip install -e .[testing]
-
-WORKDIR /code/bakerydemo

Dockerfile.frontend

@@ -1,7 +0,0 @@
-FROM node:20-slim
-LABEL maintainer="hello@wagtail.org"
-
-RUN apt-get update && apt-get install rsync make -y
-COPY ./wagtail/package.json ./wagtail/package-lock.json ./
-
-RUN npm --prefix / install --loglevel info

Makefile

@@ -8,7 +8,7 @@ help: ## ⁉️ - Display help comments for each make command
 		| sort
 
 build: ## Build the backend Docker image
-	docker compose build web
+	docker compose build app
 
 start: ## Bring the backend Docker container up
 	docker compose up
@@ -17,37 +17,37 @@ stop: ## Stop the backend Docker container
 	docker compose stop
 
 ssh: ## Enter the running backend Docker container for the wagtail bakery site
-	docker compose exec web bash
+	docker compose exec app bash
 
 ssh-shell: ## Enter the running Docker container shell
-	docker compose exec web python manage.py shell
+	docker compose exec app python manage.py shell
 
 ssh-fe: ## Open a shell to work with the frontend code (Node/NPM)
 	docker compose exec frontend bash
 
 ssh-wagtail: ## Enter the running Docker container for the wagtail development environment
-	docker compose exec -w /code/wagtail web bash
+	docker compose exec -w /code/wagtail app bash
 
 ssh-db: ## Open a PostgreSQL shell session
-	docker compose exec web python manage.py dbshell
+	docker compose exec app python manage.py dbshell
 
 down: ## Stop and remove all Docker containers
 	docker compose down
 
 migrations: ## Make migrations to the wagtail bakery site
-	docker compose exec web python manage.py makemigrations
+	docker compose exec app python manage.py makemigrations
 
 migrate: ## Migrate the wagtail bakery site migrations
-	docker compose exec web python manage.py migrate
+	docker compose exec app python manage.py migrate
 
 test: ## Run all wagtail tests or pass in a file with `make test file=wagtail.admin.tests.test_name`
-	docker compose exec -w /code/wagtail web python runtests.py $(file) $(FILE)
+	docker compose exec -w /code/wagtail app python runtests.py $(file) $(FILE)
 
 format-wagtail: ## Format Wagtail repo
-	docker compose exec -w /code/wagtail web make format-server
+	docker compose exec -w /code/wagtail app make format-server
 	docker compose exec frontend make format-client
 
 lint-wagtail: ## Lint the Wagtail repo (server, client, docs)
-	docker compose exec -w /code/wagtail web make lint-server
-	docker compose exec -w /code/wagtail web make lint-docs
+	docker compose exec -w /code/wagtail app make lint-server
+	docker compose exec -w /code/wagtail app make lint-docs
 	docker compose exec frontend make lint-client

README.md

@@ -64,7 +64,7 @@ WARNINGS:
 If you're running this on Linux you might get into some privilege issues that can be solved using this command (tested on Ubuntu):
 
 ```sh
-CURRENT_UID=$(id -u):$(id -g) docker compose -f docker-compose.yml -f docker-compose.linux.yml up
+CURRENT_UID=$(id -u):$(id -g) docker compose -f compose.yaml -f compose.linux.yaml up
 ```
 
 Alternatively, if you're using VSCode and have the "Remote - Containers" extension, you can open the command palette and select "Remote Containers - Reopen in Container" to attach VSCode to the container. This allows for much deeper debugging.
@@ -78,11 +78,10 @@ Alternatively, if you're using VSCode and have the "Remote - Containers" extensi
 
 ```sh
 $ docker compose ps
-  Name                Command               State           Ports
---------------------------------------------------------------------------
-db         docker-entrypoint.sh postgres    Up          5432/tcp
-frontend   docker-entrypoint.sh /bin/ ...   Up
-web        ./manage.py runserver 0.0. ...   Up          0.0.0.0:8000->8000/tcp
+NAME                     IMAGE                    COMMAND                  SERVICE    CREATED          STATUS                    PORTS
+wagtail-dev-app-1        wagtail-dev-app          "/bin/sh -c 'python …"   app        10 minutes ago   Up 10 minutes             0.0.0.0:8000->8000/tcp
+wagtail-dev-db-1         postgres:16.0-bookworm   "docker-entrypoint.s…"   db         10 minutes ago   Up 10 minutes (healthy)   5432/tcp
+wagtail-dev-frontend-1   wagtail-dev-frontend     "docker-entrypoint.s…"   frontend   10 minutes ago   Up 10 minutes
 ```
 
 ### Build the backend Docker image
@@ -94,7 +93,7 @@ make build
 or
 
 ```sh
-docker compose build web
+docker compose build app
 ```
 
 ### Bring the backend Docker container up
@@ -142,7 +141,7 @@ make test
 or
 
 ```sh
-docker compose exec -w /code/wagtail web python runtests.py
+docker compose exec -w /code/wagtail app python runtests.py
 ```
 
 ### Run tests for a specific file
@@ -154,7 +153,7 @@ make test file=wagtail.admin.tests.test_name.py
 or
 
 ```sh
-docker compose exec -w /code/wagtail web python runtests.py wagtail.admin.tests.{test_file_name_here}
+docker compose exec -w /code/wagtail app python runtests.py wagtail.admin.tests.{test_file_name_here}
 ```
 
 ### Format Wagtail codebase
@@ -164,7 +163,7 @@ make format-wagtail
 ```
 or
 ```sh
-docker compose exec -w /code/wagtail web make format-server
+docker compose exec -w /code/wagtail app make format-server
 docker compose exec frontend make format-client
 ```
 
@@ -175,8 +174,8 @@ make lint-wagtail
 ```
 or
 ```sh
-docker compose exec -w /code/wagtail web make lint-server
-docker compose exec -w /code/wagtail web make lint-docs
+docker compose exec -w /code/wagtail app make lint-server
+docker compose exec -w /code/wagtail app make lint-docs
 docker compose exec frontend make lint-client
 ```
 
@@ -189,7 +188,7 @@ make ssh-shell
 or
 
 ```sh
-docker compose exec web python manage.py shell
+docker compose exec app python manage.py shell
 ```
 
 ### Open a PostgreSQL shell session
@@ -201,7 +200,7 @@ make ssh-db
 or
 
 ```sh
-docker compose exec web python manage.py dbshell
+docker compose exec app python manage.py dbshell
 ```
 
 ### Open a shell on the web server
@@ -213,7 +212,7 @@ make ssh
 or
 
 ```sh
-docker compose exec web bash
+docker compose exec app bash
 ```
 
 ### Open a shell to work with the frontend code (Node/NPM)
@@ -237,7 +236,7 @@ make ssh-fe
 or
 
 ```sh
-docker compose exec -w /code/wagtail web bash
+docker compose exec -w /code/wagtail app bash
 ```
 
 ### Make migrations to the wagtail bakery site
@@ -249,7 +248,7 @@ make migrations
 or
 
 ```sh
-docker compose exec web python manage.py makemigrations
+docker compose exec app python manage.py makemigrations
 ```
 
 ### Migrate the wagtail bakery site
@@ -261,14 +260,14 @@ make migrate
 or
 
 ```sh
-docker compose exec web python manage.py migrate
+docker compose exec app python manage.py migrate
 ```
 
 ## Getting ready to contribute
 
 Here are other actions you will likely need to do to make your first contribution to Wagtail.
 
-Set up git remotes to Wagtail forks (run these lines outside of the Docker instances, on your machine):
+Set up git remotes to Wagtail forks (run these commands on your machine, not within Docker):
 
 ```sh
 cd ~/Development/wagtail-dev/wagtail
@@ -280,10 +279,8 @@ git remote add upstream git@github.com:wagtail/wagtail.git
 git pull --all
 ```
 
-## Contributing to Willow
-
-You can use the same setup to contribute to Willow.
-You simply do the same operations to fork the Willow project and point your local copy of Willow to your fork.
+You can use the same steps to contribute to any of the dependencies installed in `./libs`.
+By default, `django-modelcluster` and `Willow` are checked out from `git` into this folder.
 
 ## See also
 

compose.yaml

@@ -0,0 +1,42 @@
+volumes:
+  postgres-data:
+  node_modules:
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: docker/app.Dockerfile
+    volumes:
+      - ./wagtail:/code/wagtail
+      - ./libs:/code/libs
+      - ./bakerydemo:/code/bakerydemo
+    ports:
+      - "8000:8000"
+    depends_on:
+      db:
+        condition: service_healthy
+      frontend:
+        condition: service_started
+
+  db:
+    environment:
+      - "POSTGRES_DB=${DATABASE_NAME}"
+      - "POSTGRES_USER=${DATABASE_USER}"
+      - "POSTGRES_PASSWORD=${DATABASE_PASSWORD}"
+    restart: always
+    image: postgres:16.0-bookworm
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    healthcheck:
+      test: "pg_isready --quiet --dbname=${DATABASE_URL}"
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  frontend:
+    build:
+      context: .
+      dockerfile: docker/frontend.Dockerfile
+    volumes:
+      - ./wagtail:/code/wagtail

docker-compose.yml

@@ -1,49 +0,0 @@
-volumes:
-  postgres-data:
-  node_modules:
-  
-services:
-  web:
-    build:
-      context: .
-      dockerfile: ./Dockerfile
-    command: python manage.py runserver 0.0.0.0:8000
-    volumes:
-      - ./wagtail:/code/wagtail
-      - ./bakerydemo:/code/bakerydemo
-    ports:
-      - "8000:8000"
-    environment:
-      DATABASE_URL: "postgres://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}"
-    depends_on:
-      db:
-        condition: service_healthy
-      frontend:
-        condition: service_started
-
-  db:
-    environment:
-      - "POSTGRES_DB=${DATABASE_NAME}"
-      - "POSTGRES_USER=${DATABASE_USER}"
-      - "POSTGRES_PASSWORD=${DATABASE_PASSWORD}"
-    restart: unless-stopped
-    image: postgres:14.1
-    volumes:
-      - postgres-data:/var/lib/postgresql/data
-    healthcheck:
-      test: "pg_isready --quiet --dbname=postgres://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}"
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
-  frontend:
-    build:
-      context: .
-      dockerfile: Dockerfile.frontend
-    working_dir: /code/wagtail
-    volumes:
-      - ./wagtail:/code/wagtail
-      - node_modules:/code/wagtail/node_modules
-    command: bash -c "echo 'Copying node_modules, this may take a few minutes...' && rsync -rah --info=progress2 /node_modules /code/wagtail/ && npm run start"
-    restart: "no"
-    tty: true

docker/app.Dockerfile

@@ -0,0 +1,55 @@
+# syntax=docker/dockerfile:1
+ARG PYTHON_VERSION=3.12.0
+FROM python:${PYTHON_VERSION}-slim-bookworm as base
+
+LABEL maintainer="hello@wagtail.org"
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+RUN apt-get update -y && apt-get install -y \
+    dumb-init \
+    libenchant-2-dev \
+    postgresql-client
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/go/dockerfile-user-best-practices/
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+
+WORKDIR /code/
+
+# Download dependencies as a separate step to take advantage of Docker's caching.
+# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
+# Leverage a bind mount to requirements.txt to avoid having to copy them into
+# into this layer.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements,target=/code/requirements \
+    --mount=type=bind,source=libs,target=/code/libs,rw \
+    --mount=type=bind,source=wagtail,target=/code/wagtail,rw \
+    python -m pip install --cache-dir=/root/.cache/pip -U pip \
+    && python -m pip install --cache-dir=/root/.cache/pip -r /code/requirements/development.txt
+
+# Switch to the non-privileged user to run the application.
+USER appuser
+
+WORKDIR /code/bakerydemo
+
+# Expose the port that the application listens on.
+EXPOSE 8000
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+# Run the application.
+CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]

docker/app.Dockerfile.dockerignore

@@ -0,0 +1,12 @@
+*
+!**/.env
+!bakerydemo/requirements/*.txt
+!libs/Willow/pyproject.toml
+!libs/Willow/README.md
+!libs/Willow/willow
+!libs/django-modelcluster/setup.py
+!libs/django-modelcluster/README.rst
+!libs/django-modelcluster/modelcluster
+!requirements
+!wagtail/setup.py
+!wagtail/wagtail

docker/frontend.Dockerfile

@@ -0,0 +1,27 @@
+# syntax=docker/dockerfile:1
+ARG NODE_VERSION=20
+FROM node:${NODE_VERSION}-slim
+
+LABEL maintainer="hello@wagtail.org"
+
+RUN apt update -y \
+    && apt install -y \
+    dumb-init \
+    make
+
+# Download dependencies as a separate step to take advantage of Docker's caching.
+# Leverage a cache mount to /root/.npm to speed up subsequent builds.
+# Leverage a bind mounts to package.json and package-lock.json to avoid having to copy them into
+# into this layer.
+RUN --mount=type=bind,source=wagtail/package.json,target=package.json \
+    --mount=type=bind,source=wagtail/package-lock.json,target=package-lock.json,rw \
+    --mount=type=cache,target=/root/.npm \
+    npm install
+
+USER node
+
+WORKDIR /code/wagtail
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+CMD ["npm", "run", "start"]

docker/frontend.Dockerfile.dockerignore

@@ -0,0 +1,3 @@
+*
+!wagtail/package.json
+!wagtail/package-lock.json

requirements/development.txt

@@ -0,0 +1,12 @@
+psycopg[binary]>=3.1,<3.2
+-e ./libs/django-modelcluster
+-e ./libs/Willow[testing]
+-e ./wagtail[testing,docs]
+# Bakerydemo requirements not already satisfied by the above:
+wagtail-font-awesome-svg>=1,<2
+django-csp==3.7
+django-debug-toolbar>=4.2,<5
+django-dotenv==1.4.2
+django-extensions==3.2.3
+dj-database-url==2.1.0
+ipython

setup-db.sh

@@ -3,6 +3,6 @@
 # Fail if any command fails.
 set -e
 
-docker compose exec web python manage.py migrate --noinput
-docker compose exec web python manage.py load_initial_data
-docker compose exec web python manage.py update_index
+docker compose exec app python manage.py migrate --noinput
+docker compose exec app python manage.py load_initial_data
+docker compose exec app python manage.py update_index