diff --git a/deployment/.dockerignore b/deployment/.dockerignore new file mode 100644 index 0000000..72e8ffc --- /dev/null +++ b/deployment/.dockerignore @@ -0,0 +1 @@ +* diff --git a/deployment/.gitignore b/deployment/.gitignore new file mode 100644 index 0000000..a212a9b --- /dev/null +++ b/deployment/.gitignore @@ -0,0 +1,79 @@ +.* +!.github +!.dockerignore +!.editorconfig +!.flake8 +!.gitignore +!.isort.cfg +!/.travis.yml +!/.style.yapf +!.coveralls.yml + +poetry.lock + +# for django-dbbackup +/backups/ +!/backups/.gitkeep + +# from test projects: +/static/ +/media/ +*.sqlite3 + +# docker-compose usage: +volumes + +# Django +secret.txt + +# Coverage HTML Report files: +htmlcov + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.tox +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +db.sqlite3 +coverage_html/ +coverage.xml +*,cover + +# Translations +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + diff --git a/deployment/Dockerfile b/deployment/Dockerfile new file mode 100644 index 0000000..0edda1d --- /dev/null +++ b/deployment/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.9-slim-buster +# https://hub.docker.com/_/python + +# Install deps +RUN apt-get update \ + && apt-mark auto $(apt-mark showinstall) \ + && apt-get install -y postgresql-client-11 python3-pip \ + && apt autoremove \ + && apt -y full-upgrade \ + && rm -rf /var/lib/apt \ + && python3 -m pip install -U pip \ + && pip install -U psycopg2-binary + +WORKDIR /inventory + +RUN pip install "pyinventory>=0.4.2" + + diff --git a/deployment/Makefile b/deployment/Makefile new file mode 100644 index 0000000..aa1845d --- /dev/null +++ b/deployment/Makefile @@ -0,0 +1,131 @@ +SHELL := /bin/bash + +help: ## List all commands + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z0-9 -_]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +check-poetry: + @if [[ "$(shell poetry --version 2>/dev/null)" == *"Poetry"* ]] ; \ + then \ + echo "Poetry found, ok." ; \ + else \ + echo 'Please install poetry first, with e.g.:' ; \ + echo 'make install-poetry' ; \ + exit 1 ; \ + fi + +install-poetry: ## install or update poetry + @if [[ "$(shell poetry --version 2>/dev/null)" == *"Poetry"* ]] ; \ + then \ + echo 'Update poetry' ; \ + poetry self update ; \ + else \ + echo 'Install poetry' ; \ + curl -sSL "https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py" | python3 ; \ + fi + +install: check-poetry ## install requirements to setup project + poetry install + +update: check-poetry ## update the sources and docker containers + git fetch --all + git pull origin deployment + poetry update + ./compose.sh build --pull + $(MAKE) restart + +check-compose: + @if [[ "$(shell poetry run docker-compose --version 2>/dev/null)" = *"docker-compose version"* ]] ; \ + then \ + echo "docker-compose found, ok." ; \ + else \ + echo 'Please install extras first, with e.g.:' ; \ + echo 'make install-compose' ; \ + exit 1 ; \ + fi + +up: check-compose ## Start containers via docker-compose + ./compose.sh up -d + $(MAKE) prune + ./compose.sh logs --tail=500 --follow + +down: ## Stop all containers + ./compose.sh down + +prune: ## Cleanup docker + docker system prune --force --all --filter until=4464h + +build: check-compose ## Update docker container build + ./compose.sh build --pull + +init_postgres: ## Create postgres database + ./compose.sh exec postgres ./docker/postgres_init.sh + +############################################################################## + +createsuperuser: ## Create super user + ./compose.sh exec inventory ./manage.sh createsuperuser + +############################################################################## + +shell_inventory: ## Go into bash shell in inventory container + ./compose.sh exec inventory /bin/bash + +shell_postgres: ## Go into bash shell in postgres container + ./compose.sh exec postgres /bin/bash + +shell_caddy: ## Go into bash shell in caddy container + ./compose.sh exec caddy /bin/ash + +############################################################################## + +caddy_environ: ## Prints the caddy environment + ./compose.sh exec caddy /usr/bin/caddy environ + +############################################################################## + +logs: ## Display docker logs from all containers + ./compose.sh logs --tail=500 --follow + +logs_postgres: ## Display docker logs from postgres container + ./compose.sh logs --tail=500 --follow postgres + +logs_inventory: ## Display docker logs from inventory container + ./compose.sh logs --tail=500 --follow inventory + +logs_caddy: ## Display docker logs from caddy container + ./compose.sh logs --tail=500 --follow caddy + +############################################################################## + +dbbackup: ## Backup database + ./compose.sh exec inventory ./manage.sh dbbackup + +dbrestore: ## Restore a database backup + ./compose.sh exec inventory ./manage.sh dbrestore + +############################################################################## + +restart: down up ## Restart all containers + +upgrade_inventory: ## Upgrade "inventory" container and restart it + $(MAKE) build + ./compose.sh stop inventory + $(MAKE) up + +restart_caddy: ## Restart caddy container + ./compose.sh stop caddy + $(MAKE) up + +############################################################################## + +reload_caddy: ## Reload Caddy server + ./compose.sh exec caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile + $(MAKE) logs_caddy + +reload_inventory: ## Reload server in "inventory" container + ./compose.sh exec inventory ./docker/kill_python.sh + ./compose.sh logs --tail=500 --follow inventory + +############################################################################## + +.PHONY: help \ No newline at end of file diff --git a/deployment/README.creole b/deployment/README.creole new file mode 100644 index 0000000..34c3204 --- /dev/null +++ b/deployment/README.creole @@ -0,0 +1,221 @@ += PyInventory - deployment branch + +Web based management to catalog things including state and location etc. using Python/Django. + +**Note:** + +* This README (in git **deployment** branch) contains only the instruction to install PyInventory on a root server. +* Read [[https://github.com/jedie/PyInventory/tree/master#readme|master README]] for local develompment installation. + +Pull requests welcome! + + +== git branches + +Currently we have two main branches: + +|= git branch |= description +| **[[https://github.com/jedie/PyInventory/tree/master|master]]** | The main PyInventory source code +| **[[https://github.com/jedie/PyInventory/tree/deployment|deployment]]** | separate project to deploy PyInventory for production use case + + +== deploy + +Install PyInventory on a root server: + +**Note:** + +* Running a public web server is a lot of work and brings some risks. +* This instructions are only the basics to get PyInventory working. +* To run the server safely, more work should be done, which is not explained here. +* Run at your own risk! No warranty is given. + + +=== prepare root server + +Here some steps for a fresh created root server: + +Update all packages, e.g.: +{{{ +apt update && apt -y full-upgrade +}}} + +Setup a normal user. You may use [[https://github.com/jedie/PyInventory/blob/deployment/scripts/setup_user.sh|scripts/setup_user.sh]] + + +=== Setup SSH services + +{{{ +~# nano /etc/ssh/sshd_config +}}} + +Change e.g.: +{{{ +Port xxxx +PermitRootLogin no +PasswordAuthentication no +}}} +(Changing the Port may need to change a firewall/network settings) + +{{{ +# restart SSH deamon: +~# service ssh restart + +# Display and follow the ssh log output to see connection errors: +~# journalctl -f -u ssh +}}} + +* Keep the current SSH session (with the log output) open! +* Update your {{{~/.ssh/config}}} +* Try to connect as the new, normal user in a **separate** terminal + +Only after a working new connection: Terminate the first root SSH session ;) + + +=== setup unattended-upgrades + +{{{ +~$ sudo apt install unattended-upgrades +~$ sudo dpkg-reconfigure unattended-upgrades +~$ sudo nano /etc/apt/apt.conf.d/50unattended-upgrades +}}} + + +=== install requirements + +Install docker, see: https://docs.docker.com/engine/install/ubuntu/ + +install some base packages, e.g.: +{{{ +~$ sudo apt install git make +}}} + + + +=== install PyInventory + +{{{ +# Checkout the deployment branch: +~$ git clone -b deployment https://github.com/jedie/PyInventory.git PyInventory-Deployment +~$ cd PyInventory-Deployment + +# Theses Makefile targets exists: +~/PyInventory-Deployment$ make +help List all commands +install-poetry install or update poetry +install install requirements to setup project +update update the sources and docker containers +up Start containers via docker-compose +down Stop all containers +prune Cleanup docker +build Update docker container build +init_postgres Create postgres database +createsuperuser Create super user +shell_inventory Go into bash shell in inventory container +shell_postgres Go into bash shell in postgres container +shell_caddy Go into bash shell in caddy container +caddy_environ Prints the caddy environment +logs Display docker logs from all containers +logs_postgres Display docker logs from postgres container +logs_inventory Display docker logs from inventory container +logs_caddy Display docker logs from caddy container +dbbackup Backup database +dbrestore Restore a database backup +restart Restart all containers +upgrade_inventory Upgrade "inventory" container and restart it +restart_caddy Restart caddy container +reload_caddy Reload Caddy server +reload_inventory Reload server in "inventory" container +}}} + +Install, e.g.: + +{{{ +# install or update poetry: +~/PyInventory-Deployment$ make install-poetry +}}} + +To keep poetry running (PATH must be expand) just logout and login ;) + +{{{ +# install requirements (e.g.: docker-compose) via poetry: +~/PyInventory-Deployment$ make install +}}} + +Create a {{{.env}}} file in project root directory with these content, e.g.: +{{{ +~/PyInventory-Deployment$ touch .env +~/PyInventory-Deployment$ nano .env +~/PyInventory-Deployment$ cat .env +# Your Public domain: +HOSTNAME=domain.tld + +# eMail address for Let's encrypt: +LETSENCRYPT_EMAIL=webmaster@domain.tld +}}} + +For local testing of the docker-compose setup, used this values: +{{{ +HOSTNAME=localhost +LETSENCRYPT_EMAIL=internal +}}} +(Caddy will create a self signed https certificate) + +Start containers via docker-compose: +{{{ +~/PyInventory-Deployment$ make up +}}} + +Notes: At the first start it takes a little while until the database is created ;) + +Create first super user: +{{{ +~/PyInventory-Deployment$ make docker_createsuperuser +}}} + +* Now you should be able to connect to your PyInventory installation and login with created super user. +* Redirect from **http** to **https** should work. +* Let's Encrypt certificate should be installed and valid. +* Containers should be restarted after a server reboot + + +=== Maintenance + +TO keep everything up-to-date do the following steps: + +Update the OS call: +{{{ +~/PyInventory-Deployment$ sudo ./scripts/apt-distupgrade.sh +}}} + +Update PyInventory installation and docker containers: +{{{ +~/PyInventory-Deployment$ make update +}}} + +This will do: + +* update the source code +* update docker-compose and all requirements +* Pull and rebuild all docker containers +* restart all docker containers + +Take a look at [[https://github.com/jedie/PyInventory/blob/deployment/scripts/apt-cleanup.sh|scripts/apt-cleanup.sh]] to keep your system clean. + + +== links == + +| Homepage | http://github.com/jedie/PyInventory + +Web server stuff: + +* https://help.ubuntu.com/community/Security +* https://help.ubuntu.com/lts/serverguide/automatic-updates.html +* https://help.ubuntu.com/community/AutomaticSecurityUpdate + + +== donation == + +* [[https://www.paypal.me/JensDiemer|paypal.me/JensDiemer]] +* [[https://flattr.com/submit/auto?uid=jedie&url=https%3A%2F%2Fgithub.com%2Fjedie%2FPyInventory%2F|Flattr This!]] +* Send [[http://www.bitcoin.org/|Bitcoins]] to [[https://blockexplorer.com/address/1823RZ5Md1Q2X5aSXRC5LRPcYdveCiVX6F|1823RZ5Md1Q2X5aSXRC5LRPcYdveCiVX6F]] diff --git a/deployment/caddy/Caddyfile b/deployment/caddy/Caddyfile new file mode 100644 index 0000000..519fc3f --- /dev/null +++ b/deployment/caddy/Caddyfile @@ -0,0 +1,31 @@ +# https://caddyserver.com/docs/caddyfile + +{$HOSTNAME} { + tls {$LETSENCRYPT_EMAIL} + + log { + output stdout + format console + level WARN + } + + header { + X-Robots-Tag "none" + } + + respond /robots.txt 200 { + body "User-agent: * +Disallow: /" + close + } + + route { + file_server /static/* { + root /srv/ + } + file_server /media/* { + root /srv/ + } + reverse_proxy inventory:8000 + } +} diff --git a/deployment/common.env b/deployment/common.env new file mode 100644 index 0000000..8e324cb --- /dev/null +++ b/deployment/common.env @@ -0,0 +1,10 @@ +DB_NAME=pyinventory +DB_USER=postgres +DB_PASS=postgres +DB_HOST=postgres +DB_PORT=5432 + +PYTHONUNBUFFERED=1 + +# e.g.: pip cache: +XDG_CACHE_HOME="/var/cache" diff --git a/deployment/compose.sh b/deployment/compose.sh new file mode 100755 index 0000000..72e9c9d --- /dev/null +++ b/deployment/compose.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +if [[ -f .env ]]; then + echo "Read '.env' file..." + source .env +fi + +set -x + +exec poetry run docker-compose "$@" diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml new file mode 100644 index 0000000..7241a2b --- /dev/null +++ b/deployment/docker-compose.yml @@ -0,0 +1,55 @@ +version: "3.7" + +services: + caddy: # https://hub.docker.com/_/caddy + image: caddy:2-alpine + restart: unless-stopped + ports: + - "80:80" + - "443:443" + volumes: + - ./caddy/Caddyfile:/etc/caddy/Caddyfile + - ./volumes/static/:/srv/static/:ro + - ./volumes/media/:/srv/media/:ro + environment: + - HOSTNAME=${HOSTNAME:-localhost} + - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL:-internal} + + inventory: + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + hostname: inventory + ports: + - "8000" + env_file: ./common.env + environment: + - DJANGO_SETTINGS_MODULE=inventory_settings + - HOSTNAME=${HOSTNAME:-localhost} + links: + - postgres:postgres + depends_on: + - postgres + - caddy + volumes: + - ./inventory/:/inventory/ + - ./volumes/static/:/static/:rw + - ./volumes/media/:/media/:rw + # e.g.: pip cache must be the same value as $XDG_CACHE_HOME ! + - ./volumes/cache/:/var/cache/:rw + entrypoint: /inventory/entrypoint.sh + + postgres: + # https://hub.docker.com/_/postgres + image: postgres:11-alpine + restart: unless-stopped + hostname: postgres + ports: + - "5432" + env_file: ./common.env + environment: + - POSTGRES_HOST_AUTH_METHOD=trust + volumes: + - ./postgres/init-user-db.sh:/docker-entrypoint-initdb.d/init-user-db.sh:ro + - ./volumes/postgresql/data/:/var/lib/postgresql/data/:rw diff --git a/deployment/inventory/entrypoint.sh b/deployment/inventory/entrypoint.sh new file mode 100755 index 0000000..4fa99e1 --- /dev/null +++ b/deployment/inventory/entrypoint.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +set -e + +restart_error_handler() { + ( + echo "Restart ${0} in 3 sec..." + sleep 1 + echo "Restart ${0} in 2 sec..." + sleep 1 + echo "Restart ${0} in 1 sec..." + sleep 1 + ) + exec ${0} +} +trap restart_error_handler 0 + +echo "_______________________________________________________________________" +echo "$(date +%c) - ${0}" + +( + set -x + + pip3 install -U pyinventory + + ./manage.py collectstatic --noinput + ./manage.py migrate + + uwsgi \ + --http inventory:8000 \ + --chdir /inventory/ \ + --wsgi-file /inventory/wsgi.py \ + --master \ + --processes 2 \ + --threads 2 \ + --ignore-sigpipe \ + --ignore-write-errors \ + --disable-write-exception \ + --http-auto-chunked \ + --http-keepalive + echo "uwsgi terminated with exit code: $?" + sleep 3 + exit 1 +) + +exit 2 diff --git a/deployment/inventory/inventory_settings.py b/deployment/inventory/inventory_settings.py new file mode 100644 index 0000000..7252369 --- /dev/null +++ b/deployment/inventory/inventory_settings.py @@ -0,0 +1,42 @@ +""" + Django settings for docker usage +""" +import os as __os + +from inventory_project.settings.base import * # noqa + +HOSTNAME = __os.environ['HOSTNAME'] + + +if HOSTNAME != 'localhost': + print(f'Production mode on domain: {HOSTNAME!r}') + DEBUG = False + INTERNAL_IPS = () +else: + print('Local development mode') + DEBUG = True + INTERNAL_IPS = ('127.0.0.1', '0.0.0.0', 'localhost') + + +SERVE_FILES = False # Caddy serve static/media files + + +ALLOWED_HOSTS = (HOSTNAME,) + + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': __os.environ['DB_NAME'], + 'USER': __os.environ['DB_USER'], + 'PASSWORD': __os.environ['DB_PASS'], + 'HOST': __os.environ['DB_HOST'], + 'PORT': __os.environ['DB_PORT'], + 'DEBUG_NAME': 'default', + 'CONN_MAX_AGE': 600, + }, +} + +# docker volumes: +STATIC_ROOT = '/static/' +MEDIA_ROOT = '/media/' diff --git a/deployment/inventory/kill_python.sh b/deployment/inventory/kill_python.sh new file mode 100755 index 0000000..b67e706 --- /dev/null +++ b/deployment/inventory/kill_python.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -ex + +for pid in $(pidof python3); do kill $pid; done diff --git a/deployment/inventory/manage.py b/deployment/inventory/manage.py new file mode 100755 index 0000000..bb39550 --- /dev/null +++ b/deployment/inventory/manage.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +import os +import sys + + +def main(): + assert 'DJANGO_SETTINGS_MODULE' in os.environ, 'No "DJANGO_SETTINGS_MODULE" in environment!' + from django.core.management import execute_from_command_line + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/deployment/inventory/wsgi.py b/deployment/inventory/wsgi.py new file mode 100644 index 0000000..d9598cf --- /dev/null +++ b/deployment/inventory/wsgi.py @@ -0,0 +1,9 @@ +""" + WSGI config +""" + + +from django.core.wsgi import get_wsgi_application + + +application = get_wsgi_application() diff --git a/deployment/postgres/init-user-db.sh b/deployment/postgres/init-user-db.sh new file mode 100644 index 0000000..5e2bf63 --- /dev/null +++ b/deployment/postgres/init-user-db.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -ex + +psql -U postgres -c "CREATE DATABASE $DB_NAME OWNER $DB_USER" diff --git a/deployment/pyproject.toml b/deployment/pyproject.toml new file mode 100644 index 0000000..e5d354b --- /dev/null +++ b/deployment/pyproject.toml @@ -0,0 +1,16 @@ +[tool.poetry] +name = "pyinventory-deployment" +version = "0.1.0" +description = "production deployment for PyInventory project with docker-compose usage" +authors = ["JensDiemer "] +license = "GPL" + +[tool.poetry.dependencies] +python = ">=3.7,<4.0.0" +docker-compose = "*" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/deployment/scripts/apt-cleanup.sh b/deployment/scripts/apt-cleanup.sh new file mode 100755 index 0000000..b1ff142 --- /dev/null +++ b/deployment/scripts/apt-cleanup.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Cleanup installed packages by using apt-mark: +# +# 1. mark all packages as "auto" +# 2. install really needed packages "manual" +# 3. call "autoremove" to deinstall all not needed packages +# +# WARNING: You may need some more packages depend on your cloud provider! + + + +############################################################### +# Remove this lines: +echo "Adjust this script first, before you use it!" +exit 1 +############################################################### + + + +set -e + +if [ "$(whoami)" != "root" ]; then + echo "Please start with 'sudo' !" + exit 1 +fi + +clear + +# These packages should be installed: +PACKAGES=( + linux-image-virtual ubuntu-minimal acpid + qemu-guest-agent + command-not-found + update-manager-core + unattended-upgrades + openssh-server + rsync + lshw htop mc nano + git make + apt-transport-https curl gnupg-agent software-properties-common + docker-ce docker-ce-cli containerd.io +) + +( + set -ex + + apt update + + { echo "---------------------------------------------------"; } 2>/dev/null + + # Mark all installed packages as "auto": + apt-mark auto $(apt-mark showinstall) + + { echo "---------------------------------------------------"; } 2>/dev/null + + # Install the really needed packages: + apt -y install "${PACKAGES[@]}" + + { echo "---------------------------------------------------"; } 2>/dev/null + + # Update all installed packages: + apt -y full-upgrade + + { echo "---------------------------------------------------"; } 2>/dev/null + + # Deinstall all not needed packages: + apt autoremove +) diff --git a/deployment/scripts/apt-distupgrade.sh b/deployment/scripts/apt-distupgrade.sh new file mode 100755 index 0000000..7558646 --- /dev/null +++ b/deployment/scripts/apt-distupgrade.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +( + set -x + sudo apt update + { echo "---------------------------------------------------"; } 2>/dev/null + sudo apt -y full-upgrade + { echo "---------------------------------------------------"; } 2>/dev/null + sudo apt -y autoremove + { echo "---------------------------------------------------"; } 2>/dev/null + # Delete old entries: + sudo journalctl --vacuum-size=1G + sudo journalctl --vacuum-time=1years +) diff --git a/deployment/scripts/setup_user.sh b/deployment/scripts/setup_user.sh new file mode 100755 index 0000000..d95c466 --- /dev/null +++ b/deployment/scripts/setup_user.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +if [ "$(whoami)" != "root" ]; then + echo "Please start with 'sudo' !" + ( + set -x + id + exit 1 + ) +fi + + +set -ex + +export USERNAME=${1} + +adduser --disabled-password --gecos "" --home=/home/${USERNAME} ${USERNAME} +mkdir -p /home/${USERNAME}/.ssh +cp /root/.ssh/authorized_keys /home/${USERNAME}/.ssh/ +chown -Rfc ${USERNAME}.${USERNAME} /home/${USERNAME}/ +echo "${USERNAME} ALL=(ALL) NOPASSWD:ALL">/etc/sudoers.d/${USERNAME} \ No newline at end of file