Merge remote-tracking branch 'origin/deployment' into merge-deploy-src

pull/20/head
JensDiemer 2020-11-13 20:41:48 +01:00
commit e9ad443e06
19 zmienionych plików z 799 dodań i 0 usunięć

Wyświetl plik

@ -0,0 +1 @@
*

79
deployment/.gitignore vendored 100644
Wyświetl plik

@ -0,0 +1,79 @@
.*
!.github
!.dockerignore
!.editorconfig
!.flake8
!.gitignore
!.isort.cfg
!/.travis.yml
!/.style.yapf
!.coveralls.yml
poetry.lock
# for django-dbbackup
/backups/
!/backups/.gitkeep
# from test projects:
/static/
/media/
*.sqlite3
# docker-compose usage:
volumes
# Django
secret.txt
# Coverage HTML Report files:
htmlcov
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.tox
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
db.sqlite3
coverage_html/
coverage.xml
*,cover
# Translations
*.pot
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
# PyBuilder
target/

Wyświetl plik

@ -0,0 +1,18 @@
FROM python:3.9-slim-buster
# https://hub.docker.com/_/python
# Install deps
RUN apt-get update \
&& apt-mark auto $(apt-mark showinstall) \
&& apt-get install -y postgresql-client-11 python3-pip \
&& apt autoremove \
&& apt -y full-upgrade \
&& rm -rf /var/lib/apt \
&& python3 -m pip install -U pip \
&& pip install -U psycopg2-binary
WORKDIR /inventory
RUN pip install "pyinventory>=0.4.2"

131
deployment/Makefile 100644
Wyświetl plik

@ -0,0 +1,131 @@
SHELL := /bin/bash
help: ## List all commands
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z0-9 -_]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
check-poetry:
@if [[ "$(shell poetry --version 2>/dev/null)" == *"Poetry"* ]] ; \
then \
echo "Poetry found, ok." ; \
else \
echo 'Please install poetry first, with e.g.:' ; \
echo 'make install-poetry' ; \
exit 1 ; \
fi
install-poetry: ## install or update poetry
@if [[ "$(shell poetry --version 2>/dev/null)" == *"Poetry"* ]] ; \
then \
echo 'Update poetry' ; \
poetry self update ; \
else \
echo 'Install poetry' ; \
curl -sSL "https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py" | python3 ; \
fi
install: check-poetry ## install requirements to setup project
poetry install
update: check-poetry ## update the sources and docker containers
git fetch --all
git pull origin deployment
poetry update
./compose.sh build --pull
$(MAKE) restart
check-compose:
@if [[ "$(shell poetry run docker-compose --version 2>/dev/null)" = *"docker-compose version"* ]] ; \
then \
echo "docker-compose found, ok." ; \
else \
echo 'Please install extras first, with e.g.:' ; \
echo 'make install-compose' ; \
exit 1 ; \
fi
up: check-compose ## Start containers via docker-compose
./compose.sh up -d
$(MAKE) prune
./compose.sh logs --tail=500 --follow
down: ## Stop all containers
./compose.sh down
prune: ## Cleanup docker
docker system prune --force --all --filter until=4464h
build: check-compose ## Update docker container build
./compose.sh build --pull
init_postgres: ## Create postgres database
./compose.sh exec postgres ./docker/postgres_init.sh
##############################################################################
createsuperuser: ## Create super user
./compose.sh exec inventory ./manage.sh createsuperuser
##############################################################################
shell_inventory: ## Go into bash shell in inventory container
./compose.sh exec inventory /bin/bash
shell_postgres: ## Go into bash shell in postgres container
./compose.sh exec postgres /bin/bash
shell_caddy: ## Go into bash shell in caddy container
./compose.sh exec caddy /bin/ash
##############################################################################
caddy_environ: ## Prints the caddy environment
./compose.sh exec caddy /usr/bin/caddy environ
##############################################################################
logs: ## Display docker logs from all containers
./compose.sh logs --tail=500 --follow
logs_postgres: ## Display docker logs from postgres container
./compose.sh logs --tail=500 --follow postgres
logs_inventory: ## Display docker logs from inventory container
./compose.sh logs --tail=500 --follow inventory
logs_caddy: ## Display docker logs from caddy container
./compose.sh logs --tail=500 --follow caddy
##############################################################################
dbbackup: ## Backup database
./compose.sh exec inventory ./manage.sh dbbackup
dbrestore: ## Restore a database backup
./compose.sh exec inventory ./manage.sh dbrestore
##############################################################################
restart: down up ## Restart all containers
upgrade_inventory: ## Upgrade "inventory" container and restart it
$(MAKE) build
./compose.sh stop inventory
$(MAKE) up
restart_caddy: ## Restart caddy container
./compose.sh stop caddy
$(MAKE) up
##############################################################################
reload_caddy: ## Reload Caddy server
./compose.sh exec caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile
$(MAKE) logs_caddy
reload_inventory: ## Reload server in "inventory" container
./compose.sh exec inventory ./docker/kill_python.sh
./compose.sh logs --tail=500 --follow inventory
##############################################################################
.PHONY: help

Wyświetl plik

@ -0,0 +1,221 @@
= PyInventory - deployment branch
Web based management to catalog things including state and location etc. using Python/Django.
**Note:**
* This README (in git **deployment** branch) contains only the instruction to install PyInventory on a root server.
* Read [[https://github.com/jedie/PyInventory/tree/master#readme|master README]] for local develompment installation.
Pull requests welcome!
== git branches
Currently we have two main branches:
|= git branch |= description
| **[[https://github.com/jedie/PyInventory/tree/master|master]]** | The main PyInventory source code
| **[[https://github.com/jedie/PyInventory/tree/deployment|deployment]]** | separate project to deploy PyInventory for production use case
== deploy
Install PyInventory on a root server:
**Note:**
* Running a public web server is a lot of work and brings some risks.
* This instructions are only the basics to get PyInventory working.
* To run the server safely, more work should be done, which is not explained here.
* Run at your own risk! No warranty is given.
=== prepare root server
Here some steps for a fresh created root server:
Update all packages, e.g.:
{{{
apt update && apt -y full-upgrade
}}}
Setup a normal user. You may use [[https://github.com/jedie/PyInventory/blob/deployment/scripts/setup_user.sh|scripts/setup_user.sh]]
=== Setup SSH services
{{{
~# nano /etc/ssh/sshd_config
}}}
Change e.g.:
{{{
Port xxxx
PermitRootLogin no
PasswordAuthentication no
}}}
(Changing the Port may need to change a firewall/network settings)
{{{
# restart SSH deamon:
~# service ssh restart
# Display and follow the ssh log output to see connection errors:
~# journalctl -f -u ssh
}}}
* Keep the current SSH session (with the log output) open!
* Update your {{{~/.ssh/config}}}
* Try to connect as the new, normal user in a **separate** terminal
Only after a working new connection: Terminate the first root SSH session ;)
=== setup unattended-upgrades
{{{
~$ sudo apt install unattended-upgrades
~$ sudo dpkg-reconfigure unattended-upgrades
~$ sudo nano /etc/apt/apt.conf.d/50unattended-upgrades
}}}
=== install requirements
Install docker, see: https://docs.docker.com/engine/install/ubuntu/
install some base packages, e.g.:
{{{
~$ sudo apt install git make
}}}
=== install PyInventory
{{{
# Checkout the deployment branch:
~$ git clone -b deployment https://github.com/jedie/PyInventory.git PyInventory-Deployment
~$ cd PyInventory-Deployment
# Theses Makefile targets exists:
~/PyInventory-Deployment$ make
help List all commands
install-poetry install or update poetry
install install requirements to setup project
update update the sources and docker containers
up Start containers via docker-compose
down Stop all containers
prune Cleanup docker
build Update docker container build
init_postgres Create postgres database
createsuperuser Create super user
shell_inventory Go into bash shell in inventory container
shell_postgres Go into bash shell in postgres container
shell_caddy Go into bash shell in caddy container
caddy_environ Prints the caddy environment
logs Display docker logs from all containers
logs_postgres Display docker logs from postgres container
logs_inventory Display docker logs from inventory container
logs_caddy Display docker logs from caddy container
dbbackup Backup database
dbrestore Restore a database backup
restart Restart all containers
upgrade_inventory Upgrade "inventory" container and restart it
restart_caddy Restart caddy container
reload_caddy Reload Caddy server
reload_inventory Reload server in "inventory" container
}}}
Install, e.g.:
{{{
# install or update poetry:
~/PyInventory-Deployment$ make install-poetry
}}}
To keep poetry running (PATH must be expand) just logout and login ;)
{{{
# install requirements (e.g.: docker-compose) via poetry:
~/PyInventory-Deployment$ make install
}}}
Create a {{{.env}}} file in project root directory with these content, e.g.:
{{{
~/PyInventory-Deployment$ touch .env
~/PyInventory-Deployment$ nano .env
~/PyInventory-Deployment$ cat .env
# Your Public domain:
HOSTNAME=domain.tld
# eMail address for Let's encrypt:
LETSENCRYPT_EMAIL=webmaster@domain.tld
}}}
For local testing of the docker-compose setup, used this values:
{{{
HOSTNAME=localhost
LETSENCRYPT_EMAIL=internal
}}}
(Caddy will create a self signed https certificate)
Start containers via docker-compose:
{{{
~/PyInventory-Deployment$ make up
}}}
Notes: At the first start it takes a little while until the database is created ;)
Create first super user:
{{{
~/PyInventory-Deployment$ make docker_createsuperuser
}}}
* Now you should be able to connect to your PyInventory installation and login with created super user.
* Redirect from **http** to **https** should work.
* Let's Encrypt certificate should be installed and valid.
* Containers should be restarted after a server reboot
=== Maintenance
TO keep everything up-to-date do the following steps:
Update the OS call:
{{{
~/PyInventory-Deployment$ sudo ./scripts/apt-distupgrade.sh
}}}
Update PyInventory installation and docker containers:
{{{
~/PyInventory-Deployment$ make update
}}}
This will do:
* update the source code
* update docker-compose and all requirements
* Pull and rebuild all docker containers
* restart all docker containers
Take a look at [[https://github.com/jedie/PyInventory/blob/deployment/scripts/apt-cleanup.sh|scripts/apt-cleanup.sh]] to keep your system clean.
== links ==
| Homepage | http://github.com/jedie/PyInventory
Web server stuff:
* https://help.ubuntu.com/community/Security
* https://help.ubuntu.com/lts/serverguide/automatic-updates.html
* https://help.ubuntu.com/community/AutomaticSecurityUpdate
== donation ==
* [[https://www.paypal.me/JensDiemer|paypal.me/JensDiemer]]
* [[https://flattr.com/submit/auto?uid=jedie&url=https%3A%2F%2Fgithub.com%2Fjedie%2FPyInventory%2F|Flattr This!]]
* Send [[http://www.bitcoin.org/|Bitcoins]] to [[https://blockexplorer.com/address/1823RZ5Md1Q2X5aSXRC5LRPcYdveCiVX6F|1823RZ5Md1Q2X5aSXRC5LRPcYdveCiVX6F]]

Wyświetl plik

@ -0,0 +1,31 @@
# https://caddyserver.com/docs/caddyfile
{$HOSTNAME} {
tls {$LETSENCRYPT_EMAIL}
log {
output stdout
format console
level WARN
}
header {
X-Robots-Tag "none"
}
respond /robots.txt 200 {
body "User-agent: *
Disallow: /"
close
}
route {
file_server /static/* {
root /srv/
}
file_server /media/* {
root /srv/
}
reverse_proxy inventory:8000
}
}

Wyświetl plik

@ -0,0 +1,10 @@
DB_NAME=pyinventory
DB_USER=postgres
DB_PASS=postgres
DB_HOST=postgres
DB_PORT=5432
PYTHONUNBUFFERED=1
# e.g.: pip cache:
XDG_CACHE_HOME="/var/cache"

Wyświetl plik

@ -0,0 +1,12 @@
#!/bin/bash
set -e
if [[ -f .env ]]; then
echo "Read '.env' file..."
source .env
fi
set -x
exec poetry run docker-compose "$@"

Wyświetl plik

@ -0,0 +1,55 @@
version: "3.7"
services:
caddy: # https://hub.docker.com/_/caddy
image: caddy:2-alpine
restart: unless-stopped
ports:
- "80:80"
- "443:443"
volumes:
- ./caddy/Caddyfile:/etc/caddy/Caddyfile
- ./volumes/static/:/srv/static/:ro
- ./volumes/media/:/srv/media/:ro
environment:
- HOSTNAME=${HOSTNAME:-localhost}
- LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL:-internal}
inventory:
build:
context: .
dockerfile: Dockerfile
restart: unless-stopped
hostname: inventory
ports:
- "8000"
env_file: ./common.env
environment:
- DJANGO_SETTINGS_MODULE=inventory_settings
- HOSTNAME=${HOSTNAME:-localhost}
links:
- postgres:postgres
depends_on:
- postgres
- caddy
volumes:
- ./inventory/:/inventory/
- ./volumes/static/:/static/:rw
- ./volumes/media/:/media/:rw
# e.g.: pip cache must be the same value as $XDG_CACHE_HOME !
- ./volumes/cache/:/var/cache/:rw
entrypoint: /inventory/entrypoint.sh
postgres:
# https://hub.docker.com/_/postgres
image: postgres:11-alpine
restart: unless-stopped
hostname: postgres
ports:
- "5432"
env_file: ./common.env
environment:
- POSTGRES_HOST_AUTH_METHOD=trust
volumes:
- ./postgres/init-user-db.sh:/docker-entrypoint-initdb.d/init-user-db.sh:ro
- ./volumes/postgresql/data/:/var/lib/postgresql/data/:rw

Wyświetl plik

@ -0,0 +1,46 @@
#!/bin/sh
set -e
restart_error_handler() {
(
echo "Restart ${0} in 3 sec..."
sleep 1
echo "Restart ${0} in 2 sec..."
sleep 1
echo "Restart ${0} in 1 sec..."
sleep 1
)
exec ${0}
}
trap restart_error_handler 0
echo "_______________________________________________________________________"
echo "$(date +%c) - ${0}"
(
set -x
pip3 install -U pyinventory
./manage.py collectstatic --noinput
./manage.py migrate
uwsgi \
--http inventory:8000 \
--chdir /inventory/ \
--wsgi-file /inventory/wsgi.py \
--master \
--processes 2 \
--threads 2 \
--ignore-sigpipe \
--ignore-write-errors \
--disable-write-exception \
--http-auto-chunked \
--http-keepalive
echo "uwsgi terminated with exit code: $?"
sleep 3
exit 1
)
exit 2

Wyświetl plik

@ -0,0 +1,42 @@
"""
Django settings for docker usage
"""
import os as __os
from inventory_project.settings.base import * # noqa
HOSTNAME = __os.environ['HOSTNAME']
if HOSTNAME != 'localhost':
print(f'Production mode on domain: {HOSTNAME!r}')
DEBUG = False
INTERNAL_IPS = ()
else:
print('Local development mode')
DEBUG = True
INTERNAL_IPS = ('127.0.0.1', '0.0.0.0', 'localhost')
SERVE_FILES = False # Caddy serve static/media files
ALLOWED_HOSTS = (HOSTNAME,)
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': __os.environ['DB_NAME'],
'USER': __os.environ['DB_USER'],
'PASSWORD': __os.environ['DB_PASS'],
'HOST': __os.environ['DB_HOST'],
'PORT': __os.environ['DB_PORT'],
'DEBUG_NAME': 'default',
'CONN_MAX_AGE': 600,
},
}
# docker volumes:
STATIC_ROOT = '/static/'
MEDIA_ROOT = '/media/'

Wyświetl plik

@ -0,0 +1,5 @@
#!/bin/sh
set -ex
for pid in $(pidof python3); do kill $pid; done

Wyświetl plik

@ -0,0 +1,14 @@
#!/usr/bin/env python3
import os
import sys
def main():
assert 'DJANGO_SETTINGS_MODULE' in os.environ, 'No "DJANGO_SETTINGS_MODULE" in environment!'
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()

Wyświetl plik

@ -0,0 +1,9 @@
"""
WSGI config
"""
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()

Wyświetl plik

@ -0,0 +1,5 @@
#!/bin/bash
set -ex
psql -U postgres -c "CREATE DATABASE $DB_NAME OWNER $DB_USER"

Wyświetl plik

@ -0,0 +1,16 @@
[tool.poetry]
name = "pyinventory-deployment"
version = "0.1.0"
description = "production deployment for PyInventory project with docker-compose usage"
authors = ["JensDiemer <git@jensdiemer.de>"]
license = "GPL"
[tool.poetry.dependencies]
python = ">=3.7,<4.0.0"
docker-compose = "*"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

Wyświetl plik

@ -0,0 +1,69 @@
#!/bin/bash
# Cleanup installed packages by using apt-mark:
#
# 1. mark all packages as "auto"
# 2. install really needed packages "manual"
# 3. call "autoremove" to deinstall all not needed packages
#
# WARNING: You may need some more packages depend on your cloud provider!
###############################################################
# Remove this lines:
echo "Adjust this script first, before you use it!"
exit 1
###############################################################
set -e
if [ "$(whoami)" != "root" ]; then
echo "Please start with 'sudo' !"
exit 1
fi
clear
# These packages should be installed:
PACKAGES=(
linux-image-virtual ubuntu-minimal acpid
qemu-guest-agent
command-not-found
update-manager-core
unattended-upgrades
openssh-server
rsync
lshw htop mc nano
git make
apt-transport-https curl gnupg-agent software-properties-common
docker-ce docker-ce-cli containerd.io
)
(
set -ex
apt update
{ echo "---------------------------------------------------"; } 2>/dev/null
# Mark all installed packages as "auto":
apt-mark auto $(apt-mark showinstall)
{ echo "---------------------------------------------------"; } 2>/dev/null
# Install the really needed packages:
apt -y install "${PACKAGES[@]}"
{ echo "---------------------------------------------------"; } 2>/dev/null
# Update all installed packages:
apt -y full-upgrade
{ echo "---------------------------------------------------"; } 2>/dev/null
# Deinstall all not needed packages:
apt autoremove
)

Wyświetl plik

@ -0,0 +1,14 @@
#!/bin/bash
(
set -x
sudo apt update
{ echo "---------------------------------------------------"; } 2>/dev/null
sudo apt -y full-upgrade
{ echo "---------------------------------------------------"; } 2>/dev/null
sudo apt -y autoremove
{ echo "---------------------------------------------------"; } 2>/dev/null
# Delete old entries:
sudo journalctl --vacuum-size=1G
sudo journalctl --vacuum-time=1years
)

Wyświetl plik

@ -0,0 +1,21 @@
#!/usr/bin/env bash
if [ "$(whoami)" != "root" ]; then
echo "Please start with 'sudo' !"
(
set -x
id
exit 1
)
fi
set -ex
export USERNAME=${1}
adduser --disabled-password --gecos "" --home=/home/${USERNAME} ${USERNAME}
mkdir -p /home/${USERNAME}/.ssh
cp /root/.ssh/authorized_keys /home/${USERNAME}/.ssh/
chown -Rfc ${USERNAME}.${USERNAME} /home/${USERNAME}/
echo "${USERNAME} ALL=(ALL) NOPASSWD:ALL">/etc/sudoers.d/${USERNAME}