From 7a3f89cfa554e2af005a43334e2080f89fe45cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Sen=C3=ADn?= Date: Mon, 18 Nov 2019 10:26:26 +0100 Subject: [PATCH 01/34] Do not process empty lines or commented at Procfile --- piku.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/piku.py b/piku.py index d7ae87b..2380341 100755 --- a/piku.py +++ b/piku.py @@ -246,13 +246,17 @@ def parse_procfile(filename): workers = {} if not exists(filename): return None + with open(filename, 'r') as procfile: - for line in procfile: + for line_number, line in enumerate(procfile): + line = line.strip() + if line.startswith("#") or not line: + continue try: kind, command = map(lambda x: x.strip(), line.split(":", 1)) workers[kind] = command except: - echo("Warning: unrecognized Procfile entry '{}'".format(line), fg='yellow') + echo("Warning: unrecognized Procfile entry '{}' at line {}".format(line, line_number), fg='yellow') if not len(workers): return {} # WSGI trumps regular web workers From 5ddb3619a2236a969647589dfec859e3fb626042 Mon Sep 17 00:00:00 2001 From: Chris McCormick Date: Tue, 19 Nov 2019 17:10:13 +0800 Subject: [PATCH 02/34] Fix clashing HTTP ports in uwsgi deploy. #125 --- piku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piku.py b/piku.py index e90d1bf..7bc6d89 100755 --- a/piku.py +++ b/piku.py @@ -887,7 +887,7 @@ def spawn_worker(app, kind, command, env, ordinal=1): echo("-----> nginx will talk to uWSGI via {BIND_ADDRESS:s}:{PORT:s}".format(**env), fg='yellow') settings.extend([ ('http', '{BIND_ADDRESS:s}:{PORT:s}'.format(**env)), - ('http-socket', '{BIND_ADDRESS:s}:{PORT:s}'.format(**env)), + ('http-use-socket', '{BIND_ADDRESS:s}:{PORT:s}'.format(**env)), ]) elif kind == 'web': echo("-----> nginx will talk to the 'web' process via {BIND_ADDRESS:s}:{PORT:s}".format(**env), fg='yellow') From a93b471d671d916ba4bb8e866b438f87b87ecf74 Mon Sep 17 00:00:00 2001 From: dWiGhT Date: Wed, 20 Nov 2019 00:28:02 -0800 Subject: [PATCH 03/34] build: lite cleanup of pylint results (#141) --- piku.py | 211 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 114 insertions(+), 97 deletions(-) diff --git a/piku.py b/piku.py index e90d1bf..6ad5b1b 100755 --- a/piku.py +++ b/piku.py @@ -4,32 +4,31 @@ try: from sys import version_info - assert version_info >= (3,5) + assert version_info >= (3, 5) except AssertionError: exit("Piku requires Python 3.5 or above") -from click import argument, command, group, get_current_context, option, secho as echo, pass_context from collections import defaultdict, deque -from datetime import datetime from fcntl import fcntl, F_SETFL, F_GETFL from glob import glob -from hashlib import md5 +from grp import getgrgid from json import loads from multiprocessing import cpu_count from os import chmod, getgid, getuid, symlink, unlink, remove, stat, listdir, environ, makedirs, O_NONBLOCK from os.path import abspath, basename, dirname, exists, getmtime, join, realpath, splitext +from pwd import getpwuid from re import sub from shutil import copyfile, rmtree, which from socket import socket, AF_INET, SOCK_STREAM -from sys import argv, stdin, stdout, stderr, version_info, exit from stat import S_IRUSR, S_IWUSR, S_IXUSR -from subprocess import call, check_output, Popen, STDOUT, PIPE +from subprocess import call, check_output, Popen, STDOUT +from sys import argv, stdin, stdout, stderr, version_info, exit from tempfile import NamedTemporaryFile -from traceback import format_exc from time import sleep +from traceback import format_exc from urllib.request import urlopen -from pwd import getpwuid -from grp import getgrgid + +from click import argument, group, secho as echo, pass_context # === Make sure we can access all system binaries === @@ -38,8 +37,8 @@ if 'sbin' not in environ['PATH']: # === Globals - all tweakable settings are here === -PIKU_ROOT = environ.get('PIKU_ROOT', join(environ['HOME'],'.piku')) -PIKU_BIN = join(environ['HOME'],'bin') +PIKU_ROOT = environ.get('PIKU_ROOT', join(environ['HOME'], '.piku')) +PIKU_BIN = join(environ['HOME'], 'bin') PIKU_SCRIPT = realpath(__file__) APP_ROOT = abspath(join(PIKU_ROOT, "apps")) ENV_ROOT = abspath(join(PIKU_ROOT, "envs")) @@ -50,7 +49,7 @@ UWSGI_AVAILABLE = abspath(join(PIKU_ROOT, "uwsgi-available")) UWSGI_ENABLED = abspath(join(PIKU_ROOT, "uwsgi-enabled")) UWSGI_ROOT = abspath(join(PIKU_ROOT, "uwsgi")) UWSGI_LOG_MAXSIZE = '1048576' -ACME_ROOT = environ.get('ACME_ROOT', join(environ['HOME'],'.acme.sh')) +ACME_ROOT = environ.get('ACME_ROOT', join(environ['HOME'], '.acme.sh')) ACME_WWW = abspath(join(PIKU_ROOT, "acme")) # === Make sure we can access piku user-installed binaries === # @@ -189,12 +188,13 @@ INTERNAL_NGINX_UWSGI_SETTINGS = """ uwsgi_param SERVER_NAME $server_name; """ + # === Utility functions === def sanitize_app_name(app): """Sanitize the app name and build matching path""" - app = "".join(c for c in app if c.isalnum() or c in ('.','_')).rstrip().lstrip('/') + app = "".join(c for c in app if c.isalnum() or c in ('.', '_')).rstrip().lstrip('/') return app @@ -212,7 +212,7 @@ def get_free_port(address=""): """Find a free TCP port (entirely at random)""" s = socket(AF_INET, SOCK_STREAM) - s.bind((address,0)) + s.bind((address, 0)) port = s.getsockname()[1] s.close() return port @@ -230,7 +230,7 @@ def write_config(filename, bag, separator='='): def setup_authorized_keys(ssh_fingerprint, script_path, pubkey): """Sets up an authorized_keys file to redirect SSH commands""" - authorized_keys = join(environ['HOME'],'.ssh','authorized_keys') + authorized_keys = join(environ['HOME'], '.ssh', 'authorized_keys') if not exists(dirname(authorized_keys)): makedirs(dirname(authorized_keys)) # Restrict features and force all SSH commands to go through our script @@ -253,13 +253,13 @@ def parse_procfile(filename): workers[kind] = command except: echo("Warning: unrecognized Procfile entry '{}'".format(line), fg='yellow') - if not len(workers): + if len(workers) == 0: return {} # WSGI trumps regular web workers if 'wsgi' in workers or 'jwsgi' in workers: if 'web' in workers: echo("Warning: found both 'wsgi' and 'web' workers, disabling 'web'", fg='yellow') - del(workers['web']) + del workers['web'] return workers @@ -290,7 +290,7 @@ def parse_settings(filename, env={}): with open(filename, 'r') as settings: for line in settings: - if '#' == line[0] or len(line.strip()) == 0: # ignore comments and newlines + if line[0] == '#' or len(line.strip()) == 0: # ignore comments and newlines continue try: k, v = map(lambda x: x.strip(), line.split("=", 1)) @@ -312,10 +312,13 @@ def check_requirements(binaries): return False return True + def found_app(kind): + """Helper function to output app detected""" echo("-----> {} app detected.".format(kind), fg='green') return True + def do_deploy(app, deltas={}, newrev=None): """Deploy an app by resetting the work directory""" @@ -336,17 +339,18 @@ def do_deploy(app, deltas={}, newrev=None): if not exists(log_path): makedirs(log_path) workers = parse_procfile(procfile) - if workers and len(workers): + if workers and len(workers) > 0: settings = {} if exists(join(app_path, 'requirements.txt')) and found_app("Python"): settings.update(deploy_python(app, deltas)) - elif exists(join(app_path, 'package.json')) and found_app("Node") and (check_requirements(['nodejs', 'npm']) or check_requirements(['nodeenv'])): + elif exists(join(app_path, 'package.json')) and found_app("Node") and ( + check_requirements(['nodejs', 'npm']) or check_requirements(['nodeenv'])): settings.update(deploy_node(app, deltas)) elif exists(join(app_path, 'pom.xml')) and found_app("Java Maven") and check_requirements(['java', 'mvn']): settings.update(deploy_java(app, deltas)) elif exists(join(app_path, 'build.gradle')) and found_app("Java Gradle") and check_requirements(['java', 'gradle']): settings.update(deploy_java(app, deltas)) - elif (exists(join(app_path, 'Godeps')) or len(glob(join(app_path,'*.go')))) and found_app("Go") and check_requirements(['go']): + elif (exists(join(app_path, 'Godeps')) or len(glob(join(app_path, '*.go')))) and found_app("Go") and check_requirements(['go']): settings.update(deploy_go(app, deltas)) elif exists(join(app_path, 'project.clj')) and found_app("Clojure Lein") and check_requirements(['java', 'lein']): settings.update(deploy_clojure(app, deltas)) @@ -371,6 +375,7 @@ def do_deploy(app, deltas={}, newrev=None): else: echo("Error: app '{}' not found.".format(app), fg='red') + def deploy_gradle(app, deltas={}): """Deploy a Java application using Gradle""" java_path = join(ENV_ROOT, app) @@ -380,7 +385,7 @@ def deploy_gradle(app, deltas={}): env = { 'VIRTUAL_ENV': java_path, - "PATH": ':'.join([join(java_path, "bin"), join(app, ".bin"),environ['PATH']]) + "PATH": ':'.join([join(java_path, "bin"), join(app, ".bin"), environ['PATH']]) } if exists(env_file): @@ -400,6 +405,7 @@ def deploy_gradle(app, deltas={}): return spawn_app(app, deltas) + def deploy_java(app, deltas={}): """Deploy a Java application using Maven""" # TODO: Use jenv to isolate Java Application environments @@ -411,7 +417,7 @@ def deploy_java(app, deltas={}): env = { 'VIRTUAL_ENV': java_path, - "PATH": ':'.join([join(java_path, "bin"), join(app, ".bin"),environ['PATH']]) + "PATH": ':'.join([join(java_path, "bin"), join(app, ".bin"), environ['PATH']]) } if exists(env_file): @@ -431,6 +437,7 @@ def deploy_java(app, deltas={}): return spawn_app(app, deltas) + def deploy_clojure(app, deltas={}): """Deploy a Clojure Application""" @@ -444,7 +451,7 @@ def deploy_clojure(app, deltas={}): env = { 'VIRTUAL_ENV': virtual, "PATH": ':'.join([join(virtual, "bin"), join(app, ".bin"), environ['PATH']]), - "LEIN_HOME": environ.get('LEIN_HOME', join(environ['HOME'],'.lein')), + "LEIN_HOME": environ.get('LEIN_HOME', join(environ['HOME'], '.lein')), } if exists(env_file): env.update(parse_settings(env_file, env)) @@ -501,7 +508,7 @@ def deploy_node(app, deltas={}): 'VIRTUAL_ENV': virtualenv_path, 'NODE_PATH': node_path, 'NPM_CONFIG_PREFIX': abspath(join(node_path, "..")), - "PATH": ':'.join([join(virtualenv_path, "bin"), join(node_path, ".bin"),environ['PATH']]) + "PATH": ':'.join([join(virtualenv_path, "bin"), join(node_path, ".bin"), environ['PATH']]) } if exists(env_file): env.update(parse_settings(env_file, env)) @@ -511,7 +518,8 @@ def deploy_node(app, deltas={}): version = env.get("NODE_VERSION") node_binary = join(virtualenv_path, "bin", "node") - installed = check_output("{} -v".format(node_binary), cwd=join(APP_ROOT, app), env=env, shell=True).decode("utf8").rstrip("\n") if exists(node_binary) else "" + installed = check_output("{} -v".format(node_binary), cwd=join(APP_ROOT, app), env=env, shell=True).decode("utf8").rstrip( + "\n") if exists(node_binary) else "" if version and check_requirements(['nodeenv']): if not installed.endswith(version): @@ -520,7 +528,8 @@ def deploy_node(app, deltas={}): echo("Warning: Can't update node with app running. Stop the app & retry.", fg='yellow') else: echo("-----> Installing node version '{NODE_VERSION:s}' using nodeenv".format(**env), fg='green') - call("nodeenv --prebuilt --node={NODE_VERSION:s} --clean-src --force {VIRTUAL_ENV:s}".format(**env), cwd=virtualenv_path, env=env, shell=True) + call("nodeenv --prebuilt --node={NODE_VERSION:s} --clean-src --force {VIRTUAL_ENV:s}".format(**env), + cwd=virtualenv_path, env=env, shell=True) else: echo("-----> Node is installed at {}.".format(version)) @@ -555,7 +564,7 @@ def deploy_python(app, deltas={}): call('virtualenv --python=python{version:d} {app:s}'.format(**locals()), cwd=ENV_ROOT, shell=True) first_time = True - activation_script = join(virtualenv_path,'bin','activate_this.py') + activation_script = join(virtualenv_path, 'bin', 'activate_this.py') exec(open(activation_script).read(), dict(__file__=activation_script)) if first_time or getmtime(requirements) > getmtime(virtualenv_path): @@ -579,8 +588,8 @@ def spawn_app(app, deltas={}): procfile = join(app_path, 'Procfile') workers = parse_procfile(procfile) workers.pop("release", None) - ordinals = defaultdict(lambda:1) - worker_count = {k:1 for k in workers.keys()} + ordinals = defaultdict(lambda: 1) + worker_count = {k: 1 for k in workers.keys()} # the Python virtualenv virtualenv_path = join(ENV_ROOT, app) @@ -599,7 +608,7 @@ def spawn_app(app, deltas={}): 'LOG_ROOT': LOG_ROOT, 'HOME': environ['HOME'], 'USER': environ['USER'], - 'PATH': ':'.join([join(virtualenv_path,'bin'),environ['PATH']]), + 'PATH': ':'.join([join(virtualenv_path, 'bin'), environ['PATH']]), 'PWD': dirname(env_file), 'VIRTUAL_ENV': virtualenv_path, } @@ -614,7 +623,7 @@ def spawn_app(app, deltas={}): node_path = join(virtualenv_path, "node_modules") if exists(node_path): env["NODE_PATH"] = node_path - env["PATH"] = ':'.join([join(node_path, ".bin"),env['PATH']]) + env["PATH"] = ':'.join([join(node_path, ".bin"), env['PATH']]) # Load environment variables shipped with repo (if any) if exists(env_file): @@ -642,9 +651,9 @@ def spawn_app(app, deltas={}): nginx_ssl = "443 ssl" if "--with-http_v2_module" in nginx: nginx_ssl += " http2" - elif "--with-http_spdy_module" in nginx and "nginx/1.6.2" not in nginx: # avoid Raspbian bug + elif "--with-http_spdy_module" in nginx and "nginx/1.6.2" not in nginx: # avoid Raspbian bug nginx_ssl += " spdy" - nginx_conf = join(NGINX_ROOT,"{}.conf".format(app)) + nginx_conf = join(NGINX_ROOT, "{}.conf".format(app)) env.update({ 'NGINX_SSL': nginx_ssl, @@ -664,9 +673,8 @@ def spawn_app(app, deltas={}): env['NGINX_SOCKET'] = "{BIND_ADDRESS:s}:{PORT:s}".format(**env) echo("-----> nginx will look for app '{}' on {}".format(app, env['NGINX_SOCKET'])) - domain = env['NGINX_SERVER_NAME'].split()[0] - key, crt = [join(NGINX_ROOT, "{}.{}".format(app,x)) for x in ['key','crt']] + key, crt = [join(NGINX_ROOT, "{}.{}".format(app, x)) for x in ['key', 'crt']] if exists(join(ACME_ROOT, "acme.sh")): acme = ACME_ROOT www = ACME_WWW @@ -680,7 +688,8 @@ def spawn_app(app, deltas={}): if not exists(key) or not exists(join(ACME_ROOT, domain, domain + ".key")): echo("-----> getting letsencrypt certificate") call('{acme:s}/acme.sh --issue -d {domain:s} -w {www:s}'.format(**locals()), shell=True) - call('{acme:s}/acme.sh --install-cert -d {domain:s} --key-file {key:s} --fullchain-file {crt:s}'.format(**locals()), shell=True) + call('{acme:s}/acme.sh --install-cert -d {domain:s} --key-file {key:s} --fullchain-file {crt:s}'.format( + **locals()), shell=True) if exists(join(ACME_ROOT, domain)) and not exists(join(ACME_WWW, app)): symlink(join(ACME_ROOT, domain), join(ACME_WWW, app)) else: @@ -689,7 +698,9 @@ def spawn_app(app, deltas={}): # fall back to creating self-signed certificate if acme failed if not exists(key) or stat(crt).st_size == 0: echo("-----> generating self-signed certificate") - call('openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj "/C=US/ST=NY/L=New York/O=Piku/OU=Self-Signed/CN={domain:s}" -keyout {key:s} -out {crt:s}'.format(**locals()), shell=True) + call( + 'openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj "/C=US/ST=NY/L=New York/O=Piku/OU=Self-Signed/CN={domain:s}" -keyout {key:s} -out {crt:s}'.format( + **locals()), shell=True) # restrict access to server from CloudFlare IP addresses acl = [] @@ -706,7 +717,7 @@ def spawn_app(app, deltas={}): remote_ip = environ['SSH_CLIENT'].split()[0] echo("-----> Adding your IP ({}) to nginx ACL".format(remote_ip)) acl.append("allow {};".format(remote_ip)) - acl.extend(["allow 127.0.0.1;","deny all;"]) + acl.extend(["allow 127.0.0.1;", "deny all;"]) except Exception: cf = defaultdict() echo("-----> Could not retrieve CloudFlare IP ranges: {}".format(format_exc()), fg="red") @@ -718,7 +729,7 @@ def spawn_app(app, deltas={}): env['INTERNAL_NGINX_STATIC_MAPPINGS'] = '' # Get a mapping of /url:path1,/url2:path2 - static_paths = env.get('NGINX_STATIC_PATHS','') + static_paths = env.get('NGINX_STATIC_PATHS', '') # prepend static worker path if present if 'static' in workers: stripped = workers['static'].strip("/").rstrip("/") @@ -730,19 +741,21 @@ def spawn_app(app, deltas={}): static_url, static_path = item.split(':') if static_path[0] != '/': static_path = join(app_path, static_path) - env['INTERNAL_NGINX_STATIC_MAPPINGS'] = env['INTERNAL_NGINX_STATIC_MAPPINGS'] + expandvars(INTERNAL_NGINX_STATIC_MAPPING, locals()) + env['INTERNAL_NGINX_STATIC_MAPPINGS'] = env['INTERNAL_NGINX_STATIC_MAPPINGS'] + expandvars( + INTERNAL_NGINX_STATIC_MAPPING, locals()) except Exception as e: echo("Error {} in static path spec: should be /url1:path1[,/url2:path2], ignoring.".format(e)) env['INTERNAL_NGINX_STATIC_MAPPINGS'] = '' - env['INTERNAL_NGINX_CUSTOM_CLAUSES'] = expandvars(open(join(app_path, env["NGINX_INCLUDE_FILE"])).read(), env) if env.get("NGINX_INCLUDE_FILE") else "" + env['INTERNAL_NGINX_CUSTOM_CLAUSES'] = expandvars(open(join(app_path, env["NGINX_INCLUDE_FILE"])).read(), + env) if env.get("NGINX_INCLUDE_FILE") else "" env['INTERNAL_NGINX_PORTMAP'] = "" if 'web' in workers or 'wsgi' in workers or 'jwsgi' in workers: env['INTERNAL_NGINX_PORTMAP'] = expandvars(NGINX_PORTMAP_FRAGMENT, env) env['INTERNAL_NGINX_COMMON'] = expandvars(NGINX_COMMON_FRAGMENT, env) echo("-----> nginx will map app '{}' to hostname '{}'".format(app, env['NGINX_SERVER_NAME'])) - if('NGINX_HTTPS_ONLY' in env) or ('HTTPS_ONLY' in env): + if ('NGINX_HTTPS_ONLY' in env) or ('HTTPS_ONLY' in env): buffer = expandvars(NGINX_HTTPS_ONLY_TEMPLATE, env) echo("-----> nginx will redirect all requests to hostname '{}' to HTTPS".format(env['NGINX_SERVER_NAME'])) else: @@ -761,17 +774,17 @@ def spawn_app(app, deltas={}): # Configured worker count if exists(scaling): - worker_count.update({k: int(v) for k,v in parse_procfile(scaling).items() if k in workers}) + worker_count.update({k: int(v) for k, v in parse_procfile(scaling).items() if k in workers}) to_create = {} to_destroy = {} for k, v in worker_count.items(): - to_create[k] = range(1,worker_count[k] + 1) + to_create[k] = range(1, worker_count[k] + 1) if k in deltas and deltas[k]: to_create[k] = range(1, worker_count[k] + deltas[k] + 1) if deltas[k] < 0: to_destroy[k] = range(worker_count[k], worker_count[k] + deltas[k], -1) - worker_count[k] = worker_count[k]+deltas[k] + worker_count[k] = worker_count[k] + deltas[k] # Cleanup env for k, v in list(env.items()): @@ -819,62 +832,61 @@ def spawn_worker(app, kind, command, env, ordinal=1): log_file = join(LOG_ROOT, app, kind) settings = [ - ('chdir', join(APP_ROOT, app)), - ('master', 'true'), - ('project', app), - ('max-requests', env.get('UWSGI_MAX_REQUESTS', '1024')), - ('listen', env.get('UWSGI_LISTEN', '16')), - ('processes', env.get('UWSGI_PROCESSES', '1')), - ('procname-prefix', '{app:s}:{kind:s}'.format(**locals())), - ('enable-threads', env.get('UWSGI_ENABLE_THREADS', 'true').lower()), + ('chdir', join(APP_ROOT, app)), + ('master', 'true'), + ('project', app), + ('max-requests', env.get('UWSGI_MAX_REQUESTS', '1024')), + ('listen', env.get('UWSGI_LISTEN', '16')), + ('processes', env.get('UWSGI_PROCESSES', '1')), + ('procname-prefix', '{app:s}:{kind:s}'.format(**locals())), + ('enable-threads', env.get('UWSGI_ENABLE_THREADS', 'true').lower()), ('log-x-forwarded-for', env.get('UWSGI_LOG_X_FORWARDED_FOR', 'false').lower()), - ('log-maxsize', env.get('UWSGI_LOG_MAXSIZE', UWSGI_LOG_MAXSIZE)), - ('logto', '{log_file:s}.{ordinal:d}.log'.format(**locals())), - ('log-backupname', '{log_file:s}.{ordinal:d}.log.old'.format(**locals())), + ('log-maxsize', env.get('UWSGI_LOG_MAXSIZE', UWSGI_LOG_MAXSIZE)), + ('logto', '{log_file:s}.{ordinal:d}.log'.format(**locals())), + ('log-backupname', '{log_file:s}.{ordinal:d}.log.old'.format(**locals())), ] # only add virtualenv to uwsgi if it's a real virtualenv if exists(join(env_path, "bin", "activate_this.py")): settings.append(('virtualenv', env_path)) - if kind== 'jwsgi': + if kind == 'jwsgi': settings.extend([ ('module', command), - ('threads', env.get('UWSGI_THREADS','4')), + ('threads', env.get('UWSGI_THREADS', '4')), ('plugin', 'jvm'), ('plugin', 'jwsgi') ]) - python_version = int(env.get('PYTHON_VERSION','3')) + python_version = int(env.get('PYTHON_VERSION', '3')) if kind == 'wsgi': settings.extend([ - ('module', command), - ('threads', env.get('UWSGI_THREADS','4')), + ('module', command), + ('threads', env.get('UWSGI_THREADS', '4')), ]) if python_version == 2: settings.extend([ - ('plugin', 'python'), + ('plugin', 'python'), ]) if 'UWSGI_GEVENT' in env: settings.extend([ - ('plugin', 'gevent_python'), - ('gevent', env['UWSGI_GEVENT']), + ('plugin', 'gevent_python'), + ('gevent', env['UWSGI_GEVENT']), ]) elif 'UWSGI_ASYNCIO' in env: settings.extend([ - ('plugin', 'asyncio_python'), + ('plugin', 'asyncio_python'), ]) elif python_version == 3: settings.extend([ - ('plugin', 'python3'), + ('plugin', 'python3'), ]) if 'UWSGI_ASYNCIO' in env: settings.extend([ - ('plugin', 'asyncio_python3'), + ('plugin', 'asyncio_python3'), ]) - # If running under nginx, don't expose a port at all if 'NGINX_SERVER_NAME' in env: sock = join(NGINX_ROOT, "{}.sock".format(app)) @@ -886,7 +898,7 @@ def spawn_worker(app, kind, command, env, ordinal=1): else: echo("-----> nginx will talk to uWSGI via {BIND_ADDRESS:s}:{PORT:s}".format(**env), fg='yellow') settings.extend([ - ('http', '{BIND_ADDRESS:s}:{PORT:s}'.format(**env)), + ('http', '{BIND_ADDRESS:s}:{PORT:s}'.format(**env)), ('http-socket', '{BIND_ADDRESS:s}:{PORT:s}'.format(**env)), ]) elif kind == 'web': @@ -898,7 +910,8 @@ def spawn_worker(app, kind, command, env, ordinal=1): settings.append(('attach-daemon', command)) if kind in ['wsgi', 'web']: - settings.append(('log-format','%%(addr) - %%(user) [%%(ltime)] "%%(method) %%(uri) %%(proto)" %%(status) %%(size) "%%(referer)" "%%(uagent)" %%(msecs)ms')) + settings.append(('log-format', + '%%(addr) - %%(user) [%%(ltime)] "%%(method) %%(uri) %%(proto)" %%(status) %%(size) "%%(referer)" "%%(uagent)" %%(msecs)ms')) # remove unnecessary variables from the env in nginx.ini for k in ['NGINX_ACL']: @@ -919,10 +932,13 @@ def spawn_worker(app, kind, command, env, ordinal=1): copyfile(available, enabled) + def do_restart(app): + """Restarts a deployed app""" + config = glob(join(UWSGI_ENABLED, '{}*.ini'.format(app))) - if len(config): + if len(config) > 0: echo("Restarting app '{}'...".format(app), fg='yellow') for c in config: remove(c) @@ -966,9 +982,7 @@ def multi_tail(app, filenames, catch_up=20): # Check for updates on every file for f in filenames: line = peek(files[f]) - if not line: - continue - else: + if line: updated = True yield "{} | {}".format(prefixes[f].ljust(longest), line) @@ -987,6 +1001,8 @@ def multi_tail(app, filenames, catch_up=20): # === CLI commands === CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + + @group(context_settings=CONTEXT_SETTINGS) def piku(): """The smallest PaaS you've ever seen""" @@ -1118,12 +1134,12 @@ def cmd_destroy(app): for p in [join(x, '{}*.ini'.format(app)) for x in [UWSGI_AVAILABLE, UWSGI_ENABLED]]: g = glob(p) - if len(g): + if len(g) > 0: for f in g: echo("Removing file '{}'".format(f), fg='yellow') remove(f) - nginx_files = [join(NGINX_ROOT, "{}.{}".format(app,x)) for x in ['conf','sock','key','crt']] + nginx_files = [join(NGINX_ROOT, "{}.{}".format(app, x)) for x in ['conf', 'sock', 'key', 'crt']] for f in nginx_files: if exists(f): echo("Removing file '{}'".format(f), fg='yellow') @@ -1147,7 +1163,7 @@ def cmd_logs(app, process): app = exit_if_invalid(app) logfiles = glob(join(LOG_ROOT, app, process + '.*.log')) - if len(logfiles): + if len(logfiles) > 0: for line in multi_tail(app, logfiles): echo(line.strip(), fg='white') else: @@ -1177,12 +1193,12 @@ def cmd_ps_scale(app, settings): app = exit_if_invalid(app) config_file = join(ENV_ROOT, app, 'SCALING') - worker_count = {k:int(v) for k, v in parse_procfile(config_file).items()} + worker_count = {k: int(v) for k, v in parse_procfile(config_file).items()} deltas = {} for s in settings: try: k, v = map(lambda x: x.strip(), s.split("=", 1)) - c = int(v) # check for integer value + c = int(v) # check for integer value if c < 0: echo("Error: cannot scale type '{}' below 0".format(k), fg='red') return @@ -1209,9 +1225,10 @@ def cmd_run(app, cmd): for f in [stdout, stderr]: fl = fcntl(f, F_GETFL) fcntl(f, F_SETFL, fl | O_NONBLOCK) - p = Popen(' '.join(cmd), stdin=stdin, stdout=stdout, stderr=stderr, env=environ, cwd=join(APP_ROOT,app), shell=True) + p = Popen(' '.join(cmd), stdin=stdin, stdout=stdout, stderr=stderr, env=environ, cwd=join(APP_ROOT, app), shell=True) p.communicate() + @piku.command("restart") @argument('app') def cmd_restart(app): @@ -1226,7 +1243,7 @@ def cmd_restart(app): def cmd_setup(): """Initialize environment""" - echo("Running in Python {}".format(".".join(map(str,version_info)))) + echo("Running in Python {}".format(".".join(map(str, version_info)))) # Create required paths for p in [APP_ROOT, GIT_ROOT, ENV_ROOT, UWSGI_ROOT, UWSGI_AVAILABLE, UWSGI_ENABLED, LOG_ROOT, NGINX_ROOT]: @@ -1236,25 +1253,25 @@ def cmd_setup(): # Set up the uWSGI emperor config settings = [ - ('chdir', UWSGI_ROOT), - ('emperor', UWSGI_ENABLED), - ('log-maxsize', UWSGI_LOG_MAXSIZE), - ('logto', join(UWSGI_ROOT, 'uwsgi.log')), - ('log-backupname', join(UWSGI_ROOT, 'uwsgi.old.log')), - ('socket', join(UWSGI_ROOT, 'uwsgi.sock')), - ('uid', getpwuid(getuid()).pw_name), - ('gid', getgrgid(getgid()).gr_name), - ('enable-threads', 'true'), - ('threads', '{}'.format(cpu_count() * 2)), + ('chdir', UWSGI_ROOT), + ('emperor', UWSGI_ENABLED), + ('log-maxsize', UWSGI_LOG_MAXSIZE), + ('logto', join(UWSGI_ROOT, 'uwsgi.log')), + ('log-backupname', join(UWSGI_ROOT, 'uwsgi.old.log')), + ('socket', join(UWSGI_ROOT, 'uwsgi.sock')), + ('uid', getpwuid(getuid()).pw_name), + ('gid', getgrgid(getgid()).gr_name), + ('enable-threads', 'true'), + ('threads', '{}'.format(cpu_count() * 2)), ] - with open(join(UWSGI_ROOT,'uwsgi.ini'), 'w') as h: + with open(join(UWSGI_ROOT, 'uwsgi.ini'), 'w') as h: h.write('[uwsgi]\n') # pylint: disable=unused-variable for k, v in settings: h.write("{k:s} = {v}\n".format(**locals())) # mark this script as executable (in case we were invoked via interpreter) - if not(stat(PIKU_SCRIPT).st_mode & S_IXUSR): + if not (stat(PIKU_SCRIPT).st_mode & S_IXUSR): echo("Setting '{}' as executable.".format(PIKU_SCRIPT), fg='yellow') chmod(PIKU_SCRIPT, stat(PIKU_SCRIPT).st_mode | S_IXUSR) @@ -1273,7 +1290,7 @@ def cmd_setup_ssh(public_key_file): setup_authorized_keys(fingerprint, PIKU_SCRIPT, key) except Exception: echo("Error: invalid public key file '{}': {}".format(key_file, format_exc()), fg='red') - elif '-' == public_key_file: + elif public_key_file == '-': buffer = "".join(stdin.readlines()) with NamedTemporaryFile(mode="w") as f: f.write(buffer) @@ -1293,7 +1310,7 @@ def cmd_stop(app): app = exit_if_invalid(app) config = glob(join(UWSGI_ENABLED, '{}*.ini'.format(app))) - if len(config): + if len(config) > 0: echo("Stopping app '{}'...".format(app), fg='yellow') for c in config: remove(c) @@ -1349,7 +1366,7 @@ cat | PIKU_ROOT="{PIKU_ROOT:s}" {PIKU_SCRIPT:s} git-hook {app:s}""".format(**env @piku.command("git-upload-pack", hidden=True) @argument('app') -def cmd_git_receive_pack(app): +def cmd_git_upload_pack(app): """INTERNAL: Handle git upload pack for an app""" app = sanitize_app_name(app) env = globals() From aac4f058d03a8526031e1f9bcedc7f7aba7306e2 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 18:38:49 +0000 Subject: [PATCH 04/34] Create pylint.yml Adds a GitHub Action to run pylint --- .github/workflows/pylint.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/pylint.yml diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000..e2e4813 --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,3 @@ +- name: GitHub Action for pylint + uses: cclauss/GitHub-Action-for-pylint@0.7.0 + args: "pip install -r requirements.txt ; pylint **/*.py" From 9e3636c028fcc7d2665378bb11424c0de0a7e485 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 18:44:44 +0000 Subject: [PATCH 05/34] Update pylint.yml switch to black --- .github/workflows/pylint.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index e2e4813..b8bf043 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,3 +1,2 @@ -- name: GitHub Action for pylint - uses: cclauss/GitHub-Action-for-pylint@0.7.0 - args: "pip install -r requirements.txt ; pylint **/*.py" +- name: Python style check + uses: bulv1ne/python-style-check@v0.3 From b495a9e9bc8c9018625b54944961255e94a3846f Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 18:47:22 +0000 Subject: [PATCH 06/34] Update pylint.yml --- .github/workflows/pylint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index b8bf043..52052a3 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,2 +1,4 @@ - name: Python style check uses: bulv1ne/python-style-check@v0.3 +- name: Black Code Formatter + uses: lgeiger/black-action@v1.0.1 From 26cf27abde55a3714f2c960fad4265cabc39878d Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 18:56:37 +0000 Subject: [PATCH 07/34] Update pylint.yml --- .github/workflows/pylint.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 52052a3..f7d1346 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,4 +1,16 @@ -- name: Python style check - uses: bulv1ne/python-style-check@v0.3 -- name: Black Code Formatter - uses: lgeiger/black-action@v1.0.1 +name: Python Testing + +on: [push] + +jobs: + name: Linting + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + steps: + - name: Python style check + uses: bulv1ne/python-style-check@v0.3 + - name: Black Code Formatter + uses: lgeiger/black-action@v1.0.1 From 4108ae0f2ce7a7818a7fef45b1c31ed1552a49ca Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 18:58:36 +0000 Subject: [PATCH 08/34] Update pylint.yml --- .github/workflows/pylint.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index f7d1346..3b62f8c 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -5,12 +5,12 @@ on: [push] jobs: name: Linting runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.5, 3.6, 3.7, 3.8] - steps: - - name: Python style check - uses: bulv1ne/python-style-check@v0.3 - - name: Black Code Formatter - uses: lgeiger/black-action@v1.0.1 + strategy: + max-parallel: 4 + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + steps: + - name: Python style check + uses: bulv1ne/python-style-check@v0.3 + - name: Black Code Formatter + uses: lgeiger/black-action@v1.0.1 From 4480ca415e6c5d76c594d66c2ccace1c4cdc251d Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:00:58 +0000 Subject: [PATCH 09/34] Update pylint.yml --- .github/workflows/pylint.yml | 42 +++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 3b62f8c..6ac2924 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,16 +1,32 @@ -name: Python Testing - on: [push] jobs: - name: Linting - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.5, 3.6, 3.7, 3.8] - steps: - - name: Python style check - uses: bulv1ne/python-style-check@v0.3 - - name: Black Code Formatter - uses: lgeiger/black-action@v1.0.1 + build: + + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pip install pytest + pytest From d01242eeab6ff0a58934f317e6abbb10a68844dc Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:01:20 +0000 Subject: [PATCH 10/34] Update pylint.yml --- .github/workflows/pylint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 6ac2924..d115c02 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,3 +1,5 @@ +name: Testing + on: [push] jobs: From 2f8a573aa2968f76bd6aa6b4022e30b88f8528b8 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:03:38 +0000 Subject: [PATCH 11/34] Update pylint.yml --- .github/workflows/pylint.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index d115c02..d724bcd 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -28,7 +28,3 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pip install pytest - pytest From 8320b297e6074dfcc9073efb11bc3a64a8e49e8f Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:10:00 +0000 Subject: [PATCH 12/34] Update pylint.yml --- .github/workflows/pylint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index d724bcd..84418a6 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,4 +1,4 @@ -name: Testing +name: Linting on: [push] From d647311878579f3bc434e2b156ede404d90a015c Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:15:38 +0000 Subject: [PATCH 13/34] Update pylint.yml --- .github/workflows/pylint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 84418a6..5600a56 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,6 +1,6 @@ name: Linting -on: [push] +on: [push, pull_request] jobs: build: From 2afa655f72ddb2271f4f3b1b049e476c5e8f7bc6 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:19:50 +0000 Subject: [PATCH 14/34] Update pylint.yml --- .github/workflows/pylint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 5600a56..310a5b4 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -26,5 +26,5 @@ jobs: pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + # --exit-zero treats all errors as warnings. + flake8 . --count --max-complexity=50 --max-line-length=127 --statistics From 28ec78ad4721aa9acbc4e88ca351998c563ab46b Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:23:41 +0000 Subject: [PATCH 15/34] Remove Internet Explorer 6 gzip disable --- piku.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piku.py b/piku.py index 975c468..1a88878 100755 --- a/piku.py +++ b/piku.py @@ -119,7 +119,6 @@ NGINX_COMMON_FRAGMENT = """ gzip_comp_level 7; gzip_min_length 2048; gzip_vary on; - gzip_disable "MSIE [1-6]\.(?!.*SV1)"; # set a custom header for requests add_header X-Deployed-By Piku; From 173600aa13f17f77e8079b653f95410620f452a5 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:25:58 +0000 Subject: [PATCH 16/34] Allow longer lines for inlining SSH entries --- .github/workflows/pylint.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 310a5b4..d9f3da4 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -26,5 +26,6 @@ jobs: pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # --exit-zero treats all errors as warnings. - flake8 . --count --max-complexity=50 --max-line-length=127 --statistics + # --exit-zero treats all errors as warnings + # allow longer lines for inlining SSH entries + flake8 . --count --max-complexity=50 --max-line-length=255 --statistics From 135c771a1f71d73bfb25668213b241cc15604ff6 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:27:20 +0000 Subject: [PATCH 17/34] Keep pyflake happy for now --- piku.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piku.py b/piku.py index 1a88878..82086e9 100755 --- a/piku.py +++ b/piku.py @@ -254,7 +254,7 @@ def parse_procfile(filename): try: kind, command = map(lambda x: x.strip(), line.split(":", 1)) workers[kind] = command - except: + except Exception: echo("Warning: unrecognized Procfile entry '{}' at line {}".format(line, line_number), fg='yellow') if len(workers) == 0: return {} @@ -281,7 +281,7 @@ def command_output(cmd): try: env = environ return str(check_output(cmd, stderr=STDOUT, env=env, shell=True)) - except: + except Exception: return "" From f689992e50587e1356293f51361b949a7d4281ab Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:37:37 +0000 Subject: [PATCH 18/34] pyflake8 changes --- .github/workflows/pylint.yml | 4 ++-- piku.py | 15 +++++---------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index d9f3da4..bfece5b 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -27,5 +27,5 @@ jobs: # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # --exit-zero treats all errors as warnings - # allow longer lines for inlining SSH entries - flake8 . --count --max-complexity=50 --max-line-length=255 --statistics + # allow longer lines for inlining SSH entries and set a complexity threshold that will pass for now + flake8 . --count --max-complexity=60 --max-line-length=255 --statistics diff --git a/piku.py b/piku.py index 82086e9..31747a3 100755 --- a/piku.py +++ b/piku.py @@ -298,7 +298,7 @@ def parse_settings(filename, env={}): try: k, v = map(lambda x: x.strip(), line.split("=", 1)) env[k] = expandvars(v, env) - except: + except Exception: echo("Error: malformed setting '{}', ignoring file.".format(line), fg='red') return {} return env @@ -328,8 +328,6 @@ def do_deploy(app, deltas={}, newrev=None): app_path = join(APP_ROOT, app) procfile = join(app_path, 'Procfile') log_path = join(LOG_ROOT, app) - env_file = join(APP_ROOT, app, 'ENV') - config_file = join(ENV_ROOT, app, 'ENV') env = {'GIT_WORK_DIR': app_path} if exists(app_path): @@ -384,7 +382,6 @@ def deploy_gradle(app, deltas={}): java_path = join(ENV_ROOT, app) build_path = join(APP_ROOT, app, 'build') env_file = join(APP_ROOT, app, 'ENV') - build = join(APP_ROOT, app, 'build.gradle') env = { 'VIRTUAL_ENV': java_path, @@ -416,7 +413,6 @@ def deploy_java(app, deltas={}): java_path = join(ENV_ROOT, app) target_path = join(APP_ROOT, app, 'target') env_file = join(APP_ROOT, app, 'ENV') - pom = join(APP_ROOT, app, 'pom.xml') env = { 'VIRTUAL_ENV': java_path, @@ -447,7 +443,6 @@ def deploy_clojure(app, deltas={}): virtual = join(ENV_ROOT, app) target_path = join(APP_ROOT, app, 'target') env_file = join(APP_ROOT, app, 'ENV') - projectfile = join(APP_ROOT, app, 'project.clj') if not exists(target_path): makedirs(virtual) @@ -710,7 +705,7 @@ def spawn_app(app, deltas={}): if env.get('NGINX_CLOUDFLARE_ACL', 'false').lower() == 'true': try: cf = loads(urlopen('https://api.cloudflare.com/client/v4/ips').read().decode("utf-8")) - if cf['success'] == True: + if cf['success'] is True: for i in cf['result']['ipv4_cidrs']: acl.append("allow {};".format(i)) for i in cf['result']['ipv6_cidrs']: @@ -768,7 +763,7 @@ def spawn_app(app, deltas={}): # prevent broken config from breaking other deploys try: nginx_config_test = str(check_output("nginx -t 2>&1 | grep {}".format(app), env=environ, shell=True)) - except: + except Exception: nginx_config_test = None if nginx_config_test: echo("Error: [nginx config] {}".format(nginx_config_test), fg='red') @@ -1075,7 +1070,7 @@ def cmd_config_set(app, settings): k, v = map(lambda x: x.strip(), s.split("=", 1)) env[k] = v echo("Setting {k:s}={v} for '{app:s}'".format(**locals()), fg='white') - except: + except Exception: echo("Error: malformed setting '{}'".format(s), fg='red') return write_config(config_file, env) @@ -1209,7 +1204,7 @@ def cmd_ps_scale(app, settings): echo("Error: worker type '{}' not present in '{}'".format(k, app), fg='red') return deltas[k] = c - worker_count[k] - except: + except Exception: echo("Error: malformed setting '{}'".format(s), fg='red') return do_deploy(app, deltas) From 56d4e8bd91d560c56d746d687da2530db2b6ead9 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 20 Nov 2019 19:41:04 +0000 Subject: [PATCH 19/34] reinstate gzip_disable --- .github/workflows/pylint.yml | 2 +- piku.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index bfece5b..3736878 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -28,4 +28,4 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # --exit-zero treats all errors as warnings # allow longer lines for inlining SSH entries and set a complexity threshold that will pass for now - flake8 . --count --max-complexity=60 --max-line-length=255 --statistics + flake8 . --ignore=W605 --count --max-complexity=60 --max-line-length=255 --statistics diff --git a/piku.py b/piku.py index 31747a3..e1a40a5 100755 --- a/piku.py +++ b/piku.py @@ -119,6 +119,7 @@ NGINX_COMMON_FRAGMENT = """ gzip_comp_level 7; gzip_min_length 2048; gzip_vary on; + gzip_disable "MSIE [1-6]\.(?!.*SV1)"; # set a custom header for requests add_header X-Deployed-By Piku; From 288c8c5e7a4b4b8296c9b854ad46985dfa458690 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Thu, 21 Nov 2019 13:21:05 +0000 Subject: [PATCH 20/34] Document current flake8 settings (discussed in #95) --- .github/workflows/pylint.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 3736878..0f8836c 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -4,11 +4,12 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest strategy: max-parallel: 4 matrix: + # This is currently overkill since we are targeting 3.5, but affords us visibility + # onto syntax changes in newer Pythons python-version: [3.5, 3.6, 3.7, 3.8] steps: @@ -26,6 +27,10 @@ jobs: pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # --exit-zero treats all errors as warnings - # allow longer lines for inlining SSH entries and set a complexity threshold that will pass for now + # Notes: + # --exit-zero treats all errors as warnings, but we don't use it anymore + # Allow longer lines for inlining SSH entries and set a complexity threshold that will pass for now + # Ignore W605 (https://lintlyci.github.io/Flake8Rules/rules/W605.html) because + # flake8 does not understand escaping dots inside templates for nginx and SSH + # TODO: pare down complexity and line length as we shrink piku core flake8 . --ignore=W605 --count --max-complexity=60 --max-line-length=255 --statistics From d60f4d44da8fd4a1578bad88c1d777ac92a7d675 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Thu, 21 Nov 2019 13:35:04 +0000 Subject: [PATCH 21/34] Moved old text to prep for CONTRIBUTING and GOALS as per #134 --- README.md | 87 ++++----------------------------------------- docs/FAQ.md | 29 +++++++++++++++ docs/old/ROADMAP.md | 58 ++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 81 deletions(-) create mode 100644 docs/FAQ.md create mode 100644 docs/old/ROADMAP.md diff --git a/README.md b/README.md index c4f6643..2034a27 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ The tiniest Heroku/CloudFoundry-like PaaS you've ever seen. ### Documentation: [Procfile](docs/DESIGN.md#procfile-format) | [ENV](./docs/ENV.md) | [Examples](./examples/README.md) +## Goals and Motivation + +(New text being summarized and drafted in #134, soon to find its way here) + +I kept finding myself wanting an Heroku/CloudFoundry-like way to deploy stuff on a few remote ARM boards and [my Raspberry Pi cluster][raspi-cluster], but since [dokku][dokku] didn't work on ARM at the time and even `docker` can be overkill sometimes, I decided to roll my own. + ## Using `piku` `piku` supports a Heroku-like workflow, like so: @@ -103,67 +109,6 @@ You can also use `piku-bootstrap` to run your own Ansible playbooks like this: piku-bootstrap root@yourserver.net ./myplaybook.yml ``` -## Examples - -You can find examples for deploying various kinds of apps into a `piku` server in the [Examples folder](./examples). - -## Motivation - -I kept finding myself wanting an Heroku/CloudFoundry-like way to deploy stuff on a few remote ARM boards and [my Raspberry Pi cluster][raspi-cluster], but since [dokku][dokku] didn't work on ARM at the time and even `docker` can be overkill sometimes, I decided to roll my own. - -## Project Status/To Do: - -This is currently being used for production deployments of [my website](https://taoofmac.com) and a few other projects of mine that run on Azure and other IaaS providers. Regardless, there is still room for improvement: - -From the bottom up: - -- [ ] Prebuilt Raspbian image with everything baked in -- [ ] `chroot`/namespace isolation (tentative) -- [ ] Relay commands to other nodes -- [ ] Proxy deployments to other nodes (build on one box, deploy to many) -- [ ] Support Clojure/Java deployments through `boot` or `lein` -- [ ] Sample Go app -- [ ] Support Go deployments (in progress) -- [ ] nginx SSL optimization/cypher suites, own certificates -- [ ] Review deployment messages -- [ ] WIP: Review docs/CLI command documentation (short descriptions done, need `help ` and better descriptions) -- [ ] Lua/WSAPI support -- [x] Support for Java Apps with maven/gradle (in progress through jwsgi, by @matrixjnr) -- [x] Django and Wisp examples (by @chr15m) -- [x] Project logo (by @chr15m) -- [x] Various release/deployment improvements (by @chr15m) -- [x] Support Node deployments (by @chr15m) -- [x] Let's Encrypt support (by @chr15m) -- [x] Allow setting `nginx` IP bindings in `ENV` file (`NGINX_IPV4_ADDRESS` and `NGINX_IPV6_ADDRESS`) -- [x] Cleanups to remove 2.7 syntax internally -- [x] Change to Python 3 runtime as default, with `PYTHON_VERSION = 2` as fallback -- [x] Run in Python 3 only -- [x] (experimental) REPL in `feature/repl` -- [x] Python 3 support through `PYTHON_VERSION = 3` -- [x] static URL mapping to arbitrary paths (hat tip to @carlosefr for `nginx` tuning) -- [x] remote CLI (requires `ssh -t`) -- [x] saner uWSGI logging -- [x] `gevent` activated when `UWSGI_GEVENT = ` -- [x] enable CloudFlare ACL when `NGINX_CLOUDFLARE_ACL = True` -- [x] Autodetect SPDY/HTTPv2 support and activate it -- [x] Basic nginx SSL config with self-signed certificates and UNIX domain socket connection -- [x] nginx support - creates an nginx config file if `NGINX_SERVER_NAME` is defined -- [x] Testing with pre-packaged [uWSGI][uwsgi] versions on Debian Jessie (yes, it was painful) -- [x] Support barebones binary deployments -- [x] Complete installation instructions (see `INSTALL.md`, which also has a draft of Go installation steps) -- [x] Installation helper/SSH key setup -- [x] Worker scaling -- [x] Remote CLI commands for changing/viewing applied/live settings -- [x] Remote tailing of all logfiles for a single application -- [x] HTTP port selection (and per-app environment variables) -- [x] Sample Python app -- [X] `Procfile` support (`wsgi` and `worker` processes for now, `web` processes being tested) -- [x] Basic CLI commands to manage apps -- [x] `virtualenv` isolation -- [x] Support Python deployments -- [x] Repo creation upon first push -- [x] Basic understanding of [how `dokku` works](http://off-the-stack.moorman.nu/2013-11-23-how-dokku-works.html) - ## Internals This is an illustrated example of how `piku` works for a Python deployment: @@ -185,26 +130,6 @@ But there are already a few folk using `piku` on vanilla `x64` Linux without any `piku` currently supports deploying apps (and dependencies) written in Python, with Go, Clojure (Java) and Node (see [above](#project-statustodo)) in the works. But if it can be invoked from a shell, it can be run inside `piku`. -## FAQ - -**Q:** Why `piku`? - -**A:** Partly because it's supposed to run on a [Pi][pi], because it's Japanese onomatopeia for 'twitch' or 'jolt', and because I know the name will annoy some of my friends. - -**Q:** Why Python/why not Go? - -**A:** I actually thought about doing this in Go right off the bat, but [click][click] is so cool and I needed to have [uWSGI][uwsgi] running anyway, so I caved in. But I'm very likely to take something like [suture](https://github.com/thejerf/suture) and port this across, doing away with [uWSGI][uwsgi] altogether. - -Go also (at the time) did not have a way to vendor dependencies that I was comfortable with, and that is also why Go support fell behind. Hopefully that will change soon. - -**Q:** Does it run under Python 3? - -**A:** Right now, it _only_ runs on Python 3, even though it can deploy apps written in both major versions. It began its development using 2.7 and using`click` for abstracting the simpler stuff, and I eventually switched over to 3.5 once it was supported in Debian Stretch and Raspbian since I wanted to make installing it on the Raspberry Pi as simple as possible. - -**Q:** Why not just use `dokku`? - -**A:** I used `dokku` daily for most of my personal stuff for a good while. But it relied on a number of `x64` containers that needed to be completely rebuilt for ARM, and when I decided I needed something like this (March 2016) that was barely possible - `docker` itself was not fully baked for ARM yet, and people were at the time trying to get `herokuish` and `buildstep` to build on ARM. - [click]: http://click.pocoo.org [pi]: http://www.raspberrypi.org [dokku]: https://github.com/dokku/dokku diff --git a/docs/FAQ.md b/docs/FAQ.md new file mode 100644 index 0000000..a547486 --- /dev/null +++ b/docs/FAQ.md @@ -0,0 +1,29 @@ +# FAQ + +**Q:** Why `piku`? + +**A:** Partly because it's supposed to run on a [Pi][pi], because it's Japanese onomatopeia for 'twitch' or 'jolt', and because I know the name will annoy some of my friends. + +**Q:** Why Python/why not Go? + +**A:** I actually thought about doing this in Go right off the bat, but [click][click] is so cool and I needed to have [uWSGI][uwsgi] running anyway, so I caved in. But I'm very likely to take something like [suture](https://github.com/thejerf/suture) and port this across, doing away with [uWSGI][uwsgi] altogether. + +Go also (at the time) did not have a way to vendor dependencies that I was comfortable with, and that is also why Go support fell behind. Hopefully that will change soon. + +**Q:** Does it run under Python 3? + +**A:** Right now, it _only_ runs on Python 3, even though it can deploy apps written in both major versions. It began its development using 2.7 and using`click` for abstracting the simpler stuff, and I eventually switched over to 3.5 once it was supported in Debian Stretch and Raspbian since I wanted to make installing it on the Raspberry Pi as simple as possible. + +**Q:** Why not just use `dokku`? + +**A:** I used `dokku` daily for most of my personal stuff for a good while. But it relied on a number of `x64` containers that needed to be completely rebuilt for ARM, and when I decided I needed something like this (March 2016) that was barely possible - `docker` itself was not fully baked for ARM yet, and people were at the time trying to get `herokuish` and `buildstep` to build on ARM. + + + +[click]: http://click.pocoo.org +[pi]: http://www.raspberrypi.org +[dokku]: https://github.com/dokku/dokku +[raspi-cluster]: https://github.com/rcarmo/raspi-cluster +[cygwin]: http://www.cygwin.com +[uwsgi]: https://github.com/unbit/uwsgi +[wsl]: https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux diff --git a/docs/old/ROADMAP.md b/docs/old/ROADMAP.md new file mode 100644 index 0000000..f693e5f --- /dev/null +++ b/docs/old/ROADMAP.md @@ -0,0 +1,58 @@ +# Roadmap + +This was the original roadmap, filed here in 2019-11-21 for future reference: + +- [ ] Prebuilt Raspbian image with everything baked in +- [ ] `chroot`/namespace isolation (tentative) +- [ ] Relay commands to other nodes +- [ ] Proxy deployments to other nodes (build on one box, deploy to many) +- [ ] Support Clojure/Java deployments through `boot` or `lein` +- [ ] Sample Go app +- [ ] Support Go deployments (in progress) +- [ ] nginx SSL optimization/cypher suites, own certificates +- [ ] Review deployment messages +- [ ] WIP: Review docs/CLI command documentation (short descriptions done, need `help ` and better descriptions) +- [ ] Lua/WSAPI support +- [x] Support for Java Apps with maven/gradle (in progress through jwsgi, by @matrixjnr) +- [x] Django and Wisp examples (by @chr15m) +- [x] Project logo (by @chr15m) +- [x] Various release/deployment improvements (by @chr15m) +- [x] Support Node deployments (by @chr15m) +- [x] Let's Encrypt support (by @chr15m) +- [x] Allow setting `nginx` IP bindings in `ENV` file (`NGINX_IPV4_ADDRESS` and `NGINX_IPV6_ADDRESS`) +- [x] Cleanups to remove 2.7 syntax internally +- [x] Change to Python 3 runtime as default, with `PYTHON_VERSION = 2` as fallback +- [x] Run in Python 3 only +- [x] (experimental) REPL in `feature/repl` +- [x] Python 3 support through `PYTHON_VERSION = 3` +- [x] static URL mapping to arbitrary paths (hat tip to @carlosefr for `nginx` tuning) +- [x] remote CLI (requires `ssh -t`) +- [x] saner uWSGI logging +- [x] `gevent` activated when `UWSGI_GEVENT = ` +- [x] enable CloudFlare ACL when `NGINX_CLOUDFLARE_ACL = True` +- [x] Autodetect SPDY/HTTPv2 support and activate it +- [x] Basic nginx SSL config with self-signed certificates and UNIX domain socket connection +- [x] nginx support - creates an nginx config file if `NGINX_SERVER_NAME` is defined +- [x] Testing with pre-packaged [uWSGI][uwsgi] versions on Debian Jessie (yes, it was painful) +- [x] Support barebones binary deployments +- [x] Complete installation instructions (see `INSTALL.md`, which also has a draft of Go installation steps) +- [x] Installation helper/SSH key setup +- [x] Worker scaling +- [x] Remote CLI commands for changing/viewing applied/live settings +- [x] Remote tailing of all logfiles for a single application +- [x] HTTP port selection (and per-app environment variables) +- [x] Sample Python app +- [X] `Procfile` support (`wsgi` and `worker` processes for now, `web` processes being tested) +- [x] Basic CLI commands to manage apps +- [x] `virtualenv` isolation +- [x] Support Python deployments +- [x] Repo creation upon first push +- [x] Basic understanding of [how `dokku` works](http://off-the-stack.moorman.nu/2013-11-23-how-dokku-works.html) + +[click]: http://click.pocoo.org +[pi]: http://www.raspberrypi.org +[dokku]: https://github.com/dokku/dokku +[raspi-cluster]: https://github.com/rcarmo/raspi-cluster +[cygwin]: http://www.cygwin.com +[uwsgi]: https://github.com/unbit/uwsgi +[wsl]: https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux From 37b334509ce0547b9902fa42fa1bcc5e3fd883a3 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Thu, 21 Nov 2019 13:38:42 +0000 Subject: [PATCH 22/34] move image to DESIGN. --- README.md | 7 ------- docs/DESIGN.md | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2034a27..929f1df 100644 --- a/README.md +++ b/README.md @@ -108,13 +108,6 @@ You can also use `piku-bootstrap` to run your own Ansible playbooks like this: ```shell piku-bootstrap root@yourserver.net ./myplaybook.yml ``` - -## Internals - -This is an illustrated example of how `piku` works for a Python deployment: - -![](img/piku.png) - ## Supported Platforms `piku` is intended to work in any POSIX-like environment where you have Python, [uWSGI][uwsgi] and SSH, i.e.: diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 0bddf1a..de6a688 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -86,6 +86,12 @@ As to runtime isolation, `piku` only provides `virtualenv` support until 1.0. Py This separation makes it easier to cope with long/large deployments and restore apps to a pristine condition, since the app will only go live after the deployment clone is reset (via `git checkout -f`). +## Components + +This diagram (available as a `dot` file in the `img` folder) outlines how its components interact: + +![](../img/piku.png + [uwsgi]: https://github.com/unbit/uwsgi [emperor]: http://uwsgi-docs.readthedocs.org/en/latest/Emperor.html [12f]: http://12factor.net From b018a8b321926d44d3cf2c03890c353a1738efcb Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Thu, 21 Nov 2019 13:39:22 +0000 Subject: [PATCH 23/34] Fix typo --- docs/DESIGN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index de6a688..29d2b8d 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -90,7 +90,7 @@ This separation makes it easier to cope with long/large deployments and restore This diagram (available as a `dot` file in the `img` folder) outlines how its components interact: -![](../img/piku.png +![](../img/piku.png) [uwsgi]: https://github.com/unbit/uwsgi [emperor]: http://uwsgi-docs.readthedocs.org/en/latest/Emperor.html From fefff34127adf6e2317fbc2eb2ead75047877292 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Thu, 21 Nov 2019 13:51:10 +0000 Subject: [PATCH 24/34] Add link to new roadmap --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 929f1df..343898a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The tiniest Heroku/CloudFoundry-like PaaS you've ever seen. [![asciicast](https://asciinema.org/a/Ar31IoTkzsZmWWvlJll6p7haS.svg)](https://asciinema.org/a/Ar31IoTkzsZmWWvlJll6p7haS) -### Documentation: [Procfile](docs/DESIGN.md#procfile-format) | [ENV](./docs/ENV.md) | [Examples](./examples/README.md) +### Documentation: [Procfile](docs/DESIGN.md#procfile-format) | [ENV](./docs/ENV.md) | [Examples](./examples/README.md) | [Roadmap](https://github.com/piku/piku/projects/2) ## Goals and Motivation From 8048149bd5e8a1d04a57508f41c1d44528a236a1 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Mon, 25 Nov 2019 18:47:46 +0000 Subject: [PATCH 25/34] Create cloud-init.yml First stab at a cloud-init file for Azure/AWS/GCP --- cloud-config/cloud-init.yml | 64 +++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 cloud-config/cloud-init.yml diff --git a/cloud-config/cloud-init.yml b/cloud-config/cloud-init.yml new file mode 100644 index 0000000..a2c5675 --- /dev/null +++ b/cloud-config/cloud-init.yml @@ -0,0 +1,64 @@ +users: + - name: piku + gecos: PaaS access + primary_group: www-data + groups: www-data + +apt_update: true +apt_upgrade: true + +packages: + - ntp + - tmux + - htop + - vim + - fail2ban + - curl + - build-essential + - certbot + - git + - incron + - libjpeg-dev + - libxml2-dev + - libxslt1-dev + - zlib1g-dev + - nginx + - python-certbot-nginx + - python-dev + - python-pip + - python-virtualenv + - python3-dev + - python3-pip + - python3-click + - python3-virtualenv + - uwsgi + - uwsgi-plugin-asyncio-python3 + - uwsgi-plugin-gevent-python + - uwsgi-plugin-python + - uwsgi-plugin-python3 + - uwsgi-plugin-tornado-python + - nodejs + - npm + +write_files: + - path: /etc/nginx/sites-available/default + content: | + server { + listen 80 default_server; + listen [::]:80 default_server; + root /var/www/html; + index index.html index.htm; + server_name _; + location / { + try_files $uri $uri/ =404; + } + } + include /home/piku/.piku/nginx/*.conf; + - path: /etc/incron.d/paas + content: | + /home/piku/.piku/nginx IN_MODIFY,IN_NO_LOOP /bin/systemctl reload nginx + +runcmd: + - timedatectl set-timezone Europe/Lisbon + - ln /home/piku/.piku/uwsgi/uwsgi.ini /etc/uwsgi/apps-enabled/piku.ini + - sudo su - piku -c "wget https://raw.githubusercontent.com/piku/piku/master/piku.py && python3 ~/piku.py setup From b8b9d38d527423dd946a9e3b9e6a1b69e21a141c Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Mon, 25 Nov 2019 18:49:21 +0000 Subject: [PATCH 26/34] Fix indenting --- cloud-config/cloud-init.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cloud-config/cloud-init.yml b/cloud-config/cloud-init.yml index a2c5675..88019ac 100644 --- a/cloud-config/cloud-init.yml +++ b/cloud-config/cloud-init.yml @@ -43,20 +43,20 @@ packages: write_files: - path: /etc/nginx/sites-available/default content: | - server { - listen 80 default_server; - listen [::]:80 default_server; - root /var/www/html; - index index.html index.htm; - server_name _; - location / { - try_files $uri $uri/ =404; - } - } - include /home/piku/.piku/nginx/*.conf; + server { + listen 80 default_server; + listen [::]:80 default_server; + root /var/www/html; + index index.html index.htm; + server_name _; + location / { + try_files $uri $uri/ =404; + } + } + include /home/piku/.piku/nginx/*.conf; - path: /etc/incron.d/paas content: | - /home/piku/.piku/nginx IN_MODIFY,IN_NO_LOOP /bin/systemctl reload nginx + /home/piku/.piku/nginx IN_MODIFY,IN_NO_LOOP /bin/systemctl reload nginx runcmd: - timedatectl set-timezone Europe/Lisbon From 54b6df5b47f110b64e3c15ef0f02c94a4e44683b Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Mon, 25 Nov 2019 19:44:36 +0000 Subject: [PATCH 27/34] Fix #150 --- piku.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/piku.py b/piku.py index e1a40a5..c147339 100755 --- a/piku.py +++ b/piku.py @@ -832,6 +832,8 @@ def spawn_worker(app, kind, command, env, ordinal=1): settings = [ ('chdir', join(APP_ROOT, app)), + ('uid', getpwuid(getuid()).pw_name), + ('gid', getgrgid(getgid()).gr_name), ('master', 'true'), ('project', app), ('max-requests', env.get('UWSGI_MAX_REQUESTS', '1024')), @@ -841,10 +843,12 @@ def spawn_worker(app, kind, command, env, ordinal=1): ('enable-threads', env.get('UWSGI_ENABLE_THREADS', 'true').lower()), ('log-x-forwarded-for', env.get('UWSGI_LOG_X_FORWARDED_FOR', 'false').lower()), ('log-maxsize', env.get('UWSGI_LOG_MAXSIZE', UWSGI_LOG_MAXSIZE)), + ('logfile-chown', '%s:%s' % (getpwuid(getuid()).pw_name, getgrgid(getgid()).gr_name)), + ('logfile-chmod', '640'), ('logto', '{log_file:s}.{ordinal:d}.log'.format(**locals())), ('log-backupname', '{log_file:s}.{ordinal:d}.log.old'.format(**locals())), ] - + # only add virtualenv to uwsgi if it's a real virtualenv if exists(join(env_path, "bin", "activate_this.py")): settings.append(('virtualenv', env_path)) From 250b1ef1e64d8962a54f99d555f3fc2a98e2a607 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Mon, 25 Nov 2019 19:49:49 +0000 Subject: [PATCH 28/34] Fixes #149 --- .github/workflows/pylint.yml | 8 ++++---- piku.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 0f8836c..0f55f8d 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -6,11 +6,11 @@ jobs: build: runs-on: ubuntu-latest strategy: - max-parallel: 4 + max-parallel: 3 matrix: - # This is currently overkill since we are targeting 3.5, but affords us visibility - # onto syntax changes in newer Pythons - python-version: [3.5, 3.6, 3.7, 3.8] + # This is currently overkill since we are targeting 3.5+ with a support window back to 3.4, + # but affords us visibility onto syntax changes in newer Pythons + python-version: [3.4, 3.5, 3.6, 3.7, 3.8] steps: - uses: actions/checkout@v1 diff --git a/piku.py b/piku.py index c147339..01eea94 100755 --- a/piku.py +++ b/piku.py @@ -4,9 +4,9 @@ try: from sys import version_info - assert version_info >= (3, 5) + assert version_info >= (3, 4) except AssertionError: - exit("Piku requires Python 3.5 or above") + exit("Piku requires Python 3.4 or above") from collections import defaultdict, deque from fcntl import fcntl, F_SETFL, F_GETFL @@ -848,7 +848,7 @@ def spawn_worker(app, kind, command, env, ordinal=1): ('logto', '{log_file:s}.{ordinal:d}.log'.format(**locals())), ('log-backupname', '{log_file:s}.{ordinal:d}.log.old'.format(**locals())), ] - + # only add virtualenv to uwsgi if it's a real virtualenv if exists(join(env_path, "bin", "activate_this.py")): settings.append(('virtualenv', env_path)) From e22b2c1cb8f74fb4cb68d23fa6682c67421b42b6 Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Mon, 25 Nov 2019 19:53:36 +0000 Subject: [PATCH 29/34] revert #149 --- .github/workflows/pylint.yml | 4 ++-- piku.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 0f55f8d..934db01 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -8,9 +8,9 @@ jobs: strategy: max-parallel: 3 matrix: - # This is currently overkill since we are targeting 3.5+ with a support window back to 3.4, + # This is currently overkill since we are targeting 3.5 # but affords us visibility onto syntax changes in newer Pythons - python-version: [3.4, 3.5, 3.6, 3.7, 3.8] + python-version: [3.5, 3.6, 3.7, 3.8] steps: - uses: actions/checkout@v1 diff --git a/piku.py b/piku.py index 01eea94..e465a99 100755 --- a/piku.py +++ b/piku.py @@ -4,9 +4,9 @@ try: from sys import version_info - assert version_info >= (3, 4) + assert version_info >= (3, 5) except AssertionError: - exit("Piku requires Python 3.4 or above") + exit("Piku requires Python 3.5 or above") from collections import defaultdict, deque from fcntl import fcntl, F_SETFL, F_GETFL From 480d2858826a53d180abaf3a792aa5e35d792c1f Mon Sep 17 00:00:00 2001 From: Chris McCormick Date: Tue, 26 Nov 2019 12:20:25 +0800 Subject: [PATCH 30/34] Guard against non-existent plugins dir. --- piku.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/piku.py b/piku.py index c93a96c..f33f924 100755 --- a/piku.py +++ b/piku.py @@ -1386,13 +1386,14 @@ def _get_plugin_commands(path): sys_path.append(abspath(path)) cli_commands = [] - for item in listdir(path): - module_path = join(path, item) - if not isdir(module_path): - continue - module = import_module(item) - if hasattr(module, 'cli_commands'): - cli_commands.append(module.cli_commands()) + if isdir(path): + for item in listdir(path): + module_path = join(path, item) + if not isdir(module_path): + continue + module = import_module(item) + if hasattr(module, 'cli_commands'): + cli_commands.append(module.cli_commands()) return cli_commands From 9e5d56a4ca978e98c56a4f00221f4ecfe5d7e966 Mon Sep 17 00:00:00 2001 From: Chris McCormick Date: Tue, 26 Nov 2019 12:23:35 +0800 Subject: [PATCH 31/34] Guard against plugin modules containing errors. --- piku.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/piku.py b/piku.py index f33f924..d312277 100755 --- a/piku.py +++ b/piku.py @@ -1389,11 +1389,13 @@ def _get_plugin_commands(path): if isdir(path): for item in listdir(path): module_path = join(path, item) - if not isdir(module_path): - continue - module = import_module(item) - if hasattr(module, 'cli_commands'): - cli_commands.append(module.cli_commands()) + if isdir(module_path): + try: + module = import_module(item) + except: + module = None + if hasattr(module, 'cli_commands'): + cli_commands.append(module.cli_commands()) return cli_commands From 8882bb5f61914b16bfab8928663c1c8efc2e1ead Mon Sep 17 00:00:00 2001 From: Chris McCormick Date: Tue, 26 Nov 2019 12:26:31 +0800 Subject: [PATCH 32/34] Fix flake8 issues. --- piku.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/piku.py b/piku.py index d312277..ccb13ea 100755 --- a/piku.py +++ b/piku.py @@ -9,7 +9,6 @@ except AssertionError: exit("Piku requires Python 3.5 or above") from importlib import import_module -from click import argument, command, group, get_current_context, option, secho as echo, pass_context, CommandCollection from collections import defaultdict, deque from fcntl import fcntl, F_SETFL, F_GETFL from glob import glob @@ -20,10 +19,8 @@ from os import chmod, getgid, getuid, symlink, unlink, remove, stat, listdir, en from os.path import abspath, basename, dirname, exists, getmtime, join, realpath, splitext, isdir from re import sub from shutil import copyfile, rmtree, which -from socket import socket, AF_INET, SOCK_STREAM from sys import argv, stdin, stdout, stderr, version_info, exit, path as sys_path from pwd import getpwuid -from shutil import copyfile, rmtree, which from socket import socket, AF_INET, SOCK_STREAM from stat import S_IRUSR, S_IWUSR, S_IXUSR from subprocess import call, check_output, Popen, STDOUT @@ -32,7 +29,7 @@ from time import sleep from traceback import format_exc from urllib.request import urlopen -from click import argument, group, secho as echo, pass_context +from click import argument, command, group, get_current_context, option, secho as echo, pass_context, CommandCollection # === Make sure we can access all system binaries === @@ -258,8 +255,8 @@ def parse_procfile(filename): if line.startswith("#") or not line: continue try: - kind, command = map(lambda x: x.strip(), line.split(":", 1)) - workers[kind] = command + kind, command_line = map(lambda x: x.strip(), line.split(":", 1)) + workers[kind] = command_line except Exception: echo("Warning: unrecognized Procfile entry '{}' at line {}".format(line, line_number), fg='yellow') if len(workers) == 0: From ede54d9563a76f59aa2a1104435e285bd9c2c289 Mon Sep 17 00:00:00 2001 From: Chris McCormick Date: Tue, 26 Nov 2019 12:35:11 +0800 Subject: [PATCH 33/34] Fixed further linting issues. --- piku.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/piku.py b/piku.py index ccb13ea..d6e7ab4 100755 --- a/piku.py +++ b/piku.py @@ -29,7 +29,7 @@ from time import sleep from traceback import format_exc from urllib.request import urlopen -from click import argument, command, group, get_current_context, option, secho as echo, pass_context, CommandCollection +from click import argument, group, secho as echo, pass_context, CommandCollection # === Make sure we can access all system binaries === @@ -255,8 +255,8 @@ def parse_procfile(filename): if line.startswith("#") or not line: continue try: - kind, command_line = map(lambda x: x.strip(), line.split(":", 1)) - workers[kind] = command_line + kind, command = map(lambda x: x.strip(), line.split(":", 1)) + workers[kind] = command except Exception: echo("Warning: unrecognized Procfile entry '{}' at line {}".format(line, line_number), fg='yellow') if len(workers) == 0: @@ -1379,6 +1379,7 @@ def cmd_git_upload_pack(app): # Handle the actual receive. We'll be called with 'git-hook' after it happens call('git-shell -c "{}" '.format(argv[1] + " '{}'".format(app)), cwd=GIT_ROOT, shell=True) + def _get_plugin_commands(path): sys_path.append(abspath(path)) @@ -1396,6 +1397,7 @@ def _get_plugin_commands(path): return cli_commands + @piku.command("help") @pass_context def cmd_help(ctx): From 2dbc5d0233919913289d988598cb452f1c4c427d Mon Sep 17 00:00:00 2001 From: Chris McCormick Date: Tue, 26 Nov 2019 12:39:01 +0800 Subject: [PATCH 34/34] Remove bare exception (linting fix). --- piku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piku.py b/piku.py index d6e7ab4..f394290 100755 --- a/piku.py +++ b/piku.py @@ -1390,7 +1390,7 @@ def _get_plugin_commands(path): if isdir(module_path): try: module = import_module(item) - except: + except Exception: module = None if hasattr(module, 'cli_commands'): cli_commands.append(module.cli_commands())