mirror
/
piku
kopia lustrzana https://github.com/piku/piku
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

build: lite cleanup of pylint results (#141)

pull/136/head^2
dWiGhT 2019-11-20 00:28:02 -08:00 zatwierdzone przez Rui Carmo
rodzic 666f871bc9
commit a93b471d67
1 zmienionych plików z 114 dodań i 97 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

79
piku.py
Wyświetl plik

@ -8,28 +8,27 @@ try:
except AssertionError:
exit("Piku requires Python 3.5 or above")
from click import argument, command, group, get_current_context, option, secho as echo, pass_context
from collections import defaultdict, deque
from datetime import datetime
from fcntl import fcntl, F_SETFL, F_GETFL
from glob import glob
from hashlib import md5
from grp import getgrgid
from json import loads
from multiprocessing import cpu_count
from os import chmod, getgid, getuid, symlink, unlink, remove, stat, listdir, environ, makedirs, O_NONBLOCK
from os.path import abspath, basename, dirname, exists, getmtime, join, realpath, splitext
from pwd import getpwuid
from re import sub
from shutil import copyfile, rmtree, which
from socket import socket, AF_INET, SOCK_STREAM
from sys import argv, stdin, stdout, stderr, version_info, exit
from stat import S_IRUSR, S_IWUSR, S_IXUSR
from subprocess import call, check_output, Popen, STDOUT, PIPE
from subprocess import call, check_output, Popen, STDOUT
from sys import argv, stdin, stdout, stderr, version_info, exit
from tempfile import NamedTemporaryFile
from traceback import format_exc
from time import sleep
from traceback import format_exc
from urllib.request import urlopen
from pwd import getpwuid
from grp import getgrgid
from click import argument, group, secho as echo, pass_context
# === Make sure we can access all system binaries ===
@ -189,6 +188,7 @@ INTERNAL_NGINX_UWSGI_SETTINGS = """
uwsgi_param SERVER_NAME $server_name;
"""
# === Utility functions ===
def sanitize_app_name(app):
@ -253,13 +253,13 @@ def parse_procfile(filename):
workers[kind] = command
except:
echo("Warning: unrecognized Procfile entry '{}'".format(line), fg='yellow')
if not len(workers):
if len(workers) == 0:
return {}
# WSGI trumps regular web workers
if 'wsgi' in workers or 'jwsgi' in workers:
if 'web' in workers:
echo("Warning: found both 'wsgi' and 'web' workers, disabling 'web'", fg='yellow')
del(workers['web'])
del workers['web']
return workers
@ -290,7 +290,7 @@ def parse_settings(filename, env={}):
with open(filename, 'r') as settings:
for line in settings:
if '#' == line[0] or len(line.strip()) == 0: # ignore comments and newlines
if line[0] == '#' or len(line.strip()) == 0: # ignore comments and newlines
continue
try:
k, v = map(lambda x: x.strip(), line.split("=", 1))
@ -312,10 +312,13 @@ def check_requirements(binaries):
return False
return True
def found_app(kind):
"""Helper function to output app detected"""
echo("-----> {} app detected.".format(kind), fg='green')
return True
def do_deploy(app, deltas={}, newrev=None):
"""Deploy an app by resetting the work directory"""
@ -336,11 +339,12 @@ def do_deploy(app, deltas={}, newrev=None):
if not exists(log_path):
makedirs(log_path)
workers = parse_procfile(procfile)
if workers and len(workers):
if workers and len(workers) > 0:
settings = {}
if exists(join(app_path, 'requirements.txt')) and found_app("Python"):
settings.update(deploy_python(app, deltas))
elif exists(join(app_path, 'package.json')) and found_app("Node") and (check_requirements(['nodejs', 'npm']) or check_requirements(['nodeenv'])):
elif exists(join(app_path, 'package.json')) and found_app("Node") and (
check_requirements(['nodejs', 'npm']) or check_requirements(['nodeenv'])):
settings.update(deploy_node(app, deltas))
elif exists(join(app_path, 'pom.xml')) and found_app("Java Maven") and check_requirements(['java', 'mvn']):
settings.update(deploy_java(app, deltas))
@ -371,6 +375,7 @@ def do_deploy(app, deltas={}, newrev=None):
else:
echo("Error: app '{}' not found.".format(app), fg='red')
def deploy_gradle(app, deltas={}):
"""Deploy a Java application using Gradle"""
java_path = join(ENV_ROOT, app)
@ -400,6 +405,7 @@ def deploy_gradle(app, deltas={}):
return spawn_app(app, deltas)
def deploy_java(app, deltas={}):
"""Deploy a Java application using Maven"""
# TODO: Use jenv to isolate Java Application environments
@ -431,6 +437,7 @@ def deploy_java(app, deltas={}):
return spawn_app(app, deltas)
def deploy_clojure(app, deltas={}):
"""Deploy a Clojure Application"""
@ -511,7 +518,8 @@ def deploy_node(app, deltas={}):
version = env.get("NODE_VERSION")
node_binary = join(virtualenv_path, "bin", "node")
installed = check_output("{} -v".format(node_binary), cwd=join(APP_ROOT, app), env=env, shell=True).decode("utf8").rstrip("\n") if exists(node_binary) else ""
installed = check_output("{} -v".format(node_binary), cwd=join(APP_ROOT, app), env=env, shell=True).decode("utf8").rstrip(
"\n") if exists(node_binary) else ""
if version and check_requirements(['nodeenv']):
if not installed.endswith(version):
@ -520,7 +528,8 @@ def deploy_node(app, deltas={}):
echo("Warning: Can't update node with app running. Stop the app & retry.", fg='yellow')
else:
echo("-----> Installing node version '{NODE_VERSION:s}' using nodeenv".format(**env), fg='green')
call("nodeenv --prebuilt --node={NODE_VERSION:s} --clean-src --force {VIRTUAL_ENV:s}".format(**env), cwd=virtualenv_path, env=env, shell=True)
call("nodeenv --prebuilt --node={NODE_VERSION:s} --clean-src --force {VIRTUAL_ENV:s}".format(**env),
cwd=virtualenv_path, env=env, shell=True)
else:
echo("-----> Node is installed at {}.".format(version))
@ -664,7 +673,6 @@ def spawn_app(app, deltas={}):
env['NGINX_SOCKET'] = "{BIND_ADDRESS:s}:{PORT:s}".format(**env)
echo("-----> nginx will look for app '{}' on {}".format(app, env['NGINX_SOCKET']))
domain = env['NGINX_SERVER_NAME'].split()[0]
key, crt = [join(NGINX_ROOT, "{}.{}".format(app, x)) for x in ['key', 'crt']]
if exists(join(ACME_ROOT, "acme.sh")):
@ -680,7 +688,8 @@ def spawn_app(app, deltas={}):
if not exists(key) or not exists(join(ACME_ROOT, domain, domain + ".key")):
echo("-----> getting letsencrypt certificate")
call('{acme:s}/acme.sh --issue -d {domain:s} -w {www:s}'.format(**locals()), shell=True)
call('{acme:s}/acme.sh --install-cert -d {domain:s} --key-file {key:s} --fullchain-file {crt:s}'.format(**locals()), shell=True)
call('{acme:s}/acme.sh --install-cert -d {domain:s} --key-file {key:s} --fullchain-file {crt:s}'.format(
**locals()), shell=True)
if exists(join(ACME_ROOT, domain)) and not exists(join(ACME_WWW, app)):
symlink(join(ACME_ROOT, domain), join(ACME_WWW, app))
else:
@ -689,7 +698,9 @@ def spawn_app(app, deltas={}):
# fall back to creating self-signed certificate if acme failed
if not exists(key) or stat(crt).st_size == 0:
echo("-----> generating self-signed certificate")
call('openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj "/C=US/ST=NY/L=New York/O=Piku/OU=Self-Signed/CN={domain:s}" -keyout {key:s} -out {crt:s}'.format(**locals()), shell=True)
call(
'openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj "/C=US/ST=NY/L=New York/O=Piku/OU=Self-Signed/CN={domain:s}" -keyout {key:s} -out {crt:s}'.format(
**locals()), shell=True)
# restrict access to server from CloudFlare IP addresses
acl = []
@ -730,12 +741,14 @@ def spawn_app(app, deltas={}):
static_url, static_path = item.split(':')
if static_path[0] != '/':
static_path = join(app_path, static_path)
env['INTERNAL_NGINX_STATIC_MAPPINGS'] = env['INTERNAL_NGINX_STATIC_MAPPINGS'] + expandvars(INTERNAL_NGINX_STATIC_MAPPING, locals())
env['INTERNAL_NGINX_STATIC_MAPPINGS'] = env['INTERNAL_NGINX_STATIC_MAPPINGS'] + expandvars(
INTERNAL_NGINX_STATIC_MAPPING, locals())
except Exception as e:
echo("Error {} in static path spec: should be /url1:path1[,/url2:path2], ignoring.".format(e))
env['INTERNAL_NGINX_STATIC_MAPPINGS'] = ''
env['INTERNAL_NGINX_CUSTOM_CLAUSES'] = expandvars(open(join(app_path, env["NGINX_INCLUDE_FILE"])).read(), env) if env.get("NGINX_INCLUDE_FILE") else ""
env['INTERNAL_NGINX_CUSTOM_CLAUSES'] = expandvars(open(join(app_path, env["NGINX_INCLUDE_FILE"])).read(),
env) if env.get("NGINX_INCLUDE_FILE") else ""
env['INTERNAL_NGINX_PORTMAP'] = ""
if 'web' in workers or 'wsgi' in workers or 'jwsgi' in workers:
env['INTERNAL_NGINX_PORTMAP'] = expandvars(NGINX_PORTMAP_FRAGMENT, env)
@ -874,7 +887,6 @@ def spawn_worker(app, kind, command, env, ordinal=1):
('plugin', 'asyncio_python3'),
])
# If running under nginx, don't expose a port at all
if 'NGINX_SERVER_NAME' in env:
sock = join(NGINX_ROOT, "{}.sock".format(app))
@ -898,7 +910,8 @@ def spawn_worker(app, kind, command, env, ordinal=1):
settings.append(('attach-daemon', command))
if kind in ['wsgi', 'web']:
settings.append(('log-format','%%(addr) - %%(user) [%%(ltime)] "%%(method) %%(uri) %%(proto)" %%(status) %%(size) "%%(referer)" "%%(uagent)" %%(msecs)ms'))
settings.append(('log-format',
'%%(addr) - %%(user) [%%(ltime)] "%%(method) %%(uri) %%(proto)" %%(status) %%(size) "%%(referer)" "%%(uagent)" %%(msecs)ms'))
# remove unnecessary variables from the env in nginx.ini
for k in ['NGINX_ACL']:
@ -919,10 +932,13 @@ def spawn_worker(app, kind, command, env, ordinal=1):
copyfile(available, enabled)
def do_restart(app):
"""Restarts a deployed app"""
config = glob(join(UWSGI_ENABLED, '{}*.ini'.format(app)))
if len(config):
if len(config) > 0:
echo("Restarting app '{}'...".format(app), fg='yellow')
for c in config:
remove(c)
@ -966,9 +982,7 @@ def multi_tail(app, filenames, catch_up=20):
# Check for updates on every file
for f in filenames:
line = peek(files[f])
if not line:
continue
else:
if line:
updated = True
yield "{} | {}".format(prefixes[f].ljust(longest), line)
@ -987,6 +1001,8 @@ def multi_tail(app, filenames, catch_up=20):
# === CLI commands ===
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
@group(context_settings=CONTEXT_SETTINGS)
def piku():
"""The smallest PaaS you've ever seen"""
@ -1118,7 +1134,7 @@ def cmd_destroy(app):
for p in [join(x, '{}*.ini'.format(app)) for x in [UWSGI_AVAILABLE, UWSGI_ENABLED]]:
g = glob(p)
if len(g):
if len(g) > 0:
for f in g:
echo("Removing file '{}'".format(f), fg='yellow')
remove(f)
@ -1147,7 +1163,7 @@ def cmd_logs(app, process):
app = exit_if_invalid(app)
logfiles = glob(join(LOG_ROOT, app, process + '.*.log'))
if len(logfiles):
if len(logfiles) > 0:
for line in multi_tail(app, logfiles):
echo(line.strip(), fg='white')
else:
@ -1212,6 +1228,7 @@ def cmd_run(app, cmd):
p = Popen(' '.join(cmd), stdin=stdin, stdout=stdout, stderr=stderr, env=environ, cwd=join(APP_ROOT, app), shell=True)
p.communicate()
@piku.command("restart")
@argument('app')
def cmd_restart(app):
@ -1273,7 +1290,7 @@ def cmd_setup_ssh(public_key_file):
setup_authorized_keys(fingerprint, PIKU_SCRIPT, key)
except Exception:
echo("Error: invalid public key file '{}': {}".format(key_file, format_exc()), fg='red')
elif '-' == public_key_file:
elif public_key_file == '-':
buffer = "".join(stdin.readlines())
with NamedTemporaryFile(mode="w") as f:
f.write(buffer)
@ -1293,7 +1310,7 @@ def cmd_stop(app):
app = exit_if_invalid(app)
config = glob(join(UWSGI_ENABLED, '{}*.ini'.format(app)))
if len(config):
if len(config) > 0:
echo("Stopping app '{}'...".format(app), fg='yellow')
for c in config:
remove(c)
@ -1349,7 +1366,7 @@ cat | PIKU_ROOT="{PIKU_ROOT:s}" {PIKU_SCRIPT:s} git-hook {app:s}""".format(**env
@piku.command("git-upload-pack", hidden=True)
@argument('app')
def cmd_git_receive_pack(app):
def cmd_git_upload_pack(app):
"""INTERNAL: Handle git upload pack for an app"""
app = sanitize_app_name(app)
env = globals()