kopia lustrzana https://github.com/piku/piku
First pass at caching
Rui Carmo
GitHub
rodzic
16bed4c1d9
commit
9abbeb1f31
89
piku.py
89
piku.py
|
|
@ -44,10 +44,12 @@ PIKU_BIN = join(environ['HOME'], 'bin')
|
|||
PIKU_SCRIPT = realpath(__file__)
|
||||
PIKU_PLUGIN_ROOT = abspath(join(PIKU_ROOT, "plugins"))
|
||||
APP_ROOT = abspath(join(PIKU_ROOT, "apps"))
|
||||
DATA_ROOT = abspath(join(PIKU_ROOT, "data"))
|
||||
ENV_ROOT = abspath(join(PIKU_ROOT, "envs"))
|
||||
GIT_ROOT = abspath(join(PIKU_ROOT, "repos"))
|
||||
LOG_ROOT = abspath(join(PIKU_ROOT, "logs"))
|
||||
NGINX_ROOT = abspath(join(PIKU_ROOT, "nginx"))
|
||||
CACHE_ROOT = abspath(join(PIKU_ROOT, "cache"))
|
||||
UWSGI_AVAILABLE = abspath(join(PIKU_ROOT, "uwsgi-available"))
|
||||
UWSGI_ENABLED = abspath(join(PIKU_ROOT, "uwsgi-enabled"))
|
||||
UWSGI_ROOT = abspath(join(PIKU_ROOT, "uwsgi"))
|
||||
|
|
@ -63,6 +65,7 @@ if PIKU_BIN not in environ['PATH']:
|
|||
|
||||
# pylint: disable=anomalous-backslash-in-string
|
||||
NGINX_TEMPLATE = """
|
||||
$PIKU_INTERNAL_PROXY_CACHE_PATH
|
||||
upstream $APP {
|
||||
server $NGINX_SOCKET;
|
||||
}
|
||||
|
|
@ -74,12 +77,12 @@ server {
|
|||
allow all;
|
||||
root ${ACME_WWW};
|
||||
}
|
||||
|
||||
$PIKU_INTERNAL_NGINX_COMMON
|
||||
}
|
||||
"""
|
||||
|
||||
NGINX_HTTPS_ONLY_TEMPLATE = """
|
||||
$PIKU_INTERNAL_PROXY_CACHE_PATH
|
||||
upstream $APP {
|
||||
server $NGINX_SOCKET;
|
||||
}
|
||||
|
|
@ -110,7 +113,6 @@ NGINX_COMMON_FRAGMENT = """
|
|||
ssl_certificate $NGINX_ROOT/$APP.crt;
|
||||
ssl_certificate_key $NGINX_ROOT/$APP.key;
|
||||
server_name $NGINX_SERVER_NAME;
|
||||
|
||||
# These are not required under systemd - enable for debugging only
|
||||
# access_log $LOG_ROOT/$APP/access.log;
|
||||
# error_log $LOG_ROOT/$APP/error.log;
|
||||
|
|
@ -123,16 +125,13 @@ NGINX_COMMON_FRAGMENT = """
|
|||
gzip_min_length 2048;
|
||||
gzip_vary on;
|
||||
gzip_disable "MSIE [1-6]\.(?!.*SV1)";
|
||||
|
||||
# set a custom header for requests
|
||||
add_header X-Deployed-By Piku;
|
||||
|
||||
$PIKU_INTERNAL_NGINX_CUSTOM_CLAUSES
|
||||
|
||||
$PIKU_INTERNAL_NGINX_STATIC_MAPPINGS
|
||||
|
||||
$PIKU_INTERNAL_NGINX_CACHE_MAPPINGS
|
||||
$PIKU_INTERNAL_NGINX_BLOCK_GIT
|
||||
|
||||
$PIKU_INTERNAL_NGINX_PORTMAP
|
||||
"""
|
||||
|
||||
|
|
@ -176,6 +175,26 @@ PIKU_INTERNAL_NGINX_STATIC_MAPPING = """
|
|||
}
|
||||
"""
|
||||
|
||||
PIKU_INTERNAL_PROXY_CACHE_PATH = """
|
||||
uwsgi_cache_path $cache_path levels=1:2 keys_zone=$app:20m inactive=$cache_days max_size=$cache_size use_temp_path=off;
|
||||
"""
|
||||
|
||||
PIKU_INTERNAL_NGINX_CACHE_MAPPING = """
|
||||
location ~* ^/($cache_prefixes) {
|
||||
uwsgi_cache $APP;
|
||||
uwsgi_cache_min_uses 1;
|
||||
uwsgi_cache_key $host$uri;
|
||||
uwsgi_cache_valid 200 304 4h;
|
||||
uwsgi_cache_valid 301 307 4h;
|
||||
uwsgi_cache_valid 500 502 503 504 0s;
|
||||
uwsgi_cache_valid any 72h;
|
||||
uwsgi_hide_header Cache-Control;
|
||||
add_header Cache-Control "public, max-age=3600";
|
||||
add_header X-Cache $upstream_cache_status;
|
||||
$PIKU_INTERNAL_NGINX_UWSGI_SETTINGS
|
||||
}
|
||||
"""
|
||||
|
||||
PIKU_INTERNAL_NGINX_UWSGI_SETTINGS = """
|
||||
uwsgi_pass $APP;
|
||||
uwsgi_param QUERY_STRING $query_string;
|
||||
|
|
@ -195,7 +214,6 @@ PIKU_INTERNAL_NGINX_UWSGI_SETTINGS = """
|
|||
|
||||
CRON_REGEXP = "^((?:(?:\*\/)?\d+)|\*) ((?:(?:\*\/)?\d+)|\*) ((?:(?:\*\/)?\d+)|\*) ((?:(?:\*\/)?\d+)|\*) ((?:(?:\*\/)?\d+)|\*) (.*)$"
|
||||
|
||||
|
||||
# === Utility functions ===
|
||||
|
||||
def sanitize_app_name(app):
|
||||
|
|
@ -785,9 +803,53 @@ def spawn_app(app, deltas={}):
|
|||
|
||||
env['PIKU_INTERNAL_NGINX_BLOCK_GIT'] = "" if env.get('NGINX_ALLOW_GIT_FOLDERS') else "location ~ /\.git { deny all; }"
|
||||
|
||||
env['PIKU_INTERNAL_PROXY_CACHE_PATH'] = ''
|
||||
env['PIKU_INTERNAL_NGINX_CACHE_MAPPINGS'] = ''
|
||||
|
||||
# Get a mapping of /prefix1,/prefix2
|
||||
default_cache_path = join(CACHE_ROOT, app)
|
||||
if not exists(default_cache_path):
|
||||
makedirs(default_cache_path)
|
||||
try:
|
||||
cache_size = int(env.get('NGINX_CACHE_SIZE', '1'))
|
||||
except:
|
||||
echo("=====> Invalid cache size, defaulting to 1GB")
|
||||
cache_size = str(cache_size) + "g"
|
||||
try:
|
||||
cache_days = int(env.get('NGINX_CACHE_DAYS', '7'))
|
||||
except:
|
||||
echo("=====> Invalid cache expiry, defaulting to 7 days")
|
||||
cache_days = str(cache_days) + "d"
|
||||
|
||||
cache_prefixes = env.get('NGINX_CACHE_PREFIXES', '')
|
||||
cache_path = env.get('NGINX_CACHE_PATH', default_cache_path)
|
||||
if not exists(cache_path):
|
||||
echo("=====> Cache path {} does not exist, using default {}, be aware of disk usage.".format(cache_path, default_cache_path))
|
||||
cache_path = env.get(default_cache_path)
|
||||
if len(cache_prefixes):
|
||||
prefixes = [] # this will turn into part of /(path1|path2|path3)
|
||||
try:
|
||||
items = cache_prefixes.split(',')
|
||||
for item in items:
|
||||
if item[0] == '/':
|
||||
prefixes.append(item[1:])
|
||||
else:
|
||||
prefixes.append(item)
|
||||
cache_prefixes = "|".join(prefixes)
|
||||
echo("-----> nginx will cache /({}) prefixes up to {} or {} of disk space.".format(cache_prefixes, cache_days, cache_size))
|
||||
env['PIKU_INTERNAL_PROXY_CACHE_PATH'] = expandvars(
|
||||
PIKU_INTERNAL_PROXY_CACHE_PATH, locals())
|
||||
env['PIKU_INTERNAL_NGINX_CACHE_MAPPINGS'] = expandvars(
|
||||
PIKU_INTERNAL_NGINX_CACHE_MAPPING, locals())
|
||||
env['PIKU_INTERNAL_NGINX_CACHE_MAPPINGS'] = expandvars(
|
||||
env['PIKU_INTERNAL_NGINX_CACHE_MAPPINGS'], env)
|
||||
except Exception as e:
|
||||
echo("Error {} in cache path spec: should be /prefix1:[,/prefix2], ignoring.".format(e))
|
||||
env['PIKU_INTERNAL_NGINX_CACHE_MAPPINGS'] = ''
|
||||
|
||||
env['PIKU_INTERNAL_NGINX_STATIC_MAPPINGS'] = ''
|
||||
|
||||
# Get a mapping of /url:path1,/url2:path2
|
||||
# Get a mapping of /prefix1:path1,/prefix2:path2
|
||||
static_paths = env.get('NGINX_STATIC_PATHS', '')
|
||||
# prepend static worker path if present
|
||||
if 'static' in workers:
|
||||
|
|
@ -800,10 +862,11 @@ def spawn_app(app, deltas={}):
|
|||
static_url, static_path = item.split(':')
|
||||
if static_path[0] != '/':
|
||||
static_path = join(app_path, static_path)
|
||||
echo("-----> nginx will map {} to {}.".format(static_url, static_path))
|
||||
env['PIKU_INTERNAL_NGINX_STATIC_MAPPINGS'] = env['PIKU_INTERNAL_NGINX_STATIC_MAPPINGS'] + expandvars(
|
||||
PIKU_INTERNAL_NGINX_STATIC_MAPPING, locals())
|
||||
except Exception as e:
|
||||
echo("Error {} in static path spec: should be /url1:path1[,/url2:path2], ignoring.".format(e))
|
||||
echo("Error {} in static path spec: should be /prefix1:path1[,/prefix2:path2], ignoring.".format(e))
|
||||
env['PIKU_INTERNAL_NGINX_STATIC_MAPPINGS'] = ''
|
||||
|
||||
env['PIKU_INTERNAL_NGINX_CUSTOM_CLAUSES'] = expandvars(open(join(app_path, env["NGINX_INCLUDE_FILE"])).read(), env) if env.get("NGINX_INCLUDE_FILE") else ""
|
||||
|
|
@ -822,6 +885,9 @@ def spawn_app(app, deltas={}):
|
|||
# remove all references to IPv6 listeners (for enviroments where it's disabled)
|
||||
if env.get('DISABLE_IPV6', 'false').lower() == 'true':
|
||||
buffer = '\n'.join([line for line in buffer.split('\n') if 'NGINX_IPV6' not in line])
|
||||
# change any unecessary uWSGI specific directives to standard proxy ones
|
||||
if 'wsgi' not in workers and 'jwsgi' not in workers:
|
||||
buffer = buffer.replace("uwsgi_", "proxy_")
|
||||
|
||||
with open(nginx_conf, "w") as h:
|
||||
h.write(buffer)
|
||||
|
|
@ -1243,7 +1309,8 @@ def cmd_destroy(app):
|
|||
|
||||
app = exit_if_invalid(app)
|
||||
|
||||
for p in [join(x, app) for x in [APP_ROOT, GIT_ROOT, ENV_ROOT, LOG_ROOT]]:
|
||||
# leave DATA_ROOT, since apps may create hard to reproduce data
|
||||
for p in [join(x, app) for x in [APP_ROOT, CACHE_ROOT, GIT_ROOT, ENV_ROOT, LOG_ROOT]]:
|
||||
if exists(p):
|
||||
echo("Removing folder '{}'".format(p), fg='yellow')
|
||||
rmtree(p)
|
||||
|
|
@ -1362,7 +1429,7 @@ def cmd_setup():
|
|||
echo("Running in Python {}".format(".".join(map(str, version_info))))
|
||||
|
||||
# Create required paths
|
||||
for p in [APP_ROOT, GIT_ROOT, ENV_ROOT, UWSGI_ROOT, UWSGI_AVAILABLE, UWSGI_ENABLED, LOG_ROOT, NGINX_ROOT]:
|
||||
for p in [APP_ROOT, CACHE_ROOT, DATA_ROOT, GIT_ROOT, ENV_ROOT, UWSGI_ROOT, UWSGI_AVAILABLE, UWSGI_ENABLED, LOG_ROOT, NGINX_ROOT]:
|
||||
if not exists(p):
|
||||
echo("Creating '{}'.".format(p), fg='green')
|
||||
makedirs(p)
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue