kopia lustrzana https://github.com/jupyterhub/repo2docker
Add check-tmp step to local repo tests
- check-tmp checks disk usage in a collection of directories, and fails if they exceed a certain size, reporting on contents - re-use images and skip build stage in verify/check-tmp tests. Should save some time, even though the build cache would have been used. This way, we don't even check.pull/1126/head
rodzic
f6ecb28e5e
commit
38f5aceb45
|
@ -0,0 +1,86 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Script to check for leftover files
|
||||||
|
|
||||||
|
Checks a collection of temporary or cache directories,
|
||||||
|
to ensure we aren't wasting image size by forgetting cleanup steps.
|
||||||
|
|
||||||
|
This script is run in every local repo image we test
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from subprocess import check_output
|
||||||
|
from textwrap import indent
|
||||||
|
|
||||||
|
# directories larger than this are considered a failure
|
||||||
|
# a few little files here aren't a problem
|
||||||
|
THRESHOLD = 1 # in MB
|
||||||
|
|
||||||
|
MB = 1024 * 1024
|
||||||
|
|
||||||
|
# the paths to check
|
||||||
|
# all of these locations
|
||||||
|
# should be cleaned up
|
||||||
|
# missing is okay
|
||||||
|
PATHS = [
|
||||||
|
"/tmp/",
|
||||||
|
"~/",
|
||||||
|
"~/.cache/",
|
||||||
|
# not running with read permissions on root
|
||||||
|
# "/root/",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def du(path):
|
||||||
|
"""Return disk usage in megabytes of a path"""
|
||||||
|
# -ks: get total size, reported in kilobytes
|
||||||
|
out = check_output(["du", "-Hks", path])
|
||||||
|
return int(out.split(None, 1)[0]) / 1024
|
||||||
|
|
||||||
|
|
||||||
|
def check_dir_size(path):
|
||||||
|
"""Check the size of a directory
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
|
||||||
|
True: directory size is below THRESHOLD or is missing
|
||||||
|
False: directory is larger than THRESHOLD
|
||||||
|
"""
|
||||||
|
path = os.path.expanduser(path)
|
||||||
|
|
||||||
|
if not os.path.exists(path):
|
||||||
|
print(f"{path}: missing OK")
|
||||||
|
return True
|
||||||
|
|
||||||
|
size_mb = du(path)
|
||||||
|
print(f"{path}: {size_mb:.1f} MB", end=" ")
|
||||||
|
if size_mb <= THRESHOLD:
|
||||||
|
print("OK")
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
print("FAIL")
|
||||||
|
# check size of files one-level deep (du only reports dirs)
|
||||||
|
for name in os.listdir(path):
|
||||||
|
subpath = os.path.join(path, name)
|
||||||
|
if os.path.isfile(subpath):
|
||||||
|
file_sz = os.stat(subpath).st_size / MB
|
||||||
|
if file_sz > 0.1:
|
||||||
|
print(f" {file_sz:.1f}M {subpath}")
|
||||||
|
# get report on all subdirs that are at least 100k
|
||||||
|
print(
|
||||||
|
indent(
|
||||||
|
check_output(["du", "-Hh", "-t", "100000", path]).decode("utf8"), " "
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
results = [check_dir_size(path) for path in PATHS]
|
||||||
|
if not all(results):
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
|
@ -22,14 +22,17 @@ import shlex
|
||||||
import requests
|
import requests
|
||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from tempfile import TemporaryDirectory
|
from tempfile import TemporaryDirectory
|
||||||
|
|
||||||
|
|
||||||
|
import escapism
|
||||||
import pytest
|
import pytest
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
from repo2docker.__main__ import make_r2d
|
from repo2docker.__main__ import make_r2d
|
||||||
|
|
||||||
|
TESTS_DIR = os.path.abspath(os.path.dirname(__file__))
|
||||||
|
|
||||||
|
|
||||||
def pytest_collect_file(parent, path):
|
def pytest_collect_file(parent, path):
|
||||||
if path.basename == "verify":
|
if path.basename == "verify":
|
||||||
|
@ -38,12 +41,18 @@ def pytest_collect_file(parent, path):
|
||||||
return RemoteRepoList.from_parent(parent, fspath=path)
|
return RemoteRepoList.from_parent(parent, fspath=path)
|
||||||
|
|
||||||
|
|
||||||
def make_test_func(args):
|
def make_test_func(args, skip_build=False):
|
||||||
"""Generate a test function that runs repo2docker"""
|
"""Generate a test function that runs repo2docker"""
|
||||||
|
|
||||||
def test():
|
def test():
|
||||||
app = make_r2d(args)
|
app = make_r2d(args)
|
||||||
app.initialize()
|
app.initialize()
|
||||||
|
if skip_build:
|
||||||
|
|
||||||
|
def build_noop():
|
||||||
|
print("Skipping build")
|
||||||
|
|
||||||
|
app.skip_build = build_noop
|
||||||
if app.run_cmd:
|
if app.run_cmd:
|
||||||
# verify test, run it
|
# verify test, run it
|
||||||
app.start()
|
app.start()
|
||||||
|
@ -184,14 +193,14 @@ def repo_with_submodule():
|
||||||
class Repo2DockerTest(pytest.Function):
|
class Repo2DockerTest(pytest.Function):
|
||||||
"""A pytest.Item for running repo2docker"""
|
"""A pytest.Item for running repo2docker"""
|
||||||
|
|
||||||
def __init__(self, name, parent, args=None):
|
def __init__(self, name, parent, args=None, skip_build=False):
|
||||||
self.args = args
|
self.args = args
|
||||||
self.save_cwd = os.getcwd()
|
self.save_cwd = os.getcwd()
|
||||||
f = parent.obj = make_test_func(args)
|
f = parent.obj = make_test_func(args, skip_build=skip_build)
|
||||||
super().__init__(name, parent, callobj=f)
|
super().__init__(name, parent, callobj=f)
|
||||||
|
|
||||||
def reportinfo(self):
|
def reportinfo(self):
|
||||||
return self.parent.fspath, None, ""
|
return (self.parent.fspath, None, "")
|
||||||
|
|
||||||
def repr_failure(self, excinfo):
|
def repr_failure(self, excinfo):
|
||||||
err = excinfo.value
|
err = excinfo.value
|
||||||
|
@ -217,11 +226,34 @@ class LocalRepo(pytest.File):
|
||||||
extra_args = yaml.safe_load(f)
|
extra_args = yaml.safe_load(f)
|
||||||
args += extra_args
|
args += extra_args
|
||||||
|
|
||||||
|
print(self.fspath.basename, self.fspath.dirname, str(self.fspath))
|
||||||
|
# re-use image name for multiple tests of the same image
|
||||||
|
# so we don't run through the build twice
|
||||||
|
rel_repo_dir = os.path.relpath(self.fspath.dirname, TESTS_DIR)
|
||||||
|
image_name = f"r2d-tests-{escapism.escape(rel_repo_dir, escape_char='-').lower()}-{int(time.time())}"
|
||||||
|
args.append(f"--image-name={image_name}")
|
||||||
args.append(self.fspath.dirname)
|
args.append(self.fspath.dirname)
|
||||||
|
|
||||||
yield Repo2DockerTest.from_parent(self, name="build", args=args)
|
yield Repo2DockerTest.from_parent(self, name="build", args=args)
|
||||||
|
|
||||||
yield Repo2DockerTest.from_parent(
|
yield Repo2DockerTest.from_parent(
|
||||||
self, name=self.fspath.basename, args=args + ["./verify"]
|
self,
|
||||||
|
name=self.fspath.basename,
|
||||||
|
args=args + ["./verify"],
|
||||||
|
skip_build=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# mount the tests dir as a volume
|
||||||
|
check_tmp_args = (
|
||||||
|
args[:-1]
|
||||||
|
+ ["--volume", f"{TESTS_DIR}:/io/tests"]
|
||||||
|
+ [args[-1], "/io/tests/check-tmp"]
|
||||||
|
)
|
||||||
|
|
||||||
|
yield Repo2DockerTest.from_parent(
|
||||||
|
self,
|
||||||
|
name="check-tmp",
|
||||||
|
args=check_tmp_args,
|
||||||
|
skip_build=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
Ładowanie…
Reference in New Issue