Move argument parsing out of Repo2Docker class

Argument parsing should only be used when calling from
the command line, and not be deeply tied into the class
itself. This makes it easier for folks to just set
traitlets on an empty class and start it, without having to
deal with passing arguments.

This breaks how people might already be using Repo2Docker as
a library, so should not be part of 0.7
pull/496/head
yuvipanda 2018-12-11 09:11:01 -08:00
rodzic 7bcfa06d8f
commit e1ead75302
4 zmienionych plików z 446 dodań i 299 usunięć

Wyświetl plik

@ -1,10 +1,311 @@
import argparse
import sys
from .app import Repo2Docker
from . import __version__
from .utils import validate_and_generate_port_mapping
def validate_image_name(image_name):
"""
Validate image_name read by argparse
Note: Container names must start with an alphanumeric character and
can then use _ . or - in addition to alphanumeric.
[a-zA-Z0-9][a-zA-Z0-9_.-]+
Args:
image_name (string): argument read by the argument parser
Returns:
unmodified image_name
Raises:
ArgumentTypeError: if image_name contains characters that do not
meet the logic that container names must start
with an alphanumeric character and can then
use _ . or - in addition to alphanumeric.
[a-zA-Z0-9][a-zA-Z0-9_.-]+
"""
if not is_valid_docker_image_name(image_name):
msg = ("%r is not a valid docker image name. Image name"
"must start with an alphanumeric character and"
"can then use _ . or - in addition to alphanumeric." % image_name)
raise argparse.ArgumentTypeError(msg)
return image_name
def get_argparser():
"""Get arguments that may be used by repo2docker"""
argparser = argparse.ArgumentParser()
argparser.add_argument(
'--config',
default='repo2docker_config.py',
help="Path to config file for repo2docker"
)
argparser.add_argument(
'--json-logs',
default=False,
action='store_true',
help='Emit JSON logs instead of human readable logs'
)
argparser.add_argument(
'repo',
help=('Path to repository that should be built. Could be '
'local path or a git URL.')
)
argparser.add_argument(
'--image-name',
help=('Name of image to be built. If unspecified will be '
'autogenerated'),
type=validate_image_name
)
argparser.add_argument(
'--ref',
help=('If building a git url, which reference to check out. '
'E.g., `master`.')
)
argparser.add_argument(
'--debug',
help="Turn on debug logging",
action='store_true',
)
argparser.add_argument(
'--no-build',
dest='build',
action='store_false',
help=('Do not actually build the image. Useful in conjunction '
'with --debug.')
)
argparser.add_argument(
'--build-memory-limit',
help='Total Memory that can be used by the docker build process'
)
argparser.add_argument(
'cmd',
nargs=argparse.REMAINDER,
help='Custom command to run after building container'
)
argparser.add_argument(
'--no-run',
dest='run',
action='store_false',
help='Do not run container after it has been built'
)
argparser.add_argument(
'--publish', '-p',
dest='ports',
action='append',
help=('Specify port mappings for the image. Needs a command to '
'run in the container.')
)
argparser.add_argument(
'--publish-all', '-P',
dest='all_ports',
action='store_true',
help='Publish all exposed ports to random host ports.'
)
argparser.add_argument(
'--no-clean',
dest='clean',
action='store_false',
help="Don't clean up remote checkouts after we are done"
)
argparser.add_argument(
'--push',
dest='push',
action='store_true',
help='Push docker image to repository'
)
argparser.add_argument(
'--volume', '-v',
dest='volumes',
action='append',
help='Volumes to mount inside the container, in form src:dest',
default=[]
)
argparser.add_argument(
'--user-id',
help='User ID of the primary user in the image',
type=int
)
argparser.add_argument(
'--user-name',
help='Username of the primary user in the image',
)
argparser.add_argument(
'--env', '-e',
dest='environment',
action='append',
help='Environment variables to define at container run time',
default=[]
)
argparser.add_argument(
'--editable', '-E',
dest='editable',
action='store_true',
help='Use the local repository in edit mode',
)
argparser.add_argument(
'--appendix',
type=str,
#help=self.traits()['appendix'].help,
)
argparser.add_argument(
'--subdir',
type=str,
#help=self.traits()['subdir'].help,
)
argparser.add_argument(
'--version',
dest='version',
action='store_true',
help='Print the repo2docker version and exit.'
)
argparser.add_argument(
'--cache-from',
action='append',
default=[],
#help=self.traits()['cache_from'].help
)
return argparser
def make_r2d(argv=None):
if argv is None:
argv = sys.argv[1:]
# version must be checked before parse, as repo/cmd are required and
# will spit out an error if allowed to be parsed first.
if '--version' in argv:
print(__version__)
sys.exit(0)
args = get_argparser().parse_args(argv)
r2d = Repo2Docker()
if args.debug:
r2d.log_level = logging.DEBUG
r2d.load_config_file(args.config)
if args.appendix:
r2d.appendix = args.appendix
r2d.repo = args.repo
r2d.ref = args.ref
# user wants to mount a local directory into the container for
# editing
if args.editable:
# the user has to point at a directory, not just a path for us
# to be able to mount it. We might have content providers that can
# provide content from a local `something.zip` file, which we
# couldn't mount in editable mode
if os.path.isdir(args.repo):
r2d.volumes[os.path.abspath(args.repo)] = '.'
else:
r2d.log.error('Can not mount "{}" in editable mode '
'as it is not a directory'.format(args.repo),
extra=dict(phase='failed'))
sys.exit(1)
if args.image_name:
r2d.output_image_spec = args.image_name
r2d.push = args.push
r2d.run = args.run
r2d.json_logs = args.json_logs
r2d.build = args.build
if not r2d.build:
# Can't push nor run if we aren't building
r2d.run = False
r2d.push = False
# check against r2d.run and not args.run as r2d.run is false on
# --no-build
if args.volumes and not r2d.run:
# Can't mount if we aren't running
print('To Mount volumes with -v, you also need to run the '
'container')
sys.exit(1)
for v in args.volumes:
src, dest = v.split(':')
r2d.volumes[src] = dest
r2d.run_cmd = args.cmd
if args.all_ports and not r2d.run:
print('To publish user defined port mappings, the container must '
'also be run')
sys.exit(1)
if args.ports and not r2d.run:
print('To publish user defined port mappings, the container must '
'also be run')
sys.exit(1)
if args.ports and not r2d.run_cmd:
print('To publish user defined port mapping, user must specify '
'the command to run in the container')
sys.exit(1)
r2d.ports = validate_and_generate_port_mapping(args.ports)
r2d.all_ports = args.all_ports
if args.user_id:
r2d.user_id = args.user_id
if args.user_name:
r2d.user_name = args.user_name
if args.build_memory_limit:
r2d.build_memory_limit = args.build_memory_limit
if args.environment and not r2d.run:
print('To specify environment variables, you also need to run '
'the container')
sys.exit(1)
if args.subdir:
r2d.subdir = args.subdir
if args.cache_from:
r2d.cache_from = args.cache_from
r2d.environment = args.environment
return r2d
def main():
f = Repo2Docker()
f.initialize()
f.start()
r2d = make_r2d()
r2d.initialize()
r2d.start()
if __name__ == '__main__':

Wyświetl plik

@ -25,7 +25,7 @@ from docker.errors import DockerException
import escapism
from pythonjsonlogger import jsonlogger
from traitlets import Any, Dict, Int, List, Unicode, default
from traitlets import Any, Dict, Int, List, Unicode, Bool, default
from traitlets.config import Application
from . import __version__
@ -206,6 +206,129 @@ class Repo2Docker(Application):
"""
)
json_logs = Bool(
False,
help="""
Log output in structured JSON format.
Useful when stdout is consumed by other tools
""",
config=True
)
repo = Unicode(
".",
help="""
Specification of repository to build image for.
Could be local path or git URL.
""",
config=True
)
ref = Unicode(
None,
help="""
Git ref that should be built.
If repo is a git repository, this ref is checked out
in a local clone before repository is built.
""",
config=True,
allow_none=True
)
cleanup_checkout = Bool(
False,
help="""
Delete source repository after building is done.
Useful when repo2docker is doing the git cloning
""",
config=True
)
output_image_spec = Unicode(
"",
help="""
Docker Image name:tag to tag the built image with.
Required parameter.
""",
config=True
)
push = Bool(
False,
help="""
Set to true to push docker image after building
""",
config=True
)
run = Bool(
False,
help="""
Run docker image after building
""",
config=True
)
# FIXME: Refactor class to be able to do --no-build without needing
# deep support for it inside other code
build = Bool(
True,
help="""
Actually build the docker image.
Can be set to false to do a dry run of rep2docker
""",
config=True
)
# FIXME: Refactor classes to separate build & run steps
run_cmd = List(
[],
help="""
Command to run when running the container
When left empty, a jupyter notebook is run.
""",
config=True
)
all_ports = Bool(
False,
help="""
Publish all declared ports from container whiel running.
Equivalent to -P option to docker run
""",
config=True
)
ports = Dict(
{},
help="""
Port mappings to establish when running the container.
Equivalent to -p {key}:{value} options to docker run.
{key} refers to port inside container, and {value}
refers to port / host:port in the host
""",
config=True
)
environment = List(
[],
help="""
Environment variables to set when running the built image.
Each item must be a string formatted as KEY=VALUE
""",
config=True
)
def fetch(self, url, ref, checkout_path):
"""Fetch the contents of `url` and place it in `checkout_path`.
@ -233,192 +356,7 @@ class Repo2Docker(Application):
spec, checkout_path, yield_output=self.json_logs):
self.log.info(log_line, extra=dict(phase='fetching'))
def validate_image_name(self, image_name):
"""
Validate image_name read by argparse
Note: Container names must start with an alphanumeric character and
can then use _ . or - in addition to alphanumeric.
[a-zA-Z0-9][a-zA-Z0-9_.-]+
Args:
image_name (string): argument read by the argument parser
Returns:
unmodified image_name
Raises:
ArgumentTypeError: if image_name contains characters that do not
meet the logic that container names must start
with an alphanumeric character and can then
use _ . or - in addition to alphanumeric.
[a-zA-Z0-9][a-zA-Z0-9_.-]+
"""
if not is_valid_docker_image_name(image_name):
msg = ("%r is not a valid docker image name. Image name"
"must start with an alphanumeric character and"
"can then use _ . or - in addition to alphanumeric." % image_name)
raise argparse.ArgumentTypeError(msg)
return image_name
def get_argparser(self):
"""Get arguments that may be used by repo2docker"""
argparser = argparse.ArgumentParser()
argparser.add_argument(
'--config',
default='repo2docker_config.py',
help="Path to config file for repo2docker"
)
argparser.add_argument(
'--json-logs',
default=False,
action='store_true',
help='Emit JSON logs instead of human readable logs'
)
argparser.add_argument(
'repo',
help=('Path to repository that should be built. Could be '
'local path or a git URL.')
)
argparser.add_argument(
'--image-name',
help=('Name of image to be built. If unspecified will be '
'autogenerated'),
type=self.validate_image_name
)
argparser.add_argument(
'--ref',
help=('If building a git url, which reference to check out. '
'E.g., `master`.')
)
argparser.add_argument(
'--debug',
help="Turn on debug logging",
action='store_true',
)
argparser.add_argument(
'--no-build',
dest='build',
action='store_false',
help=('Do not actually build the image. Useful in conjunction '
'with --debug.')
)
argparser.add_argument(
'--build-memory-limit',
help='Total Memory that can be used by the docker build process'
)
argparser.add_argument(
'cmd',
nargs=argparse.REMAINDER,
help='Custom command to run after building container'
)
argparser.add_argument(
'--no-run',
dest='run',
action='store_false',
help='Do not run container after it has been built'
)
argparser.add_argument(
'--publish', '-p',
dest='ports',
action='append',
help=('Specify port mappings for the image. Needs a command to '
'run in the container.')
)
argparser.add_argument(
'--publish-all', '-P',
dest='all_ports',
action='store_true',
help='Publish all exposed ports to random host ports.'
)
argparser.add_argument(
'--no-clean',
dest='clean',
action='store_false',
help="Don't clean up remote checkouts after we are done"
)
argparser.add_argument(
'--push',
dest='push',
action='store_true',
help='Push docker image to repository'
)
argparser.add_argument(
'--volume', '-v',
dest='volumes',
action='append',
help='Volumes to mount inside the container, in form src:dest',
default=[]
)
argparser.add_argument(
'--user-id',
help='User ID of the primary user in the image',
type=int
)
argparser.add_argument(
'--user-name',
help='Username of the primary user in the image',
)
argparser.add_argument(
'--env', '-e',
dest='environment',
action='append',
help='Environment variables to define at container run time',
default=[]
)
argparser.add_argument(
'--editable', '-E',
dest='editable',
action='store_true',
help='Use the local repository in edit mode',
)
argparser.add_argument(
'--appendix',
type=str,
help=self.traits()['appendix'].help,
)
argparser.add_argument(
'--subdir',
type=str,
help=self.traits()['subdir'].help,
)
argparser.add_argument(
'--version',
dest='version',
action='store_true',
help='Print the repo2docker version and exit.'
)
argparser.add_argument(
'--cache-from',
action='append',
default=[],
help=self.traits()['cache_from'].help
)
return argparser
def json_excepthook(self, etype, evalue, traceback):
"""Called on an uncaught exception when using json logging
@ -429,50 +367,10 @@ class Repo2Docker(Application):
exc_info=(etype, evalue, traceback),
extra=dict(phase='failed'))
def initialize(self, argv=None):
def initialize(self):
"""Init repo2docker configuration before start"""
if argv is None:
argv = sys.argv[1:]
# version must be checked before parse, as repo/cmd are required and
# will spit out an error if allowed to be parsed first.
if '--version' in argv:
print(self.version)
sys.exit(0)
args = self.get_argparser().parse_args(argv)
if args.debug:
self.log_level = logging.DEBUG
self.load_config_file(args.config)
if args.appendix:
self.appendix = args.appendix
self.repo = args.repo
self.ref = args.ref
# if the source exists locally we don't want to delete it at the end
if os.path.exists(args.repo):
self.cleanup_checkout = False
else:
self.cleanup_checkout = args.clean
# user wants to mount a local directory into the container for
# editing
if args.editable:
# the user has to point at a directory, not just a path for us
# to be able to mount it. We might have content providers that can
# provide content from a local `something.zip` file, which we
# couldn't mount in editable mode
if os.path.isdir(args.repo):
self.volumes[os.path.abspath(args.repo)] = '.'
else:
self.log.error('Can not mount "{}" in editable mode '
'as it is not a directory'.format(args.repo),
extra=dict(phase='failed'))
sys.exit(1)
if args.json_logs:
# FIXME: Remove this function, move it to setters / traitlet reactors
if self.json_logs:
# register JSON excepthook to avoid non-JSON output on errors
sys.excepthook = self.json_excepthook
# Need to reset existing handlers, or we repeat messages
@ -493,9 +391,13 @@ class Repo2Docker(Application):
fmt='%(message)s'
)
if args.image_name:
self.output_image_spec = args.image_name
# if the source exists locally we don't want to delete it at the end
if os.path.exists(self.repo):
self.cleanup_checkout = False
else:
self.cleanup_checkout = args.clean
if self.output_image_spec == "":
# Attempt to set a sane default!
# HACK: Provide something more descriptive?
self.output_image_spec = (
@ -504,68 +406,11 @@ class Repo2Docker(Application):
str(int(time.time()))
)
self.push = args.push
self.run = args.run
self.json_logs = args.json_logs
if not self.build and (self.run or self.push):
raise ValueError("Can not push or run image if we are not building it")
self.build = args.build
if not self.build:
# Can't push nor run if we aren't building
self.run = False
self.push = False
# check against self.run and not args.run as self.run is false on
# --no-build
if args.volumes and not self.run:
# Can't mount if we aren't running
print('To Mount volumes with -v, you also need to run the '
'container')
sys.exit(1)
for v in args.volumes:
src, dest = v.split(':')
self.volumes[src] = dest
self.run_cmd = args.cmd
if args.all_ports and not self.run:
print('To publish user defined port mappings, the container must '
'also be run')
sys.exit(1)
if args.ports and not self.run:
print('To publish user defined port mappings, the container must '
'also be run')
sys.exit(1)
if args.ports and not self.run_cmd:
print('To publish user defined port mapping, user must specify '
'the command to run in the container')
sys.exit(1)
self.ports = validate_and_generate_port_mapping(args.ports)
self.all_ports = args.all_ports
if args.user_id:
self.user_id = args.user_id
if args.user_name:
self.user_name = args.user_name
if args.build_memory_limit:
self.build_memory_limit = args.build_memory_limit
if args.environment and not self.run:
print('To specify environment variables, you also need to run '
'the container')
sys.exit(1)
if args.subdir:
self.subdir = args.subdir
if args.cache_from:
self.cache_from = args.cache_from
self.environment = args.environment
if self.volumes and not self.run:
raise ValueError("Can not mount volumes if container is not run")
def push_image(self):
"""Push docker image to registry"""

Wyświetl plik

@ -122,8 +122,8 @@ def validate_and_generate_port_mapping(port_mapping):
)$
""", re.VERBOSE)
ports = {}
if not port_mapping:
return None
if port_mapping is None:
return ports
for p in port_mapping:
if reg_regex.match(p) is None:
raise Exception('Invalid port mapping ' + str(p))

Wyświetl plik

@ -18,6 +18,7 @@ import pytest
import yaml
from repo2docker.app import Repo2Docker
from repo2docker.__main__ import make_r2d
def pytest_collect_file(parent, path):
@ -30,8 +31,8 @@ def pytest_collect_file(parent, path):
def make_test_func(args):
"""Generate a test function that runs repo2docker"""
def test():
app = Repo2Docker()
app.initialize(args)
app = make_r2d(args)
app.initialize()
if app.run_cmd:
# verify test, run it
app.start()