From dbbe707841973b50a76d2703003ae2c40e7ad1fd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 25 Jul 2018 22:15:59 -0700 Subject: [PATCH] publish_subcommand hook + default plugins mechanism, used for publish heroku/now (#349) This change introduces a new plugin hook, publish_subcommand, which can be used to implement new subcommands for the "datasette publish" command family. I've used this new hook to refactor out the "publish now" and "publish heroku" implementations into separate modules. I've also added unit tests for these two publishers, mocking the subprocess.call and subprocess.check_output functions. As part of this, I introduced a mechanism for loading default plugins. These are defined in the new "default_plugins" list inside datasette/app.py Closes #217 (Plugin support for datasette publish) Closes #348 (Unit tests for "datasette publish") Refs #14, #59, #102, #103, #146, #236, #347 --- datasette/app.py | 11 + datasette/cli.py | 221 +----------------- datasette/hookspecs.py | 5 + datasette/publish/__init__.py | 0 datasette/publish/common.py | 68 ++++++ datasette/publish/heroku.py | 99 ++++++++ datasette/publish/now.py | 80 +++++++ datasette/utils.py | 20 +- docs/datasette-publish-heroku-help.txt | 20 ++ ...elp.txt => datasette-publish-now-help.txt} | 20 +- docs/plugins.rst | 9 + docs/publish.rst | 7 +- tests/test_docs.py | 15 +- update-docs-help.py | 7 +- 14 files changed, 343 insertions(+), 239 deletions(-) create mode 100644 datasette/publish/__init__.py create mode 100644 datasette/publish/common.py create mode 100644 datasette/publish/heroku.py create mode 100644 datasette/publish/now.py create mode 100644 docs/datasette-publish-heroku-help.txt rename docs/{datasette-publish-help.txt => datasette-publish-now-help.txt} (74%) diff --git a/datasette/app.py b/datasette/app.py index 66a7573a..052131d0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2,6 +2,7 @@ import asyncio import click import collections import hashlib +import importlib import itertools import os import sqlite3 @@ -41,6 +42,11 @@ from .utils import ( from .inspect import inspect_hash, inspect_views, inspect_tables from .version import __version__ +default_plugins = ( + "datasette.publish.heroku", + "datasette.publish.now", +) + app_root = Path(__file__).parent.parent connections = threading.local() @@ -49,6 +55,11 @@ pm = pluggy.PluginManager("datasette") pm.add_hookspecs(hookspecs) pm.load_setuptools_entrypoints("datasette") +# Load default plugins +for plugin in default_plugins: + mod = importlib.import_module(plugin) + pm.register(mod, plugin) + ConfigOption = collections.namedtuple( "ConfigOption", ("name", "default", "help") diff --git a/datasette/cli.py b/datasette/cli.py index 72770326..820367ac 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -4,32 +4,17 @@ from click_default_group import DefaultGroup import json import os import shutil -from subprocess import call, check_output +from subprocess import call import sys -from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS +from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS, pm from .utils import ( temporary_docker_directory, - temporary_heroku_directory, value_as_boolean, + StaticMount, ValueAsBooleanError, ) -class StaticMount(click.ParamType): - name = "static mount" - - def convert(self, value, param, ctx): - if ":" not in value: - self.fail( - '"{}" should be of format mountpoint:directory'.format(value), - param, ctx - ) - path, dirpath = value.split(":") - if not os.path.exists(dirpath) or not os.path.isdir(dirpath): - self.fail("%s is not a valid directory path" % value, param, ctx) - return path, dirpath - - class Config(click.ParamType): name = "config" @@ -93,202 +78,14 @@ def inspect(files, inspect_file, sqlite_extensions): open(inspect_file, "w").write(json.dumps(app.inspect(), indent=2)) -@cli.command() -@click.argument("publisher", type=click.Choice(["now", "heroku"])) -@click.argument("files", type=click.Path(exists=True), nargs=-1) -@click.option( - "-n", - "--name", - default="datasette", - help="Application name to use when deploying", -) -@click.option( - "-m", - "--metadata", - type=click.File(mode="r"), - help="Path to JSON file containing metadata to publish", -) -@click.option("--extra-options", help="Extra options to pass to datasette serve") -@click.option("--force", is_flag=True, help="Pass --force option to now") -@click.option("--branch", help="Install datasette from a GitHub branch e.g. master") -@click.option("--token", help="Auth token to use for deploy (Now only)") -@click.option( - "--template-dir", - type=click.Path(exists=True, file_okay=False, dir_okay=True), - help="Path to directory containing custom templates", -) -@click.option( - "--plugins-dir", - type=click.Path(exists=True, file_okay=False, dir_okay=True), - help="Path to directory containing custom plugins", -) -@click.option( - "--static", - type=StaticMount(), - help="mountpoint:path-to-directory for serving static files", - multiple=True, -) -@click.option( - "--install", - help="Additional packages (e.g. plugins) to install", - multiple=True, -) -@click.option( - "--spatialite", is_flag=True, help="Enable SpatialLite extension" -) -@click.option("--version-note", help="Additional note to show on /-/versions") -@click.option("--title", help="Title for metadata") -@click.option("--license", help="License label for metadata") -@click.option("--license_url", help="License URL for metadata") -@click.option("--source", help="Source label for metadata") -@click.option("--source_url", help="Source URL for metadata") -def publish( - publisher, - files, - name, - metadata, - extra_options, - force, - branch, - token, - template_dir, - plugins_dir, - static, - install, - spatialite, - version_note, - **extra_metadata -): - """ - Publish specified SQLite database files to the internet along with a datasette API. +@cli.group() +def publish(): + "Publish specified SQLite database files to the internet along with a Datasette-powered interface and API" + pass - Options for PUBLISHER: - * 'now' - You must have Zeit Now installed: https://zeit.co/now - * 'heroku' - You must have Heroku installed: https://cli.heroku.com/ - Example usage: datasette publish now my-database.db - """ - - def _fail_if_publish_binary_not_installed(binary, publish_target, install_link): - """Exit (with error message) if ``binary` isn't installed""" - if not shutil.which(binary): - click.secho( - "Publishing to {publish_target} requires {binary} to be installed and configured".format( - publish_target=publish_target, binary=binary - ), - bg="red", - fg="white", - bold=True, - err=True, - ) - click.echo( - "Follow the instructions at {install_link}".format( - install_link=install_link - ), - err=True, - ) - sys.exit(1) - - if publisher == "now": - _fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now") - if extra_options: - extra_options += " " - else: - extra_options = "" - extra_options += "--config force_https_urls:on" - - with temporary_docker_directory( - files, - name, - metadata, - extra_options, - branch, - template_dir, - plugins_dir, - static, - install, - spatialite, - version_note, - extra_metadata, - ): - args = [] - if force: - args.append("--force") - if token: - args.append("--token={}".format(token)) - if args: - call(["now"] + args) - else: - call("now") - - elif publisher == "heroku": - _fail_if_publish_binary_not_installed( - "heroku", "Heroku", "https://cli.heroku.com" - ) - if spatialite: - click.secho( - "The --spatialite option is not yet supported for Heroku", - bg="red", - fg="white", - bold=True, - err=True, - ) - click.echo( - "See https://github.com/simonw/datasette/issues/301", - err=True, - ) - sys.exit(1) - - # Check for heroku-builds plugin - plugins = [ - line.split()[0] for line in check_output(["heroku", "plugins"]).splitlines() - ] - if b"heroku-builds" not in plugins: - click.echo( - "Publishing to Heroku requires the heroku-builds plugin to be installed." - ) - click.confirm( - "Install it? (this will run `heroku plugins:install heroku-builds`)", - abort=True, - ) - call(["heroku", "plugins:install", "heroku-builds"]) - - with temporary_heroku_directory( - files, - name, - metadata, - extra_options, - branch, - template_dir, - plugins_dir, - static, - install, - extra_metadata, - ): - - app_name = None - if name: - # Check to see if this app already exists - list_output = check_output(["heroku", "apps:list", "--json"]).decode('utf8') - apps = json.loads(list_output) - - for app in apps: - if app['name'] == name: - app_name = name - break - - if not app_name: - # Create a new app - cmd = ["heroku", "apps:create"] - if name: - cmd.append(name) - cmd.append("--json") - create_output = check_output(cmd).decode( - "utf8" - ) - app_name = json.loads(create_output)["name"] - - call(["heroku", "builds:create", "-a", app_name]) +# Register publish plugins +pm.hook.publish_subcommand(publish=publish) @cli.command() diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 240b58db..9546eebf 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -23,3 +23,8 @@ def extra_css_urls(): @hookspec def extra_js_urls(): "Extra JavaScript URLs added by this plugin" + + +@hookspec +def publish_subcommand(publish): + "Subcommands for 'datasette publish'" diff --git a/datasette/publish/__init__.py b/datasette/publish/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/datasette/publish/common.py b/datasette/publish/common.py new file mode 100644 index 00000000..9dd2ae9e --- /dev/null +++ b/datasette/publish/common.py @@ -0,0 +1,68 @@ +from ..utils import StaticMount +import click +import shutil +import sys + + +def add_common_publish_arguments_and_options(subcommand): + for decorator in reversed(( + click.argument("files", type=click.Path(exists=True), nargs=-1), + click.option( + "-m", + "--metadata", + type=click.File(mode="r"), + help="Path to JSON file containing metadata to publish", + ), + click.option("--extra-options", help="Extra options to pass to datasette serve"), + click.option("--branch", help="Install datasette from a GitHub branch e.g. master"), + click.option( + "--template-dir", + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Path to directory containing custom templates", + ), + click.option( + "--plugins-dir", + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Path to directory containing custom plugins", + ), + click.option( + "--static", + type=StaticMount(), + help="mountpoint:path-to-directory for serving static files", + multiple=True, + ), + click.option( + "--install", + help="Additional packages (e.g. plugins) to install", + multiple=True, + ), + click.option("--version-note", help="Additional note to show on /-/versions"), + click.option("--title", help="Title for metadata"), + click.option("--license", help="License label for metadata"), + click.option("--license_url", help="License URL for metadata"), + click.option("--source", help="Source label for metadata"), + click.option("--source_url", help="Source URL for metadata"), + )): + subcommand = decorator(subcommand) + return subcommand + + +def fail_if_publish_binary_not_installed(binary, publish_target, install_link): + """Exit (with error message) if ``binary` isn't installed""" + if not shutil.which(binary): + click.secho( + "Publishing to {publish_target} requires {binary} to be installed and configured".format( + publish_target=publish_target, binary=binary + ), + bg="red", + fg="white", + bold=True, + err=True, + ) + click.echo( + "Follow the instructions at {install_link}".format( + install_link=install_link + ), + err=True, + ) + sys.exit(1) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py new file mode 100644 index 00000000..af53a37a --- /dev/null +++ b/datasette/publish/heroku.py @@ -0,0 +1,99 @@ +from datasette import hookimpl +import click +import json +from subprocess import call, check_output + +from .common import ( + add_common_publish_arguments_and_options, + fail_if_publish_binary_not_installed, +) +from ..utils import temporary_heroku_directory + + +@hookimpl +def publish_subcommand(publish): + @publish.command() + @add_common_publish_arguments_and_options + @click.option( + "-n", + "--name", + default="datasette", + help="Application name to use when deploying", + ) + def heroku( + files, + metadata, + extra_options, + branch, + template_dir, + plugins_dir, + static, + install, + version_note, + title, + license, + license_url, + source, + source_url, + name, + ): + fail_if_publish_binary_not_installed( + "heroku", "Heroku", "https://cli.heroku.com" + ) + + # Check for heroku-builds plugin + plugins = [ + line.split()[0] for line in check_output(["heroku", "plugins"]).splitlines() + ] + if b"heroku-builds" not in plugins: + click.echo( + "Publishing to Heroku requires the heroku-builds plugin to be installed." + ) + click.confirm( + "Install it? (this will run `heroku plugins:install heroku-builds`)", + abort=True, + ) + call(["heroku", "plugins:install", "heroku-builds"]) + + with temporary_heroku_directory( + files, + name, + metadata, + extra_options, + branch, + template_dir, + plugins_dir, + static, + install, + version_note, + { + "title": title, + "license": license, + "license_url": license_url, + "source": source, + "source_url": source_url, + }, + ): + app_name = None + if name: + # Check to see if this app already exists + list_output = check_output(["heroku", "apps:list", "--json"]).decode( + "utf8" + ) + apps = json.loads(list_output) + + for app in apps: + if app["name"] == name: + app_name = name + break + + if not app_name: + # Create a new app + cmd = ["heroku", "apps:create"] + if name: + cmd.append(name) + cmd.append("--json") + create_output = check_output(cmd).decode("utf8") + app_name = json.loads(create_output)["name"] + + call(["heroku", "builds:create", "-a", app_name]) diff --git a/datasette/publish/now.py b/datasette/publish/now.py new file mode 100644 index 00000000..fd081111 --- /dev/null +++ b/datasette/publish/now.py @@ -0,0 +1,80 @@ +from datasette import hookimpl +import click +from subprocess import call + +from .common import ( + add_common_publish_arguments_and_options, + fail_if_publish_binary_not_installed, +) +from ..utils import temporary_docker_directory + + +@hookimpl +def publish_subcommand(publish): + @publish.command() + @add_common_publish_arguments_and_options + @click.option( + "-n", + "--name", + default="datasette", + help="Application name to use when deploying", + ) + @click.option("--force", is_flag=True, help="Pass --force option to now") + @click.option("--token", help="Auth token to use for deploy (Now only)") + @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") + def now( + files, + metadata, + extra_options, + branch, + template_dir, + plugins_dir, + static, + install, + version_note, + title, + license, + license_url, + source, + source_url, + name, + force, + token, + spatialite, + ): + fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now") + if extra_options: + extra_options += " " + else: + extra_options = "" + extra_options += "--config force_https_urls:on" + + with temporary_docker_directory( + files, + name, + metadata, + extra_options, + branch, + template_dir, + plugins_dir, + static, + install, + spatialite, + version_note, + { + "title": title, + "license": license, + "license_url": license_url, + "source": source, + "source_url": source_url, + }, + ): + args = [] + if force: + args.append("--force") + if token: + args.append("--token={}".format(token)) + if args: + call(["now"] + args) + else: + call("now") diff --git a/datasette/utils.py b/datasette/utils.py index 7419f9ae..f95d0695 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -1,6 +1,7 @@ from contextlib import contextmanager from collections import OrderedDict import base64 +import click import hashlib import imp import json @@ -376,6 +377,7 @@ def temporary_heroku_directory( plugins_dir, static, install, + version_note, extra_metadata=None ): # FIXME: lots of duplicated code from above @@ -430,7 +432,8 @@ def temporary_heroku_directory( os.path.join(tmp.name, 'plugins') ) extras.extend(['--plugins-dir', 'plugins/']) - + if version_note: + extras.extend(['--version-note', version_note]) if metadata: extras.extend(['--metadata', 'metadata.json']) if extra_options: @@ -876,3 +879,18 @@ def remove_infinites(row): for c in row ] return row + + +class StaticMount(click.ParamType): + name = "static mount" + + def convert(self, value, param, ctx): + if ":" not in value: + self.fail( + '"{}" should be of format mountpoint:directory'.format(value), + param, ctx + ) + path, dirpath = value.split(":") + if not os.path.exists(dirpath) or not os.path.isdir(dirpath): + self.fail("%s is not a valid directory path" % value, param, ctx) + return path, dirpath diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt new file mode 100644 index 00000000..f82eaf3e --- /dev/null +++ b/docs/datasette-publish-heroku-help.txt @@ -0,0 +1,20 @@ +$ datasette publish heroku --help + +Usage: datasette publish heroku [OPTIONS] [FILES]... + +Options: + -m, --metadata FILENAME Path to JSON file containing metadata to publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. master + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --install TEXT Additional packages (e.g. plugins) to install + --version-note TEXT Additional note to show on /-/versions + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + -n, --name TEXT Application name to use when deploying + --help Show this message and exit. diff --git a/docs/datasette-publish-help.txt b/docs/datasette-publish-now-help.txt similarity index 74% rename from docs/datasette-publish-help.txt rename to docs/datasette-publish-now-help.txt index 04cda361..ce09030f 100644 --- a/docs/datasette-publish-help.txt +++ b/docs/datasette-publish-now-help.txt @@ -1,31 +1,23 @@ -$ datasette publish --help +$ datasette publish now --help -Usage: datasette publish [OPTIONS] PUBLISHER [FILES]... - - Publish specified SQLite database files to the internet along with a datasette API. - - Options for PUBLISHER: * 'now' - You must have Zeit Now installed: - https://zeit.co/now * 'heroku' - You must have Heroku installed: - https://cli.heroku.com/ - - Example usage: datasette publish now my-database.db +Usage: datasette publish now [OPTIONS] [FILES]... Options: - -n, --name TEXT Application name to use when deploying -m, --metadata FILENAME Path to JSON file containing metadata to publish --extra-options TEXT Extra options to pass to datasette serve - --force Pass --force option to now --branch TEXT Install datasette from a GitHub branch e.g. master - --token TEXT Auth token to use for deploy (Now only) --template-dir DIRECTORY Path to directory containing custom templates --plugins-dir DIRECTORY Path to directory containing custom plugins --static STATIC MOUNT mountpoint:path-to-directory for serving static files --install TEXT Additional packages (e.g. plugins) to install - --spatialite Enable SpatialLite extension --version-note TEXT Additional note to show on /-/versions --title TEXT Title for metadata --license TEXT License label for metadata --license_url TEXT License URL for metadata --source TEXT Source label for metadata --source_url TEXT Source URL for metadata + -n, --name TEXT Application name to use when deploying + --force Pass --force option to now + --token TEXT Auth token to use for deploy (Now only) + --spatialite Enable SpatialLite extension --help Show this message and exit. diff --git a/docs/plugins.rst b/docs/plugins.rst index 24635191..fc351bf6 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -258,3 +258,12 @@ you have one: return [ '/-/static-plugins/your_plugin/app.js' ] + +publish_subcommand(publish) +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This hook allows you to create new providers for the ``datasette publish`` +command. Datasette uses this hook internally to implement the default ``now`` +and ``heroku`` subcommands, so you can read +`their source `_ +to see examples of this hook in action. diff --git a/docs/publish.rst b/docs/publish.rst index 1abe8881..8350afe0 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -36,6 +36,8 @@ You can use ``anything-you-like.now.sh``, provided no one else has already regis You can also use custom domains, if you `first register them with Zeit Now `_. +.. literalinclude:: datasette-publish-now-help.txt + Publishing to Heroku -------------------- @@ -51,6 +53,8 @@ This will output some details about the new deployment, including a URL like thi You can specify a custom app name by passing ``-n my-app-name`` to the publish command. This will also allow you to overwrite an existing app. +.. literalinclude:: datasette-publish-heroku-help.txt + Custom metadata and plugins --------------------------- @@ -71,9 +75,6 @@ You can also specify plugins you would like to install. For example, if you want datasette publish now mydatabase.db --install=datasette-vega -A full list of options can be seen by running ``datasette publish --help``: - -.. literalinclude:: datasette-publish-help.txt datasette package ================= diff --git a/tests/test_docs.py b/tests/test_docs.py index ffbb7ca1..b8581e17 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -22,21 +22,22 @@ def test_config_options_are_documented(config): assert config.name in get_headings("config.rst") -@pytest.mark.parametrize('name,filename', ( - ('serve', 'datasette-serve-help.txt'), - ('package', 'datasette-package-help.txt'), - ('publish', 'datasette-publish-help.txt'), +@pytest.mark.parametrize("name,filename", ( + ("serve", "datasette-serve-help.txt"), + ("package", "datasette-package-help.txt"), + ("publish now", "datasette-publish-now-help.txt"), + ("publish heroku", "datasette-publish-heroku-help.txt"), )) def test_help_includes(name, filename): expected = open(str(docs_path / filename)).read() runner = CliRunner() - result = runner.invoke(cli, [name, '--help'], terminal_width=88) - actual = '$ datasette {} --help\n\n{}'.format( + result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88) + actual = "$ datasette {} --help\n\n{}".format( name, result.output ) # actual has "Usage: cli package [OPTIONS] FILES" # because it doesn't know that cli will be aliased to datasette - expected = expected.replace('Usage: datasette', 'Usage: cli') + expected = expected.replace("Usage: datasette", "Usage: cli") assert expected == actual diff --git a/update-docs-help.py b/update-docs-help.py index 8f6e8956..ea311c57 100644 --- a/update-docs-help.py +++ b/update-docs-help.py @@ -7,14 +7,17 @@ docs_path = Path(__file__).parent / "docs" includes = ( ("serve", "datasette-serve-help.txt"), ("package", "datasette-package-help.txt"), - ("publish", "datasette-publish-help.txt"), + ("publish now", "datasette-publish-now-help.txt"), + ("publish heroku", "datasette-publish-heroku-help.txt"), ) def update_help_includes(): for name, filename in includes: runner = CliRunner() - result = runner.invoke(cli, [name, "--help"], terminal_width=88) + result = runner.invoke( + cli, name.split() + ["--help"], terminal_width=88 + ) actual = "$ datasette {} --help\n\n{}".format( name, result.output )