diff --git a/datasette/app.py b/datasette/app.py index 66a7573a..052131d0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2,6 +2,7 @@ import asyncio import click import collections import hashlib +import importlib import itertools import os import sqlite3 @@ -41,6 +42,11 @@ from .utils import ( from .inspect import inspect_hash, inspect_views, inspect_tables from .version import __version__ +default_plugins = ( + "datasette.publish.heroku", + "datasette.publish.now", +) + app_root = Path(__file__).parent.parent connections = threading.local() @@ -49,6 +55,11 @@ pm = pluggy.PluginManager("datasette") pm.add_hookspecs(hookspecs) pm.load_setuptools_entrypoints("datasette") +# Load default plugins +for plugin in default_plugins: + mod = importlib.import_module(plugin) + pm.register(mod, plugin) + ConfigOption = collections.namedtuple( "ConfigOption", ("name", "default", "help") diff --git a/datasette/cli.py b/datasette/cli.py index 72770326..820367ac 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -4,32 +4,17 @@ from click_default_group import DefaultGroup import json import os import shutil -from subprocess import call, check_output +from subprocess import call import sys -from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS +from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS, pm from .utils import ( temporary_docker_directory, - temporary_heroku_directory, value_as_boolean, + StaticMount, ValueAsBooleanError, ) -class StaticMount(click.ParamType): - name = "static mount" - - def convert(self, value, param, ctx): - if ":" not in value: - self.fail( - '"{}" should be of format mountpoint:directory'.format(value), - param, ctx - ) - path, dirpath = value.split(":") - if not os.path.exists(dirpath) or not os.path.isdir(dirpath): - self.fail("%s is not a valid directory path" % value, param, ctx) - return path, dirpath - - class Config(click.ParamType): name = "config" @@ -93,202 +78,14 @@ def inspect(files, inspect_file, sqlite_extensions): open(inspect_file, "w").write(json.dumps(app.inspect(), indent=2)) -@cli.command() -@click.argument("publisher", type=click.Choice(["now", "heroku"])) -@click.argument("files", type=click.Path(exists=True), nargs=-1) -@click.option( - "-n", - "--name", - default="datasette", - help="Application name to use when deploying", -) -@click.option( - "-m", - "--metadata", - type=click.File(mode="r"), - help="Path to JSON file containing metadata to publish", -) -@click.option("--extra-options", help="Extra options to pass to datasette serve") -@click.option("--force", is_flag=True, help="Pass --force option to now") -@click.option("--branch", help="Install datasette from a GitHub branch e.g. master") -@click.option("--token", help="Auth token to use for deploy (Now only)") -@click.option( - "--template-dir", - type=click.Path(exists=True, file_okay=False, dir_okay=True), - help="Path to directory containing custom templates", -) -@click.option( - "--plugins-dir", - type=click.Path(exists=True, file_okay=False, dir_okay=True), - help="Path to directory containing custom plugins", -) -@click.option( - "--static", - type=StaticMount(), - help="mountpoint:path-to-directory for serving static files", - multiple=True, -) -@click.option( - "--install", - help="Additional packages (e.g. plugins) to install", - multiple=True, -) -@click.option( - "--spatialite", is_flag=True, help="Enable SpatialLite extension" -) -@click.option("--version-note", help="Additional note to show on /-/versions") -@click.option("--title", help="Title for metadata") -@click.option("--license", help="License label for metadata") -@click.option("--license_url", help="License URL for metadata") -@click.option("--source", help="Source label for metadata") -@click.option("--source_url", help="Source URL for metadata") -def publish( - publisher, - files, - name, - metadata, - extra_options, - force, - branch, - token, - template_dir, - plugins_dir, - static, - install, - spatialite, - version_note, - **extra_metadata -): - """ - Publish specified SQLite database files to the internet along with a datasette API. +@cli.group() +def publish(): + "Publish specified SQLite database files to the internet along with a Datasette-powered interface and API" + pass - Options for PUBLISHER: - * 'now' - You must have Zeit Now installed: https://zeit.co/now - * 'heroku' - You must have Heroku installed: https://cli.heroku.com/ - Example usage: datasette publish now my-database.db - """ - - def _fail_if_publish_binary_not_installed(binary, publish_target, install_link): - """Exit (with error message) if ``binary` isn't installed""" - if not shutil.which(binary): - click.secho( - "Publishing to {publish_target} requires {binary} to be installed and configured".format( - publish_target=publish_target, binary=binary - ), - bg="red", - fg="white", - bold=True, - err=True, - ) - click.echo( - "Follow the instructions at {install_link}".format( - install_link=install_link - ), - err=True, - ) - sys.exit(1) - - if publisher == "now": - _fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now") - if extra_options: - extra_options += " " - else: - extra_options = "" - extra_options += "--config force_https_urls:on" - - with temporary_docker_directory( - files, - name, - metadata, - extra_options, - branch, - template_dir, - plugins_dir, - static, - install, - spatialite, - version_note, - extra_metadata, - ): - args = [] - if force: - args.append("--force") - if token: - args.append("--token={}".format(token)) - if args: - call(["now"] + args) - else: - call("now") - - elif publisher == "heroku": - _fail_if_publish_binary_not_installed( - "heroku", "Heroku", "https://cli.heroku.com" - ) - if spatialite: - click.secho( - "The --spatialite option is not yet supported for Heroku", - bg="red", - fg="white", - bold=True, - err=True, - ) - click.echo( - "See https://github.com/simonw/datasette/issues/301", - err=True, - ) - sys.exit(1) - - # Check for heroku-builds plugin - plugins = [ - line.split()[0] for line in check_output(["heroku", "plugins"]).splitlines() - ] - if b"heroku-builds" not in plugins: - click.echo( - "Publishing to Heroku requires the heroku-builds plugin to be installed." - ) - click.confirm( - "Install it? (this will run `heroku plugins:install heroku-builds`)", - abort=True, - ) - call(["heroku", "plugins:install", "heroku-builds"]) - - with temporary_heroku_directory( - files, - name, - metadata, - extra_options, - branch, - template_dir, - plugins_dir, - static, - install, - extra_metadata, - ): - - app_name = None - if name: - # Check to see if this app already exists - list_output = check_output(["heroku", "apps:list", "--json"]).decode('utf8') - apps = json.loads(list_output) - - for app in apps: - if app['name'] == name: - app_name = name - break - - if not app_name: - # Create a new app - cmd = ["heroku", "apps:create"] - if name: - cmd.append(name) - cmd.append("--json") - create_output = check_output(cmd).decode( - "utf8" - ) - app_name = json.loads(create_output)["name"] - - call(["heroku", "builds:create", "-a", app_name]) +# Register publish plugins +pm.hook.publish_subcommand(publish=publish) @cli.command() diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 240b58db..9546eebf 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -23,3 +23,8 @@ def extra_css_urls(): @hookspec def extra_js_urls(): "Extra JavaScript URLs added by this plugin" + + +@hookspec +def publish_subcommand(publish): + "Subcommands for 'datasette publish'" diff --git a/datasette/publish/__init__.py b/datasette/publish/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/datasette/publish/common.py b/datasette/publish/common.py new file mode 100644 index 00000000..9dd2ae9e --- /dev/null +++ b/datasette/publish/common.py @@ -0,0 +1,68 @@ +from ..utils import StaticMount +import click +import shutil +import sys + + +def add_common_publish_arguments_and_options(subcommand): + for decorator in reversed(( + click.argument("files", type=click.Path(exists=True), nargs=-1), + click.option( + "-m", + "--metadata", + type=click.File(mode="r"), + help="Path to JSON file containing metadata to publish", + ), + click.option("--extra-options", help="Extra options to pass to datasette serve"), + click.option("--branch", help="Install datasette from a GitHub branch e.g. master"), + click.option( + "--template-dir", + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Path to directory containing custom templates", + ), + click.option( + "--plugins-dir", + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Path to directory containing custom plugins", + ), + click.option( + "--static", + type=StaticMount(), + help="mountpoint:path-to-directory for serving static files", + multiple=True, + ), + click.option( + "--install", + help="Additional packages (e.g. plugins) to install", + multiple=True, + ), + click.option("--version-note", help="Additional note to show on /-/versions"), + click.option("--title", help="Title for metadata"), + click.option("--license", help="License label for metadata"), + click.option("--license_url", help="License URL for metadata"), + click.option("--source", help="Source label for metadata"), + click.option("--source_url", help="Source URL for metadata"), + )): + subcommand = decorator(subcommand) + return subcommand + + +def fail_if_publish_binary_not_installed(binary, publish_target, install_link): + """Exit (with error message) if ``binary` isn't installed""" + if not shutil.which(binary): + click.secho( + "Publishing to {publish_target} requires {binary} to be installed and configured".format( + publish_target=publish_target, binary=binary + ), + bg="red", + fg="white", + bold=True, + err=True, + ) + click.echo( + "Follow the instructions at {install_link}".format( + install_link=install_link + ), + err=True, + ) + sys.exit(1) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py new file mode 100644 index 00000000..af53a37a --- /dev/null +++ b/datasette/publish/heroku.py @@ -0,0 +1,99 @@ +from datasette import hookimpl +import click +import json +from subprocess import call, check_output + +from .common import ( + add_common_publish_arguments_and_options, + fail_if_publish_binary_not_installed, +) +from ..utils import temporary_heroku_directory + + +@hookimpl +def publish_subcommand(publish): + @publish.command() + @add_common_publish_arguments_and_options + @click.option( + "-n", + "--name", + default="datasette", + help="Application name to use when deploying", + ) + def heroku( + files, + metadata, + extra_options, + branch, + template_dir, + plugins_dir, + static, + install, + version_note, + title, + license, + license_url, + source, + source_url, + name, + ): + fail_if_publish_binary_not_installed( + "heroku", "Heroku", "https://cli.heroku.com" + ) + + # Check for heroku-builds plugin + plugins = [ + line.split()[0] for line in check_output(["heroku", "plugins"]).splitlines() + ] + if b"heroku-builds" not in plugins: + click.echo( + "Publishing to Heroku requires the heroku-builds plugin to be installed." + ) + click.confirm( + "Install it? (this will run `heroku plugins:install heroku-builds`)", + abort=True, + ) + call(["heroku", "plugins:install", "heroku-builds"]) + + with temporary_heroku_directory( + files, + name, + metadata, + extra_options, + branch, + template_dir, + plugins_dir, + static, + install, + version_note, + { + "title": title, + "license": license, + "license_url": license_url, + "source": source, + "source_url": source_url, + }, + ): + app_name = None + if name: + # Check to see if this app already exists + list_output = check_output(["heroku", "apps:list", "--json"]).decode( + "utf8" + ) + apps = json.loads(list_output) + + for app in apps: + if app["name"] == name: + app_name = name + break + + if not app_name: + # Create a new app + cmd = ["heroku", "apps:create"] + if name: + cmd.append(name) + cmd.append("--json") + create_output = check_output(cmd).decode("utf8") + app_name = json.loads(create_output)["name"] + + call(["heroku", "builds:create", "-a", app_name]) diff --git a/datasette/publish/now.py b/datasette/publish/now.py new file mode 100644 index 00000000..fd081111 --- /dev/null +++ b/datasette/publish/now.py @@ -0,0 +1,80 @@ +from datasette import hookimpl +import click +from subprocess import call + +from .common import ( + add_common_publish_arguments_and_options, + fail_if_publish_binary_not_installed, +) +from ..utils import temporary_docker_directory + + +@hookimpl +def publish_subcommand(publish): + @publish.command() + @add_common_publish_arguments_and_options + @click.option( + "-n", + "--name", + default="datasette", + help="Application name to use when deploying", + ) + @click.option("--force", is_flag=True, help="Pass --force option to now") + @click.option("--token", help="Auth token to use for deploy (Now only)") + @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") + def now( + files, + metadata, + extra_options, + branch, + template_dir, + plugins_dir, + static, + install, + version_note, + title, + license, + license_url, + source, + source_url, + name, + force, + token, + spatialite, + ): + fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now") + if extra_options: + extra_options += " " + else: + extra_options = "" + extra_options += "--config force_https_urls:on" + + with temporary_docker_directory( + files, + name, + metadata, + extra_options, + branch, + template_dir, + plugins_dir, + static, + install, + spatialite, + version_note, + { + "title": title, + "license": license, + "license_url": license_url, + "source": source, + "source_url": source_url, + }, + ): + args = [] + if force: + args.append("--force") + if token: + args.append("--token={}".format(token)) + if args: + call(["now"] + args) + else: + call("now") diff --git a/datasette/utils.py b/datasette/utils.py index 7419f9ae..f95d0695 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -1,6 +1,7 @@ from contextlib import contextmanager from collections import OrderedDict import base64 +import click import hashlib import imp import json @@ -376,6 +377,7 @@ def temporary_heroku_directory( plugins_dir, static, install, + version_note, extra_metadata=None ): # FIXME: lots of duplicated code from above @@ -430,7 +432,8 @@ def temporary_heroku_directory( os.path.join(tmp.name, 'plugins') ) extras.extend(['--plugins-dir', 'plugins/']) - + if version_note: + extras.extend(['--version-note', version_note]) if metadata: extras.extend(['--metadata', 'metadata.json']) if extra_options: @@ -876,3 +879,18 @@ def remove_infinites(row): for c in row ] return row + + +class StaticMount(click.ParamType): + name = "static mount" + + def convert(self, value, param, ctx): + if ":" not in value: + self.fail( + '"{}" should be of format mountpoint:directory'.format(value), + param, ctx + ) + path, dirpath = value.split(":") + if not os.path.exists(dirpath) or not os.path.isdir(dirpath): + self.fail("%s is not a valid directory path" % value, param, ctx) + return path, dirpath diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt new file mode 100644 index 00000000..f82eaf3e --- /dev/null +++ b/docs/datasette-publish-heroku-help.txt @@ -0,0 +1,20 @@ +$ datasette publish heroku --help + +Usage: datasette publish heroku [OPTIONS] [FILES]... + +Options: + -m, --metadata FILENAME Path to JSON file containing metadata to publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. master + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --install TEXT Additional packages (e.g. plugins) to install + --version-note TEXT Additional note to show on /-/versions + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + -n, --name TEXT Application name to use when deploying + --help Show this message and exit. diff --git a/docs/datasette-publish-help.txt b/docs/datasette-publish-now-help.txt similarity index 74% rename from docs/datasette-publish-help.txt rename to docs/datasette-publish-now-help.txt index 04cda361..ce09030f 100644 --- a/docs/datasette-publish-help.txt +++ b/docs/datasette-publish-now-help.txt @@ -1,31 +1,23 @@ -$ datasette publish --help +$ datasette publish now --help -Usage: datasette publish [OPTIONS] PUBLISHER [FILES]... - - Publish specified SQLite database files to the internet along with a datasette API. - - Options for PUBLISHER: * 'now' - You must have Zeit Now installed: - https://zeit.co/now * 'heroku' - You must have Heroku installed: - https://cli.heroku.com/ - - Example usage: datasette publish now my-database.db +Usage: datasette publish now [OPTIONS] [FILES]... Options: - -n, --name TEXT Application name to use when deploying -m, --metadata FILENAME Path to JSON file containing metadata to publish --extra-options TEXT Extra options to pass to datasette serve - --force Pass --force option to now --branch TEXT Install datasette from a GitHub branch e.g. master - --token TEXT Auth token to use for deploy (Now only) --template-dir DIRECTORY Path to directory containing custom templates --plugins-dir DIRECTORY Path to directory containing custom plugins --static STATIC MOUNT mountpoint:path-to-directory for serving static files --install TEXT Additional packages (e.g. plugins) to install - --spatialite Enable SpatialLite extension --version-note TEXT Additional note to show on /-/versions --title TEXT Title for metadata --license TEXT License label for metadata --license_url TEXT License URL for metadata --source TEXT Source label for metadata --source_url TEXT Source URL for metadata + -n, --name TEXT Application name to use when deploying + --force Pass --force option to now + --token TEXT Auth token to use for deploy (Now only) + --spatialite Enable SpatialLite extension --help Show this message and exit. diff --git a/docs/plugins.rst b/docs/plugins.rst index 24635191..fc351bf6 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -258,3 +258,12 @@ you have one: return [ '/-/static-plugins/your_plugin/app.js' ] + +publish_subcommand(publish) +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This hook allows you to create new providers for the ``datasette publish`` +command. Datasette uses this hook internally to implement the default ``now`` +and ``heroku`` subcommands, so you can read +`their source `_ +to see examples of this hook in action. diff --git a/docs/publish.rst b/docs/publish.rst index 1abe8881..8350afe0 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -36,6 +36,8 @@ You can use ``anything-you-like.now.sh``, provided no one else has already regis You can also use custom domains, if you `first register them with Zeit Now `_. +.. literalinclude:: datasette-publish-now-help.txt + Publishing to Heroku -------------------- @@ -51,6 +53,8 @@ This will output some details about the new deployment, including a URL like thi You can specify a custom app name by passing ``-n my-app-name`` to the publish command. This will also allow you to overwrite an existing app. +.. literalinclude:: datasette-publish-heroku-help.txt + Custom metadata and plugins --------------------------- @@ -71,9 +75,6 @@ You can also specify plugins you would like to install. For example, if you want datasette publish now mydatabase.db --install=datasette-vega -A full list of options can be seen by running ``datasette publish --help``: - -.. literalinclude:: datasette-publish-help.txt datasette package ================= diff --git a/tests/test_docs.py b/tests/test_docs.py index ffbb7ca1..b8581e17 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -22,21 +22,22 @@ def test_config_options_are_documented(config): assert config.name in get_headings("config.rst") -@pytest.mark.parametrize('name,filename', ( - ('serve', 'datasette-serve-help.txt'), - ('package', 'datasette-package-help.txt'), - ('publish', 'datasette-publish-help.txt'), +@pytest.mark.parametrize("name,filename", ( + ("serve", "datasette-serve-help.txt"), + ("package", "datasette-package-help.txt"), + ("publish now", "datasette-publish-now-help.txt"), + ("publish heroku", "datasette-publish-heroku-help.txt"), )) def test_help_includes(name, filename): expected = open(str(docs_path / filename)).read() runner = CliRunner() - result = runner.invoke(cli, [name, '--help'], terminal_width=88) - actual = '$ datasette {} --help\n\n{}'.format( + result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88) + actual = "$ datasette {} --help\n\n{}".format( name, result.output ) # actual has "Usage: cli package [OPTIONS] FILES" # because it doesn't know that cli will be aliased to datasette - expected = expected.replace('Usage: datasette', 'Usage: cli') + expected = expected.replace("Usage: datasette", "Usage: cli") assert expected == actual diff --git a/update-docs-help.py b/update-docs-help.py index 8f6e8956..ea311c57 100644 --- a/update-docs-help.py +++ b/update-docs-help.py @@ -7,14 +7,17 @@ docs_path = Path(__file__).parent / "docs" includes = ( ("serve", "datasette-serve-help.txt"), ("package", "datasette-package-help.txt"), - ("publish", "datasette-publish-help.txt"), + ("publish now", "datasette-publish-now-help.txt"), + ("publish heroku", "datasette-publish-heroku-help.txt"), ) def update_help_includes(): for name, filename in includes: runner = CliRunner() - result = runner.invoke(cli, [name, "--help"], terminal_width=88) + result = runner.invoke( + cli, name.split() + ["--help"], terminal_width=88 + ) actual = "$ datasette {} --help\n\n{}".format( name, result.output )