mirror
/
datasette
kopia lustrzana https://github.com/simonw/datasette
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

publish_subcommand hook + default plugins mechanism, used for publish heroku/now (#349) 

This change introduces a new plugin hook, publish_subcommand, which can be
used to implement new subcommands for the "datasette publish" command family.

I've used this new hook to refactor out the "publish now" and "publish heroku"
implementations into separate modules. I've also added unit tests for these
two publishers, mocking the subprocess.call and subprocess.check_output
functions.

As part of this, I introduced a mechanism for loading default plugins. These
are defined in the new "default_plugins" list inside datasette/app.py

Closes #217 (Plugin support for datasette publish)
Closes #348 (Unit tests for "datasette publish")
Refs #14, #59, #102, #103, #146, #236, #347
starlette
Simon Willison 2018-07-25 22:15:59 -07:00 zatwierdzone przez GitHub
rodzic 3ac21c7498
commit dbbe707841
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
14 zmienionych plików z 343 dodań i 239 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

11
datasette/app.py
Wyświetl plik

@ -2,6 +2,7 @@ import asyncio
import click
import collections
import hashlib
import importlib
import itertools
import os
import sqlite3
@ -41,6 +42,11 @@ from .utils import (
from .inspect import inspect_hash, inspect_views, inspect_tables
from .version import __version__
default_plugins = (
"datasette.publish.heroku",
"datasette.publish.now",
)
app_root = Path(__file__).parent.parent
connections = threading.local()
@ -49,6 +55,11 @@ pm = pluggy.PluginManager("datasette")
pm.add_hookspecs(hookspecs)
pm.load_setuptools_entrypoints("datasette")
# Load default plugins
for plugin in default_plugins:
mod = importlib.import_module(plugin)
pm.register(mod, plugin)
ConfigOption = collections.namedtuple(
"ConfigOption", ("name", "default", "help")

221
datasette/cli.py
Wyświetl plik

@ -4,32 +4,17 @@ from click_default_group import DefaultGroup
import json
import os
import shutil
from subprocess import call, check_output
from subprocess import call
import sys
from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS
from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS, pm
from .utils import (
temporary_docker_directory,
temporary_heroku_directory,
value_as_boolean,
StaticMount,
ValueAsBooleanError,
)
class StaticMount(click.ParamType):
name = "static mount"
def convert(self, value, param, ctx):
if ":" not in value:
self.fail(
'"{}" should be of format mountpoint:directory'.format(value),
param, ctx
)
path, dirpath = value.split(":")
if not os.path.exists(dirpath) or not os.path.isdir(dirpath):
self.fail("%s is not a valid directory path" % value, param, ctx)
return path, dirpath
class Config(click.ParamType):
name = "config"
@ -93,202 +78,14 @@ def inspect(files, inspect_file, sqlite_extensions):
open(inspect_file, "w").write(json.dumps(app.inspect(), indent=2))
@cli.command()
@click.argument("publisher", type=click.Choice(["now", "heroku"]))
@click.argument("files", type=click.Path(exists=True), nargs=-1)
@click.option(
"-n",
"--name",
default="datasette",
help="Application name to use when deploying",
)
@click.option(
"-m",
"--metadata",
type=click.File(mode="r"),
help="Path to JSON file containing metadata to publish",
)
@click.option("--extra-options", help="Extra options to pass to datasette serve")
@click.option("--force", is_flag=True, help="Pass --force option to now")
@click.option("--branch", help="Install datasette from a GitHub branch e.g. master")
@click.option("--token", help="Auth token to use for deploy (Now only)")
@click.option(
"--template-dir",
type=click.Path(exists=True, file_okay=False, dir_okay=True),
help="Path to directory containing custom templates",
)
@click.option(
"--plugins-dir",
type=click.Path(exists=True, file_okay=False, dir_okay=True),
help="Path to directory containing custom plugins",
)
@click.option(
"--static",
type=StaticMount(),
help="mountpoint:path-to-directory for serving static files",
multiple=True,
)
@click.option(
"--install",
help="Additional packages (e.g. plugins) to install",
multiple=True,
)
@click.option(
"--spatialite", is_flag=True, help="Enable SpatialLite extension"
)
@click.option("--version-note", help="Additional note to show on /-/versions")
@click.option("--title", help="Title for metadata")
@click.option("--license", help="License label for metadata")
@click.option("--license_url", help="License URL for metadata")
@click.option("--source", help="Source label for metadata")
@click.option("--source_url", help="Source URL for metadata")
def publish(
publisher,
files,
name,
metadata,
extra_options,
force,
branch,
token,
template_dir,
plugins_dir,
static,
install,
spatialite,
version_note,
**extra_metadata
):
"""
Publish specified SQLite database files to the internet along with a datasette API.
@cli.group()
def publish():
"Publish specified SQLite database files to the internet along with a Datasette-powered interface and API"
pass
Options for PUBLISHER:
* 'now' - You must have Zeit Now installed: https://zeit.co/now
* 'heroku' - You must have Heroku installed: https://cli.heroku.com/
Example usage: datasette publish now my-database.db
"""
def _fail_if_publish_binary_not_installed(binary, publish_target, install_link):
"""Exit (with error message) if ``binary` isn't installed"""
if not shutil.which(binary):
click.secho(
"Publishing to {publish_target} requires {binary} to be installed and configured".format(
publish_target=publish_target, binary=binary
),
bg="red",
fg="white",
bold=True,
err=True,
)
click.echo(
"Follow the instructions at {install_link}".format(
install_link=install_link
),
err=True,
)
sys.exit(1)
if publisher == "now":
_fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now")
if extra_options:
extra_options += " "
else:
extra_options = ""
extra_options += "--config force_https_urls:on"
with temporary_docker_directory(
files,
name,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
spatialite,
version_note,
extra_metadata,
):
args = []
if force:
args.append("--force")
if token:
args.append("--token={}".format(token))
if args:
call(["now"] + args)
else:
call("now")
elif publisher == "heroku":
_fail_if_publish_binary_not_installed(
"heroku", "Heroku", "https://cli.heroku.com"
)
if spatialite:
click.secho(
"The --spatialite option is not yet supported for Heroku",
bg="red",
fg="white",
bold=True,
err=True,
)
click.echo(
"See https://github.com/simonw/datasette/issues/301",
err=True,
)
sys.exit(1)
# Check for heroku-builds plugin
plugins = [
line.split()[0] for line in check_output(["heroku", "plugins"]).splitlines()
]
if b"heroku-builds" not in plugins:
click.echo(
"Publishing to Heroku requires the heroku-builds plugin to be installed."
)
click.confirm(
"Install it? (this will run `heroku plugins:install heroku-builds`)",
abort=True,
)
call(["heroku", "plugins:install", "heroku-builds"])
with temporary_heroku_directory(
files,
name,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
extra_metadata,
):
app_name = None
if name:
# Check to see if this app already exists
list_output = check_output(["heroku", "apps:list", "--json"]).decode('utf8')
apps = json.loads(list_output)
for app in apps:
if app['name'] == name:
app_name = name
break
if not app_name:
# Create a new app
cmd = ["heroku", "apps:create"]
if name:
cmd.append(name)
cmd.append("--json")
create_output = check_output(cmd).decode(
"utf8"
)
app_name = json.loads(create_output)["name"]
call(["heroku", "builds:create", "-a", app_name])
# Register publish plugins
pm.hook.publish_subcommand(publish=publish)
@cli.command()

5
datasette/hookspecs.py
Wyświetl plik

@ -23,3 +23,8 @@ def extra_css_urls():
@hookspec
def extra_js_urls():
"Extra JavaScript URLs added by this plugin"
@hookspec
def publish_subcommand(publish):
"Subcommands for 'datasette publish'"

0
datasette/publish/__init__.py 100644
Wyświetl plik

68
datasette/publish/common.py 100644
Wyświetl plik

@ -0,0 +1,68 @@
from ..utils import StaticMount
import click
import shutil
import sys
def add_common_publish_arguments_and_options(subcommand):
for decorator in reversed((
click.argument("files", type=click.Path(exists=True), nargs=-1),
click.option(
"-m",
"--metadata",
type=click.File(mode="r"),
help="Path to JSON file containing metadata to publish",
),
click.option("--extra-options", help="Extra options to pass to datasette serve"),
click.option("--branch", help="Install datasette from a GitHub branch e.g. master"),
click.option(
"--template-dir",
type=click.Path(exists=True, file_okay=False, dir_okay=True),
help="Path to directory containing custom templates",
),
click.option(
"--plugins-dir",
type=click.Path(exists=True, file_okay=False, dir_okay=True),
help="Path to directory containing custom plugins",
),
click.option(
"--static",
type=StaticMount(),
help="mountpoint:path-to-directory for serving static files",
multiple=True,
),
click.option(
"--install",
help="Additional packages (e.g. plugins) to install",
multiple=True,
),
click.option("--version-note", help="Additional note to show on /-/versions"),
click.option("--title", help="Title for metadata"),
click.option("--license", help="License label for metadata"),
click.option("--license_url", help="License URL for metadata"),
click.option("--source", help="Source label for metadata"),
click.option("--source_url", help="Source URL for metadata"),
)):
subcommand = decorator(subcommand)
return subcommand
def fail_if_publish_binary_not_installed(binary, publish_target, install_link):
"""Exit (with error message) if ``binary` isn't installed"""
if not shutil.which(binary):
click.secho(
"Publishing to {publish_target} requires {binary} to be installed and configured".format(
publish_target=publish_target, binary=binary
),
bg="red",
fg="white",
bold=True,
err=True,
)
click.echo(
"Follow the instructions at {install_link}".format(
install_link=install_link
),
err=True,
)
sys.exit(1)

99
datasette/publish/heroku.py 100644
Wyświetl plik

@ -0,0 +1,99 @@
from datasette import hookimpl
import click
import json
from subprocess import call, check_output
from .common import (
add_common_publish_arguments_and_options,
fail_if_publish_binary_not_installed,
)
from ..utils import temporary_heroku_directory
@hookimpl
def publish_subcommand(publish):
@publish.command()
@add_common_publish_arguments_and_options
@click.option(
"-n",
"--name",
default="datasette",
help="Application name to use when deploying",
)
def heroku(
files,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
version_note,
title,
license,
license_url,
source,
source_url,
name,
):
fail_if_publish_binary_not_installed(
"heroku", "Heroku", "https://cli.heroku.com"
)
# Check for heroku-builds plugin
plugins = [
line.split()[0] for line in check_output(["heroku", "plugins"]).splitlines()
]
if b"heroku-builds" not in plugins:
click.echo(
"Publishing to Heroku requires the heroku-builds plugin to be installed."
)
click.confirm(
"Install it? (this will run `heroku plugins:install heroku-builds`)",
abort=True,
)
call(["heroku", "plugins:install", "heroku-builds"])
with temporary_heroku_directory(
files,
name,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
version_note,
{
"title": title,
"license": license,
"license_url": license_url,
"source": source,
"source_url": source_url,
},
):
app_name = None
if name:
# Check to see if this app already exists
list_output = check_output(["heroku", "apps:list", "--json"]).decode(
"utf8"
)
apps = json.loads(list_output)
for app in apps:
if app["name"] == name:
app_name = name
break
if not app_name:
# Create a new app
cmd = ["heroku", "apps:create"]
if name:
cmd.append(name)
cmd.append("--json")
create_output = check_output(cmd).decode("utf8")
app_name = json.loads(create_output)["name"]
call(["heroku", "builds:create", "-a", app_name])

80
datasette/publish/now.py 100644
Wyświetl plik

@ -0,0 +1,80 @@
from datasette import hookimpl
import click
from subprocess import call
from .common import (
add_common_publish_arguments_and_options,
fail_if_publish_binary_not_installed,
)
from ..utils import temporary_docker_directory
@hookimpl
def publish_subcommand(publish):
@publish.command()
@add_common_publish_arguments_and_options
@click.option(
"-n",
"--name",
default="datasette",
help="Application name to use when deploying",
)
@click.option("--force", is_flag=True, help="Pass --force option to now")
@click.option("--token", help="Auth token to use for deploy (Now only)")
@click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension")
def now(
files,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
version_note,
title,
license,
license_url,
source,
source_url,
name,
force,
token,
spatialite,
):
fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now")
if extra_options:
extra_options += " "
else:
extra_options = ""
extra_options += "--config force_https_urls:on"
with temporary_docker_directory(
files,
name,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
spatialite,
version_note,
{
"title": title,
"license": license,
"license_url": license_url,
"source": source,
"source_url": source_url,
},
):
args = []
if force:
args.append("--force")
if token:
args.append("--token={}".format(token))
if args:
call(["now"] + args)
else:
call("now")

20
datasette/utils.py
Wyświetl plik

@ -1,6 +1,7 @@
from contextlib import contextmanager
from collections import OrderedDict
import base64
import click
import hashlib
import imp
import json
@ -376,6 +377,7 @@ def temporary_heroku_directory(
plugins_dir,
static,
install,
version_note,
extra_metadata=None
):
# FIXME: lots of duplicated code from above
@ -430,7 +432,8 @@ def temporary_heroku_directory(
os.path.join(tmp.name, 'plugins')
)
extras.extend(['--plugins-dir', 'plugins/'])
if version_note:
extras.extend(['--version-note', version_note])
if metadata:
extras.extend(['--metadata', 'metadata.json'])
if extra_options:
@ -876,3 +879,18 @@ def remove_infinites(row):
for c in row
]
return row
class StaticMount(click.ParamType):
name = "static mount"
def convert(self, value, param, ctx):
if ":" not in value:
self.fail(
'"{}" should be of format mountpoint:directory'.format(value),
param, ctx
)
path, dirpath = value.split(":")
if not os.path.exists(dirpath) or not os.path.isdir(dirpath):
self.fail("%s is not a valid directory path" % value, param, ctx)
return path, dirpath

20
docs/datasette-publish-heroku-help.txt 100644
Wyświetl plik

@ -0,0 +1,20 @@
$ datasette publish heroku --help
Usage: datasette publish heroku [OPTIONS] [FILES]...
Options:
-m, --metadata FILENAME Path to JSON file containing metadata to publish
--extra-options TEXT Extra options to pass to datasette serve
--branch TEXT Install datasette from a GitHub branch e.g. master
--template-dir DIRECTORY Path to directory containing custom templates
--plugins-dir DIRECTORY Path to directory containing custom plugins
--static STATIC MOUNT mountpoint:path-to-directory for serving static files
--install TEXT Additional packages (e.g. plugins) to install
--version-note TEXT Additional note to show on /-/versions
--title TEXT Title for metadata
--license TEXT License label for metadata
--license_url TEXT License URL for metadata
--source TEXT Source label for metadata
--source_url TEXT Source URL for metadata
-n, --name TEXT Application name to use when deploying
--help Show this message and exit.

20
docs/datasette-publish-help.txt → docs/datasette-publish-now-help.txt
Wyświetl plik

@ -1,31 +1,23 @@
$ datasette publish --help
$ datasette publish now --help
Usage: datasette publish [OPTIONS] PUBLISHER [FILES]...
Publish specified SQLite database files to the internet along with a datasette API.
Options for PUBLISHER: * 'now' - You must have Zeit Now installed:
https://zeit.co/now * 'heroku' - You must have Heroku installed:
https://cli.heroku.com/
Example usage: datasette publish now my-database.db
Usage: datasette publish now [OPTIONS] [FILES]...
Options:
-n, --name TEXT Application name to use when deploying
-m, --metadata FILENAME Path to JSON file containing metadata to publish
--extra-options TEXT Extra options to pass to datasette serve
--force Pass --force option to now
--branch TEXT Install datasette from a GitHub branch e.g. master
--token TEXT Auth token to use for deploy (Now only)
--template-dir DIRECTORY Path to directory containing custom templates
--plugins-dir DIRECTORY Path to directory containing custom plugins
--static STATIC MOUNT mountpoint:path-to-directory for serving static files
--install TEXT Additional packages (e.g. plugins) to install
--spatialite Enable SpatialLite extension
--version-note TEXT Additional note to show on /-/versions
--title TEXT Title for metadata
--license TEXT License label for metadata
--license_url TEXT License URL for metadata
--source TEXT Source label for metadata
--source_url TEXT Source URL for metadata
-n, --name TEXT Application name to use when deploying
--force Pass --force option to now
--token TEXT Auth token to use for deploy (Now only)
--spatialite Enable SpatialLite extension
--help Show this message and exit.

9
docs/plugins.rst
Wyświetl plik

@ -258,3 +258,12 @@ you have one:
return [
'/-/static-plugins/your_plugin/app.js'
]
publish_subcommand(publish)
~~~~~~~~~~~~~~~~~~~~~~~~~~~
This hook allows you to create new providers for the ``datasette publish``
command. Datasette uses this hook internally to implement the default ``now``
and ``heroku`` subcommands, so you can read
`their source <https://github.com/simonw/datasette/tree/master/datasette/publish>`_
to see examples of this hook in action.

7
docs/publish.rst
Wyświetl plik

@ -36,6 +36,8 @@ You can use ``anything-you-like.now.sh``, provided no one else has already regis
You can also use custom domains, if you `first register them with Zeit Now <https://zeit.co/docs/features/aliases>`_.
.. literalinclude:: datasette-publish-now-help.txt
Publishing to Heroku
--------------------
@ -51,6 +53,8 @@ This will output some details about the new deployment, including a URL like thi
You can specify a custom app name by passing ``-n my-app-name`` to the publish command. This will also allow you to overwrite an existing app.
.. literalinclude:: datasette-publish-heroku-help.txt
Custom metadata and plugins
---------------------------
@ -71,9 +75,6 @@ You can also specify plugins you would like to install. For example, if you want
datasette publish now mydatabase.db --install=datasette-vega
A full list of options can be seen by running ``datasette publish --help``:
.. literalinclude:: datasette-publish-help.txt
datasette package
=================

15
tests/test_docs.py
Wyświetl plik

@ -22,21 +22,22 @@ def test_config_options_are_documented(config):
assert config.name in get_headings("config.rst")
@pytest.mark.parametrize('name,filename', (
('serve', 'datasette-serve-help.txt'),
('package', 'datasette-package-help.txt'),
('publish', 'datasette-publish-help.txt'),
@pytest.mark.parametrize("name,filename", (
("serve", "datasette-serve-help.txt"),
("package", "datasette-package-help.txt"),
("publish now", "datasette-publish-now-help.txt"),
("publish heroku", "datasette-publish-heroku-help.txt"),
))
def test_help_includes(name, filename):
expected = open(str(docs_path / filename)).read()
runner = CliRunner()
result = runner.invoke(cli, [name, '--help'], terminal_width=88)
actual = '$ datasette {} --help\n\n{}'.format(
result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88)
actual = "$ datasette {} --help\n\n{}".format(
name, result.output
)
# actual has "Usage: cli package [OPTIONS] FILES"
# because it doesn't know that cli will be aliased to datasette
expected = expected.replace('Usage: datasette', 'Usage: cli')
expected = expected.replace("Usage: datasette", "Usage: cli")
assert expected == actual

7
update-docs-help.py
Wyświetl plik

@ -7,14 +7,17 @@ docs_path = Path(__file__).parent / "docs"
includes = (
("serve", "datasette-serve-help.txt"),
("package", "datasette-package-help.txt"),
("publish", "datasette-publish-help.txt"),
("publish now", "datasette-publish-now-help.txt"),
("publish heroku", "datasette-publish-heroku-help.txt"),
)
def update_help_includes():
for name, filename in includes:
runner = CliRunner()
result = runner.invoke(cli, [name, "--help"], terminal_width=88)
result = runner.invoke(
cli, name.split() + ["--help"], terminal_width=88
)
actual = "$ datasette {} --help\n\n{}".format(
name, result.output
)