diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index 9dfa8e46..1abb6e0a 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -1,3 +1,7 @@ img.logo { width:100% } + +div.body p.caption { + font-size: 1.5em; +} diff --git a/docs/source/conf.py b/docs/source/conf.py index a786a9b7..af59a319 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -44,7 +44,7 @@ source_parsers = { } def setup(app): - app.add_stylesheet('custom.css') # may also be an URL + app.add_stylesheet('_static/custom.css') # may also be an URL # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: @@ -187,39 +187,3 @@ epub_copyright = copyright # A list of files that should not be packed into the epub file. epub_exclude_files = ['search.html'] - -# Build the repo2docker test syntax -import os -s = '' -FILE_ORDER_PREFERENCE = ['System', 'Python', 'Julia', 'Docker', 'Misc'] -SKIP_FOLDERS = ['binder-dir'] -FILES_DICT = {ii: [] for ii in FILE_ORDER_PREFERENCE} -for root, _, files in os.walk(os.path.join('..', '..', 'tests')): - depth = len(root.split('/')) - 3 - if 'README.rst' not in files or depth > 2: - continue - if any(folder in root for folder in SKIP_FOLDERS): - continue - header = files.pop(files.index('README.rst')) - with open(os.path.join(root, header), 'r') as ff: - this_s = ff.read() + '\n\n' - title = this_s.split('\n')[0] - for ifile in files: - filename = os.path.basename(ifile) - if filename == 'verify': - continue - with open(os.path.join(root, ifile), 'r') as ff: - lines = ff.readlines() - lines = [' ' + line for line in lines] - this_s += 'File: ``{}``\n{}\n\n**Contents**::\n\n'.format( - filename, '~' * (len(filename) + 10)) - this_s += '\n'.join(lines) - this_s += '\n\n' - usename = [ii for ii in FILE_ORDER_PREFERENCE if ii in title] - usename = 'Misc' if len(usename) == 0 else usename[0] - FILES_DICT[usename].append(this_s) -for key in FILE_ORDER_PREFERENCE: - for istring in FILES_DICT[key]: - s += istring -with open('./test_file_text.txt', 'w') as ff: - ff.write(s) diff --git a/docs/source/config_files.rst b/docs/source/config_files.rst index d354487b..e5d88a12 100644 --- a/docs/source/config_files.rst +++ b/docs/source/config_files.rst @@ -1,9 +1,8 @@ -.. NOTE: the header characters are different in this file because it is 'included' in another site -.. see https://raw.githubusercontent.com/jupyterhub/binder/master/doc/using.rst .. _config-files: +=================== Configuration Files -------------------- +=================== ``repo2docker`` looks for configuration files in the repository being built to determine how to build it. In general, ``repo2docker`` uses the same @@ -13,16 +12,8 @@ rather than creating new custom configuration files. A number of ``repo2docker`` configuration files can be combined to compose more complex setups. -``repo2docker`` will look for configuration files in either: - -* A folder named ``binder/`` in the root of the repository. -* The root directory of the repository. - -If the folder ``binder/`` is located at the top level of the repository, - **only configuration files in the** ``binder/`` **folder will be considered**. - The `binder examples `_ organization on -GitHub contains a list of sample repositories for common configurations +GitHub contains a list of sample repositories for common configurations that ``repo2docker`` can build with various configuration files such as Python and R installation in a repository. @@ -32,64 +23,71 @@ Below is a list of supported configuration files (roughly in the order of build :local: :depth: 1 -``Dockerfile`` -~~~~~~~~~~~~~~ - -In the majority of cases, providing your own Dockerfile is not necessary as the base -images provide core functionality, compact image sizes, and efficient builds. We recommend -trying the other configuration files before deciding to use your own Dockerfile. - -With Dockerfiles, a regular Docker build will be performed. -**If a Dockerfile is present, all other configuration files will be ignored.** - -See the `Binder Documentation `_ for -best-practices with Dockerfiles. - .. _environment-yml: -``environment.yml`` -~~~~~~~~~~~~~~~~~~~ +``environment.yml`` - Install a Python environment +================================================== ``environment.yml`` is the standard configuration file used by Anaconda, conda, -and miniconda that lets you install Python packages. -You can also install files from pip in your ``environment.yml`` as well. -Our example `environment.yml `_ -shows how one can specify a conda environment for repo2docker. +and miniconda that lets you install packages in the data analytics stack (it +primarily installs Python packages, though can be used to install a range of +non-Python packages as well). + +.. note:: + + You can install files from pip in your ``environment.yml`` as well. + For example, see the `binder-examples environment.yml + `_ + file. You can also specify which Python version to install in your built environment with ``environment.yml``. By default, ``repo2docker`` **installs Python 3.6** with your ``environment.yml`` unless you include the version of Python in the file. ``conda`` supports Python versions 3.6, 3.5, 3.4, and 2.7. -``repo2docker`` support is best with Python 3.6, 3.5, and 2.7. If you include -a Python version in a ``runtime.txt`` file in addition to your -``environment.yml``, your ``runtime.txt`` **will be ignored**. +``repo2docker`` support is best with Python 3.6, 3.5, and 2.7. -``requirements.txt`` -~~~~~~~~~~~~~~~~~~~~ +.. warning:: + If you include a Python version in a ``runtime.txt`` file in addition to your + ``environment.yml``, your ``runtime.txt`` **will be ignored**. + + +``requirements.txt`` - Install a Python environment +=================================================== This specifies a list of Python packages that should be installed in your environment. Our `requirements.txt example `_ on GitHub shows a typical requirements file. -``REQUIRE`` -~~~~~~~~~~~ + +``setup.py`` - Install Python packages +====================================== + +To install your repository like a Python package, you may include a +``setup.py`` file. repo2docker installs ``setup.py`` files by running +``pip install -e .``. + + +``REQUIRE`` - Install a Julia environment +========================================= This specifies a list of Julia packages. Repositories with a ``REQUIRE`` file **must also contain an** ``environment.yml`` **file**. To see an example of a Julia repository with ``REQUIRE`` and ``environment.yml``, visit `binder-examples/julia-python `_. -``install.R`` -~~~~~~~~~~~~~ + +``install.R`` - Install an R/RStudio environment +================================================ This is used to install R libraries pinned to a specific snapshot on `MRAN `_. To set the date of the snapshot add a runtime.txt_. For an example ``install.R`` file, visit our `example install.R file `_. -``apt.txt`` -~~~~~~~~~~~ + +``apt.txt`` - Install packages with apt-get +=========================================== A list of Debian packages that should be installed. The base image used is usually the latest released version of Ubuntu. @@ -97,7 +95,6 @@ version of Ubuntu. We use ``apt.txt``, for example, to install LaTeX in our `example apt.txt for LaTeX `_. - ``setup.py`` ~~~~~~~~~~~~ @@ -122,8 +119,8 @@ To install your repository like an R package, you may include a .. _postBuild: -``postBuild`` -~~~~~~~~~~~~~ +``postBuild`` - Run code after installing the environment +========================================================= A script that can contain arbitrary commands to be run after the whole repository has been built. If you want this to be a shell script, make sure the first line is ```#!/bin/bash``. @@ -132,10 +129,11 @@ An example use-case of ``postBuild`` file is JupyterLab's demo on mybinder.org. It uses a ``postBuild`` file in a folder called ``binder`` to `prepare their demo for binder `_. + .. _start: -``start`` -^^^^^^^^^ +``start`` - Run code before the user sessions starts +==================================================== A script that can contain simple commands to be run at runtime (as an `ENTRYPOINT ` @@ -149,17 +147,18 @@ should at most take a few seconds to run. If you only need to run things once during the build phase use :ref:`postBuild`. + .. TODO: Discuss runtime limits, best practices, etc. Also, point to an example. .. _runtime.txt: -``runtime.txt`` -~~~~~~~~~~~~~~~ +``runtime.txt`` - Specifying runtimes +===================================== This allows you to control the runtime of Python or R. -To use python-2.7: add python-2.7 in runtime.txt file. +To use python-2.7: add ``python-2.7`` in runtime.txt file. The repository will run in a virtualenv with Python 2 installed. To see a full example repository, visit our `Python2 example `_. @@ -174,3 +173,17 @@ used for installing libraries. To see an example R repository, visit our `R example in binder-examples `_. + + +``Dockerfile`` - Advanced environments +====================================== + +In the majority of cases, providing your own Dockerfile is not necessary as the base +images provide core functionality, compact image sizes, and efficient builds. We recommend +trying the other configuration files before deciding to use your own Dockerfile. + +With Dockerfiles, a regular Docker build will be performed. +**If a Dockerfile is present, all other configuration files will be ignored.** + +See the `Advanced Binder Documentation `_ for +best-practices with Dockerfiles. diff --git a/docs/source/howto/jupyterhub_images.rst b/docs/source/howto/jupyterhub_images.rst new file mode 100644 index 00000000..6697f1fb --- /dev/null +++ b/docs/source/howto/jupyterhub_images.rst @@ -0,0 +1,25 @@ +.. _jupyterhub_images: + +============================= +Build JupyterHub-ready images +============================= + +JupyterHub_ allows multiple +users to collaborate on a shared Jupyter server. ``repo2docker`` can build +Docker images that can be shared within a JupyterHub deployment. For example, +`mybinder.org `_ uses JupyterHub and ``repo2docker`` +to allow anyone to build a Docker image of a git repository online and +share an executable version of the repository with a URL to the built image. + +To build JupyterHub_-ready Docker images with ``repo2docker``, the +version of your JupterHub deployment must be included in the +``environment.yml`` or ``requirements.txt`` of the git repositories you +build. + +If your instance of JupyterHub uses ``DockerSpawner``, you will need to set its +command to run ``jupyterhub-singleuser`` by adding this line in your +configuration file:: + + c.DockerSpawner.cmd = ['jupyterhub-singleuser'] + +.. _JupyterHub: https://github.com/jupyterhub/jupyterhub diff --git a/docs/source/howto/languages.rst b/docs/source/howto/languages.rst new file mode 100644 index 00000000..27680995 --- /dev/null +++ b/docs/source/howto/languages.rst @@ -0,0 +1,84 @@ +.. _languages: + +===================================== +Choose languages for your environment +===================================== + +You can define many different languages in your configuration files. This +page describes how to use some of the more common ones. + +Python +====== + +Your environment will have Python (and specified dependencies) installed when +you use one of the following configuration files: + +* ``requirements.txt`` +* ``environment.yml`` + +Note that by default, the environment will have **Python 3** installed. + +Specifying a version of Python +------------------------------ + +To specify a specific version of Python, you have two options: + +* **use ``runtime.txt``**. Include a line that specifies the Python version in + this file. This line takes the following form:: + + python=X.X + + For example,:: + + python=2.7 +* **Use ``environment.yml``**. The Anaconda distribution also lets you define + the Python environment within ``environment.yml``. To do so, add ``python=X.X`` + to your dependencies section, like so:: + + name: python 2.7 + dependencies: + - python=2.7 + - numpy + +The R Language +============== + +To ensure that R is installed, you must specify a version of R in a ``runtime.txt`` +file. This takes the following form:: + + r-YYYY-MM-DD + +The date corresponds to the state of the MRAN repository at this day. Make sure +that you choose a day with the desired version of your packages. For example, +to use the MRAN repository on January 1st, 2018, add this line to ``runtime.txt``:: + + r-2018-01-01 + +Note that to install specific packages with the R environment, you should +use the ``install.R`` configuration file. + +Julia +===== + +To build an environment with Julia, include a configuration file called +``REQUIRE``. Each line of this file should include a package that you wish +to have installed with Julia. For example, the following contents of ``REQURE`` +would install the ``PyPlot`` package with your Julia environment.:: + + PyPlot + +Languages not covered here +========================== + +If a language is not "officially" supported by a build pack, it can often be +installed with a ``postBuild`` script. This will run arbitrary ``bash`` commands, +and can be used to download / install a language. + +Using multiple languages at once +================================ + +It may also be possible to combine multiple languages in a single environment. +The details on how to accomplish this with all possible combinations are outside +the scope of this guide. However we recommend that you take a look at the +`Multi-Language Demo `_ +repository for some inspiration. diff --git a/docs/source/howto/user_interface.rst b/docs/source/howto/user_interface.rst new file mode 100644 index 00000000..dd9d6eab --- /dev/null +++ b/docs/source/howto/user_interface.rst @@ -0,0 +1,40 @@ +.. _user_interface: + +============================ +Configure the user interface +============================ + +You can build several user interfaces into the resulting Docker image. +This is controlled with various :ref:`configuration files `. + +JupyterLab +---------- + +You do not need any extra configuration in order to allow the use +of the JupyterLab interface. You can launch JupyterLab from within a user +session by opening the Jupyter Notebook and appending ``/lab`` to the end of the URL +like so: + +.. code-block:: none + + http(s):///lab + +To switch back to the classic notebook, add ``/tree`` to the URL like so: + +.. code-block:: none + + http(s):///tree + +To learn more about URLs in JupyterLab and Jupyter Notebook, visit +`starting JupyterLab `_. + +RStudio +------- + +The RStudio user interface is automatically enabled a configuration file for +R is detected (an R version specified in ``runtime.txt``). If this is detected, +RStudio will be accessible by appending ``/rstudio`` to the URL, like so: + +.. code-block:: none + + http(s):///rstudio diff --git a/docs/source/index.rst b/docs/source/index.rst index 84d2975f..61930917 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,26 +1,45 @@ jupyter-repo2docker =================== -**jupyter-repo2docker** is a tool to build, run, and push Docker -images from source code repositories. repo2docker fetches a repository +``jupyter-repo2docker`` is a tool to **build, run, and push Docker +images from source code repositories** that run via a Jupyter server. + +``repo2docker`` fetches a repository (e.g., from GitHub or other locations) and builds a container image based on the configuration files found in the repository. It can be used to explore a repository locally by building and executing the -constructed image of the repository. +constructed image of the repository, or as a means of building images that +are pushed to a Docker registry. Please report `Bugs `_, `ask questions `_ or `contribute to the project `_. - -Site Contents -------------- .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Getting started with repo2docker install usage + +.. toctree:: + :maxdepth: 1 + :caption: How-to... + + howto/user_interface + howto/languages + howto/jupyterhub_images + +.. toctree:: + :maxdepth: 2 + :caption: Complete list of configuration files + config_files + +.. toctree:: + :maxdepth: 1 + :caption: Advanced and developer information + faq deploy design diff --git a/docs/source/install.rst b/docs/source/install.rst index 864edc27..79fec223 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -6,7 +6,7 @@ Installing ``repo2docker`` repo2docker requires Python 3.4 and above on Linux and macOS. See :ref:`below ` for more information about Windows support. -Prerequisite: docker +Prerequisite: Docker -------------------- Install `Docker `_ as it is required @@ -56,28 +56,3 @@ Bash on Windows) provides additional information about Windows and docker. .. _using Windows and the WSL: https://nickjanetakis.com/blog/setting-up-docker-for-windows-and-wsl-to-work-flawlessly - -.. _jupyterhub_ready: - -JupyterHub-ready images ------------------------ - -JupyterHub_ allows multiple -users to collaborate on a shared Jupyter server. ``repo2docker`` can build -Docker images that can be shared within a JupyterHub deployment. For example, -`mybinder.org `_ uses JupyterHub and ``repo2docker`` -to allow anyone to build a Docker image of a git repository online and -share an executable version of the repository with a URL to the built image. - -To build JupyterHub_-ready Docker images with ``repo2docker``, the -version of your JupterHub deployment must be included in the -``environment.yml`` or ``requirements.txt`` of the git repositories you -build. - -If your instance of JupyterHub uses ``DockerSpawner``, you will need to set its -command to run ``jupyterhub-singleuser`` by adding this line in your -configuration file:: - - c.DockerSpawner.cmd = ['jupyterhub-singleuser'] - -.. _JupyterHub: https://github.com/jupyterhub/jupyterhub diff --git a/docs/source/usage.rst b/docs/source/usage.rst index e11514f9..a6010cd7 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -1,60 +1,50 @@ .. _usage: +===================== Using ``repo2docker`` ===================== -`Docker `_ **must be running** in -order to run ``repo2docker``. For more information on installing -``repo2docker``, see :ref:`install`. +.. note:: -``repo2docker`` performs two steps: + `Docker `_ **must be running** in + order to run ``repo2docker``. For more information on installing + ``repo2docker``, see :ref:`install`. -1. builds a Docker image from a git repo -2. runs a Jupyter server within the image to explore the repository +``repo2docker`` is called with a URL/path to a git repository. It then +performs these steps: -.. To ensure you can run the software in your repository, you must +1. Inspects the repository for :ref:`configuration files `. These will be used to build + the environment needed to run the repository. +2. Builds a Docker image with an environment specified in these :ref:`configuration files `. +3. Runs a Jupyter server within the image that lets you explore the + repository interactively (optional) +4. Pushes the images to a Docker registry so that it may be accessed remotely + (optional) + +Calling repo2docker +=================== repo2docker is called with this command:: jupyter-repo2docker -where ```` is a URL or path to the source repository. +where ```` is a URL or path to the source repository +for which you'd like to build an image. -For example, use the following to build an image of Peter Norvig's -Pytudes_:: +For example, the following command will build an image of Peter Norvig's +Pytudes_ repository:: jupyter-repo2docker https://github.com/norvig/pytudes -To build a particular branch and commit, use the argument ``--ref`` to -specify the ``branch-name`` or ``commit-hash``:: - - jupyter-repo2docker https://github.com/norvig/pytudes --ref 9ced85dd9a84859d0767369e58f33912a214a3cf - -.. tip:: - For reproducible research, we recommend specifying a commit-hash to - deterministically build a fixed version of a repository. Not specifying a - commit-hash will result in the latest commit of the repository being built. - Building the image may take a few minutes. -During building, ``repo2docker`` -clones the repository to obtain its contents and inspects the repository for -:ref:`configuration files `. - -By default, ``repo2docker`` will assume you are using -Python 3.6 unless you include the version of Python in your -:ref:`configuration files `. ``repo2docker`` support is best with -Python 2.7, 3.5, and 3.6. In the case of this repository, a Python version is not -specified in their configuration files and Python 3.6 is installed. - Pytudes_ uses a `requirements.txt file `_ -to specify its Python environment. ``repo2docker`` uses ``pip`` to install -dependencies listed in the ``requirement.txt`` in the image. To learn more about +to specify its Python environment. Because of this, ``repo2docker`` will use +``pip`` to install dependencies listed in this ``requirement.txt`` file, and +these will be present in the generated Docker image. To learn more about configuration files in ``repo2docker`` visit :ref:`config-files`. -.. _Pytudes: https://github.com/norvig/pytudes - When the image is built, a message will be output to your terminal:: Copy/paste this URL into your browser when you connect for the first time, @@ -64,25 +54,41 @@ When the image is built, a message will be output to your terminal:: Pasting the URL into your browser will open Jupyter Notebook with the dependencies and contents of the source repository in the built image. -Because JupyterLab is a server extension of the classic Jupyter Notebook server, -you can launch JupyterLab by opening Jupyter Notebook and visiting the -```/lab`` to the end of the URL: -.. code-block:: none +Building a specific branch / commit / tag +========================================= - http(s):////lab +To build a particular branch and commit, use the argument ``--ref`` and +specify the ``branch-name`` or ``commit-hash``. For example:: -To switch back to the classic notebook, add ``/tree`` to the URL: + jupyter-repo2docker https://github.com/norvig/pytudes --ref 9ced85dd9a84859d0767369e58f33912a214a3cf -.. code-block:: none +.. tip:: + For reproducible research, we recommend specifying a commit-hash to + deterministically build a fixed version of a repository. Not specifying a + commit-hash will result in the latest commit of the repository being built. - http(s):////tree -To learn more about URLs in JupyterLab and Jupyter Notebook, visit -`starting JupyterLab `_. +Where to put configuration files +================================ -``--debug`` and ``--no-build`` ------------------------------- +``repo2docker`` will look for configuration files in either: + +* A folder named ``binder/`` in the root of the repository. +* The root directory of the repository. + +If the folder ``binder/`` is located at the top level of the repository, + **only configuration files in the** ``binder/`` **folder will be considered**. + +.. note:: + + ``repo2docker`` builds an environment with Python 3.6 by default. If you'd + like a different version, you can specify this in your + :ref:`configuration files `. + + +Debugging repo2docker with ``--debug`` and ``--no-build`` +========================================================= To debug the docker image being built, pass the ``--debug`` parameter: @@ -100,3 +106,5 @@ be used by docker directly. .. code-block:: bash jupyter-repo2docker --no-build --debug https://github.com/norvig/pytudes + +.. _Pytudes: https://github.com/norvig/pytudes