From a4063e3d8db99a4724105a2cb6de7e4c0489c7df Mon Sep 17 00:00:00 2001 From: Tim Head Date: Sat, 2 Mar 2019 08:40:35 +0100 Subject: [PATCH 1/4] Add FAQ entry and Howto on exporting conda envs --- docs/source/faq.rst | 30 +++++++++++++- docs/source/howto/export_environment.rst | 50 ++++++++++++++++++++++++ docs/source/index.rst | 1 + 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 docs/source/howto/export_environment.rst diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 9fbfcfbb..4d676415 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -55,6 +55,34 @@ R Only R 3.4.4 is currently supported, which is installed via ``apt`` from the `ubuntu bionic repository `_. + +Why is my repository is failing to build with ``ResolvePackageNotFound`` ? +-------------------------------------------------------------------------- + +If you used ``conda env export`` to generate your ``environment.yml`` it will +generate a list of packages and versions of packages that is pinned to platform +specific versions. These very specific versions are not available in the linux +docker image used by ``repo2docker``. A typical error message will look like +the following:: + + Step 39/44 : RUN conda env update -n root -f "environment.yml" && conda clean -tipsy && conda list -n root + ---> Running in ebe9a67762e4 + Solving environment: ...working... failed + + ResolvePackageNotFound: + - jsonschema==2.6.0=py36hb385e00_0 + - libedit==3.1.20181209=hb402a30_0 + - tornado==5.1.1=py36h1de35cc_0 + ... + +We recommend to use ``conda env export --no-builds -f environment.yml`` to export +your environment and then edit the file by hand to remove platform specific +packages like ``appnope``. + +See :ref:`export-environment` for a recipe on how to create strict exports of +your environment that will work with ``repo2docker``. + + Can I add executable files to the user's PATH? ---------------------------------------------- @@ -98,7 +126,7 @@ Yes: use the ``--editable`` or ``-E`` flag (don't confuse this with the ``-e`` flag for environment variables), and run repo2docker on a local repository:: - repo2docker -E my-repository/. + repo2docker -E my-repository/ This builds a Docker container from the files in that repository (using, for example, a ``requirements.txt`` or ``install.R`` file), diff --git a/docs/source/howto/export_environment.rst b/docs/source/howto/export_environment.rst new file mode 100644 index 00000000..b38a84c0 --- /dev/null +++ b/docs/source/howto/export_environment.rst @@ -0,0 +1,50 @@ +.. _export-environment: + +============================================================================= +How to automatically create a ``environment.yml`` that works with repo2docker +============================================================================= + +This how-to explains how to create a ``environment.yml`` that specifies all +installed packages and their precise versions from your environment. + + +The challenge +============= + +``conda env export -f environment.yml`` creates a strict export of all packages. +This is the most robust for reproducibility, but it does bake in potential +platform-specific packages, so you can only use an exported environment on the +same platform. + +``repo2docker`` uses a linux based image as the starting point for every docker +image it creates. However a lot of people use OSX or Windows as their day to +day operating system. This means that the ``environment.yml`` created by a strict +export will not work with error messages saying that certain packages can not +be resolved (``ResolvePackageNotFound``). + + +The solution +============ + +Follow this procedure to create a strict export of your environment that will +work with ``repo2docker`` and sites like `mybinder.org `_. + +We will launch a terminal inside a basic docker image, install the packages +you need and then perform a strict export of the environment. + +#. install repo2docker on your computer by following :ref:`install` +#. in a terminal launch a basic repository + ``repo2docker https://github.com/binder-examples/conda-freeze`` + inside repo2docker +#. open the URL printed at the end in a browser, the URL should look like + ``http://127.0.0.1:61037/?token=30e61ec80bda6dd0d14805ea76bb59e7b0cd78b5d6b436f0`` +#. open a terminal by clicking "New -> Terminal" next to the "Upload" button on the + right hand side of the webpage +#. install the packages your project requires with ``conda install `` +#. use ``conda env export -n root`` to print the environment +#. copy and paste the environment you just printed into a ``environment.yml`` in + your projects repository +#. close your browser tabs and exit the repo2docker session by pressing Ctrl-C. + +This will give you a strict export of your environment that precisely pins the +versions of packages in your environment based on a linux environment. diff --git a/docs/source/index.rst b/docs/source/index.rst index b3e598c6..f150f352 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -32,6 +32,7 @@ Please report `Bugs `_, howto/user_interface howto/languages + howto/export_environment howto/lab_workspaces howto/jupyterhub_images howto/deploy From 92239e20c3374e5927ec601c78fbb84b2e293bf6 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 4 Mar 2019 16:01:24 +0100 Subject: [PATCH 2/4] make sure we don't touch user $HOME during install-miniconda.bash which is run as root --- repo2docker/buildpacks/conda/install-miniconda.bash | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index ada8e0b3..54fcd55d 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -8,6 +8,10 @@ CONDA_VERSION=4.5.11 URL="https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" INSTALLER_PATH=/tmp/miniconda-installer.sh +# make sure we don't do anything funky with user's $HOME +# since this is run as root +unset HOME + wget --quiet $URL -O ${INSTALLER_PATH} chmod +x ${INSTALLER_PATH} From e0fcd4a55d1c2fc4253de83c5371281cbebefb80 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Mon, 4 Mar 2019 21:32:53 +0530 Subject: [PATCH 3/4] Add test for environment.yml + --target-repo-dir conda sometimes puts stuff in $HOME - specifically under .conda/ - that might cause package installs to fail due to permission mismatches. Ref https://github.com/jupyter/repo2docker/issues/604 --- tests/{venv => conda}/repo-path/README.rst | 0 tests/conda/repo-path/environment.yml | 2 ++ tests/{venv => conda}/repo-path/extra-args.yaml | 0 tests/{venv => conda}/repo-path/verify | 5 ++++- 4 files changed, 6 insertions(+), 1 deletion(-) rename tests/{venv => conda}/repo-path/README.rst (100%) create mode 100644 tests/conda/repo-path/environment.yml rename tests/{venv => conda}/repo-path/extra-args.yaml (100%) rename tests/{venv => conda}/repo-path/verify (61%) diff --git a/tests/venv/repo-path/README.rst b/tests/conda/repo-path/README.rst similarity index 100% rename from tests/venv/repo-path/README.rst rename to tests/conda/repo-path/README.rst diff --git a/tests/conda/repo-path/environment.yml b/tests/conda/repo-path/environment.yml new file mode 100644 index 00000000..6a91d667 --- /dev/null +++ b/tests/conda/repo-path/environment.yml @@ -0,0 +1,2 @@ +dependencies: + - numpy diff --git a/tests/venv/repo-path/extra-args.yaml b/tests/conda/repo-path/extra-args.yaml similarity index 100% rename from tests/venv/repo-path/extra-args.yaml rename to tests/conda/repo-path/extra-args.yaml diff --git a/tests/venv/repo-path/verify b/tests/conda/repo-path/verify similarity index 61% rename from tests/venv/repo-path/verify rename to tests/conda/repo-path/verify index 63c7200c..6df74cbd 100755 --- a/tests/venv/repo-path/verify +++ b/tests/conda/repo-path/verify @@ -7,4 +7,7 @@ assert sys.executable == '/srv/conda/bin/python' # Repo should be in /srv/repo assert os.path.exists('/srv/repo/verify') -assert os.path.abspath(__file__) == '/srv/repo/verify' \ No newline at end of file +assert os.path.abspath(__file__) == '/srv/repo/verify' + +# We should be able to import the package in environment.yml +import numpy From 1427cd1047fd804e1eb5e61a1d044cf0cd953ccd Mon Sep 17 00:00:00 2001 From: Tim Head Date: Mon, 4 Mar 2019 19:50:37 +0100 Subject: [PATCH 4/4] Reopen change log --- docs/source/changelog.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 7cc4df72..09ea1ef3 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -2,6 +2,21 @@ Changelog ========= +Version 0.x.x +============= + +Release date: TBD + +New features +------------ + +API changes +----------- + +Bug fixes +--------- + + Version 0.8.0 =============