Merge pull request #448 from betatim/nix-experiment

[MRG] Switch to BaseImage for nix build pack

docs/source/config_files.rst

@@ -180,6 +180,27 @@ used for installing libraries.
 To see an example R repository, visit our `R
 example in binder-examples <https://github.com/binder-examples/r/blob/master/runtime.txt>`_.
 
+.. _default.nix:
+
+``default.nix`` - the nix package manager
+=========================================
+
+Specify packages to be installed by the `nix package manager <https://github.com/NixOS/nixpkgs>`_.
+When you use this config file all other configuration files (like ``requirements.txt``)
+that specify packages are ignored. When using ``nix`` you have to specify all
+packages and dependencies explicitly, including the Jupyter notebook package that
+repo2docker expects to be installed. If you do not install Jupyter explicitly
+repo2docker will no be able to start your container.
+
+`nix-shell <https://nixos.org/nix/manual/#sec-nix-shell>`_ is used to evaluate
+a ``nix`` expression written in a ``default.nix`` file. Make sure to
+`pin your nixpkgs <https://discourse.nixos.org/t/nixops-pinning-nixpkgs/734>`_
+to produce a reproducible environment.
+
+To see an example repository visit
+`nix binder example <https://github.com/binder-examples/nix>`_.
+
+
 ``Dockerfile`` - Advanced environments
 ======================================
 
@@ -194,26 +215,3 @@ With Dockerfiles, a regular Docker build will be performed.
 
 See the `Advanced Binder Documentation <https://mybinder.readthedocs.io/en/latest/tutorials/dockerfile.html>`_ for
 best-practices with Dockerfiles.
-
-.. _default.nix:
-
-``default.nix``
-~~~~~~~~~~~~~~~
-
-This allows you to use the `nix package manager <https://github.com/NixOS/nixpkgs>`_. It is hard to explain what nix
-is to new users and why it is usefull. If you are inclined please read
-more at the `nix homepage <https://nixos.org/nix/>`_. It is currently
-the largest package repository, offers reproducible builds, multiple
-versions of same package coexisting, source and binary based, and
-packages many languages such as python, R, go, javascript, haskell,
-ruby, etc. .
-
-A ``default.nix`` file allows you to use `nix-shell <https://nixos.org/nix/manual/#sec-nix-shell>`_
-to evaluate a ``nix`` expression to define a reproducible nix environment.
-The only requirement is that you expose a ``jupyter`` command within the shell
-(since jupyterlab is currently what ``repo2docker`` is designed
-around). While the ``nix`` environment does have ``NIX_PATH`` set with
-``nixpkgs=...`` you should not rely on it and make sure to
-`pin your nixpkgs <https://discourse.nixos.org/t/nixops-pinning-nixpkgs/734>`_.
-By doing this you are truley producing a reproducible environment. To see an
-example repository visit a `nix binder example <https://gitlab.com/costrouc/nix-binder-example>`_.

repo2docker/buildpacks/nix/__init__.py

@@ -1,10 +1,10 @@
 """BuildPack for nixpkgs environments"""
 import os
 
-from ..base import BuildPack
+from ..base import BuildPack, BaseImage
 
 
-class NixBuildPack(BuildPack):
+class NixBuildPack(BaseImage):
     """A nix Package Manager BuildPack"""
 
     def get_path(self):
@@ -57,12 +57,15 @@ class NixBuildPack(BuildPack):
             ('${NB_USER}', """
             nix-channel --add https://nixos.org/channels/nixpkgs-unstable nixpkgs && \
             nix-channel --update && \
-            nix-shell default.nix --command "command -v jupyter"
+            nix-shell {}
-            """)
+            """.format(self.binder_path('default.nix')))
         ]
 
     def get_start_script(self):
         """The path to a script to be executed as ENTRYPOINT"""
+        # the shell wrapper script duplicates the behaviour of other buildpacks
+        # when it comes to the `start` script as well as handling a binder/
+        # sub-directory when it exists
         return "/usr/local/bin/nix-shell-wrapper"
 
     def detect(self):

repo2docker/buildpacks/nix/install-nix.bash

@@ -5,7 +5,7 @@ set -ex
 NIX_VERSION="2.1.1"
 NIX_SHA256="ad10b4da69035a585fe89d7330037c4a5d867a372bb0e52a1542ab95aec67999"
 
-wget https://nixos.org/releases/nix/nix-$NIX_VERSION/nix-$NIX_VERSION-x86_64-linux.tar.bz2
+wget --quiet https://nixos.org/releases/nix/nix-$NIX_VERSION/nix-$NIX_VERSION-x86_64-linux.tar.bz2
 echo "$NIX_SHA256  nix-2.1.1-x86_64-linux.tar.bz2" | sha256sum -c
 tar xjf nix-*-x86_64-linux.tar.bz2
 sh nix-*-x86_64-linux/install

repo2docker/buildpacks/nix/nix-shell-wrapper

@@ -8,8 +8,32 @@ _term() {
 
 trap _term SIGTERM
 
-echo "$*"
+# if there is a binder/ sub-directory it takes precedence
-nix-shell default.nix --command "$*" &
+# files outside it are ignored
+if [ -e ./binder ]; then
+  nixpath="./binder/default.nix";
+  if [ -f ./binder/start ]; then
+    chmod u+x ./binder/start
+    # Using `$@`` here which is what the internet recommends leads to
+    # errors when the command is something like `jupyter --ip=0.0.0.0 ...`
+    # as nix-shell picks that up as an argument to it instead of the command.
+    # There are several issues on the nix repos discussing this and adding support
+    # for -- to indicate "all arguments after this are for the command, not nix-shell"
+    # but it seems they have stalled/not yet produced an implementation.
+    # So let's use `$*` for now.
+    nix-shell $nixpath --command "./binder/start $*" &
+  else
+    nix-shell $nixpath --command "$*" &
+  fi
+else
+  nixpath="./default.nix";
+  if [ -f ./start ]; then
+    chmod u+x ./start
+    nix-shell $nixpath --command "./start $*" &
+  else
+    nix-shell $nixpath --command "$*" &
+  fi
+fi
 
 PID=$!
 wait "$PID"

tests/nix/binder-dir/README.rst

@@ -0,0 +1,4 @@
+default.nix in a binder/ directory
+----------------------------------
+
+Check we find and use ``default.nix`` when it is in a ``binder/`` sub-directory.

tests/nix/binder-dir/binder/default.nix

@@ -0,0 +1,20 @@
+let
+  # Pinning nixpkgs to specific release
+  # To get sha256 use "nix-prefetch-git <url> --rev <commit>"
+  commitRev="5574b6a152b1b3ae5f93ba37c4ffd1981f62bf5a";
+  nixpkgs = builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${commitRev}.tar.gz";
+    sha256 = "1pqdddp4aiz726c7qs1dwyfzixi14shp0mbzi1jhapl9hrajfsjg";
+  };
+  pkgs = import nixpkgs { config = { allowUnfree = true; }; };
+in
+pkgs.mkShell {
+  buildInputs = with pkgs; [
+    python36Packages.numpy
+    python36Packages.notebook
+  ];
+
+  shellHook = ''
+    export NIX_PATH="nixpkgs=${nixpkgs}:."
+  '';
+}

tests/nix/binder-dir/verify

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+import numpy

tests/nix/ignore-outside/README.rst

@@ -0,0 +1,5 @@
+Check `start` works with nix
+----------------------------
+
+In this example we set a environment variable in the `start` script and check
+it works when using the nix build pack.

tests/nix/ignore-outside/binder/default.nix

@@ -0,0 +1,19 @@
+let
+  # Pinning nixpkgs to specific release
+  # To get sha256 use "nix-prefetch-git <url> --rev <commit>"
+  commitRev="5574b6a152b1b3ae5f93ba37c4ffd1981f62bf5a";
+  nixpkgs = builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${commitRev}.tar.gz";
+    sha256 = "1pqdddp4aiz726c7qs1dwyfzixi14shp0mbzi1jhapl9hrajfsjg";
+  };
+  pkgs = import nixpkgs { config = { allowUnfree = true; }; };
+in
+pkgs.mkShell {
+  buildInputs = with pkgs; [
+    python36Packages.notebook
+  ];
+
+  shellHook = ''
+    export NIX_PATH="nixpkgs=${nixpkgs}:."
+  '';
+}

tests/nix/ignore-outside/default.nix

@@ -0,0 +1,21 @@
+let
+  # Pinning nixpkgs to specific release
+  # To get sha256 use "nix-prefetch-git <url> --rev <commit>"
+  commitRev="5574b6a152b1b3ae5f93ba37c4ffd1981f62bf5a";
+  nixpkgs = builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${commitRev}.tar.gz";
+    sha256 = "1pqdddp4aiz726c7qs1dwyfzixi14shp0mbzi1jhapl9hrajfsjg";
+  };
+  pkgs = import nixpkgs { config = { allowUnfree = true; }; };
+in
+pkgs.mkShell {
+  buildInputs = with pkgs; [
+    python36Packages.numpy
+    __THIS_IS_A_SYNTAX_ERROR_THAT_SHOULD_NOT_MATTER_AS_IT_ISNT_EXECUTED__
+    python36Packages.notebook
+  ];
+
+  shellHook = ''
+    export NIX_PATH="nixpkgs=${nixpkgs}:."
+  '';
+}

tests/nix/ignore-outside/start

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# this script should not be executed
+echo "The start script in the top level directory should not be executed"
+exit 1

tests/nix/ignore-outside/verify

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -euo pipefail
+# check that numpy isn't installed
+test -z $(pip list | grep numpy | awk '{print $1}')

tests/nix/simple/default.nix

@@ -11,8 +11,7 @@ in
 pkgs.mkShell {
   buildInputs = with pkgs; [
     python36Packages.numpy
-    python36Packages.scipy
+    python36Packages.notebook
-    python36Packages.jupyterlab
   ];
 
   shellHook = ''

tests/nix/simple/verify

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
+
 import numpy
-import scipy

tests/nix/start/README.rst

@@ -0,0 +1,5 @@
+Check `start` works with nix
+----------------------------
+
+In this example we set a environment variable in the `start` script and check
+it works when using the nix build pack.

tests/nix/start/default.nix

@@ -0,0 +1,20 @@
+let
+  # Pinning nixpkgs to specific release
+  # To get sha256 use "nix-prefetch-git <url> --rev <commit>"
+  commitRev="5574b6a152b1b3ae5f93ba37c4ffd1981f62bf5a";
+  nixpkgs = builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${commitRev}.tar.gz";
+    sha256 = "1pqdddp4aiz726c7qs1dwyfzixi14shp0mbzi1jhapl9hrajfsjg";
+  };
+  pkgs = import nixpkgs { config = { allowUnfree = true; }; };
+in
+pkgs.mkShell {
+  buildInputs = with pkgs; [
+    python36Packages.numpy
+    python36Packages.notebook
+  ];
+
+  shellHook = ''
+    export NIX_PATH="nixpkgs=${nixpkgs}:."
+  '';
+}

tests/nix/start/start

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+export TEST_START_VAR="var is set"
+
+exec "$@"

tests/nix/start/verify

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+
+# set value of TEST_START_VAR to empty string when it is not defined
+if [ "${TEST_START_VAR:-}" != "var is set" ]
+then
+  echo "TEST_START_VAR is not set"
+  exit 1
+fi