Merge pull request #639 from djhoese/git-update-submodules-alimanfoo-20181224-merry-xmas

[MRG] Ensure git submodules are updated and initialized

repo2docker/contentproviders/git.py

@@ -45,6 +45,12 @@ class Git(ContentProvider):
                                     capture=yield_output):
                 yield line
 
+        # ensure that git submodules are initialised and updated
+        for line in execute_cmd(['git', 'submodule', 'update', '--init', '--recursive'],
+                                cwd=output_dir,
+                                capture=yield_output):
+            yield line
+
         cmd = ['git', 'rev-parse', 'HEAD']
         sha1 = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=output_dir)
         self._sha1 = sha1.stdout.read().decode().strip()

tests/conftest.py

@@ -80,6 +80,24 @@ def run_repo2docker():
     return run_test
 
 
+def _add_content_to_git(repo_dir):
+    """Add content to file 'test' in git repository and commit."""
+    # use append mode so this can be called multiple times
+    with open(os.path.join(repo_dir, 'test'), 'a') as f:
+        f.write("Hello")
+
+    subprocess.check_call(['git', 'add', 'test'], cwd=repo_dir)
+    subprocess.check_call(['git', 'commit', '-m', 'Test commit'],
+                          cwd=repo_dir)
+
+
+def _get_sha1(repo_dir):
+    """Get repository's current commit SHA1."""
+    sha1 = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
+                            stdout=subprocess.PIPE, cwd=repo_dir)
+    return sha1.stdout.read().decode().strip()
+
+
 @pytest.fixture()
 def git_repo():
     """
@@ -95,20 +113,52 @@ def git_repo():
 @pytest.fixture()
 def repo_with_content(git_repo):
     """Create a git repository with content"""
-    with open(os.path.join(git_repo, 'test'), 'w') as f:
-        f.write("Hello")
-
-    subprocess.check_call(['git', 'add', 'test'], cwd=git_repo)
-    subprocess.check_call(['git', 'commit', '-m', 'Test commit'],
-                          cwd=git_repo)
-    # get the commit's SHA1
-    sha1 = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
-                            stdout=subprocess.PIPE, cwd=git_repo)
-    sha1 = sha1.stdout.read().decode().strip()
+    _add_content_to_git(git_repo)
+    sha1 = _get_sha1(git_repo)
 
     yield git_repo, sha1
 
 
+@pytest.fixture()
+def repo_with_submodule():
+    """Create a git repository with a git submodule in a non-master branch.
+
+    Provides parent repository directory, current parent commit SHA1, and
+    the submodule's current commit. The submodule will be added under the
+    name "submod" in the parent repository.
+
+    Creating the submodule in a separate branch is useful to prove that
+    submodules are initialized properly when the master branch doesn't have
+    the submodule yet.
+
+    """
+    with TemporaryDirectory() as git_a_dir, TemporaryDirectory() as git_b_dir:
+        # create "parent" repository
+        subprocess.check_call(['git', 'init'], cwd=git_a_dir)
+        _add_content_to_git(git_a_dir)
+        # create repository with 2 commits that will be the submodule
+        subprocess.check_call(['git', 'init'], cwd=git_b_dir)
+        _add_content_to_git(git_b_dir)
+        submod_sha1_b = _get_sha1(git_b_dir)
+        _add_content_to_git(git_b_dir)
+
+        # create a new branch in the parent to add the submodule
+        subprocess.check_call(['git', 'checkout', '-b', 'branch-with-submod'],
+                              cwd=git_a_dir)
+        subprocess.check_call(['git', 'submodule', 'add', git_b_dir, 'submod'],
+                              cwd=git_a_dir)
+        # checkout the first commit for the submod, not the latest
+        subprocess.check_call(['git', 'checkout', submod_sha1_b],
+                              cwd=os.path.join(git_a_dir, 'submod'))
+        subprocess.check_call(['git', 'add', git_a_dir, ".gitmodules"],
+                              cwd=git_a_dir)
+        subprocess.check_call(['git', 'commit', '-m', 'Add B repos submod'],
+                              cwd=git_a_dir)
+
+        sha1_a = _get_sha1(git_a_dir)
+        yield git_a_dir, sha1_a, submod_sha1_b
+
+
 class Repo2DockerTest(pytest.Function):
     """A pytest.Item for running repo2docker"""
     def __init__(self, name, parent, args):

tests/unit/contentproviders/test_git.py

@@ -1,5 +1,6 @@
 import os
 import pytest
+import subprocess
 from tempfile import TemporaryDirectory
 from repo2docker.contentproviders import Git
 
@@ -18,6 +19,28 @@ def test_clone(repo_with_content):
         assert git_content.content_id == sha1[:7]
 
 
+def test_submodule_clone(repo_with_submodule):
+    """Test git clone containing a git submodule."""
+    upstream, expected_sha1_upstream, expected_sha1_submod = repo_with_submodule
+
+    with TemporaryDirectory() as clone_dir:
+        submod_dir = os.path.join(clone_dir, 'submod')  # set by fixture
+        spec = {'repo': upstream}
+        git_content = Git()
+        for _ in git_content.fetch(spec, clone_dir):
+            pass
+        assert os.path.exists(os.path.join(clone_dir, 'test'))
+        assert os.path.exists(os.path.join(submod_dir, 'test'))
+
+        # get current sha1 of submodule
+        cmd = ['git', 'rev-parse', 'HEAD']
+        sha1 = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=submod_dir)
+        submod_sha1 = sha1.stdout.read().decode().strip()
+
+        assert git_content.content_id == expected_sha1_upstream[:7]
+        assert submod_sha1[:7] == expected_sha1_submod[:7]
+
+
 def test_bad_ref(repo_with_content):
     """
     Test trying to checkout a ref that doesn't exist