diff --git a/datasette/utils.py b/datasette/utils.py index e5ab1462..f6430934 100644 --- a/datasette/utils.py +++ b/datasette/utils.py @@ -8,6 +8,7 @@ import shlex import sqlite3 import tempfile import time +import shutil import urllib @@ -186,8 +187,8 @@ def temporary_docker_directory(files, name, metadata, extra_options, branch=None open('metadata.json', 'w').write(json.dumps(metadata_content, indent=2)) open('Dockerfile', 'w').write(dockerfile) for path, filename in zip(file_paths, file_names): - os.link(path, os.path.join(datasette_dir, filename)) - yield + link_or_copy(path, os.path.join(datasette_dir, filename)) + yield datasette_dir finally: tmp.cleanup() os.chdir(saved_cwd) @@ -241,7 +242,7 @@ def temporary_heroku_directory(files, name, metadata, extra_options, branch=None open('Procfile', 'w').write(procfile_cmd) for path, filename in zip(file_paths, file_names): - os.link(path, os.path.join(tmp.name, filename)) + link_or_copy(path, os.path.join(tmp.name, filename)) yield @@ -494,3 +495,14 @@ def to_css_class(s): # Attach the md5 suffix bits = [b for b in (s, md5_suffix) if b] return '-'.join(bits) + + +def link_or_copy(src, dst): + # Intended for use in populating a temp directory. We link if possible, + # but fall back to copying if the temp directory is on a different device + # https://github.com/simonw/datasette/issues/141 + try: + os.link(src, dst) + except OSError as e: + print('Got OSError {} linking {} to {}'.format(e, src, dst)) + shutil.copyfile(src, dst) diff --git a/tests/test_utils.py b/tests/test_utils.py index 7f7a472d..2ad12d2a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,9 +3,12 @@ Tests for various datasette helper functions. """ from datasette import utils +import json +import os import pytest import sqlite3 -import json +import tempfile +from unittest.mock import patch @pytest.mark.parametrize('path,expected', [ @@ -178,3 +181,40 @@ def test_is_url(url, expected): ]) def test_to_css_class(s, expected): assert expected == utils.to_css_class(s) + + +def test_temporary_docker_directory_uses_hard_link(): + with tempfile.TemporaryDirectory() as td: + os.chdir(td) + open('hello', 'w').write('world') + # Default usage of this should use symlink + with utils.temporary_docker_directory( + files=['hello'], + name='t', + metadata=None, + extra_options=None + ) as temp_docker: + hello = os.path.join(temp_docker, 'hello') + assert 'world' == open(hello).read() + # It should be a hard link + assert 2 == os.stat(hello).st_nlink + + +@patch('os.link') +def test_temporary_docker_directory_uses_copy_if_hard_link_fails(mock_link): + # Copy instead if os.link raises OSError (normally due to different device) + mock_link.side_effect = OSError + with tempfile.TemporaryDirectory() as td: + os.chdir(td) + open('hello', 'w').write('world') + # Default usage of this should use symlink + with utils.temporary_docker_directory( + files=['hello'], + name='t', + metadata=None, + extra_options=None + ) as temp_docker: + hello = os.path.join(temp_docker, 'hello') + assert 'world' == open(hello).read() + # It should be a copy, not a hard link + assert 1 == os.stat(hello).st_nlink