Added entwine module, add Pillow, opensfm update

Former-commit-id: 4db777915f

Dockerfile

@@ -76,7 +76,8 @@ RUN pip install -U \
   xmltodict \
   rasterio \
   attrs==19.1.0 \
-  pyodm==1.5.1
+  pyodm==1.5.1 \
+  Pillow
 
 RUN pip install --upgrade cryptography && python -m easy_install --upgrade pyOpenSSL
 

SuperBuild/cmake/External-Entwine.cmake

@@ -8,7 +8,7 @@ ExternalProject_Add(${_proj_name}
   STAMP_DIR         ${_SB_BINARY_DIR}/stamp
   #--Download step--------------
   DOWNLOAD_DIR      ${SB_DOWNLOAD_DIR}
-  URL               https://github.com/connormanning/entwine/archive/098c1ee82cf5b08dcd15ce9b6b910a179c85c6fb.zip
+  URL               https://github.com/connormanning/entwine/archive/d0ed334514a00b234a6fd60b97e1b04353b560f0.zip
   #--Update/Patch step----------
   UPDATE_COMMAND    ""
   #--Configure step-------------

configure.sh

@@ -80,7 +80,8 @@ install() {
                         repoze.lru \
                         rasterio \
                         attrs==19.1.0 \
-                        pyodm==1.5.1
+                        pyodm==1.5.1 \
+                        Pillow
 
     # Fix:  /usr/local/lib/python2.7/dist-packages/requests/__init__.py:83: RequestsDependencyWarning: Old version of cryptography ([1, 2, 3]) may cause slowdown.
     pip install --upgrade cryptography

opendm/entwine.py

@@ -0,0 +1,26 @@
+import os
+import shutil
+from pipes import quote
+from opendm import io
+from opendm import log
+from opendm import system
+from opendm import concurrency
+
+def build(input_point_cloud_files, output_path, max_concurrency=8):
+    if len(input_point_cloud_files) == 0:
+        log.ODM_WARNING("No input point cloud files to process")
+        return
+
+    tmpdir = io.related_file_path(output_path, postfix="-tmp")
+
+    kwargs = {
+        'threads': max_concurrency,
+        'tmpdir': tmpdir,
+        'input': "-i " + ", -i ".join(map(quote, input_point_cloud_files)),
+        'outputdir': output_path
+    }
+
+    system.run("entwine build --threads {threads} --tmp {tmpdir} {input} -o {outputdir}".format(**kwargs))
+
+    if os.path.exists(tmpdir):
+        shutil.rmtree(tmpdir)

portable.Dockerfile

@@ -76,7 +76,8 @@ RUN pip install -U \
   xmltodict \
   rasterio \
   attrs==19.1.0 \
-  pyodm==1.5.1
+  pyodm==1.5.1 \
+  Pillow
 
 RUN pip install --upgrade cryptography && python -m easy_install --upgrade pyOpenSSL
 

stages/splitmerge.py

@@ -13,6 +13,7 @@ from opensfm.large import metadataset
 from opendm.cropper import Cropper
 from opendm.concurrency import get_max_memory
 from opendm.remote import LocalRemoteExecutor
+from opendm import entwine
 from pipes import quote
 
 class ODMSplitStage(types.ODM_Stage):
@@ -177,10 +178,8 @@ class ODMMergeStage(types.ODM_Stage):
                     all_point_clouds = get_submodel_paths(tree.submodels_path, "odm_georeferencing", "odm_georeferenced_model.laz")
                     
                     try:
-                        # TODO: use entwine to create a tileset instead of 
-                        # merging, which is memory inefficient and creates
-                        # monster files.
-                        pdal.merge_point_clouds(all_point_clouds, tree.odm_georeferencing_model_laz, args.verbose)
+                        # pdal.merge_point_clouds(all_point_clouds, tree.odm_georeferencing_model_laz, args.verbose)
+                        entwine.build(all_point_clouds, tree.path("entwine_pointcloud"), max_concurrency=args.max_concurrency)
                     except Exception as e:
                         log.ODM_WARNING("Could not merge point cloud: %s (skipping)" % str(e))
                 else: