Started migration to 18.04, Python3.6

pull/1156/head
Piero Toffanin 2020-09-08 13:08:57 -04:00
rodzic e0a3384d28
commit c769dd2807
21 zmienionych plików z 105 dodań i 272 usunięć

Wyświetl plik

@ -1,61 +1,10 @@
FROM phusion/baseimage:0.10.2 as base
FROM ubuntu:18.04
# Env variables
ENV DEBIAN_FRONTEND noninteractive
#Install dependencies and required requisites
RUN add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable \
&& add-apt-repository -y ppa:george-edison55/cmake-3.x \
&& apt-get update -y \
&& apt-get install --no-install-recommends -y \
build-essential \
cmake \
gdal-bin \
git \
libatlas-base-dev \
libavcodec-dev \
libavformat-dev \
libboost-date-time-dev \
libboost-filesystem-dev \
libboost-iostreams-dev \
libboost-log-dev \
libboost-python-dev \
libboost-regex-dev \
libboost-thread-dev \
libeigen3-dev \
libflann-dev \
libgdal-dev \
libgeotiff-dev \
libgoogle-glog-dev \
libgtk2.0-dev \
libjasper-dev \
libjpeg-dev \
libjsoncpp-dev \
liblapack-dev \
liblas-bin \
libpng-dev \
libproj-dev \
libsuitesparse-dev \
libswscale-dev \
libtbb2 \
libtbb-dev \
libtiff-dev \
libvtk6-dev \
libxext-dev \
python-dev \
python-gdal \
python-matplotlib \
python-pip \
python-software-properties \
python-wheel \
software-properties-common \
swig2.0 \
grass-core \
libssl-dev \
&& apt-get remove libdc1394-22-dev \
&& pip install --upgrade pip \
&& pip install setuptools
ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages"
ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/src/opensfm"
ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"
# Prepare directories
WORKDIR /code
@ -63,24 +12,7 @@ WORKDIR /code
# Copy everything
COPY . ./
RUN pip install -r requirements.txt
ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages"
ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm"
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"
# Compile code in SuperBuild and root directories
RUN rm -fr docker \
&& cd SuperBuild \
&& mkdir build \
&& cd build \
&& cmake .. \
&& make -j$(nproc) \
&& cd ../.. \
&& mkdir build \
&& cd build \
&& cmake .. \
&& make -j$(nproc)
RUN bash configure.sh install
# Cleanup APT
RUN apt-get clean \
@ -98,4 +30,4 @@ RUN rm -rf \
/code/SuperBuild/src/pdal
# Entry point
ENTRYPOINT ["python", "/code/run.py"]
ENTRYPOINT ["python3", "/code/run.py"]

Wyświetl plik

@ -16,7 +16,7 @@ ExternalProject_Add(${_proj_name}
CMAKE_ARGS
-DBUILD_TESTS=OFF
-DBUILD_PYTHON=ON
-DPYBIND11_PYTHON_VERSION=2.7
-DPYBIND11_PYTHON_VERSION=3.6
-DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
#--Build step-----------------
BINARY_DIR ${_SB_BINARY_DIR}

Wyświetl plik

@ -18,7 +18,7 @@ ExternalProject_Add(${_proj_name}
-DCERES_ROOT_DIR=${SB_INSTALL_DIR}
-DOpenCV_DIR=${SB_INSTALL_DIR}/share/OpenCV
-DOPENSFM_BUILD_TESTS=off
-DPYTHON_EXECUTABLE=/usr/bin/python
-DPYTHON_EXECUTABLE=/usr/bin/python3
#--Build step-----------------
BINARY_DIR ${_SB_BINARY_DIR}
#--Install step---------------

Wyświetl plik

@ -1 +1 @@
1.0.2
2.0.0

93
configure.sh 100755 → 100644
Wyświetl plik

@ -1,5 +1,21 @@
#!/bin/bash
check_version(){
UBUNTU_VERSION=$(lsb_release -r)
if [[ $UBUNTU_VERSION = *"18.04"* ]]; then
echo "Ubuntu: $UBUNTU_VERSION, good!"
elif [[ $UBUNTU_VERSION = *"16.04" ]]; then
echo "ODM 2.0 has upgraded to Ubuntu 18.04, but you're on 16.04"
echo "The last version of ODM that supports Ubuntu 16.04 is v1.0.2. We recommend you upgrade to Ubuntu 18.04, or better yet, use docker."
exit 1
else
echo "You are not on Ubuntu 18.04 (detected: $UBUNTU_VERSION)"
echo "It might be possible to run ODM on a newer version of Ubuntu, however, you cannot rely on this script."
exit 1
fi
}
check_version
if [[ $2 =~ ^[0-9]+$ ]] ; then
processes=$2
else
@ -7,63 +23,69 @@ else
fi
install() {
cd /code
## Set up library paths
export PYTHONPATH=$RUNPATH/SuperBuild/install/lib/python2.7/dist-packages:$RUNPATH/SuperBuild/src/opensfm:$PYTHONPATH
export DEBIAN_FRONTEND=noninteractive
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$RUNPATH/SuperBuild/install/lib
## Before installing
## Before installing
echo "Updating the system"
add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
apt-get update
if ! command -v sudo &> /dev/null
then
echo "Installing sudo"
apt-get update && apt-get install -y sudo
fi
sudo apt-get update && sudo apt-get install software-properties-common -y --no-install-recommends
sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
sudo apt-get update
echo "Installing Required Requisites"
apt-get install -y -qq build-essential \
sudo apt-get install -y -qq --no-install-recommends \
build-essential \
git \
cmake \
python-pip \
python3-pip \
libgdal-dev \
gdal-bin \
libgeotiff-dev \
pkg-config \
libjsoncpp-dev \
python-gdal \
python3-gdal \
python3-setuptools \
grass-core \
libssl-dev \
liblas-bin \
swig2.0 \
python-wheel \
swig3.0 \
python3-wheel \
libboost-log-dev
sudo pip3 install -U pip
echo "Getting CMake 3.1 for MVS-Texturing"
apt-get install -y software-properties-common python-software-properties
add-apt-repository -y ppa:george-edison55/cmake-3.x
apt-get update -y
apt-get install -y --only-upgrade cmake
echo "Installing OpenCV Dependencies"
apt-get install -y -qq libgtk2.0-dev \
sudo apt-get install -y -qq --no-install-recommends libgtk2.0-dev \
libavcodec-dev \
libavformat-dev \
libswscale-dev \
python-dev \
python3-dev \
libtbb2 \
libtbb-dev \
libjpeg-dev \
libpng-dev \
libtiff-dev \
libjasper-dev \
libflann-dev \
libproj-dev \
libxext-dev \
liblapack-dev \
libeigen3-dev \
libvtk6-dev
echo "Removing libdc1394-22-dev due to python opencv issue"
apt-get remove libdc1394-22-dev
sudo add-apt-repository "deb http://security.ubuntu.com/ubuntu xenial-security main"
sudo apt-get update
sudo apt-get install -y -qq --no-install-recommends libjasper1 \
libjasper-dev
echo "Installing OpenSfM Dependencies"
apt-get install -y -qq libgoogle-glog-dev \
sudo apt-get install -y -qq --no-install-recommends libgoogle-glog-dev \
libsuitesparse-dev \
libboost-filesystem-dev \
libboost-iostreams-dev \
@ -72,22 +94,18 @@ install() {
libboost-date-time-dev \
libboost-thread-dev
pip install -r "${RUNPATH}/requirements.txt"
# Fix: /usr/local/lib/python2.7/dist-packages/requests/__init__.py:83: RequestsDependencyWarning: Old version of cryptography ([1, 2, 3]) may cause slowdown.
pip install --upgrade cryptography
python -m easy_install --upgrade pyOpenSSL
pip install -r requirements.txt
echo "Compiling SuperBuild"
cd ${RUNPATH}/SuperBuild
mkdir -p build && cd build
cmake .. && make -j$processes
# cd ${RUNPATH}/SuperBuild
# mkdir -p build && cd build
# cmake .. && make -j$processes
echo "Compiling build"
cd ${RUNPATH}
mkdir -p build && cd build
cmake .. && make -j$processes
# cd ${RUNPATH}
# mkdir -p build && cd build
# cmake .. && make -j$processes
echo "Configuration Finished"
}
@ -104,7 +122,6 @@ reinstall() {
uninstall
install
}
usage() {
echo "Usage:"
echo "bash configure.sh <install|update|uninstall|help> [nproc]"
@ -120,7 +137,7 @@ usage() {
echo "[nproc] is an optional argument that can set the number of processes for the make -j tag. By default it uses $(nproc)"
}
if [[ $1 =~ ^(install|reinstall|uninstall|usage)$ ]]; then
if [[ $1 =~ ^(install|reinstall|uninstall)$ ]]; then
RUNPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
"$1"
else

Wyświetl plik

@ -1,128 +0,0 @@
#!/bin/bash
if [[ $2 =~ ^[0-9]+$ ]] ; then
processes=$2
else
processes=$(nproc)
fi
install() {
cd /code
## Set up library paths
export PYTHONPATH=$RUNPATH/SuperBuild/install/lib/python2.7/dist-packages:$RUNPATH/SuperBuild/src/opensfm:$PYTHONPATH
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$RUNPATH/SuperBuild/install/lib
## Before installing
echo "Updating the system"
sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
echo "Installing Required Requisites"
sudo apt-get install -y -qq build-essential \
git \
cmake \
python-pip \
libgdal-dev \
gdal-bin \
libgeotiff-dev \
pkg-config \
libjsoncpp-dev \
python-gdal \
grass-core \
libssl-dev \
liblas-bin \
swig3.0 \
python-wheel \
libboost-log-dev
echo "Installing OpenCV Dependencies"
sudo apt-get install -y -qq libgtk2.0-dev \
libavcodec-dev \
libavformat-dev \
libswscale-dev \
python-dev \
libtbb2 \
libtbb-dev \
libjpeg-dev \
libpng-dev \
libtiff-dev \
libflann-dev \
libproj-dev \
libxext-dev \
liblapack-dev \
libeigen3-dev \
libvtk6-dev
sudo add-apt-repository "deb http://security.ubuntu.com/ubuntu xenial-security main"
sudo apt-get update
sudo apt-get install -y -qq libjasper1 \
libjasper-dev
echo "Installing OpenSfM Dependencies"
sudo apt-get install -y -qq libgoogle-glog-dev \
libsuitesparse-dev \
libboost-filesystem-dev \
libboost-iostreams-dev \
libboost-regex-dev \
libboost-python-dev \
libboost-date-time-dev \
libboost-thread-dev
pip install -r "/code/requirements.txt"
# Fix: /usr/local/lib/python2.7/dist-packages/requests/__init__.py:83: RequestsDependencyWarning: Old version of cryptography ([1, 2, 3]) may cause slowdown.
pip install --upgrade cryptography
python -m easy_install --upgrade pyOpenSSL
echo "Compiling SuperBuild"
cd ${RUNPATH}/SuperBuild
mkdir -p build && cd build
cmake .. && make -j$processes
echo "Compiling build"
cd ${RUNPATH}
mkdir -p build && cd build
cmake .. && make -j$processes
echo "Configuration Finished"
}
uninstall() {
echo "Removing SuperBuild and build directories"
cd ${RUNPATH}/SuperBuild
rm -rfv build src download install
cd ../
rm -rfv build
}
reinstall() {
echo "Reinstalling ODM modules"
uninstall
install
}
usage() {
echo "Usage:"
echo "bash configure.sh <install|update|uninstall|help> [nproc]"
echo "Subcommands:"
echo " install"
echo " Installs all dependencies and modules for running OpenDroneMap"
echo " reinstall"
echo " Removes SuperBuild and build modules, then re-installs them. Note this does not update OpenDroneMap to the latest version. "
echo " uninstall"
echo " Removes SuperBuild and build modules. Does not uninstall dependencies"
echo " help"
echo " Displays this message"
echo "[nproc] is an optional argument that can set the number of processes for the make -j tag. By default it uses $(nproc)"
}
if [[ $1 =~ ^(install|reinstall|uninstall)$ ]]; then
RUNPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
"$1"
else
echo "Invalid instructions." >&2
usage
exit 1
fi

Wyświetl plik

@ -60,9 +60,6 @@ class StoreValue(argparse.Action):
setattr(namespace, self.dest, values)
setattr(namespace, self.dest + '_is_set', True)
parser = SettingsParser(description='OpenDroneMap',
usage='%(prog)s [options] <project name>',
yaml_file=open(context.settings_path))
args = None
def config(argv=None):
@ -70,6 +67,10 @@ def config(argv=None):
if args is not None and argv is None:
return args
parser = SettingsParser(description='OpenDroneMap',
usage='%(prog)s [options] <project name>',
yaml_file=open(context.settings_path))
parser.add_argument('--project-path',
metavar='<path>',
@ -742,7 +743,7 @@ def config(argv=None):
if not args.project_path:
log.ODM_ERROR('You need to set the project path in the '
'settings.yaml file before you can run ODM, '
'or use `--project-path <path>`. Run `python '
'or use `--project-path <path>`. Run `python3 '
'run.py --help` for more information. ')
sys.exit(1)

Wyświetl plik

@ -12,9 +12,11 @@ superbuild_bin_path = os.path.join(superbuild_path, 'install', 'bin')
tests_path = os.path.join(root_path, 'tests')
tests_data_path = os.path.join(root_path, 'tests/test_data')
# add opencv to python path
pyopencv_path = os.path.join(superbuild_path, 'install/lib/python2.7/dist-packages')
sys.path.append(pyopencv_path)
# add opencv,opensfm to python path
sys.path.append(os.path.join(superbuild_path, 'install/lib/python3.6/dist-packages'))
sys.path.append(os.path.join(superbuild_path, 'install/lib/python3/dist-packages'))
sys.path.append(os.path.join(superbuild_path, 'src/opensfm'))
# define opensfm path
opensfm_path = os.path.join(superbuild_path, "src/opensfm")

Wyświetl plik

@ -15,9 +15,9 @@ class GCPFile:
def read(self):
if self.exists():
with open(self.gcp_path, 'r') as f:
contents = f.read().decode('utf-8-sig').encode('utf-8').strip()
contents = f.read().strip()
lines = map(str.strip, contents.split('\n'))
lines = list(map(str.strip, contents.split('\n')))
if lines:
self.raw_srs = lines[0] # SRS
self.srs = location.parse_srs_header(self.raw_srs)
@ -104,7 +104,7 @@ class GCPFile:
if os.path.exists(gcp_file_output):
os.remove(gcp_file_output)
files = map(os.path.basename, glob.glob(os.path.join(images_dir, "*")))
files = list(map(os.path.basename, glob.glob(os.path.join(images_dir, "*"))))
output = [self.raw_srs]
files_found = 0

Wyświetl plik

@ -74,6 +74,8 @@ def proj_srs_convert(srs):
proj4 = srs.to_proj4()
res.ImportFromProj4(proj4)
res.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
return res
def transformer(from_srs, to_srs):

Wyświetl plik

@ -18,8 +18,7 @@ class OSFMContext:
self.opensfm_project_path = opensfm_project_path
def run(self, command):
# Use Python 2.x by default, otherwise OpenSfM uses Python 3.x
system.run('/usr/bin/env python2 %s/bin/opensfm %s "%s"' %
system.run('/usr/bin/env python3 %s/bin/opensfm %s "%s"' %
(context.opensfm_path, command, self.opensfm_project_path))
def is_reconstruction_done(self):

Wyświetl plik

@ -8,8 +8,8 @@ from six import string_types
from datetime import datetime, timedelta
import pytz
import log
import system
from opendm import log
from opendm import system
import xmltodict as x2d
from opendm import get_image_size
from xml.parsers.expat import ExpatError

Wyświetl plik

@ -10,7 +10,7 @@ from pipes import quote
def ply_info(input_ply):
if not os.path.exists(input_ply):
return False
raise IOError("%s does not exist" % input_ply)
# Read PLY header, check if point cloud has normals
has_normals = False

Wyświetl plik

@ -20,6 +20,7 @@ def add_pseudo_georeferencing(geotiff):
dst_ds = gdal.Open(geotiff, GA_Update)
srs = osr.SpatialReference()
srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
srs.ImportFromProj4(get_pseudogeo_utm())
dst_ds.SetProjection( srs.ExportToWkt() )
dst_ds.SetGeoTransform( [ 0.0, get_pseudogeo_scale(), 0.0, 0.0, 0.0, -get_pseudogeo_scale() ] )

Wyświetl plik

@ -12,7 +12,7 @@ from opendm import config
from pyodm import Node, exceptions
from pyodm.utils import AtomicCounter
from pyodm.types import TaskStatus
from osfm import OSFMContext, get_submodel_args_dict, get_submodel_argv
from opendm.osfm import OSFMContext, get_submodel_args_dict, get_submodel_argv
from pipes import quote
try:

Wyświetl plik

@ -8,11 +8,11 @@ from pyproj import CRS
import xmltodict as x2d
from six import string_types
import log
import io
import system
import context
import logging
from opendm import log
from opendm import io
from opendm import system
from opendm import context
from opendm.progress import progressbc
from opendm.photo import ODM_Photo

Wyświetl plik

@ -25,3 +25,4 @@ scikit-learn==0.20
laspy==1.6.0
beautifulsoup4==4.9.1
lxml==4.5.1
matplotlib==1.5.1

8
run.py
Wyświetl plik

@ -1,4 +1,10 @@
#!/usr/bin/python
#!/usr/bin/python3
# Basic check
import sys
if sys.version_info.major < 3:
print("Ups! ODM needs to run with Python 3. It seems you launched it with Python 2. Try using: python3 run.py ... ")
sys.exit(1)
from opendm import log
from opendm import config

Wyświetl plik

@ -1,5 +1,5 @@
if [ ! -z "$1" ]; then
python -m unittest discover tests "test_$1.py"
python3 -m unittest discover tests "test_$1.py"
else
python -m unittest discover tests "test_*.py"
python3 -m unittest discover tests "test_*.py"
fi

Wyświetl plik

@ -15,7 +15,7 @@ class TestCamera(unittest.TestCase):
def test_camera(self):
c = camera.get_cameras_from_opensfm("tests/assets/reconstruction.json")
self.assertEqual(len(c.keys()), 1)
camera_id = c.keys()[0]
camera_id = list(c.keys())[0]
self.assertTrue('v2 ' not in camera_id)
self.assertRaises(RuntimeError, camera.get_cameras_from_opensfm, 'tests/assets/nonexistant.json')
@ -27,7 +27,7 @@ class TestCamera(unittest.TestCase):
osfm_c = camera.get_opensfm_camera_models(c)
self.assertEqual(len(osfm_c.keys()), 1)
c1 = osfm_c[osfm_c.keys()[0]]
c1 = osfm_c[list(osfm_c.keys())[0]]
self.assertTrue('k1_prior' in c1)
self.assertTrue('k2_prior' in c1)
self.assertFalse('test' in c1)

Wyświetl plik

@ -33,8 +33,8 @@ class TestGcp(unittest.TestCase):
copy = GCPFile(gcp.create_utm_copy("tests/assets/output/gcp_utm.txt"))
self.assertTrue(copy.exists())
self.assertEqual(copy.raw_srs, "WGS84 UTM 16N")
self.assertEqual(copy.get_entry(0).x, 609865.707705)
self.assertEqual(copy.get_entry(0).y, 4950688.36182)
self.assertEqual(copy.get_entry(0).x, 609865.7077054137)
self.assertEqual(copy.get_entry(0).y, 4950688.361817497)
def test_utm_conversion_feet(self):
gcp = GCPFile("tests/assets/gcp_michigan_feet_valid.txt")
@ -43,7 +43,7 @@ class TestGcp(unittest.TestCase):
self.assertEqual(copy.raw_srs, "WGS84 UTM 16N")
self.assertEqual(round(copy.get_entry(0).x, 3), 609925.818)
self.assertEqual(round(copy.get_entry(0).y, 3), 4950688.772)
self.assertEqual(round(copy.get_entry(0).z, 3), 171.663)
self.assertEqual(round(copy.get_entry(0).z, 3), 563.199)
def test_filtered_copy(self):
gcp = GCPFile('tests/assets/gcp_latlon_valid.txt')