From 0998872ba5de181f95012f58b0d6dc8faf0205a1 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 20 May 2019 16:29:51 -0400 Subject: [PATCH] sm-cluster address check at startup, pyodm 1.5.1 update --- Dockerfile | 2 +- configure.sh | 2 +- opendm/config.py | 10 +++++++++- opendm/progress.py | 4 ++-- opendm/remote.py | 4 ++-- opendm/types.py | 2 +- portable.Dockerfile | 2 +- 7 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6b2959dc..6d9fecfb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,7 +76,7 @@ RUN pip install -U \ xmltodict \ rasterio \ attrs==19.1.0 \ - pyodm==1.5.0 + pyodm==1.5.1 RUN pip install --upgrade cryptography && python -m easy_install --upgrade pyOpenSSL diff --git a/configure.sh b/configure.sh index deb90563..f11767a9 100755 --- a/configure.sh +++ b/configure.sh @@ -80,7 +80,7 @@ install() { repoze.lru \ rasterio \ attrs==19.1.0 \ - pyodm==1.5.0 + pyodm==1.5.1 # Fix: /usr/local/lib/python2.7/dist-packages/requests/__init__.py:83: RequestsDependencyWarning: Old version of cryptography ([1, 2, 3]) may cause slowdown. pip install --upgrade cryptography diff --git a/opendm/config.py b/opendm/config.py index 22a70285..5ce77a49 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -3,6 +3,7 @@ from opendm import context from opendm import io from opendm import log from appsettings import SettingsParser +from pyodm import Node, exceptions import sys @@ -27,7 +28,7 @@ def url_string(string): import re regex = re.compile( r'^(?:http|ftp)s?://' # http:// or https:// - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.?)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... r'localhost|' #localhost... r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip r'(?::\d+)?' # optional port @@ -575,4 +576,11 @@ def config(): log.ODM_WARNING("--orthophoto-cutline is set, but --crop is not. --crop will be set to 0.01") args.crop = 0.01 + if args.sm_cluster: + try: + Node.from_url(args.sm_cluster).info() + except exceptions.NodeConnectionError as e: + log.ODM_ERROR("Cluster node seems to be offline: %s" % str(e)) + sys.exit(1) + return args diff --git a/opendm/progress.py b/opendm/progress.py index 3dd02523..cc030461 100644 --- a/opendm/progress.py +++ b/opendm/progress.py @@ -18,7 +18,7 @@ class Broadcaster: def set_project_name(self, project_name): self.project_name = project_name - def send_update(self, global_progress, stage_progress, stage): + def send_update(self, global_progress): """ Update any listener on the pipeline progress (in percentage terms) """ @@ -32,7 +32,7 @@ class Broadcaster: global_progress = 100 try: - sock.sendto("PGUP/{}/{}/{}/{}/{}".format(self.pid, self.project_name, float(global_progress), float(stage_progress), stage).encode('utf-8'), (UDP_IP, self.port)) + sock.sendto("PGUP/{}/{}/{}".format(self.pid, self.project_name, float(global_progress)).encode('utf-8'), (UDP_IP, self.port)) except: log.ODM_WARNING("Failed to broadcast progress update on UDP port %s" % str(self.port)) diff --git a/opendm/remote.py b/opendm/remote.py index 656729d6..15c7e63d 100644 --- a/opendm/remote.py +++ b/opendm/remote.py @@ -37,8 +37,8 @@ class LocalRemoteExecutor: log.ODM_INFO("LRE: Initializing using cluster node %s:%s" % (self.node.host, self.node.port)) try: - odm_version = self.node.info().odm_version - log.ODM_INFO("LRE: Node is online and running ODM version: %s" % odm_version) + info = self.node.info() + log.ODM_INFO("LRE: Node is online and running %s version %s" % (info.engine, info.engine_version)) except exceptions.NodeConnectionError: log.ODM_WARNING("LRE: The node seems to be offline! We'll still process the dataset, but it's going to run entirely locally.") self.node_online = False diff --git a/opendm/types.py b/opendm/types.py index 163352c2..e169db60 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -394,7 +394,7 @@ class ODM_Stage: def update_progress(self, progress): progress = max(0.0, min(100.0, progress)) progressbc.send_update(self.previous_stages_progress() + - (self.delta_progress() / 100.0) * float(progress), progress, self.name) + (self.delta_progress() / 100.0) * float(progress)) def process(self, args, outputs): raise NotImplementedError diff --git a/portable.Dockerfile b/portable.Dockerfile index 6f2b728c..86835ed8 100644 --- a/portable.Dockerfile +++ b/portable.Dockerfile @@ -76,7 +76,7 @@ RUN pip install -U \ xmltodict \ rasterio \ attrs==19.1.0 \ - pyodm==1.5.0 + pyodm==1.5.1 RUN pip install --upgrade cryptography && python -m easy_install --upgrade pyOpenSSL