From 84356f1ce790d318e176fffe4e3de0907786529a Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 21 Aug 2023 11:43:50 -0400 Subject: [PATCH] External auth PoC, add task sizes --- app/api/tasks.py | 2 +- app/auth/backends.py | 73 +++++++++++++++++++ app/migrations/0036_task_size.py | 50 +++++++++++++ app/models/task.py | 14 ++++ app/static/app/js/classes/Utils.js | 10 +++ app/static/app/js/components/TaskListItem.jsx | 6 ++ .../app/js/components/UploadProgressBar.jsx | 13 +--- package.json | 2 +- webodm/settings.py | 5 ++ 9 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 app/auth/backends.py create mode 100644 app/migrations/0036_task_size.py diff --git a/app/api/tasks.py b/app/api/tasks.py index 8e4d9f2e..4e4da2da 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -75,7 +75,7 @@ class TaskSerializer(serializers.ModelSerializer): class Meta: model = models.Task exclude = ('console_output', 'orthophoto_extent', 'dsm_extent', 'dtm_extent', ) - read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', ) + read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', 'size', ) class TaskViewSet(viewsets.ViewSet): """ diff --git a/app/auth/backends.py b/app/auth/backends.py new file mode 100644 index 00000000..95fcb2c1 --- /dev/null +++ b/app/auth/backends.py @@ -0,0 +1,73 @@ +import requests +from django.contrib.auth.backends import ModelBackend +from django.contrib.auth.models import User +from nodeodm.models import ProcessingNode +from webodm.settings import EXTERNAL_AUTH_ENDPOINT, USE_EXTERNAL_AUTH +from guardian.shortcuts import assign_perm +import logging + +logger = logging.getLogger('app.logger') + +class ExternalBackend(ModelBackend): + def authenticate(self, request, username=None, password=None): + if not USE_EXTERNAL_AUTH: + return None + + try: + r = requests.post(EXTERNAL_AUTH_ENDPOINT, { + 'username': username, + 'password': password + }, headers={'Accept': 'application/json'}) + res = r.json() + + if 'message' in res or 'error' in res: + return None + + logger.info(res) + + if 'user_id' in res: + try: + user = User.objects.get(pk=res['user_id']) + + # Update user info + if user.username != username: + user.username = username + user.save() + except User.DoesNotExist: + user = User(pk=res['user_id'], username=username) + user.save() + + # Setup/update processing node + if ('api_key' in res or 'token' in res) and 'node' in res: + hostname = res['node']['hostname'] + port = res['node']['port'] + token = res['api_key'] if 'api_key' in res else res['token'] + + try: + node = ProcessingNode.objects.get(token=token) + if node.hostname != hostname or node.port != port: + node.hostname = hostname + node.port = port + node.save() + + except ProcessingNode.DoesNotExist: + node = ProcessingNode(hostname=hostname, port=port, token=token) + node.save() + + if not user.has_perm('view_processingnode', node): + assign_perm('view_processingnode', user, node) + + return user + else: + return None + except: + return None + + def get_user(self, user_id): + if not USE_EXTERNAL_AUTH: + return None + + try: + return User.objects.get(pk=user_id) + except User.DoesNotExist: + return None \ No newline at end of file diff --git a/app/migrations/0036_task_size.py b/app/migrations/0036_task_size.py new file mode 100644 index 00000000..1fe42195 --- /dev/null +++ b/app/migrations/0036_task_size.py @@ -0,0 +1,50 @@ +# Generated by Django 2.2.27 on 2023-08-21 14:50 +import os +from django.db import migrations, models +from webodm import settings + +def task_path(project_id, task_id, *args): + return os.path.join(settings.MEDIA_ROOT, + "project", + str(project_id), + "task", + str(task_id), + *args) + +def update_size(task): + try: + total_bytes = 0 + for dirpath, _, filenames in os.walk(task_path(task.project.id, task.id)): + for f in filenames: + fp = os.path.join(dirpath, f) + if not os.path.islink(fp): + total_bytes += os.path.getsize(fp) + task.size = (total_bytes / 1024 / 1024) + task.save() + print("Updated {} with size {}".format(task, task.size)) + except Exception as e: + print("Cannot update size for task {}: {}".format(task, str(e))) + + + +def update_task_sizes(apps, schema_editor): + Task = apps.get_model('app', 'Task') + + for t in Task.objects.all(): + update_size(t) + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0035_task_orthophoto_bands'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='size', + field=models.FloatField(blank=True, default=0.0, help_text='Size of the task on disk in megabytes', verbose_name='Size'), + ), + + migrations.RunPython(update_task_sizes), + ] diff --git a/app/models/task.py b/app/models/task.py index be2c4987..09d51aa8 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -279,6 +279,7 @@ class Task(models.Model): epsg = models.IntegerField(null=True, default=None, blank=True, help_text=_("EPSG code of the dataset (if georeferenced)"), verbose_name="EPSG") tags = models.TextField(db_index=True, default="", blank=True, help_text=_("Task tags"), verbose_name=_("Tags")) orthophoto_bands = fields.JSONField(default=list, blank=True, help_text=_("List of orthophoto bands"), verbose_name=_("Orthophoto Bands")) + size = models.FloatField(default=0.0, blank=True, help_text=_("Size of the task on disk in megabytes"), verbose_name=_("Size")) class Meta: verbose_name = _("Task") @@ -1161,3 +1162,16 @@ class Task(models.Model): else: with open(file.temporary_file_path(), 'rb') as f: shutil.copyfileobj(f, fd) + + def update_size(self, commit=False): + try: + total_bytes = 0 + for dirpath, _, filenames in os.walk(self.task_path()): + for f in filenames: + fp = os.path.join(dirpath, f) + if not os.path.islink(fp): + total_bytes += os.path.getsize(fp) + self.size = (total_bytes / 1024 / 1024) + if commit: self.save() + except Exception as e: + logger.warn("Cannot update size for task {}: {}".format(self, str(e))) diff --git a/app/static/app/js/classes/Utils.js b/app/static/app/js/classes/Utils.js index 989da530..e843275d 100644 --- a/app/static/app/js/classes/Utils.js +++ b/app/static/app/js/classes/Utils.js @@ -93,6 +93,16 @@ export default { saveAs: function(text, filename){ var blob = new Blob([text], {type: "text/plain;charset=utf-8"}); FileSaver.saveAs(blob, filename); + }, + + // http://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript + bytesToSize: function(bytes, decimals = 2){ + if(bytes == 0) return '0 byte'; + var k = 1000; // or 1024 for binary + var dm = decimals || 3; + var sizes = ['bytes', 'Kb', 'Mb', 'Gb', 'Tb', 'Pb', 'Eb', 'Zb', 'Yb']; + var i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; } }; diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index 25611138..c995e49f 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -14,6 +14,7 @@ import PipelineSteps from '../classes/PipelineSteps'; import Css from '../classes/Css'; import Tags from '../classes/Tags'; import Trans from './Trans'; +import Utils from '../classes/Utils'; import { _, interpolate } from '../classes/gettext'; class TaskListItem extends React.Component { @@ -572,6 +573,11 @@ class TaskListItem extends React.Component { {_("Reconstructed Points:")} {stats.pointcloud.points.toLocaleString()} } + {task.size > 0 && + + {_("Size:")} + {Utils.bytesToSize(task.size * 1024 * 1024)} + } {_("Task Output:")}
diff --git a/app/static/app/js/components/UploadProgressBar.jsx b/app/static/app/js/components/UploadProgressBar.jsx index 689b22a1..e0bbe5d7 100644 --- a/app/static/app/js/components/UploadProgressBar.jsx +++ b/app/static/app/js/components/UploadProgressBar.jsx @@ -2,6 +2,7 @@ import '../css/UploadProgressBar.scss'; import React from 'react'; import PropTypes from 'prop-types'; import { _, interpolate } from '../classes/gettext'; +import Utils from '../classes/Utils'; class UploadProgressBar extends React.Component { static propTypes = { @@ -11,22 +12,12 @@ class UploadProgressBar extends React.Component { totalCount: PropTypes.number // number of files } - // http://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript - bytesToSize(bytes, decimals = 2){ - if(bytes == 0) return '0 byte'; - var k = 1000; // or 1024 for binary - var dm = decimals || 3; - var sizes = ['bytes', 'Kb', 'Mb', 'Gb', 'Tb', 'Pb', 'Eb', 'Zb', 'Yb']; - var i = Math.floor(Math.log(bytes) / Math.log(k)); - return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; - } - render() { let percentage = (this.props.progress !== undefined ? this.props.progress : 0).toFixed(2); let bytes = this.props.totalBytesSent !== undefined && this.props.totalBytes !== undefined ? - ' ' + interpolate(_("remaining to upload: %(bytes)s"), { bytes: this.bytesToSize(this.props.totalBytes - this.props.totalBytesSent)}) : + ' ' + interpolate(_("remaining to upload: %(bytes)s"), { bytes: Utils.bytesToSize(this.props.totalBytes - this.props.totalBytesSent)}) : ""; let active = percentage < 100 ? "active" : ""; diff --git a/package.json b/package.json index 05af3481..3887114c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "2.0.3", + "version": "2.1.0", "description": "User-friendly, extendable application and API for processing aerial imagery.", "main": "index.js", "scripts": { diff --git a/webodm/settings.py b/webodm/settings.py index aff1e75d..943a4705 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -169,6 +169,7 @@ AUTH_PASSWORD_VALIDATORS = [ AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend', # this is default 'guardian.backends.ObjectPermissionBackend', + 'app.auth.backends.ExternalBackend', ) # Internationalization @@ -380,6 +381,10 @@ CELERY_WORKER_HIJACK_ROOT_LOGGER = False # before it should be considered offline NODE_OFFLINE_MINUTES = 5 +USE_EXTERNAL_AUTH = True # TODO: change +EXTERNAL_AUTH_ENDPOINT = "http://192.168.2.253:5000/r/auth/login" +# TODO: make these env vars? + if TESTING or FLUSHING: CELERY_TASK_ALWAYS_EAGER = True