External auth PoC, add task sizes

app/api/tasks.py

@@ -75,7 +75,7 @@ class TaskSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.Task
         exclude = ('console_output', 'orthophoto_extent', 'dsm_extent', 'dtm_extent', )
-        read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', )
+        read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', 'size', )
 
 class TaskViewSet(viewsets.ViewSet):
     """

app/auth/backends.py

@@ -0,0 +1,73 @@
+import requests
+from django.contrib.auth.backends import ModelBackend
+from django.contrib.auth.models import User
+from nodeodm.models import ProcessingNode
+from webodm.settings import EXTERNAL_AUTH_ENDPOINT, USE_EXTERNAL_AUTH
+from guardian.shortcuts import assign_perm
+import logging
+
+logger = logging.getLogger('app.logger')
+
+class ExternalBackend(ModelBackend):
+    def authenticate(self, request, username=None, password=None):
+        if not USE_EXTERNAL_AUTH:
+            return None
+        
+        try:
+            r = requests.post(EXTERNAL_AUTH_ENDPOINT, {
+                'username': username,
+                'password': password
+            }, headers={'Accept': 'application/json'})
+            res = r.json()
+
+            if 'message' in res or 'error' in res:
+                return None
+            
+            logger.info(res)
+
+            if 'user_id' in res:
+                try:
+                    user = User.objects.get(pk=res['user_id'])
+
+                    # Update user info
+                    if user.username != username:
+                        user.username = username
+                        user.save()
+                except User.DoesNotExist:
+                    user = User(pk=res['user_id'], username=username)
+                    user.save()
+
+                # Setup/update processing node
+                if ('api_key' in res or 'token' in res) and 'node' in res:
+                    hostname = res['node']['hostname']
+                    port = res['node']['port']
+                    token = res['api_key'] if 'api_key' in res else res['token']
+
+                    try:
+                        node = ProcessingNode.objects.get(token=token)
+                        if node.hostname != hostname or node.port != port:
+                            node.hostname = hostname
+                            node.port = port
+                            node.save()
+                        
+                    except ProcessingNode.DoesNotExist:
+                        node = ProcessingNode(hostname=hostname, port=port, token=token)
+                        node.save()
+                    
+                    if not user.has_perm('view_processingnode', node):
+                        assign_perm('view_processingnode', user, node)
+
+                return user
+            else:
+                return None
+        except:
+            return None
+    
+    def get_user(self, user_id):
+        if not USE_EXTERNAL_AUTH:
+            return None
+
+        try:
+            return User.objects.get(pk=user_id)
+        except User.DoesNotExist:
+            return None

app/migrations/0036_task_size.py

@@ -0,0 +1,50 @@
+# Generated by Django 2.2.27 on 2023-08-21 14:50
+import os
+from django.db import migrations, models
+from webodm import settings
+
+def task_path(project_id, task_id, *args):
+    return os.path.join(settings.MEDIA_ROOT,
+                        "project",
+                        str(project_id),
+                        "task",
+                        str(task_id),
+                        *args)
+
+def update_size(task):
+    try:
+        total_bytes = 0
+        for dirpath, _, filenames in os.walk(task_path(task.project.id, task.id)):
+            for f in filenames:
+                fp = os.path.join(dirpath, f)
+                if not os.path.islink(fp):
+                    total_bytes += os.path.getsize(fp)
+        task.size = (total_bytes / 1024 / 1024)
+        task.save()
+        print("Updated {} with size {}".format(task, task.size))
+    except Exception as e:
+        print("Cannot update size for task {}: {}".format(task, str(e)))
+
+
+
+def update_task_sizes(apps, schema_editor):
+    Task = apps.get_model('app', 'Task')
+
+    for t in Task.objects.all():
+        update_size(t)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0035_task_orthophoto_bands'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='size',
+            field=models.FloatField(blank=True, default=0.0, help_text='Size of the task on disk in megabytes', verbose_name='Size'),
+        ),
+
+        migrations.RunPython(update_task_sizes),
+    ]

app/models/task.py

@@ -279,6 +279,7 @@ class Task(models.Model):
     epsg = models.IntegerField(null=True, default=None, blank=True, help_text=_("EPSG code of the dataset (if georeferenced)"), verbose_name="EPSG")
     tags = models.TextField(db_index=True, default="", blank=True, help_text=_("Task tags"), verbose_name=_("Tags"))
     orthophoto_bands = fields.JSONField(default=list, blank=True, help_text=_("List of orthophoto bands"), verbose_name=_("Orthophoto Bands"))
+    size = models.FloatField(default=0.0, blank=True, help_text=_("Size of the task on disk in megabytes"), verbose_name=_("Size"))
     
     class Meta:
         verbose_name = _("Task")
@@ -1161,3 +1162,16 @@ class Task(models.Model):
                 else:
                     with open(file.temporary_file_path(), 'rb') as f:
                         shutil.copyfileobj(f, fd)
+
+    def update_size(self, commit=False):
+        try:
+            total_bytes = 0
+            for dirpath, _, filenames in os.walk(self.task_path()):
+                for f in filenames:
+                    fp = os.path.join(dirpath, f)
+                    if not os.path.islink(fp):
+                        total_bytes += os.path.getsize(fp)
+            self.size = (total_bytes / 1024 / 1024)
+            if commit: self.save()
+        except Exception as e:
+            logger.warn("Cannot update size for task {}: {}".format(self, str(e)))

app/static/app/js/classes/Utils.js

@@ -93,6 +93,16 @@ export default {
     saveAs: function(text, filename){
       var blob = new Blob([text], {type: "text/plain;charset=utf-8"});
       FileSaver.saveAs(blob, filename);
+    },
+
+    // http://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript
+    bytesToSize: function(bytes, decimals = 2){
+      if(bytes == 0) return '0 byte';
+      var k = 1000; // or 1024 for binary
+      var dm = decimals || 3;
+      var sizes = ['bytes', 'Kb', 'Mb', 'Gb', 'Tb', 'Pb', 'Eb', 'Zb', 'Yb'];
+      var i = Math.floor(Math.log(bytes) / Math.log(k));
+      return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
     }
 };
 

app/static/app/js/components/TaskListItem.jsx

@@ -14,6 +14,7 @@ import PipelineSteps from '../classes/PipelineSteps';
 import Css from '../classes/Css';
 import Tags from '../classes/Tags';
 import Trans from './Trans';
+import Utils from '../classes/Utils';
 import { _, interpolate } from '../classes/gettext';
 
 class TaskListItem extends React.Component {
@@ -572,6 +573,11 @@ class TaskListItem extends React.Component {
                     <td><strong>{_("Reconstructed Points:")}</strong></td>
                     <td>{stats.pointcloud.points.toLocaleString()}</td>
                   </tr>}
+                  {task.size > 0 && 
+                  <tr>
+                    <td><strong>{_("Size:")}</strong></td>
+                    <td>{Utils.bytesToSize(task.size * 1024 * 1024)}</td>
+                  </tr>}
                   <tr>
                       <td><strong>{_("Task Output:")}</strong></td>
                       <td><div className="btn-group btn-toggle"> 

app/static/app/js/components/UploadProgressBar.jsx

@@ -2,6 +2,7 @@ import '../css/UploadProgressBar.scss';
 import React from 'react';
 import PropTypes from 'prop-types';
 import { _, interpolate } from '../classes/gettext';
+import Utils from '../classes/Utils';
 
 class UploadProgressBar extends React.Component {
   static propTypes = {
@@ -11,22 +12,12 @@ class UploadProgressBar extends React.Component {
     totalCount: PropTypes.number // number of files
   }
 
-  // http://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript
-  bytesToSize(bytes, decimals = 2){
-   if(bytes == 0) return '0 byte';
-   var k = 1000; // or 1024 for binary
-   var dm = decimals || 3;
-   var sizes = ['bytes', 'Kb', 'Mb', 'Gb', 'Tb', 'Pb', 'Eb', 'Zb', 'Yb'];
-   var i = Math.floor(Math.log(bytes) / Math.log(k));
-   return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
-  }
-
   render() {
     let percentage = (this.props.progress !== undefined ? 
                      this.props.progress : 
                      0).toFixed(2);
     let bytes = this.props.totalBytesSent !== undefined && this.props.totalBytes !== undefined ? 
-              ' ' + interpolate(_("remaining to upload: %(bytes)s"), { bytes: this.bytesToSize(this.props.totalBytes - this.props.totalBytesSent)}) : 
+              ' ' + interpolate(_("remaining to upload: %(bytes)s"), { bytes: Utils.bytesToSize(this.props.totalBytes - this.props.totalBytesSent)}) : 
                "";
 
     let active = percentage < 100 ? "active" : "";

package.json

@@ -1,6 +1,6 @@
 {
   "name": "WebODM",
-  "version": "2.0.3",
+  "version": "2.1.0",
   "description": "User-friendly, extendable application and API for processing aerial imagery.",
   "main": "index.js",
   "scripts": {

webodm/settings.py

@@ -169,6 +169,7 @@ AUTH_PASSWORD_VALIDATORS = [
 AUTHENTICATION_BACKENDS = (
     'django.contrib.auth.backends.ModelBackend', # this is default
     'guardian.backends.ObjectPermissionBackend',
+    'app.auth.backends.ExternalBackend',
 )
 
 # Internationalization
@@ -380,6 +381,10 @@ CELERY_WORKER_HIJACK_ROOT_LOGGER = False
 # before it should be considered offline
 NODE_OFFLINE_MINUTES = 5 
 
+USE_EXTERNAL_AUTH = True # TODO: change
+EXTERNAL_AUTH_ENDPOINT = "http://192.168.2.253:5000/r/auth/login"
+# TODO: make these env vars?
+
 if TESTING or FLUSHING:
     CELERY_TASK_ALWAYS_EAGER = True