Expose profiles API, quota update endpoint

app/api/admin.py

@@ -1,7 +1,10 @@
 from django.contrib.auth.models import User, Group
-from rest_framework import serializers, viewsets, generics, status
+from app.models import Profile
+from rest_framework import serializers, viewsets, generics, status, exceptions
+from rest_framework.decorators import action
 from rest_framework.permissions import IsAdminUser
 from rest_framework.response import Response
+from django.core.exceptions import ObjectDoesNotExist
 from django.contrib.auth.hashers import make_password
 from app import models
 
@@ -20,6 +23,7 @@ class AdminUserViewSet(viewsets.ModelViewSet):
         if email is not None:
             queryset = queryset.filter(email=email)
         return queryset
+
     def create(self, request):
         data = request.data.copy()
         password = data.get('password')
@@ -44,3 +48,36 @@ class AdminGroupViewSet(viewsets.ModelViewSet):
         if name is not None:
             queryset = queryset.filter(name=name)
         return queryset
+
+
+class ProfileSerializer(serializers.ModelSerializer):
+    class Meta:
+        model  = Profile
+        exclude = ('id', ) 
+
+        read_only_fields = ('user', )
+
+class AdminProfileViewSet(viewsets.ModelViewSet):
+    serializer_class = ProfileSerializer
+    permission_classes = [IsAdminUser]
+    lookup_field = 'user'
+    
+    def get_queryset(self):
+        return Profile.objects.all()
+
+    
+    @action(detail=True, methods=['post'])
+    def update_quota_deadline(self, request, user=None):
+        try:
+            hours = float(request.data.get('hours', ''))
+            if hours < 0:
+                raise ValueError("hours must be >= 0")
+        except ValueError as e:
+            raise exceptions.ValidationError(str(e))
+
+        try:
+            p = Profile.objects.get(user=user)
+        except ObjectDoesNotExist:
+            raise exceptions.NotFound()
+        
+        return Response({'deadline': p.set_quota_deadline(hours)}, status=status.HTTP_200_OK)

app/api/urls.py

@@ -6,7 +6,7 @@ from .projects import ProjectViewSet
 from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskAssetsImport
 from .imageuploads import Thumbnail, ImageDownload
 from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView
-from .admin import AdminUserViewSet, AdminGroupViewSet
+from .admin import AdminUserViewSet, AdminGroupViewSet, AdminProfileViewSet
 from rest_framework_nested import routers
 from rest_framework_jwt.views import obtain_jwt_token
 from .tiler import TileJson, Bounds, Metadata, Tiles, Export
@@ -26,6 +26,7 @@ tasks_router.register(r'tasks', TaskViewSet, basename='projects-tasks')
 admin_router = routers.DefaultRouter()
 admin_router.register(r'admin/users', AdminUserViewSet, basename='admin-users')
 admin_router.register(r'admin/groups', AdminGroupViewSet, basename='admin-groups')
+admin_router.register(r'admin/profiles', AdminProfileViewSet, basename='admin-groups')
 
 urlpatterns = [
     url(r'processingnodes/options/$', ProcessingNodeOptionsView.as_view()),
@@ -56,7 +57,7 @@ urlpatterns = [
     url(r'^auth/', include('rest_framework.urls')),
     url(r'^token-auth/', obtain_jwt_token),
 
-    url(r'^plugins/(?P<plugin_name>[^/.]+)/(.*)$', api_view_handler)
+    url(r'^plugins/(?P<plugin_name>[^/.]+)/(.*)$', api_view_handler),
 ]
 
 if settings.ENABLE_USERS_API:

app/models/profile.py

@@ -1,3 +1,4 @@
+import time
 from django.contrib.auth.models import User
 from django.db import models
 from django.utils.translation import gettext_lazy as _
@@ -6,6 +7,8 @@ from django.dispatch import receiver
 from app.models import Task
 from django.db.models import Sum
 from django.core.cache import cache
+from webodm import settings
+
 
 class Profile(models.Model):
     user = models.OneToOneField(User, on_delete=models.CASCADE)
@@ -17,6 +20,13 @@ class Profile(models.Model):
     def used_quota(self):
         return Task.objects.filter(project__owner=self.user).aggregate(total=Sum('size'))['total']
 
+    def has_exceeded_quota(self):
+        if not self.has_quota():
+            return False
+        
+        q = self.used_quota()
+        return q > self.quota
+
     def used_quota_cached(self):
         k = f'used_quota_{self.user.id}'
         cached = cache.get(k)
@@ -36,6 +46,20 @@ class Profile(models.Model):
 
     def clear_used_quota_cache(self):
         cache.delete(f'used_quota_{self.user.id}')
+
+    def get_quota_deadline(self):
+        return cache.get(f'quota_deadline_{self.user.id}')
+
+    def set_quota_deadline(self, hours):
+        k = f'quota_deadline_{self.user.id}'
+        seconds = (hours * 60 * 60)
+        v = time.time() + seconds
+        cache.set(k, v, int(max(seconds * 10, settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60)))
+        return v
+    
+    def clear_quota_deadline(self):
+        cache.delete(f'quota_deadline_{self.user.id}')
+
     
 @receiver(post_save, sender=User)
 def create_user_profile(sender, instance, created, **kwargs):
@@ -44,4 +68,4 @@ def create_user_profile(sender, instance, created, **kwargs):
 
 @receiver(post_save, sender=User)
 def save_user_profile(sender, instance, **kwargs):
-    instance.profile.save()
+    instance.profile.save()

app/templates/app/dashboard.html

@@ -42,9 +42,9 @@
 
 		{% if user.profile.has_exceeded_quota_cached %}
 			{% with total=user.profile.quota|disk_size used=user.profile.used_quota_cached|disk_size %}
-			{% quota_exceeded_grace_period as hours %}
+			{% quota_exceeded_grace_period as when %}
 			<div class="alert alert-warning alert-dismissible">
-				<i class="fas fa-exclamation-triangle"></i> {% blocktrans %}The disk quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted within {{ hours }} hours, until usage falls below {{ total }}.{% endblocktrans %}
+				<i class="fas fa-info-circle"></i> {% blocktrans %}The disk quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted {{ when }}, until usage falls below {{ total }}.{% endblocktrans %}
 			</div>
 			{% endwith %}
 		{% endif %}

app/templatetags/settings.py

@@ -1,8 +1,10 @@
 import datetime
 import math
 import logging
+import time
 from django import template
 from webodm import settings
+from django.utils.translation import gettext as _
 
 register = template.Library()
 logger = logging.getLogger('app.logger')
@@ -28,9 +30,22 @@ def percentage(num, den, maximum=None):
         perc = min(perc, maximum)
     return perc
 
-@register.simple_tag
-def quota_exceeded_grace_period():
-    return settings.QUOTA_EXCEEDED_GRACE_PERIOD
+@register.simple_tag(takes_context=True)
+def quota_exceeded_grace_period(context):
+    deadline = context.request.user.profile.get_quota_deadline()
+    now = time.time()
+    if deadline is None:
+        deadline = now + settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60
+    diff = max(0, deadline - now)
+    if diff >= 60*60*24*2:
+        return _("within %(num)s days") % {"num": math.ceil(diff / (60*60*24))}
+    elif diff >= 60*60:
+        return _("within %(num)s hours") % {"num": math.ceil(diff / (60*60))}
+    elif diff > 0:
+        return _("within %(num)s minutes") % {"num": math.ceil(diff / 60)}
+    else:
+        return _("very soon")
+    
 
 @register.simple_tag
 def is_single_user_mode():

worker/celery.py

@@ -44,6 +44,14 @@ app.conf.beat_schedule = {
         	'retry': False
         }
     },
+    'check-quotas': {
+        'task': 'worker.tasks.check_quotas',
+        'schedule': 3600,
+        'options': {
+        	'expires': 1799,
+        	'retry': False
+        }
+    },
 }
 
 # Mock class for handling async results during testing

worker/tasks.py

@@ -11,6 +11,7 @@ from celery.utils.log import get_task_logger
 from django.core.exceptions import ObjectDoesNotExist
 from django.db.models import Count
 from django.db.models import Q
+from app.models import Profile
 
 from app.models import Project
 from app.models import Task
@@ -202,4 +203,17 @@ def export_pointcloud(self, input, **opts):
         return result
     except Exception as e:
         logger.error(str(e))
-        return {'error': str(e)}
+        return {'error': str(e)}
+
+@app.task
+def check_quotas():
+    profiles = Profile.objects.filter(quota__gt=-1)
+    # for p in profiles:
+    #     deadline_key = "%s_quota_exceeded_deadline" % p.user.id
+
+    #     if p.has_exceeded_quota():
+    #         now = time.time()
+    #         deadline = redis_client.getset(deadline_key, now + (settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60))
+    #         # if deadline < now: TODO..
+    #     else:
+    #         redis_client.delete(deadline_key)