diff --git a/app/api/admin.py b/app/api/admin.py index 329e00ef..2de55e4d 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,7 +1,10 @@ from django.contrib.auth.models import User, Group -from rest_framework import serializers, viewsets, generics, status +from app.models import Profile +from rest_framework import serializers, viewsets, generics, status, exceptions +from rest_framework.decorators import action from rest_framework.permissions import IsAdminUser from rest_framework.response import Response +from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth.hashers import make_password from app import models @@ -20,6 +23,7 @@ class AdminUserViewSet(viewsets.ModelViewSet): if email is not None: queryset = queryset.filter(email=email) return queryset + def create(self, request): data = request.data.copy() password = data.get('password') @@ -44,3 +48,36 @@ class AdminGroupViewSet(viewsets.ModelViewSet): if name is not None: queryset = queryset.filter(name=name) return queryset + + +class ProfileSerializer(serializers.ModelSerializer): + class Meta: + model = Profile + exclude = ('id', ) + + read_only_fields = ('user', ) + +class AdminProfileViewSet(viewsets.ModelViewSet): + serializer_class = ProfileSerializer + permission_classes = [IsAdminUser] + lookup_field = 'user' + + def get_queryset(self): + return Profile.objects.all() + + + @action(detail=True, methods=['post']) + def update_quota_deadline(self, request, user=None): + try: + hours = float(request.data.get('hours', '')) + if hours < 0: + raise ValueError("hours must be >= 0") + except ValueError as e: + raise exceptions.ValidationError(str(e)) + + try: + p = Profile.objects.get(user=user) + except ObjectDoesNotExist: + raise exceptions.NotFound() + + return Response({'deadline': p.set_quota_deadline(hours)}, status=status.HTTP_200_OK) diff --git a/app/api/urls.py b/app/api/urls.py index bebfccd4..a29f0e0a 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -6,7 +6,7 @@ from .projects import ProjectViewSet from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskAssetsImport from .imageuploads import Thumbnail, ImageDownload from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView -from .admin import AdminUserViewSet, AdminGroupViewSet +from .admin import AdminUserViewSet, AdminGroupViewSet, AdminProfileViewSet from rest_framework_nested import routers from rest_framework_jwt.views import obtain_jwt_token from .tiler import TileJson, Bounds, Metadata, Tiles, Export @@ -26,6 +26,7 @@ tasks_router.register(r'tasks', TaskViewSet, basename='projects-tasks') admin_router = routers.DefaultRouter() admin_router.register(r'admin/users', AdminUserViewSet, basename='admin-users') admin_router.register(r'admin/groups', AdminGroupViewSet, basename='admin-groups') +admin_router.register(r'admin/profiles', AdminProfileViewSet, basename='admin-groups') urlpatterns = [ url(r'processingnodes/options/$', ProcessingNodeOptionsView.as_view()), @@ -56,7 +57,7 @@ urlpatterns = [ url(r'^auth/', include('rest_framework.urls')), url(r'^token-auth/', obtain_jwt_token), - url(r'^plugins/(?P[^/.]+)/(.*)$', api_view_handler) + url(r'^plugins/(?P[^/.]+)/(.*)$', api_view_handler), ] if settings.ENABLE_USERS_API: diff --git a/app/models/profile.py b/app/models/profile.py index d77d5932..1a1cf971 100644 --- a/app/models/profile.py +++ b/app/models/profile.py @@ -1,3 +1,4 @@ +import time from django.contrib.auth.models import User from django.db import models from django.utils.translation import gettext_lazy as _ @@ -6,6 +7,8 @@ from django.dispatch import receiver from app.models import Task from django.db.models import Sum from django.core.cache import cache +from webodm import settings + class Profile(models.Model): user = models.OneToOneField(User, on_delete=models.CASCADE) @@ -17,6 +20,13 @@ class Profile(models.Model): def used_quota(self): return Task.objects.filter(project__owner=self.user).aggregate(total=Sum('size'))['total'] + def has_exceeded_quota(self): + if not self.has_quota(): + return False + + q = self.used_quota() + return q > self.quota + def used_quota_cached(self): k = f'used_quota_{self.user.id}' cached = cache.get(k) @@ -36,6 +46,20 @@ class Profile(models.Model): def clear_used_quota_cache(self): cache.delete(f'used_quota_{self.user.id}') + + def get_quota_deadline(self): + return cache.get(f'quota_deadline_{self.user.id}') + + def set_quota_deadline(self, hours): + k = f'quota_deadline_{self.user.id}' + seconds = (hours * 60 * 60) + v = time.time() + seconds + cache.set(k, v, int(max(seconds * 10, settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60))) + return v + + def clear_quota_deadline(self): + cache.delete(f'quota_deadline_{self.user.id}') + @receiver(post_save, sender=User) def create_user_profile(sender, instance, created, **kwargs): @@ -44,4 +68,4 @@ def create_user_profile(sender, instance, created, **kwargs): @receiver(post_save, sender=User) def save_user_profile(sender, instance, **kwargs): - instance.profile.save() \ No newline at end of file + instance.profile.save() diff --git a/app/templates/app/dashboard.html b/app/templates/app/dashboard.html index 8fa7a548..668af63e 100644 --- a/app/templates/app/dashboard.html +++ b/app/templates/app/dashboard.html @@ -42,9 +42,9 @@ {% if user.profile.has_exceeded_quota_cached %} {% with total=user.profile.quota|disk_size used=user.profile.used_quota_cached|disk_size %} - {% quota_exceeded_grace_period as hours %} + {% quota_exceeded_grace_period as when %}
- {% blocktrans %}The disk quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted within {{ hours }} hours, until usage falls below {{ total }}.{% endblocktrans %} + {% blocktrans %}The disk quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted {{ when }}, until usage falls below {{ total }}.{% endblocktrans %}
{% endwith %} {% endif %} diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index 2cb0d723..8ed581fa 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -1,8 +1,10 @@ import datetime import math import logging +import time from django import template from webodm import settings +from django.utils.translation import gettext as _ register = template.Library() logger = logging.getLogger('app.logger') @@ -28,9 +30,22 @@ def percentage(num, den, maximum=None): perc = min(perc, maximum) return perc -@register.simple_tag -def quota_exceeded_grace_period(): - return settings.QUOTA_EXCEEDED_GRACE_PERIOD +@register.simple_tag(takes_context=True) +def quota_exceeded_grace_period(context): + deadline = context.request.user.profile.get_quota_deadline() + now = time.time() + if deadline is None: + deadline = now + settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60 + diff = max(0, deadline - now) + if diff >= 60*60*24*2: + return _("within %(num)s days") % {"num": math.ceil(diff / (60*60*24))} + elif diff >= 60*60: + return _("within %(num)s hours") % {"num": math.ceil(diff / (60*60))} + elif diff > 0: + return _("within %(num)s minutes") % {"num": math.ceil(diff / 60)} + else: + return _("very soon") + @register.simple_tag def is_single_user_mode(): diff --git a/worker/celery.py b/worker/celery.py index 083edd81..cb9209e1 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -44,6 +44,14 @@ app.conf.beat_schedule = { 'retry': False } }, + 'check-quotas': { + 'task': 'worker.tasks.check_quotas', + 'schedule': 3600, + 'options': { + 'expires': 1799, + 'retry': False + } + }, } # Mock class for handling async results during testing diff --git a/worker/tasks.py b/worker/tasks.py index b220b3e3..7b8ef84e 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -11,6 +11,7 @@ from celery.utils.log import get_task_logger from django.core.exceptions import ObjectDoesNotExist from django.db.models import Count from django.db.models import Q +from app.models import Profile from app.models import Project from app.models import Task @@ -202,4 +203,17 @@ def export_pointcloud(self, input, **opts): return result except Exception as e: logger.error(str(e)) - return {'error': str(e)} \ No newline at end of file + return {'error': str(e)} + +@app.task +def check_quotas(): + profiles = Profile.objects.filter(quota__gt=-1) + # for p in profiles: + # deadline_key = "%s_quota_exceeded_deadline" % p.user.id + + # if p.has_exceeded_quota(): + # now = time.time() + # deadline = redis_client.getset(deadline_key, now + (settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60)) + # # if deadline < now: TODO.. + # else: + # redis_client.delete(deadline_key)