Expand console output API, tie UI, add missing cluster command

app/api/tasks.py

@@ -158,8 +158,14 @@ class TaskViewSet(viewsets.ViewSet):
     def output(self, request, pk=None, project_pk=None):
         """
         Retrieve the console output for this task.
+
         An optional "line" query param can be passed to retrieve
         only the output starting from a certain line number.
+
+        An optional "limit" query param can be passed to limit
+        the number of lines to be returned
+
+        An optional "f" query param can be either: "text" (default) or "json"
         """
         get_and_check_project(request, project_pk)
         try:
@@ -167,8 +173,36 @@ class TaskViewSet(viewsets.ViewSet):
         except (ObjectDoesNotExist, ValidationError):
             raise exceptions.NotFound()
 
-        line_num = max(0, int(request.query_params.get('line', 0)))
+        try:
-        return Response('\n'.join(task.console.output().rstrip().split('\n')[line_num:]))
+            line_num = max(0, int(request.query_params.get('line', 0)))
+            limit = int(request.query_params.get('limit', 0)) or None
+            fmt = request.query_params.get('f', 'text')
+            if fmt not in ['text', 'json', 'raw']:
+                raise ValueError("Invalid format")
+        except ValueError:
+            raise exceptions.ValidationError("Invalid parameter")
+
+        lines = task.console.output().rstrip().split('\n')
+        count = len(lines)
+        line_start = min(line_num, count)
+        line_end = None
+
+        if limit is not None:
+            if limit > 0:
+                line_end = line_num + limit
+            else:
+                line_start = line_start if count - line_start <= abs(limit) else count - abs(limit) 
+                line_end = None 
+
+        if fmt == 'text':
+            return Response('\n'.join(lines[line_start:line_end]))
+        elif fmt == 'raw':
+            return HttpResponse('\n'.join(lines[line_start:line_end]), content_type="text/plain; charset=utf-8")
+        else:
+            return Response({
+                'lines': lines[line_start:line_end],
+                'count': count
+            })
 
     def list(self, request, project_pk=None):
         get_and_check_project(request, project_pk)

app/management/commands/cluster.py

@@ -0,0 +1,78 @@
+import os
+import json
+import math
+from django.core.management.base import BaseCommand
+from django.core.management import call_command
+from app.models import Project
+from webodm import settings
+from django.db import connection
+
+
+class Command(BaseCommand):
+    requires_system_checks = []
+
+    def add_arguments(self, parser):
+        parser.add_argument("action", type=str, choices=['stagger', 'getref'])
+        parser.add_argument("--refs", required=False, help="JSON array of reference dictionaries")
+        parser.add_argument("--id-buffer", required=False, default=1000, help="ID increment buffer when assigning next seq IDs")
+        parser.add_argument("--dry-run", required=False, action="store_true", help="Don't actually modify tables, just test")
+        
+        
+        super(Command, self).add_arguments(parser)
+
+    def handle(self, **options):
+        if settings.CLUSTER_ID is None:
+            print("CLUSTER_ID is not set")
+            exit(1)
+
+        dry_run = options.get('dry_run', False)
+
+        if options.get('action') == 'stagger':
+            refs = json.loads(options.get('refs'))
+            id_buffer = int(options.get('id_buffer'))
+
+            if not isinstance(refs, list):
+                print("Invalid refs, must be an array")
+                exit(1)
+            if len(refs) <= 1:
+                print("Invalid refs, must have 2 or more items")
+                exit(1)
+            
+            max_project_id = max([r['next_project_id'] for r in refs])
+            start_project_id = max_project_id + id_buffer
+            start_project_id = math.ceil(start_project_id / id_buffer) * id_buffer
+            start_project_id += (settings.CLUSTER_ID - 1)
+            increment_by = len(refs)
+
+            print("Number of clusters/increment: %s" % increment_by)
+            print("Max project ID: %s" % max_project_id)
+            print("New start project ID: %s" % start_project_id)
+
+            project_sql = "ALTER SEQUENCE app_project_id_seq RESTART WITH %s INCREMENT BY %s;" % (start_project_id, increment_by)
+            print(project_sql)
+
+            if not dry_run:
+                with connection.cursor() as c:
+                    c.execute(project_sql)
+                print("Done!")
+            else:
+                print("Dry run, not executing")
+
+
+        elif options.get('action') == 'getref':
+            with connection.cursor() as c:
+                c.execute("SELECT last_value FROM app_project_id_seq")
+                next_project_id = c.fetchone()[0]
+
+                ref = {
+                    'cluster_id': settings.CLUSTER_ID,
+                    'next_project_id': next_project_id,
+                }
+
+                print(json.dumps(ref))
+
+
+
+            
+            
+

app/static/app/js/Console.jsx

@@ -12,7 +12,8 @@ class Console extends React.Component {
     super();
 
     this.state = {
-      lines: []
+      lines: [],
+      sourceLinesCount: null // as reported by source
     };
 
     if (typeof props.children === "string"){
@@ -40,15 +41,23 @@ class Console extends React.Component {
     if (this.props.source !== undefined){
       const updateFromSource = () => {
         let sourceUrl = typeof this.props.source === 'function' ?
-                       this.props.source(this.state.lines.length) :
+                       this.props.source(this.state.sourceLinesCount !== null ? this.state.sourceLinesCount : this.state.lines.length) :
                        this.props.source;
 
         // Fetch
-        this.sourceRequest = $.get(sourceUrl, text => {
+        this.sourceRequest = $.get(sourceUrl, res => {
-          if (text !== ""){
+          let lines = [];
-            let lines = text.split("\n");
+
-            this.addLines(lines);
+          if (typeof res === "string"){
+            if (res !== ""){
+              lines = res.split("\n");
+            }
+          }else if (typeof res === "object"){
+            lines = res.lines;
+            this.setState({sourceLinesCount: res.count});
           }
+
+          if (lines.length > 0) this.addLines(lines);
         })
         .always((_, textStatus) => {
           if (textStatus !== "abort" && this.props.refreshInterval !== undefined){
@@ -69,7 +78,15 @@ class Console extends React.Component {
   }
 
   downloadTxt(filename="console.txt"){
-    Utils.saveAs(this.state.lines.join("\n"), filename);
+    if (this.state.sourceLinesCount !== null){
+      if (this.state.lines.length !== this.state.sourceLinesCount && typeof this.props.source === 'function'){
+        Utils.downloadAs(this.props.source(undefined, true), filename);
+      }else{
+        Utils.saveAs(this.state.lines.join("\n"), filename);
+      }
+    }else{
+      Utils.saveAs(this.state.lines.join("\n"), filename);
+    }
   }
 
   enterFullscreen(){

app/static/app/js/classes/Utils.js

@@ -103,6 +103,15 @@ export default {
       FileSaver.saveAs(blob, filename);
     },
 
+    downloadAs: function(url, filename){
+      const a = document.createElement('a');
+      a.href = url;
+      a.download = filename;
+      document.body.appendChild(a);
+      a.click();
+      document.body.removeChild(a);
+    },
+
     // http://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript
     bytesToSize: function(bytes, decimals = 2){
       if(bytes == 0) return '0 byte';

app/static/app/js/components/TaskListItem.jsx

@@ -168,8 +168,20 @@ class TaskListItem extends React.Component {
     });
   }
 
-  consoleOutputUrl(line){
+  consoleOutputUrl(line, download){
-    return `/api/projects/${this.state.task.project}/tasks/${this.state.task.id}/output/?line=${line}`;
+    let url = `/api/projects/${this.state.task.project}/tasks/${this.state.task.id}/output/`;
+    
+    if (download !== undefined){
+      url += `?f=raw`;
+    }else{
+      url += `?f=json`;
+    }
+
+    if (line !== undefined){
+      url += `&line=${line}&limit=-502`;
+    }
+
+    return url;
   }
 
   thumbnailUrl = () => {

app/tests/test_api.py

@@ -162,6 +162,24 @@ class TestApi(BootTestCase):
         res = client.get('/api/projects/{}/tasks/{}/output/?line=-1'.format(project.id, task.id))
         self.assertEqual(res.data, task.console.output())
 
+        # Console output with limit
+        res = client.get('/api/projects/{}/tasks/{}/output/?line=0&limit=2'.format(project.id, task.id))
+        self.assertEqual(res.data, "line1\nline2")
+
+        # Console output with negative limit
+        res = client.get('/api/projects/{}/tasks/{}/output/?line=0&limit=-2'.format(project.id, task.id))
+        self.assertEqual(res.data, "line2\nline3")
+        
+        # Console output json/raw format
+        res = client.get('/api/projects/{}/tasks/{}/output/?line=0&limit=-2&f=json'.format(project.id, task.id))
+        j = res.json()
+        self.assertEqual(j['lines'][0], "line2")
+        self.assertEqual(j['lines'][1], "line3")
+        self.assertEqual(j['count'], 3)
+
+        res = client.get('/api/projects/{}/tasks/{}/output/?line=0&limit=-2&f=raw'.format(project.id, task.id))
+        self.assertEqual(res.content.decode("utf-8"), "line2\nline3")
+
         # Cannot list task details for a task belonging to a project we don't have access to
         res = client.get('/api/projects/{}/tasks/{}/'.format(other_project.id, other_task.id))
         self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)