diff --git a/app/api/tasks.py b/app/api/tasks.py index 00f3fd45..c586b0ac 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -158,8 +158,14 @@ class TaskViewSet(viewsets.ViewSet): def output(self, request, pk=None, project_pk=None): """ Retrieve the console output for this task. + An optional "line" query param can be passed to retrieve only the output starting from a certain line number. + + An optional "limit" query param can be passed to limit + the number of lines to be returned + + An optional "f" query param can be either: "text" (default) or "json" """ get_and_check_project(request, project_pk) try: @@ -167,8 +173,36 @@ class TaskViewSet(viewsets.ViewSet): except (ObjectDoesNotExist, ValidationError): raise exceptions.NotFound() - line_num = max(0, int(request.query_params.get('line', 0))) - return Response('\n'.join(task.console.output().rstrip().split('\n')[line_num:])) + try: + line_num = max(0, int(request.query_params.get('line', 0))) + limit = int(request.query_params.get('limit', 0)) or None + fmt = request.query_params.get('f', 'text') + if fmt not in ['text', 'json', 'raw']: + raise ValueError("Invalid format") + except ValueError: + raise exceptions.ValidationError("Invalid parameter") + + lines = task.console.output().rstrip().split('\n') + count = len(lines) + line_start = min(line_num, count) + line_end = None + + if limit is not None: + if limit > 0: + line_end = line_num + limit + else: + line_start = line_start if count - line_start <= abs(limit) else count - abs(limit) + line_end = None + + if fmt == 'text': + return Response('\n'.join(lines[line_start:line_end])) + elif fmt == 'raw': + return HttpResponse('\n'.join(lines[line_start:line_end]), content_type="text/plain; charset=utf-8") + else: + return Response({ + 'lines': lines[line_start:line_end], + 'count': count + }) def list(self, request, project_pk=None): get_and_check_project(request, project_pk) diff --git a/app/management/commands/cluster.py b/app/management/commands/cluster.py new file mode 100644 index 00000000..a683bfda --- /dev/null +++ b/app/management/commands/cluster.py @@ -0,0 +1,78 @@ +import os +import json +import math +from django.core.management.base import BaseCommand +from django.core.management import call_command +from app.models import Project +from webodm import settings +from django.db import connection + + +class Command(BaseCommand): + requires_system_checks = [] + + def add_arguments(self, parser): + parser.add_argument("action", type=str, choices=['stagger', 'getref']) + parser.add_argument("--refs", required=False, help="JSON array of reference dictionaries") + parser.add_argument("--id-buffer", required=False, default=1000, help="ID increment buffer when assigning next seq IDs") + parser.add_argument("--dry-run", required=False, action="store_true", help="Don't actually modify tables, just test") + + + super(Command, self).add_arguments(parser) + + def handle(self, **options): + if settings.CLUSTER_ID is None: + print("CLUSTER_ID is not set") + exit(1) + + dry_run = options.get('dry_run', False) + + if options.get('action') == 'stagger': + refs = json.loads(options.get('refs')) + id_buffer = int(options.get('id_buffer')) + + if not isinstance(refs, list): + print("Invalid refs, must be an array") + exit(1) + if len(refs) <= 1: + print("Invalid refs, must have 2 or more items") + exit(1) + + max_project_id = max([r['next_project_id'] for r in refs]) + start_project_id = max_project_id + id_buffer + start_project_id = math.ceil(start_project_id / id_buffer) * id_buffer + start_project_id += (settings.CLUSTER_ID - 1) + increment_by = len(refs) + + print("Number of clusters/increment: %s" % increment_by) + print("Max project ID: %s" % max_project_id) + print("New start project ID: %s" % start_project_id) + + project_sql = "ALTER SEQUENCE app_project_id_seq RESTART WITH %s INCREMENT BY %s;" % (start_project_id, increment_by) + print(project_sql) + + if not dry_run: + with connection.cursor() as c: + c.execute(project_sql) + print("Done!") + else: + print("Dry run, not executing") + + + elif options.get('action') == 'getref': + with connection.cursor() as c: + c.execute("SELECT last_value FROM app_project_id_seq") + next_project_id = c.fetchone()[0] + + ref = { + 'cluster_id': settings.CLUSTER_ID, + 'next_project_id': next_project_id, + } + + print(json.dumps(ref)) + + + + + + diff --git a/app/static/app/js/Console.jsx b/app/static/app/js/Console.jsx index 2f0a3aa6..c946f3b7 100644 --- a/app/static/app/js/Console.jsx +++ b/app/static/app/js/Console.jsx @@ -12,7 +12,8 @@ class Console extends React.Component { super(); this.state = { - lines: [] + lines: [], + sourceLinesCount: null // as reported by source }; if (typeof props.children === "string"){ @@ -40,15 +41,23 @@ class Console extends React.Component { if (this.props.source !== undefined){ const updateFromSource = () => { let sourceUrl = typeof this.props.source === 'function' ? - this.props.source(this.state.lines.length) : + this.props.source(this.state.sourceLinesCount !== null ? this.state.sourceLinesCount : this.state.lines.length) : this.props.source; // Fetch - this.sourceRequest = $.get(sourceUrl, text => { - if (text !== ""){ - let lines = text.split("\n"); - this.addLines(lines); + this.sourceRequest = $.get(sourceUrl, res => { + let lines = []; + + if (typeof res === "string"){ + if (res !== ""){ + lines = res.split("\n"); + } + }else if (typeof res === "object"){ + lines = res.lines; + this.setState({sourceLinesCount: res.count}); } + + if (lines.length > 0) this.addLines(lines); }) .always((_, textStatus) => { if (textStatus !== "abort" && this.props.refreshInterval !== undefined){ @@ -69,7 +78,15 @@ class Console extends React.Component { } downloadTxt(filename="console.txt"){ - Utils.saveAs(this.state.lines.join("\n"), filename); + if (this.state.sourceLinesCount !== null){ + if (this.state.lines.length !== this.state.sourceLinesCount && typeof this.props.source === 'function'){ + Utils.downloadAs(this.props.source(undefined, true), filename); + }else{ + Utils.saveAs(this.state.lines.join("\n"), filename); + } + }else{ + Utils.saveAs(this.state.lines.join("\n"), filename); + } } enterFullscreen(){ diff --git a/app/static/app/js/classes/Utils.js b/app/static/app/js/classes/Utils.js index 0a2a8de5..c6c7f801 100644 --- a/app/static/app/js/classes/Utils.js +++ b/app/static/app/js/classes/Utils.js @@ -103,6 +103,15 @@ export default { FileSaver.saveAs(blob, filename); }, + downloadAs: function(url, filename){ + const a = document.createElement('a'); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + }, + // http://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript bytesToSize: function(bytes, decimals = 2){ if(bytes == 0) return '0 byte'; diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index 7c9b4a96..1fdb2e32 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -168,8 +168,20 @@ class TaskListItem extends React.Component { }); } - consoleOutputUrl(line){ - return `/api/projects/${this.state.task.project}/tasks/${this.state.task.id}/output/?line=${line}`; + consoleOutputUrl(line, download){ + let url = `/api/projects/${this.state.task.project}/tasks/${this.state.task.id}/output/`; + + if (download !== undefined){ + url += `?f=raw`; + }else{ + url += `?f=json`; + } + + if (line !== undefined){ + url += `&line=${line}&limit=-502`; + } + + return url; } thumbnailUrl = () => { diff --git a/app/tests/test_api.py b/app/tests/test_api.py index 99ccaf7c..3fd45518 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -162,6 +162,24 @@ class TestApi(BootTestCase): res = client.get('/api/projects/{}/tasks/{}/output/?line=-1'.format(project.id, task.id)) self.assertEqual(res.data, task.console.output()) + # Console output with limit + res = client.get('/api/projects/{}/tasks/{}/output/?line=0&limit=2'.format(project.id, task.id)) + self.assertEqual(res.data, "line1\nline2") + + # Console output with negative limit + res = client.get('/api/projects/{}/tasks/{}/output/?line=0&limit=-2'.format(project.id, task.id)) + self.assertEqual(res.data, "line2\nline3") + + # Console output json/raw format + res = client.get('/api/projects/{}/tasks/{}/output/?line=0&limit=-2&f=json'.format(project.id, task.id)) + j = res.json() + self.assertEqual(j['lines'][0], "line2") + self.assertEqual(j['lines'][1], "line3") + self.assertEqual(j['count'], 3) + + res = client.get('/api/projects/{}/tasks/{}/output/?line=0&limit=-2&f=raw'.format(project.id, task.id)) + self.assertEqual(res.content.decode("utf-8"), "line2\nline3") + # Cannot list task details for a task belonging to a project we don't have access to res = client.get('/api/projects/{}/tasks/{}/'.format(other_project.id, other_task.id)) self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)