From ef52c954bdbe40967ddefac220415cfe913654c4 Mon Sep 17 00:00:00 2001 From: Luca Di Leo Date: Fri, 14 Jan 2022 10:59:28 -0800 Subject: [PATCH] Working on ddb import --- coreplugins/dronedb/api_views.py | 48 +++++++++++------ coreplugins/dronedb/ddb.py | 32 +++++------ coreplugins/dronedb/public/ImportView.jsx | 53 +++++++++---------- .../components/ConfigureNewTaskDialog.jsx | 17 +++--- .../public/components/SelectUrlDialog.jsx | 6 +-- 5 files changed, 83 insertions(+), 73 deletions(-) diff --git a/coreplugins/dronedb/api_views.py b/coreplugins/dronedb/api_views.py index dcb60355..12d8950f 100644 --- a/coreplugins/dronedb/api_views.py +++ b/coreplugins/dronedb/api_views.py @@ -3,12 +3,12 @@ import requests import os from os import path -from requests.structures import CaseInsensitiveDict +#from requests.structures import CaseInsensitiveDict from app import models, pending_actions from app.plugins.views import TaskView from app.plugins.worker import run_function_async from app.plugins import get_current_plugin -from coreplugins.dronedb.ddb import DroneDB, verify_url +from coreplugins.dronedb.ddb import DroneDB, parse_url, verify_url from worker.celery import app from rest_framework.response import Response @@ -16,33 +16,43 @@ from rest_framework import status #from .platform_helper import get_all_platforms, get_platform_by_name +VALID_IMAGE_EXTENSIONS = ['.tiff', '.tif', '.png', '.jpeg', '.jpg'] + +def is_valid(file): + _, file_extension = path.splitext(file) + return file_extension.lower() in VALID_IMAGE_EXTENSIONS or file == 'gcp_list.txt' + class ImportDatasetTaskView(TaskView): def post(self, request, project_pk=None, pk=None): + task = self.get_and_check_task(request, pk) + # Read form data ddb_url = request.data.get('ddb_url', None) #platform_name = request.data.get('platform', None) ds = get_current_plugin().get_user_data_store(request.user) - registry_url = ds.get_string('registry_url', default="") - username = ds.get_string('username', default="") - password = ds.get_string('password', default="") + registry_url = ds.get_string('registry_url') + username = ds.get_string('username') + password = ds.get_string('password') - # Make sure both values are set if ddb_url == None: return Response({'error': 'DroneDB url must be set.'}, status=status.HTTP_400_BAD_REQUEST) - # Fetch the platform by name ddb = DroneDB(registry_url, username, password) + + info = parse_url(ddb_url) + + # Get the files from the folder + files = ddb.get_files_list(info['orgSlug'], info['dsSlug'], info['folder']) + + files = [file for file in files if is_valid(file['path'])] # Verify that the folder url is valid - if ddb.verify_folder_url(ddb_url) == None: - return Response({'error': 'Invalid URL'}, status=status.HTTP_400_BAD_REQUEST) - - # Get the files from the folder - files = ddb.import_from_folder(ddb_url) - + if len(files) == 0: + return Response({'error': 'Empty dataset or folder.'}, status=status.HTTP_400_BAD_REQUEST) + # Update the task with the new information task.console_output += "Importing {} images...\n".format(len(files)) task.images_count = len(files) @@ -51,10 +61,13 @@ class ImportDatasetTaskView(TaskView): # Associate the folder url with the project and task combined_id = "{}_{}".format(project_pk, pk) - get_current_plugin().get_global_data_store().set_string(combined_id, ddb_url) + + datastore = get_current_plugin().get_global_data_store() + datastore.set_string(combined_id, {'ddbUrl': ddb_url, 'token': ddb.token}) + # Start importing the files in the background - serialized = {'token': ddb.token, 'files': [file.serialize() for file in files]} + serialized = {'token': ddb.token, 'files': files} run_function_async(import_files, task.id, serialized) return Response({}, status=status.HTTP_200_OK) @@ -199,9 +212,10 @@ def import_files(task_id, carrier): from app import models from app.plugins import logger - files = carrier.files + files = carrier['files'] - headers = CaseInsensitiveDict() + #headers = CaseInsensitiveDict() + headers = {} if carrier.token != None: headers['Authorization'] = 'Token ' + carrier['token'] diff --git a/coreplugins/dronedb/ddb.py b/coreplugins/dronedb/ddb.py index 88214866..d87b47dd 100644 --- a/coreplugins/dronedb/ddb.py +++ b/coreplugins/dronedb/ddb.py @@ -224,23 +224,23 @@ def parse_url(url): # return None - # def import_from_folder(self, folder_url): - # # Verify the url - # if self.verify_folder_url(folder_url) == None: - # raise Exception('Invalid URL') +# def import_from_folder(folder_url): +# # Verify the url +# if self.verify_folder_url(folder_url) == None: +# raise Exception('Invalid URL') - # # Parse the url and get all necessary information - # information = self.parse_url(folder_url) - # # Define the API url we will call to get all the files in the folder - # folder_api_url = self.build_list_files_in_folder_api_url(information) - # # Call the API - # payload = self.call_api(folder_api_url) - # # Parse the payload into File instances - # files = self.parse_payload_into_files(payload) - # # Let the specific platform do some processing with the files (if necessary) - # files = self.platform_file_processing(files) - # # Return all the valid files - # return [file for file in files if file.is_valid()] +# # Parse the url and get all necessary information +# information = self.parse_url(folder_url) +# # Define the API url we will call to get all the files in the folder +# folder_api_url = self.build_list_files_in_folder_api_url(information) +# # Call the API +# payload = self.call_api(folder_api_url) +# # Parse the payload into File instances +# files = self.parse_payload_into_files(payload) +# # Let the specific platform do some processing with the files (if necessary) +# files = self.platform_file_processing(files) +# # Return all the valid files +# return [file for file in files if file.is_valid()] class Folder: diff --git a/coreplugins/dronedb/public/ImportView.jsx b/coreplugins/dronedb/public/ImportView.jsx index ba1022e9..bd08dcd9 100644 --- a/coreplugins/dronedb/public/ImportView.jsx +++ b/coreplugins/dronedb/public/ImportView.jsx @@ -20,41 +20,32 @@ export default class TaskView extends Component { state = { error: "", - ddbUrl: "", - selectedFolder: "", + ddbRes: null, isDialogOpen: false }; componentDidMount() { - - console.log(this.props.apiURL); - - /* $.getJSON(`${this.props.apiURL}/platforms/`) - .done(data => { - this.setState({platforms: data.platforms}); - }) - .fail(() => { - this.onErrorInDialog("Failed to find available platforms") - }) */ + } //onSelectPlatform = platform => this.setState({ currentPlatform: platform }); - onHideDialog = () => this.setState({ ddbUrl: null, taskId: null, isDialogOpen: false }); - onSelectFolder = url => this.setState({ ddbUrl: url }); - /* - onSelectFolder = folder => this.setState({ selectedFolder: folder }); + onHideDialog = () => this.setState({ ddbRes: null, taskId: null, isDialogOpen: false }); + onSelectDdbRes = res => { + console.log("Result", res); + this.setState({ ddbRes: res, isDialogOpen: false }); + } + - onSaveTask = taskInfo => { // Create task const formData = { - name: taskInfo.name, - options: taskInfo.options, - processing_node: taskInfo.selectedNode.id, - auto_processing_node: taskInfo.selectedNode.key == "auto", - partial: true + name: taskInfo.name, + options: taskInfo.options, + processing_node: taskInfo.selectedNode.id, + auto_processing_node: taskInfo.selectedNode.key == "auto", + partial: true }; if (taskInfo.resizeMode === ResizeModes.YES) { @@ -71,7 +62,7 @@ export default class TaskView extends Component { $.ajax({ url: `${this.props.apiURL}/projects/${this.props.projectId}/tasks/${task.id}/import`, contentType: 'application/json', - data: JSON.stringify({ddb_url: this.state.ddbUrl}), + data: JSON.stringify({ddb_url: this.state.ddbRes.url}), dataType: 'json', type: 'POST' }).done(() => { @@ -88,7 +79,7 @@ export default class TaskView extends Component { onErrorInDialog = msg => { this.setState({ error: msg }); this.onHideDialog(); - };*/ + }; handleClick = () => { this.setState({ isDialogOpen: true }); @@ -97,7 +88,7 @@ export default class TaskView extends Component { render() { const { error, - ddbUrl, + ddbRes, isDialogOpen } = this.state; return ( @@ -115,9 +106,17 @@ export default class TaskView extends Component { + /> + {ddbRes ? + : ""} + ); diff --git a/coreplugins/dronedb/public/components/ConfigureNewTaskDialog.jsx b/coreplugins/dronedb/public/components/ConfigureNewTaskDialog.jsx index b84a158d..d41a8814 100644 --- a/coreplugins/dronedb/public/components/ConfigureNewTaskDialog.jsx +++ b/coreplugins/dronedb/public/components/ConfigureNewTaskDialog.jsx @@ -7,39 +7,36 @@ import "./ConfigureNewTaskDialog.scss"; export default class ConfigureNewTaskDialog extends Component { static defaultProps = { - platform: null, + }; static propTypes = { onHide: PropTypes.func.isRequired, onSaveTask: PropTypes.func.isRequired, - folder: PropTypes.object, - platform: PropTypes.object, - } + ddbRes: PropTypes.object, + } render() { const { onHide, onSaveTask, - platform, - folder, + ddbRes, } = this.props; - const title = "Import from " + (platform !== null ? platform.name : "Platform"); return ( - {title} + Import from DroneDB []} showResize={false} - suggestedTaskName={folder ? folder.name : null} + suggestedTaskName={ddbRes ? ddbRes.name : null} /> diff --git a/coreplugins/dronedb/public/components/SelectUrlDialog.jsx b/coreplugins/dronedb/public/components/SelectUrlDialog.jsx index 3062a143..cf02a3c3 100644 --- a/coreplugins/dronedb/public/components/SelectUrlDialog.jsx +++ b/coreplugins/dronedb/public/components/SelectUrlDialog.jsx @@ -34,7 +34,7 @@ export default class SelectUrlDialog extends Component { selectedDataset: null, selectedFolder: null, info: null, - + images_count: 0, // verifyStatus: null (not started), 'loading', 'success', 'error' verifyStatus: null }; @@ -91,7 +91,7 @@ export default class SelectUrlDialog extends Component { this.setState({verifyStatus: 'loading'}); $.post(`${this.props.apiURL}/verifyurl`, { url: this.state.ddbUrl }).done(result => { - this.setState({verifyStatus: result != null && result.success ? 'success' : 'error'}); + this.setState({verifyStatus: result != null && result.success ? 'success' : 'error', images_count: result != null ? result.count : 0}); }) .fail((error) => { this.setState({verifyStatus: 'error'}); @@ -206,7 +206,7 @@ export default class SelectUrlDialog extends Component { handleSubmit = e => { console.log("Submit"); - //this.props.onSubmit(this.state.selectedFolder); + this.props.onSubmit({name: "ddb", url: this.state.ddbUrl, images_count: this.state.images_count}); }; render() {