Merge branch 'master' of https://github.com/OpenDroneMap/WebODM into kathenae-task_notification_plugin

pull/1343/head
Piero Toffanin 2023-05-19 13:13:49 -04:00
commit 26dee3b023
10 zmienionych plików z 335 dodań i 101 usunięć

Wyświetl plik

@ -115,13 +115,13 @@ algos = {
'help': _('Atmospherically Resistant Vegetation Index. Useful when working with imagery for regions with high atmospheric aerosol content.'),
'range': (-1, 1)
},
'Thermal C': {
'Celsius': {
'expr': 'L',
'help': _('Thermal temperature in Celsius degrees.')
'help': _('Temperature in Celsius degrees.')
},
'Thermal K': {
'Kelvin': {
'expr': 'L * 100 + 27315',
'help': _('Thermal temperature in Centikelvin degrees.')
'help': _('Temperature in Centikelvin degrees.')
},
# more?
@ -154,6 +154,8 @@ camera_filters = [
'BGRNReL',
'BGRReNL',
'L', # FLIR camera has a single LWIR band
# more?
# TODO: certain cameras have only two bands? eg. MAPIR NDVI BLUE+NIR
]

Wyświetl plik

@ -0,0 +1,44 @@
# Generated by Django 2.2.27 on 2023-05-19 15:38
import rasterio
import os
import django.contrib.postgres.fields.jsonb
from django.db import migrations
from webodm import settings
def update_orthophoto_bands_fields(apps, schema_editor):
Task = apps.get_model('app', 'Task')
for t in Task.objects.all():
bands = []
orthophoto_path = os.path.join(settings.MEDIA_ROOT, "project", str(t.project.id), "task", str(t.id), "assets", "odm_orthophoto", "odm_orthophoto.tif")
if os.path.isfile(orthophoto_path):
try:
with rasterio.open(orthophoto_path) as f:
bands = [c.name for c in f.colorinterp]
except Exception as e:
print(e)
print("Updating {} (with orthophoto bands: {})".format(t, str(bands)))
t.orthophoto_bands = bands
t.save()
class Migration(migrations.Migration):
dependencies = [
('app', '0034_delete_imageupload'),
]
operations = [
migrations.AddField(
model_name='task',
name='orthophoto_bands',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, help_text='List of orthophoto bands', verbose_name='Orthophoto Bands'),
),
migrations.RunPython(update_orthophoto_bands_fields),
]

Wyświetl plik

@ -278,6 +278,7 @@ class Task(models.Model):
potree_scene = fields.JSONField(default=dict, blank=True, help_text=_("Serialized potree scene information used to save/load measurements and camera view angle"), verbose_name=_("Potree Scene"))
epsg = models.IntegerField(null=True, default=None, blank=True, help_text=_("EPSG code of the dataset (if georeferenced)"), verbose_name="EPSG")
tags = models.TextField(db_index=True, default="", blank=True, help_text=_("Task tags"), verbose_name=_("Tags"))
orthophoto_bands = fields.JSONField(default=list, blank=True, help_text=_("List of orthophoto bands"), verbose_name=_("Orthophoto Bands"))
class Meta:
verbose_name = _("Task")
@ -883,6 +884,7 @@ class Task(models.Model):
self.update_available_assets_field()
self.update_epsg_field()
self.update_orthophoto_bands_field()
self.potree_scene = {}
self.running_progress = 1.0
self.console_output += gettext("Done!") + "\n"
@ -904,8 +906,9 @@ class Task(models.Model):
def get_map_items(self):
types = []
if 'orthophoto.tif' in self.available_assets: types.append('orthophoto')
if 'orthophoto.tif' in self.available_assets: types.append('plant')
if 'orthophoto.tif' in self.available_assets:
types.append('orthophoto')
types.append('plant')
if 'dsm.tif' in self.available_assets: types.append('dsm')
if 'dtm.tif' in self.available_assets: types.append('dtm')
@ -924,7 +927,8 @@ class Task(models.Model):
'public': self.public,
'camera_shots': camera_shots,
'ground_control_points': ground_control_points,
'epsg': self.epsg
'epsg': self.epsg,
'orthophoto_bands': self.orthophoto_bands,
}
}
}
@ -998,6 +1002,22 @@ class Task(models.Model):
if commit: self.save()
def update_orthophoto_bands_field(self, commit=False):
"""
Updates the orthophoto bands field with the correct value
:param commit: when True also saves the model, otherwise the user should manually call save()
"""
bands = []
orthophoto_path = self.assets_path(self.ASSETS_MAP['orthophoto.tif'])
if os.path.isfile(orthophoto_path):
with rasterio.open(orthophoto_path) as f:
bands = [c.name for c in f.colorinterp]
self.orthophoto_bands = bands
if commit: self.save()
def delete(self, using=None, keep_parents=False):
task_id = self.id
from app.plugins import signals as plugin_signals

Wyświetl plik

@ -126,8 +126,17 @@ class Map extends React.Component {
let metaUrl = url + "metadata";
if (type == "plant") metaUrl += "?formula=NDVI&bands=RGN&color_map=rdylgn";
if (type == "dsm" || type == "dtm") metaUrl += "?hillshade=6&color_map=viridis";
if (type == "plant"){
if (meta.task && meta.task.orthophoto_bands && meta.task.orthophoto_bands.length === 2){
// Single band, probably thermal dataset, in any case we can't render NDVI
// because it requires 3 bands
metaUrl += "?formula=Celsius&bands=L&color_map=magma";
}else{
metaUrl += "?formula=NDVI&bands=RGN&color_map=rdylgn";
}
}else if (type == "dsm" || type == "dtm"){
metaUrl += "?hillshade=6&color_map=viridis";
}
this.tileJsonRequests.push($.getJSON(metaUrl)
.done(mres => {

Wyświetl plik

@ -246,6 +246,9 @@ class TestApiTask(BootTransactionTestCase):
# EPSG should be null
self.assertTrue(task.epsg is None)
# Orthophoto bands field should be an empty list
self.assertEqual(len(task.orthophoto_bands), 0)
# tiles.json, bounds, metadata should not be accessible at this point
tile_types = ['orthophoto', 'dsm', 'dtm']
endpoints = ['tiles.json', 'bounds', 'metadata']
@ -378,6 +381,9 @@ class TestApiTask(BootTransactionTestCase):
res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Orthophoto bands field should be populated
self.assertEqual(len(task.orthophoto_bands), 4)
# Can export orthophoto (when formula and bands are specified)
res = client.post("/api/projects/{}/tasks/{}/orthophoto/export".format(project.id, task.id), {
'formula': 'NDVI'
@ -916,6 +922,9 @@ class TestApiTask(BootTransactionTestCase):
# EPSG should be populated
self.assertEqual(task.epsg, 32615)
# Orthophoto bands should not be populated
self.assertEqual(len(task.orthophoto_bands), 0)
# Can access only tiles of available assets
res = client.get("/api/projects/{}/tasks/{}/dsm/tiles.json".format(project.id, task.id))
self.assertEqual(res.status_code, status.HTTP_200_OK)

Wyświetl plik

@ -1,4 +1,8 @@
from app.plugins import PluginBase, Menu, MountPoint
from rest_framework.response import Response
from rest_framework import status, permissions
from rest_framework.decorators import api_view, permission_classes
from app.plugins import PluginBase, Menu, MountPoint, get_current_plugin
from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from django.utils.translation import gettext as _
@ -30,30 +34,58 @@ def get_memory_stats():
except:
return {}
def get_diagnostic_stats():
plugin = get_current_plugin()
with plugin.python_imports():
import psutil
# Disk space
total_disk_space, used_disk_space, free_disk_space = shutil.disk_usage('./')
# CPU Stats
cpu_percent_used = psutil.cpu_percent()
cpu_percent_free = 100 - cpu_percent_used
cpu_freq = psutil.cpu_freq()
diagnostic_stats = {
'total_disk_space': total_disk_space,
'used_disk_space': used_disk_space,
'free_disk_space': free_disk_space,
'cpu_percent_used': round(cpu_percent_used, 2),
'cpu_percent_free': round(cpu_percent_free, 2),
'cpu_freq_current': round(cpu_freq.current / 1000, 2),
}
# Memory (Linux only)
memory_stats = get_memory_stats()
if 'free' in memory_stats:
diagnostic_stats['free_memory'] = memory_stats['free']
diagnostic_stats['used_memory'] = memory_stats['used']
diagnostic_stats['total_memory'] = memory_stats['total']
return diagnostic_stats
class Plugin(PluginBase):
def main_menu(self):
return [Menu(_("Diagnostic"), self.public_url(""), "fa fa-chart-pie fa-fw")]
def api_mount_points(self):
@api_view()
@permission_classes((permissions.IsAuthenticated,))
def diagnostic(request):
diagnostic_stats = get_diagnostic_stats()
return Response(diagnostic_stats)
return [
MountPoint('/', diagnostic)
]
def app_mount_points(self):
@login_required
def diagnostic(request):
# Disk space
total_disk_space, used_disk_space, free_disk_space = shutil.disk_usage('./')
template_args = {
'title': 'Diagnostic',
'total_disk_space': total_disk_space,
'used_disk_space': used_disk_space,
'free_disk_space': free_disk_space
}
# Memory (Linux only)
memory_stats = get_memory_stats()
if 'free' in memory_stats:
template_args['free_memory'] = memory_stats['free']
template_args['used_memory'] = memory_stats['used']
template_args['total_memory'] = memory_stats['total']
template_args = get_diagnostic_stats()
template_args['title'] = 'Diagnostic'
return render(request, self.template_path("diagnostic.html"), template_args)

Wyświetl plik

@ -0,0 +1 @@
psutil==5.9.5

Wyświetl plik

@ -11,20 +11,34 @@
<div style="width: 80%; margin-left: 10%;">
<canvas id="diskChart" width="200" height="200" style="margin-bottom: 12px;"></canvas>
</div>
<p><b>{% trans 'Free' context 'Megabytes of storage space' %}:</b> {{ free_disk_space|filesizeformat }} |
<p id="storageStatsLabel">
<b>{% trans 'Free' context 'Megabytes of storage space' %}:</b> {{ free_disk_space|filesizeformat }} |
<b>{% trans 'Used' context 'Megabytes of storage space' %}:</b> {{ used_disk_space|filesizeformat }} |
<b>{% trans 'Total' context 'Megabytes of storage space' %}:</b> {{ total_disk_space|filesizeformat }}</p>
<b>{% trans 'Total' context 'Megabytes of storage space' %}:</b> {{ total_disk_space|filesizeformat }}
</p>
</div>
<div class="col-md-4 col-sm-12">
{% if total_memory %}
<h4>{% trans 'Memory' context 'Computer memory (RAM)' %}</h4>
<div style="width: 80%; margin-left: 10%;">
<canvas id="memoryChart" width="200" height="200" style="margin-bottom: 12px;"></canvas>
{% if total_memory %}
<div class="col-md-4 col-sm-12">
<h4>{% trans 'Memory' context 'Computer memory (RAM)' %}</h4>
<div style="width: 80%; margin-left: 10%;">
<canvas id="memoryChart" width="200" height="200" style="margin-bottom: 12px;"></canvas>
</div>
<p id="memoryStatsLabel">
<b>{% trans 'Free' context 'Megabytes of memory space' %}:</b> {{ free_memory|filesizeformat }} |
<b>{% trans 'Used' context 'Megabytes of memory space' %}:</b> {{ used_memory|filesizeformat }} |
<b>{% trans 'Total' context 'Megabytes of memory space'%}:</b> {{ total_memory|filesizeformat }}
</p>
</div>
<p><b>{% trans 'Free' context 'Megabytes of memory space' %}:</b> {{ free_memory|filesizeformat }} |
<b>{% trans 'Used' context 'Megabytes of memory space' %}:</b> {{ used_memory|filesizeformat }} |
<b>{% trans 'Total' context 'Megabytes of memory space'%}:</b> {{ total_memory|filesizeformat }}</p>
{% endif %}
{% endif %}
<div class="col-md-4 col-sm-12">
<h4>{% trans 'CPU Usage' context 'Computer CPU Usage' %}</h4>
<div style="width: 80%; margin-left: 10%;">
<canvas id="cpuChart" width="200" height="200" style="margin-bottom: 12px;"></canvas>
</div>
<p id="cpuStatsLabel">
<b>{% trans 'CPU Usage' context 'CPU usage percentage' %}:</b> {{ cpu_percent_used }}% |
<b>{% trans 'CPU Frequency' context 'CPU frequenzy in Heartz' %}:</b> {{ cpu_freq_current }} GHz
</p>
</div>
</div>
@ -33,74 +47,178 @@
<div style="margin-top: 20px;"><strong>{% trans 'Note!' %}</strong> {% blocktrans with win_hyperv_link="<a href='https://docs.docker.com/desktop/settings/windows/#resources'>Windows (Hyper-V)</a>" win_wsl2_link="<a href='https://learn.microsoft.com/en-us/windows/wsl/wsl-config#configuration-setting-for-wslconfig'>Windows (WSL2)</a>" mac_link="<a href='https://docs.docker.com/desktop/settings/mac/#resources'>MacOS</a>" %}These values might be relative to the virtualization environment in which the application is running, not necessarily the values of the your machine. See instructions for {{ win_hyperv_link }}, {{ win_wsl2_link }}, and {{ mac_link }} for changing these values in a Docker setup.{% endblocktrans %}</div>
<script>
(function(){
var ctx = document.getElementById('diskChart').getContext('2d');
var labels = {
"{% trans 'Used' context 'Megabytes of storage space' %}": '{{ used_disk_space|filesizeformat }}',
"{% trans 'Free' context 'Megabytes of storage space' %}": '{{ free_disk_space|filesizeformat }}'
};
var chart = new Chart(ctx, {
type: 'doughnut',
data: {
labels: ["{% trans 'Used' context 'Megabytes of storage space' %}", "{% trans 'Free' context 'Megabytes of storage space' %}"],
datasets: [{
label: "{% trans 'Disk Space' %}",
backgroundColor:[
"rgb(255, 99, 132)",
"rgb(54, 162, 235)"
],
data: [ {{ used_disk_space }}, {{ free_disk_space }} ],
}]
},
options: {
legend:{
reverse: true
(async function(){
function initializeStorageChart(){
let ctx = document.getElementById('diskChart').getContext('2d');
let chart = new Chart(ctx, {
type: 'doughnut',
data: {
labels: ["{% trans 'Used' context 'Megabytes of storage space' %}", "{% trans 'Free' context 'Megabytes of storage space' %}"],
datasets: [{
label: "{% trans 'Disk Space' %}",
backgroundColor:[
"rgb(255, 99, 132)",
"rgb(54, 162, 235)"
],
data: [ {{ used_disk_space }}, {{ free_disk_space }} ],
}]
},
tooltips: {
callbacks: {
label: function(tooltipItem, data) {
return labels[data.labels[tooltipItem.index]];
options: {
legend:{
reverse: true
},
tooltips: {
callbacks: {
label: function(tooltipItem, data) {
let = used_disk_space = data.datasets[0].data[0]
let = free_disk_space = data.datasets[0].data[1]
let labels = {
"{% trans 'Used' context 'Megabytes of storage space' %}": `${filesizeformat(used_disk_space)}`,
"{% trans 'Free' context 'Megabytes of storage space' %}": `${filesizeformat(free_disk_space)}`
};
return labels[data.labels[tooltipItem.index]];
}
}
}
}
});
return chart;
}
function initializeMemoryChart(){
let ctx = document.getElementById('memoryChart').getContext('2d');
let chart = new Chart(ctx, {
type: 'doughnut',
data: {
labels: ["{% trans 'Used' context 'Megabytes of memory space' %}", "{% trans 'Free' context 'Megabytes of memory space' %}"],
datasets: [{
label: "{% trans 'Disk Space' %}",
backgroundColor:[
"rgb(255, 99, 132)",
"rgb(54, 162, 235)"
],
data: [ {{ used_memory }}, {{ free_memory }} ],
}]
},
options: {
legend:{
reverse: true
},
tooltips: {
callbacks: {
label: function(tooltipItem, data) {
let used_memory = data.datasets[0].data[0]
let free_memory = data.datasets[0].data[1]
let labels = {
"{% trans 'Used' context 'Megabytes of memory space' %}": `${filesizeformat(used_memory)}`,
"{% trans 'Free' context 'Megabytes of memory space' %}": `${filesizeformat(free_memory)}`
};
return labels[data.labels[tooltipItem.index]];
}
}
}
}
});
return chart;
}
function initializeCPUChart(){
let cpuPercent = "{{cpu_percent_used}}".replace(",", ".")
cpuPercent = Number(cpuPercent)
let cpuFreePercent = "{{cpu_percent_free}}".replace(",", ".")
cpuFreePercent = Number(cpuFreePercent)
var ctx = document.getElementById('cpuChart').getContext('2d');
var chart = new Chart(ctx, {
type: 'doughnut',
data: {
labels: ["{% trans 'Used' context 'CPU Usage percent' %}", "{% trans 'Free' context 'CPU Usage percent' %}"],
datasets: [{
label: "{% trans 'CPU Usage' %}",
backgroundColor:[
"rgb(255, 99, 132)",
"rgb(54, 162, 235)"
],
data: [ cpuPercent, cpuFreePercent ],
}]
},
options: {
legend:{
reverse: true
},
tooltips: {
callbacks: {
label: function(tooltipItem, data) {
let cpu_percent_used = data.datasets[0].data[0]
let cpu_percent_free = data.datasets[0].data[1]
let labels = {
"{% trans 'Used' context 'CPU Usage percent' %}": cpu_percent_used + '%',
"{% trans 'Free' context 'CPU Usage percent' %}": cpu_percent_free + '%'
};
return labels[data.labels[tooltipItem.index]];
}
}
}
}
});
return chart;
}
let storageChart = initializeStorageChart();
let cpuChart = initializeCPUChart();
{% if total_memory %}
let memoryChart = initializeMemoryChart();
{% endif %}
setInterval(async () => {
try{
let diagnosticStats = await $.ajax({
url: '/api/plugins/diagnostic/',
contentType: 'application/json',
type: 'GET'
});
storageChart.data.datasets[0].data = [diagnosticStats.used_disk_space, diagnosticStats.free_disk_space];
storageChart.update();
$('#storageStatsLabel').html(`
<b>{% trans 'Free' context 'Megabytes of storage space' %}:</b> ${filesizeformat(diagnosticStats.free_disk_space)} |
<b>{% trans 'Used' context 'Megabytes of storage space' %}:</b> ${filesizeformat(diagnosticStats.used_disk_space)} |
<b>{% trans 'Total' context 'Megabytes of storage space' %}:</b> ${filesizeformat(diagnosticStats.total_disk_space)}
`)
{% if total_memory %}
memoryChart.data.datasets[0].data = [diagnosticStats.used_memory, diagnosticStats.free_memory];
memoryChart.update()
$('#memoryStatsLabel').html(`
<b>{% trans 'Free' context 'Megabytes of memory space' %}:</b> ${filesizeformat(diagnosticStats.free_memory)} |
<b>{% trans 'Used' context 'Megabytes of memory space' %}:</b> ${filesizeformat(diagnosticStats.used_memory)} |
<b>{% trans 'Total' context 'Megabytes of memory space'%}:</b> ${filesizeformat(diagnosticStats.total_memory)}
`);
{% endif %}
cpuChart.data.datasets[0].data = [diagnosticStats.cpu_percent_used, diagnosticStats.cpu_percent_free];
cpuChart.update();
$('#cpuStatsLabel').html(`
<b>{% trans 'CPU Usage' context 'CPU usage percentage' %}:</b> ${diagnosticStats.cpu_percent_used}% |
<b>{% trans 'CPU Frequency' context 'CPU frequenzy in Heartz' %}:</b> ${diagnosticStats.cpu_freq_current} GHz
`);
}
});
catch(error){
console.error(error)
}
}, 5000);
function filesizeformat(bytes) {
var sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB'];
if (bytes == 0) return '0 Bytes';
var i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)));
return parseFloat((bytes / Math.pow(1024, i)).toFixed(1)) + ' ' + sizes[i];
}
})();
{% if total_memory %}
(function(){
var ctx = document.getElementById('memoryChart').getContext('2d');
var labels = {
"{% trans 'Used' context 'Megabytes of memory space' %}": '{{ used_memory|filesizeformat }}',
"{% trans 'Free' context 'Megabytes of memory space' %}": '{{ free_memory|filesizeformat }}'
};
var chart = new Chart(ctx, {
type: 'doughnut',
data: {
labels: ["{% trans 'Used' context 'Megabytes of memory space' %}", "{% trans 'Free' context 'Megabytes of memory space' %}"],
datasets: [{
label: "{% trans 'Disk Space' %}",
backgroundColor:[
"rgb(255, 99, 132)",
"rgb(54, 162, 235)"
],
data: [ {{ used_memory }}, {{ free_memory }} ],
}]
},
options: {
legend:{
reverse: true
},
tooltips: {
callbacks: {
label: function(tooltipItem, data) {
return labels[data.labels[tooltipItem.index]];
}
}
}
}
});
})();
{% endif %}
</script>
{% endblock %}

Wyświetl plik

@ -1,6 +1,6 @@
{
"name": "WebODM",
"version": "2.0.2",
"version": "2.0.3",
"description": "User-friendly, extendable application and API for processing aerial imagery.",
"main": "index.js",
"scripts": {

Wyświetl plik

@ -63,4 +63,3 @@ eventlet==0.32.0 ; sys_platform == "win32"
pyopenssl==19.1.0 ; sys_platform == "win32"
numpy==1.21.1
drf-yasg==1.20.0