diff --git a/app/main/bokeh_utils.py b/app/main/bokeh_utils.py new file mode 100644 index 0000000..e4b01da --- /dev/null +++ b/app/main/bokeh_utils.py @@ -0,0 +1,73 @@ +import os +from flask import current_app + +from bokeh.plotting import figure +from bokeh.models import ColumnDataSource, HoverTool, WheelZoomTool, PanTool, ResetTool +from bokeh.resources import CDN +from bokeh.embed import file_html + +import pandas as pd +import numpy as np + +COMMON_FREQUENCIES = [ + [84.015, 87.2250, 'BOS 4m'], + [87.500, 108.0000, 'UKW Rundfunk'], + [108.000, 111.9750, 'ILS'], + [112.000, 117.9750, 'VOR'], + [117.975, 137.0000, 'Flugfunk'], + [143.000, 146.0000, 'Amateurfunk 2m'], + [165.210, 173.9800, 'BOS 2m'], + [177.500, 226.5000, 'DVB-T VHF'], + [273.000, 312.0000, 'Militär'], + [390.000, 399.9000, 'BOS Digital'], + [430.000, 440.0000, 'Amateurfunk 70cm'], + [448.600, 449.9625, 'BOS 70cm'], + [474.000, 786.0000, 'DVB-T UHF'], + [791.000, 821.0000, 'LTE downlink'], + [832.000, 862.0000, 'LTE uplink'], + [868.000, 868.6000, 'Flarm 868.3MHz'], + [880.000, 915.0000, 'GSM 900 uplink'], + [925.000, 960.0000, 'GSM 900 downlink'], + [1025.000, 1095.0000, 'Funknavigation'], + [1164.000, 1215.0000, 'Funknavigation (DME,TACAN)'], + [1429.000, 1452.0000, 'Militär'], +] + + +def get_bokeh_frequency_scan(frequency_scan_file): + # Read the frequency scan file + df_scan = pd.read_csv(os.path.join(current_app.config['UPLOAD_PATH'], frequency_scan_file.name), header=None) + df_scan.columns = ['date', 'time', 'hz_low', 'hz_high', 'hz_step', 'samples'] + [f"signal{c:02}" for c in range(1, len(df_scan.columns) - 5)] + + xval = df_scan['hz_low'] / 1000000 + yval = df_scan['signal01'] + + # Read the common frequences + df_freq = pd.DataFrame(COMMON_FREQUENCIES, columns=['hz_low', 'hz_high', 'description'], dtype=float) + + N = len(df_freq.index) + low = df_freq['hz_low'] + high = df_freq['hz_high'] + + x = high - (high - low) / 2.0 + y = 0 * np.ones(N) + width = high - low + height = 50 * np.ones(N) + desc = df_freq['description'] + + frequency_source = ColumnDataSource(data=dict(low=low, high=high, x=x, y=y, width=width, height=height, desc=desc)) + + # Create the figure with tool tips + fig = figure(plot_width=900, plot_height=500, title=f"Signalauswertung {frequency_scan_file.receiver.name}", tools=[PanTool(), WheelZoomTool(), ResetTool()]) + r1 = fig.rect(x='x', y='y', width='width', height='height', color="lightgrey", source=frequency_source, legend="Gängige Frequenzen") + r2 = fig.line(xval, yval, legend=f"Messung (gain={frequency_scan_file.gain})") + r3 = fig.line(x=[868.3, 868.3], y=[-25, 25], color="red", legend="Flarm") + + fig.add_tools(HoverTool(renderers=[r1], tooltips={"info": "@desc @low-@high MHz"})) + fig.add_tools(HoverTool(renderers=[r2], tooltips={"f [MHz]": "$x", "P [dB]": "$y"})) + + fig.xaxis.axis_label = "Frequenz [MHz]" + fig.yaxis.axis_label = "Signalstärke [dB]" + fig.legend.click_policy = 'hide' + + return file_html(fig, CDN) diff --git a/app/main/routes.py b/app/main/routes.py index 70d0dda..18cd9da 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -1,13 +1,17 @@ +import os +import re from datetime import date, time, datetime -from flask import request, render_template, send_file +from flask import request, render_template, send_file, abort, current_app, make_response +from sqlalchemy.orm.exc import NoResultFound from app import db from app import cache -from app.model import Airport, Country, Sender, SenderInfo, TakeoffLanding, Logbook, Receiver, SenderPosition, RelationStatistic, ReceiverStatistic, SenderStatistic +from app.model import Airport, Country, Sender, SenderInfo, TakeoffLanding, Logbook, Receiver, SenderPosition, RelationStatistic, ReceiverStatistic, SenderStatistic, FrequencyScanFile from app.main import bp from app.main.matplotlib_service import create_range_figure +from app.main.bokeh_utils import get_bokeh_frequency_scan @cache.cached() @@ -204,17 +208,6 @@ def logbooks(): return render_template("logbooks.html", title="Logbook", sel_country=sel_country, countries=countries, sel_airport_id=sel_airport_id, airports=airports, sel_date=sel_date, dates=dates, logbooks=logbooks) -@bp.route("/download.html") -def download_flight(): - from io import StringIO - - buffer = StringIO() - buffer.write("Moin moin\nAlter Verwalter") - buffer.seek(0) - - return send_file(buffer, as_attachment=True, attachment_filename="wtf.igc", mimetype="text/plain") - - @bp.route("/sender_ranking.html") @cache.cached() def sender_ranking(): @@ -241,3 +234,47 @@ def receiver_ranking(): "receiver_ranking.html", title="Receiver Ranking", ranking=receiver_statistics) + + +@bp.route("/upload_file", methods=["POST"]) +def upload_file(): + """For uploading frequency scans. Example: + curl -X POST -F file=@Sonnblick_g49.6.csv http://localhost:5000/upload_file + """ + + if 'file' not in request.files: + abort(400, "Missing parameter 'file'") + + file = request.files['file'] + filename = file.filename + match = re.match(r'^(?P([A-Za-z0-9]+))\_g(?P([0-9]{1,2}(\.[0-9])?))\.csv$', filename) + if match is None: + abort(400, f"No valid filename '{filename}'.") + + try: + receiver = db.session.query(Receiver).filter(Receiver.name == match.group('receiver_name')).one() + except NoResultFound as e: + abort(400, f"No receiver found with name '{match.group('receiver_name')}'.") + + file.save(os.path.join(current_app.config['UPLOAD_PATH'], filename)) + + uploaded_file = FrequencyScanFile(name=filename, gain=match.group('gain'), upload_ip_address=request.remote_addr, upload_timestamp=datetime.utcnow(), receiver=receiver) + db.session.add(uploaded_file) + db.session.commit() + + return 'OK', 202 + + +@bp.route("/frequency_scan", methods=["GET"]) +def frequency_scan(): + frequency_scan_file_id = request.args.get("frequency_scan_file_id") + try: + frequency_scan_file = db.session.query(FrequencyScanFile).filter(FrequencyScanFile.id == frequency_scan_file_id).one() + except NoResultFound as e: + abort(400, f"No frequency_scan_file found id '{frequency_scan_file_id}'.") + + html = get_bokeh_frequency_scan(frequency_scan_file) + + resp = make_response(html) + resp.mimetype = 'text/html' + return resp diff --git a/app/model/__init__.py b/app/model/__init__.py index e6d6889..d7d2fd4 100644 --- a/app/model/__init__.py +++ b/app/model/__init__.py @@ -12,6 +12,7 @@ from .receiver_state import ReceiverState from .takeoff_landing import TakeoffLanding from .airport import Airport from .logbook import Logbook +from .frequency_scan_file import FrequencyScanFile from .geo import Location diff --git a/app/model/frequency_scan_file.py b/app/model/frequency_scan_file.py new file mode 100644 index 0000000..2d40f0d --- /dev/null +++ b/app/model/frequency_scan_file.py @@ -0,0 +1,18 @@ +from app import db + + +class FrequencyScanFile(db.Model): + __tablename__ = "frequency_scan_files" + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String, nullable=False) + gain = db.Column(db.Float(precision=2), nullable=False) + upload_ip_address = db.Column(db.String, nullable=False) + upload_timestamp = db.Column(db.DateTime, nullable=False, index=True) + + # Relations + receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True) + receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("frequency_scan_files", order_by=upload_timestamp.desc())) + + def __repr__(self): + return "" % (self.name, self.upload_ip_address, self.upload_timestamp) diff --git a/app/templates/receiver_detail.html b/app/templates/receiver_detail.html index 21df8b2..8a500e8 100644 --- a/app/templates/receiver_detail.html +++ b/app/templates/receiver_detail.html @@ -45,6 +45,30 @@ +{% if receiver.frequency_scan_files %} +
+

Frequency Scans

+ + + + + + + + + {% for file in receiver.frequency_scan_files %} + + + + + + + + {% endfor %} +
#NameGainUpload TimestampAnalysis
{{ loop.index }}{{ file.name }}{{ file.gain }}{{ file.upload_timestamp }}Plot
+
+{% endif %} + {% endblock %} \ No newline at end of file diff --git a/config.py b/config.py index 7e82e94..99d0d1f 100644 --- a/config.py +++ b/config.py @@ -17,6 +17,11 @@ class BaseConfig: APRS_USER = "OGNPYTHON" + # Upload configuration + MAX_CONTENT_LENGTH = 1024 * 1024 # max. 1MB + UPLOAD_EXTENSIONS = ['.csv'] + UPLOAD_PATH = 'uploads' + class DefaultConfig(BaseConfig): SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", "postgresql://postgres:postgres@localhost:5432/ogn") diff --git a/migrations/versions/c53fdb39f5a5_added_frequencyscanfiles.py b/migrations/versions/c53fdb39f5a5_added_frequencyscanfiles.py new file mode 100644 index 0000000..14108d2 --- /dev/null +++ b/migrations/versions/c53fdb39f5a5_added_frequencyscanfiles.py @@ -0,0 +1,41 @@ +"""Added UploadedFile + +Revision ID: c53fdb39f5a5 +Revises: 002656878233 +Create Date: 2020-12-01 18:18:43.404091 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c53fdb39f5a5' +down_revision = '002656878233' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('frequency_scan_files', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('gain', sa.Float(precision=2), nullable=False), + sa.Column('upload_ip_address', sa.String(), nullable=False), + sa.Column('upload_timestamp', sa.DateTime(), nullable=False), + sa.Column('receiver_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['receiver_id'], ['receivers.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_frequency_scan_files_receiver_id'), 'frequency_scan_files', ['receiver_id'], unique=False) + op.create_index(op.f('ix_frequency_scan_files_upload_timestamp'), 'frequency_scan_files', ['upload_timestamp'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_frequency_scan_files_upload_timestamp'), table_name='frequency_scan_files') + op.drop_index(op.f('ix_frequency_scan_files_receiver_id'), table_name='frequency_scan_files') + op.drop_table('frequency_scan_files') + # ### end Alembic commands ### diff --git a/setup.py b/setup.py index 44a8711..4591bd4 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,9 @@ setup( 'flower==0.9.5', 'tqdm==4.53.0', 'requests==2.25.0', - 'matplotlib==3.3.3' + 'matplotlib==3.3.3', + 'bokeh==2.2.3', + 'pandas==1.1.4' ], test_require=[ 'pytest==5.0.1',