Frequency scans (#156)

* Added FrequencyScanFiles

* Added route for upload_file and frequency_scan

app/main/bokeh_utils.py

@@ -0,0 +1,73 @@
+import os
+from flask import current_app
+
+from bokeh.plotting import figure
+from bokeh.models import ColumnDataSource, HoverTool, WheelZoomTool, PanTool, ResetTool
+from bokeh.resources import CDN
+from bokeh.embed import file_html
+
+import pandas as pd
+import numpy as np
+
+COMMON_FREQUENCIES = [
+    [84.015, 87.2250, 'BOS 4m'],
+    [87.500, 108.0000, 'UKW Rundfunk'],
+    [108.000, 111.9750, 'ILS'],
+    [112.000, 117.9750, 'VOR'],
+    [117.975, 137.0000, 'Flugfunk'],
+    [143.000, 146.0000, 'Amateurfunk 2m'],
+    [165.210, 173.9800, 'BOS 2m'],
+    [177.500, 226.5000, 'DVB-T VHF'],
+    [273.000, 312.0000, 'Militär'],
+    [390.000, 399.9000, 'BOS Digital'],
+    [430.000, 440.0000, 'Amateurfunk 70cm'],
+    [448.600, 449.9625, 'BOS 70cm'],
+    [474.000, 786.0000, 'DVB-T UHF'],
+    [791.000, 821.0000, 'LTE downlink'],
+    [832.000, 862.0000, 'LTE uplink'],
+    [868.000, 868.6000, 'Flarm 868.3MHz'],
+    [880.000, 915.0000, 'GSM 900 uplink'],
+    [925.000, 960.0000, 'GSM 900 downlink'],
+    [1025.000, 1095.0000, 'Funknavigation'],
+    [1164.000, 1215.0000, 'Funknavigation (DME,TACAN)'],
+    [1429.000, 1452.0000, 'Militär'],
+]
+
+
+def get_bokeh_frequency_scan(frequency_scan_file):
+    # Read the frequency scan file
+    df_scan = pd.read_csv(os.path.join(current_app.config['UPLOAD_PATH'], frequency_scan_file.name), header=None)
+    df_scan.columns = ['date', 'time', 'hz_low', 'hz_high', 'hz_step', 'samples'] + [f"signal{c:02}" for c in range(1, len(df_scan.columns) - 5)]
+
+    xval = df_scan['hz_low'] / 1000000
+    yval = df_scan['signal01']
+
+    # Read the common frequences
+    df_freq = pd.DataFrame(COMMON_FREQUENCIES, columns=['hz_low', 'hz_high', 'description'], dtype=float)
+
+    N = len(df_freq.index)
+    low = df_freq['hz_low']
+    high = df_freq['hz_high']
+
+    x = high - (high - low) / 2.0
+    y = 0 * np.ones(N)
+    width = high - low
+    height = 50 * np.ones(N)
+    desc = df_freq['description']
+
+    frequency_source = ColumnDataSource(data=dict(low=low, high=high, x=x, y=y, width=width, height=height, desc=desc))
+
+    # Create the figure with tool tips
+    fig = figure(plot_width=900, plot_height=500, title=f"Signalauswertung {frequency_scan_file.receiver.name}", tools=[PanTool(), WheelZoomTool(), ResetTool()])
+    r1 = fig.rect(x='x', y='y', width='width', height='height', color="lightgrey", source=frequency_source, legend="Gängige Frequenzen")
+    r2 = fig.line(xval, yval, legend=f"Messung (gain={frequency_scan_file.gain})")
+    r3 = fig.line(x=[868.3, 868.3], y=[-25, 25], color="red", legend="Flarm")
+
+    fig.add_tools(HoverTool(renderers=[r1], tooltips={"info": "@desc @low-@high MHz"}))
+    fig.add_tools(HoverTool(renderers=[r2], tooltips={"f [MHz]": "$x", "P [dB]": "$y"}))
+
+    fig.xaxis.axis_label = "Frequenz [MHz]"
+    fig.yaxis.axis_label = "Signalstärke [dB]"
+    fig.legend.click_policy = 'hide'
+
+    return file_html(fig, CDN)

app/main/routes.py

@@ -1,13 +1,17 @@
+import os
+import re
 from datetime import date, time, datetime
 
-from flask import request, render_template, send_file
+from flask import request, render_template, send_file, abort, current_app, make_response
+from sqlalchemy.orm.exc import NoResultFound
 
 from app import db
 from app import cache
-from app.model import Airport, Country, Sender, SenderInfo, TakeoffLanding, Logbook, Receiver, SenderPosition, RelationStatistic, ReceiverStatistic, SenderStatistic
+from app.model import Airport, Country, Sender, SenderInfo, TakeoffLanding, Logbook, Receiver, SenderPosition, RelationStatistic, ReceiverStatistic, SenderStatistic, FrequencyScanFile
 
 from app.main import bp
 from app.main.matplotlib_service import create_range_figure
+from app.main.bokeh_utils import get_bokeh_frequency_scan
 
 
 @cache.cached()
@@ -204,17 +208,6 @@ def logbooks():
     return render_template("logbooks.html", title="Logbook", sel_country=sel_country, countries=countries, sel_airport_id=sel_airport_id, airports=airports, sel_date=sel_date, dates=dates, logbooks=logbooks)
 
 
-@bp.route("/download.html")
-def download_flight():
-    from io import StringIO
-
-    buffer = StringIO()
-    buffer.write("Moin moin\nAlter Verwalter")
-    buffer.seek(0)
-
-    return send_file(buffer, as_attachment=True, attachment_filename="wtf.igc", mimetype="text/plain")
-
-
 @bp.route("/sender_ranking.html")
 @cache.cached()
 def sender_ranking():
@@ -241,3 +234,47 @@ def receiver_ranking():
         "receiver_ranking.html",
         title="Receiver Ranking",
         ranking=receiver_statistics)
+
+
+@bp.route("/upload_file", methods=["POST"])
+def upload_file():
+    """For uploading frequency scans. Example:
+    curl -X POST -F file=@Sonnblick_g49.6.csv http://localhost:5000/upload_file
+    """
+
+    if 'file' not in request.files:
+        abort(400, "Missing parameter 'file'")
+
+    file = request.files['file']
+    filename = file.filename
+    match = re.match(r'^(?P<receiver_name>([A-Za-z0-9]+))\_g(?P<gain>([0-9]{1,2}(\.[0-9])?))\.csv$', filename)
+    if match is None:
+        abort(400, f"No valid filename '{filename}'.")
+
+    try:
+        receiver = db.session.query(Receiver).filter(Receiver.name == match.group('receiver_name')).one()
+    except NoResultFound as e:
+        abort(400, f"No receiver found with name '{match.group('receiver_name')}'.")
+
+    file.save(os.path.join(current_app.config['UPLOAD_PATH'], filename))
+
+    uploaded_file = FrequencyScanFile(name=filename, gain=match.group('gain'), upload_ip_address=request.remote_addr, upload_timestamp=datetime.utcnow(), receiver=receiver)
+    db.session.add(uploaded_file)
+    db.session.commit()
+
+    return 'OK', 202
+
+
+@bp.route("/frequency_scan", methods=["GET"])
+def frequency_scan():
+    frequency_scan_file_id = request.args.get("frequency_scan_file_id")
+    try:
+        frequency_scan_file = db.session.query(FrequencyScanFile).filter(FrequencyScanFile.id == frequency_scan_file_id).one()
+    except NoResultFound as e:
+        abort(400, f"No frequency_scan_file found id '{frequency_scan_file_id}'.")
+
+    html = get_bokeh_frequency_scan(frequency_scan_file)
+
+    resp = make_response(html)
+    resp.mimetype = 'text/html'
+    return resp

app/model/__init__.py

@@ -12,6 +12,7 @@ from .receiver_state import ReceiverState
 from .takeoff_landing import TakeoffLanding
 from .airport import Airport
 from .logbook import Logbook
+from .frequency_scan_file import FrequencyScanFile
 
 from .geo import Location
 

app/model/frequency_scan_file.py

@@ -0,0 +1,18 @@
+from app import db
+
+
+class FrequencyScanFile(db.Model):
+    __tablename__ = "frequency_scan_files"
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String, nullable=False)
+    gain = db.Column(db.Float(precision=2), nullable=False)
+    upload_ip_address = db.Column(db.String, nullable=False)
+    upload_timestamp = db.Column(db.DateTime, nullable=False, index=True)
+
+    # Relations
+    receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True)
+    receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=db.backref("frequency_scan_files", order_by=upload_timestamp.desc()))
+
+    def __repr__(self):
+        return "<FrequencyScanFile: %s,%s,%s>" % (self.name, self.upload_ip_address, self.upload_timestamp)

app/templates/receiver_detail.html

@@ -45,6 +45,30 @@
 	</table>
 	</div>
 
+{% if receiver.frequency_scan_files %}
+<div class="panel panel-success">
+	<div class="panel-heading"><h3 class="panel-title">Frequency Scans</h3></div>
+	<table class="datatable table table-striped table-bordered">
+		<tr>	
+			<th>#</th>
+			<th>Name</th>
+			<th>Gain</th>
+			<th>Upload Timestamp</th>
+			<th>Analysis</th>
+		</tr>
+		{% for file in receiver.frequency_scan_files %}
+		<tr>
+			<td>{{ loop.index }}</td>
+			<td>{{ file.name }}</td>
+			<td>{{ file.gain }}</td>
+			<td>{{ file.upload_timestamp }}</td>
+			<td><a href="{{ url_for('main.frequency_scan', frequency_scan_file_id=file.id) }}" target="_blank">Plot</a></td>
+		</tr>
+		{% endfor %}
+	</table>
+</div>
+{% endif %}
+
 </div>
 
 {% endblock %}

config.py

@@ -17,6 +17,11 @@ class BaseConfig:
 
     APRS_USER = "OGNPYTHON"
 
+    # Upload configuration
+    MAX_CONTENT_LENGTH = 1024 * 1024    # max. 1MB
+    UPLOAD_EXTENSIONS = ['.csv']
+    UPLOAD_PATH = 'uploads'
+
 
 class DefaultConfig(BaseConfig):
     SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", "postgresql://postgres:postgres@localhost:5432/ogn")

migrations/versions/c53fdb39f5a5_added_frequencyscanfiles.py

@@ -0,0 +1,41 @@
+"""Added UploadedFile
+
+Revision ID: c53fdb39f5a5
+Revises: 002656878233
+Create Date: 2020-12-01 18:18:43.404091
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c53fdb39f5a5'
+down_revision = '002656878233'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('frequency_scan_files',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(), nullable=False),
+        sa.Column('gain', sa.Float(precision=2), nullable=False),
+        sa.Column('upload_ip_address', sa.String(), nullable=False),
+        sa.Column('upload_timestamp', sa.DateTime(), nullable=False),
+        sa.Column('receiver_id', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['receiver_id'], ['receivers.id'], ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_frequency_scan_files_receiver_id'), 'frequency_scan_files', ['receiver_id'], unique=False)
+    op.create_index(op.f('ix_frequency_scan_files_upload_timestamp'), 'frequency_scan_files', ['upload_timestamp'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_frequency_scan_files_upload_timestamp'), table_name='frequency_scan_files')
+    op.drop_index(op.f('ix_frequency_scan_files_receiver_id'), table_name='frequency_scan_files')
+    op.drop_table('frequency_scan_files')
+    # ### end Alembic commands ###

setup.py

@@ -56,7 +56,9 @@ setup(
         'flower==0.9.5',
         'tqdm==4.53.0',
         'requests==2.25.0',
-        'matplotlib==3.3.3'
+        'matplotlib==3.3.3',
+        'bokeh==2.2.3',
+        'pandas==1.1.4'
     ],
     test_require=[
         'pytest==5.0.1',