mirror
/
changedetection.io
kopia lustrzana https://github.com/dgtlmoon/changedetection.io
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Linting fixups

pull/11/head
Leigh Morresi 2021-02-21 14:21:14 +01:00
rodzic 22bc8fabd1
commit 63eea2d6db
4 zmienionych plików z 32 dodań i 51 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

55
backend/__init__.py
Wyświetl plik

@ -13,7 +13,7 @@
# https://distill.io/features # https://distill.io/features
# proxy per check # proxy per check
# - flask_cors, itsdangerous,MarkupSafe # - flask_cors, itsdangerous,MarkupSafe
import json
import time import time
import os import os
import timeago import timeago
@ -21,10 +21,9 @@ import timeago
import threading import threading
import queue import queue
from flask import Flask, render_template, request, send_file, send_from_directory, abort, redirect, url_for
from flask import Flask, render_template, request, send_file, send_from_directory, safe_join, abort, redirect, url_for datastore = None
datastore=None
# Local # Local
running_update_threads = [] running_update_threads = []
@ -35,13 +34,12 @@ extra_stylesheets = []
update_q = queue.Queue() update_q = queue.Queue()
app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static") app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
# Stop browser caching of assets # Stop browser caching of assets
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
app.config['STOP_THREADS']= False app.config['STOP_THREADS'] = False
# Disables caching of the templates # Disables caching of the templates
app.config['TEMPLATES_AUTO_RELOAD'] = True app.config['TEMPLATES_AUTO_RELOAD'] = True
@ -74,20 +72,19 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
# return timeago.format(timestamp, time.time()) # return timeago.format(timestamp, time.time())
# return datetime.datetime.utcfromtimestamp(timestamp).strftime(format) # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
def changedetection_app(config=None, datastore_o=None):
def changedetection_app(config=None, datastore_o=None):
global datastore global datastore
datastore = datastore_o datastore = datastore_o
# Hmm # Hmm
app.config.update(dict(DEBUG=True)) app.config.update(dict(DEBUG=True))
app.config.update(config or {}) app.config.update(config or {})
# Setup cors headers to allow all domains # Setup cors headers to allow all domains
# https://flask-cors.readthedocs.io/en/latest/ # https://flask-cors.readthedocs.io/en/latest/
# CORS(app) # CORS(app)
#https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39 # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
# You can divide up the stuff like this # You can divide up the stuff like this
@app.route("/", methods=['GET']) @app.route("/", methods=['GET'])
@ -100,7 +97,6 @@ def changedetection_app(config=None, datastore_o=None):
sorted_watches = [] sorted_watches = []
for uuid, watch in datastore.data['watching'].items(): for uuid, watch in datastore.data['watching'].items():
if limit_tag != None: if limit_tag != None:
# Support for comma separated list of tags. # Support for comma separated list of tags.
for tag_in_watch in watch['tag'].split(','): for tag_in_watch in watch['tag'].split(','):
@ -113,7 +109,6 @@ def changedetection_app(config=None, datastore_o=None):
watch['uuid'] = uuid watch['uuid'] = uuid
sorted_watches.append(watch) sorted_watches.append(watch)
sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True) sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
existing_tags = datastore.get_all_tags() existing_tags = datastore.get_all_tags()
@ -156,7 +151,6 @@ def changedetection_app(config=None, datastore_o=None):
return render_template("scrub.html") return render_template("scrub.html")
@app.route("/edit", methods=['GET', 'POST']) @app.route("/edit", methods=['GET', 'POST'])
def edit_page(): def edit_page():
global messages global messages
@ -193,7 +187,6 @@ def changedetection_app(config=None, datastore_o=None):
return output return output
@app.route("/settings", methods=['GET', "POST"]) @app.route("/settings", methods=['GET', "POST"])
def settings_page(): def settings_page():
global messages global messages
@ -210,10 +203,12 @@ def changedetection_app(config=None, datastore_o=None):
messages.append({'class': 'ok', 'message': "Updated"}) messages.append({'class': 'ok', 'message': "Updated"})
else: else:
messages.append({'class': 'error', 'message': "Must be equal to or greater than 5 and less than 600 minutes"}) messages.append(
{'class': 'error', 'message': "Must be equal to or greater than 5 and less than 600 minutes"})
output = render_template("settings.html", messages=messages, minutes=datastore.data['settings']['requests']['minutes_between_check']) output = render_template("settings.html", messages=messages,
messages =[] minutes=datastore.data['settings']['requests']['minutes_between_check'])
messages = []
return output return output
@ -221,7 +216,7 @@ def changedetection_app(config=None, datastore_o=None):
def import_page(): def import_page():
import validators import validators
global messages global messages
remaining_urls=[] remaining_urls = []
good = 0 good = 0
@ -250,12 +245,11 @@ def changedetection_app(config=None, datastore_o=None):
messages = [] messages = []
return output return output
@app.route("/diff/<string:uuid>", methods=['GET']) @app.route("/diff/<string:uuid>", methods=['GET'])
def diff_history_page(uuid): def diff_history_page(uuid):
global messages global messages
extra_stylesheets=['/static/css/diff.css'] extra_stylesheets = ['/static/css/diff.css']
watch = datastore.data['watching'][uuid] watch = datastore.data['watching'][uuid]
@ -299,7 +293,6 @@ def changedetection_app(config=None, datastore_o=None):
def favicon(): def favicon():
return send_from_directory("/app/static/images", filename="favicon.ico") return send_from_directory("/app/static/images", filename="favicon.ico")
# We're good but backups are even better! # We're good but backups are even better!
@app.route("/backup", methods=['GET']) @app.route("/backup", methods=['GET'])
def get_backup(): def get_backup():
@ -313,7 +306,8 @@ def changedetection_app(config=None, datastore_o=None):
# We only care about UUIDS from the current index file # We only care about UUIDS from the current index file
uuids = list(datastore.data['watching'].keys()) uuids = list(datastore.data['watching'].keys())
with zipfile.ZipFile(os.path.join(app.config['datastore_path'], backupname), 'w', compression=zipfile.ZIP_DEFLATED, with zipfile.ZipFile(os.path.join(app.config['datastore_path'], backupname), 'w',
compression=zipfile.ZIP_DEFLATED,
compresslevel=6) as zipObj: compresslevel=6) as zipObj:
# Be sure we're written fresh # Be sure we're written fresh
@ -332,7 +326,6 @@ def changedetection_app(config=None, datastore_o=None):
mimetype="application/zip", mimetype="application/zip",
attachment_filename=backupname) attachment_filename=backupname)
@app.route("/static/<string:group>/<string:filename>", methods=['GET']) @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
def static_content(group, filename): def static_content(group, filename):
# These files should be in our subdirectory # These files should be in our subdirectory
@ -344,7 +337,6 @@ def changedetection_app(config=None, datastore_o=None):
except FileNotFoundError: except FileNotFoundError:
abort(404) abort(404)
@app.route("/api/add", methods=['POST']) @app.route("/api/add", methods=['POST'])
def api_watch_add(): def api_watch_add():
global messages global messages
@ -357,7 +349,6 @@ def changedetection_app(config=None, datastore_o=None):
messages.append({'class': 'ok', 'message': 'Watch added.'}) messages.append({'class': 'ok', 'message': 'Watch added.'})
return redirect(url_for('index')) return redirect(url_for('index'))
@app.route("/api/delete", methods=['GET']) @app.route("/api/delete", methods=['GET'])
def api_delete(): def api_delete():
global messages global messages
@ -367,7 +358,6 @@ def changedetection_app(config=None, datastore_o=None):
return redirect(url_for('index')) return redirect(url_for('index'))
@app.route("/api/checknow", methods=['GET']) @app.route("/api/checknow", methods=['GET'])
def api_watch_checknow(): def api_watch_checknow():
@ -375,9 +365,9 @@ def changedetection_app(config=None, datastore_o=None):
tag = request.args.get('tag') tag = request.args.get('tag')
uuid = request.args.get('uuid') uuid = request.args.get('uuid')
i=0 i = 0
running_uuids=[] running_uuids = []
for t in running_update_threads: for t in running_update_threads:
running_uuids.append(t.current_uuid) running_uuids.append(t.current_uuid)
@ -405,20 +395,16 @@ def changedetection_app(config=None, datastore_o=None):
messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)}) messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)})
return redirect(url_for('index', tag=tag)) return redirect(url_for('index', tag=tag))
# @todo handle ctrl break # @todo handle ctrl break
ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
return app return app
# Requests for checking on the site use a pool of thread Workers managed by a Queue. # Requests for checking on the site use a pool of thread Workers managed by a Queue.
class Worker(threading.Thread): class Worker(threading.Thread):
current_uuid = None current_uuid = None
def __init__(self, q, *args, **kwargs): def __init__(self, q, *args, **kwargs):
self.q = q self.q = q
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
@ -461,14 +447,12 @@ class Worker(threading.Thread):
# No change # No change
x = 1 x = 1
self.current_uuid = None # Done self.current_uuid = None # Done
self.q.task_done() self.q.task_done()
# Thread runner to check every minute, look for new watches to feed into the Queue. # Thread runner to check every minute, look for new watches to feed into the Queue.
def ticker_thread_check_time_launch_checks(): def ticker_thread_check_time_launch_checks():
# Spin up Workers. # Spin up Workers.
for _ in range(datastore.data['settings']['requests']['workers']): for _ in range(datastore.data['settings']['requests']['workers']):
new_worker = Worker(update_q) new_worker = Worker(update_q)
@ -481,7 +465,7 @@ def ticker_thread_check_time_launch_checks():
if app.config['STOP_THREADS']: if app.config['STOP_THREADS']:
return return
running_uuids=[] running_uuids = []
for t in running_update_threads: for t in running_update_threads:
running_uuids.append(t.current_uuid) running_uuids.append(t.current_uuid)
@ -496,4 +480,3 @@ def ticker_thread_check_time_launch_checks():
# Should be low so we can break this out in testing # Should be low so we can break this out in testing
time.sleep(1) time.sleep(1)

4
backend/fetch_site_status.py
Wyświetl plik

@ -1,8 +1,6 @@
import time import time
import requests import requests
import hashlib import hashlib
import os
import re
from inscriptis import get_text from inscriptis import get_text
@ -13,7 +11,6 @@ class perform_site_check():
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.datastore = datastore self.datastore = datastore
def run(self, uuid): def run(self, uuid):
timestamp = int(time.time()) # used for storage etc too timestamp = int(time.time()) # used for storage etc too
stripped_text_from_html = False stripped_text_from_html = False
@ -88,7 +85,6 @@ class perform_site_check():
if self.datastore.get_val(uuid, 'previous_md5'): if self.datastore.get_val(uuid, 'previous_md5'):
update_obj["last_changed"] = timestamp update_obj["last_changed"] = timestamp
update_obj["previous_md5"] = fetched_md5 update_obj["previous_md5"] = fetched_md5
return update_obj, stripped_text_from_html return update_obj, stripped_text_from_html

22
backend/store.py
Wyświetl plik

@ -11,13 +11,14 @@ import logging
import time import time
import threading import threading
# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods? # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
# Open a github issue if you know something :) # Open a github issue if you know something :)
# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
class ChangeDetectionStore: class ChangeDetectionStore:
lock = Lock() lock = Lock()
def __init__(self, datastore_path="/datastore"): def __init__(self, datastore_path="/datastore", include_default_watches=True):
self.needs_write = False self.needs_write = False
self.datastore_path = datastore_path self.datastore_path = datastore_path
self.json_store_path = "{}/url-watches.json".format(self.datastore_path) self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
@ -90,11 +91,13 @@ class ChangeDetectionStore:
# First time ran, doesnt exist. # First time ran, doesnt exist.
except (FileNotFoundError, json.decoder.JSONDecodeError): except (FileNotFoundError, json.decoder.JSONDecodeError):
print("Creating JSON store at", self.datastore_path) if include_default_watches:
#self.add_watch(url='http://www.quotationspage.com/random.php', tag='test') print("Creating JSON store at", self.datastore_path)
#self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
#self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid') self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
#self.add_watch(url='https://changedetection.io', tag='Tech news') self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
self.add_watch(url='https://changedetection.io', tag='Tech news')
# Finally start the thread that will manage periodic data saves to JSON # Finally start the thread that will manage periodic data saves to JSON
save_data_thread = threading.Thread(target=self.save_datastore).start() save_data_thread = threading.Thread(target=self.save_datastore).start()
@ -146,7 +149,7 @@ class ChangeDetectionStore:
# Support for comma separated list of tags. # Support for comma separated list of tags.
for tag in watch['tag'].split(','): for tag in watch['tag'].split(','):
tag = tag.strip() tag = tag.strip()
if not tag in tags: if tag not in tags:
tags.append(tag) tags.append(tag)
tags.sort() tags.sort()
@ -210,7 +213,7 @@ class ChangeDetectionStore:
return fname return fname
def sync_to_json(self): def sync_to_json(self):
print ("Saving..") print("Saving..")
with open(self.json_store_path, 'w') as json_file: with open(self.json_store_path, 'w') as json_file:
json.dump(self.__data, json_file, indent=4) json.dump(self.__data, json_file, indent=4)
logging.info("Re-saved index") logging.info("Re-saved index")
@ -223,11 +226,10 @@ class ChangeDetectionStore:
while True: while True:
if self.stop_thread: if self.stop_thread:
print ("Shutting down datastore thread") print("Shutting down datastore thread")
return return
if self.needs_write: if self.needs_write:
self.sync_to_json() self.sync_to_json()
time.sleep(1) time.sleep(1)
# body of the constructor # body of the constructor

2
backend/tests/conftest.py
Wyświetl plik

@ -33,7 +33,7 @@ def app(request):
app_config = {'datastore_path': datastore_path} app_config = {'datastore_path': datastore_path}
datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path']) datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
app = changedetection_app(app_config, datastore) app = changedetection_app(app_config, datastore)
# Establish an application context before running the tests. # Establish an application context before running the tests.