kopia lustrzana https://github.com/dgtlmoon/changedetection.io
Tidy up thread logic and version check
Leigh Morresi
rodzic
187523d8d6
commit
e0578acca2
|
|
@ -17,6 +17,8 @@ import os
|
|||
import timeago
|
||||
|
||||
import threading
|
||||
from threading import Event
|
||||
|
||||
import queue
|
||||
|
||||
from flask import Flask, render_template, request, send_file, send_from_directory, abort, redirect, url_for
|
||||
|
|
@ -42,7 +44,9 @@ app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
|
|||
# Stop browser caching of assets
|
||||
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
|
||||
|
||||
app.config['STOP_THREADS'] = False
|
||||
app.config.exit = Event()
|
||||
|
||||
app.config['NEW_VERSION_AVAILABLE'] = False
|
||||
|
||||
# Disables caching of the templates
|
||||
app.config['TEMPLATES_AUTO_RELOAD'] = True
|
||||
|
|
@ -79,10 +83,12 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
|
|||
def changedetection_app(config=None, datastore_o=None):
|
||||
global datastore
|
||||
datastore = datastore_o
|
||||
# Hmm
|
||||
|
||||
app.config.update(dict(DEBUG=True))
|
||||
app.config.update(config or {})
|
||||
|
||||
|
||||
|
||||
# Setup cors headers to allow all domains
|
||||
# https://flask-cors.readthedocs.io/en/latest/
|
||||
# CORS(app)
|
||||
|
|
@ -90,6 +96,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
# https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
|
||||
# You can divide up the stuff like this
|
||||
|
||||
|
||||
@app.route("/", methods=['GET'])
|
||||
def index():
|
||||
global messages
|
||||
|
|
@ -318,12 +325,14 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
|
||||
if len(remaining_urls) == 0:
|
||||
return redirect(url_for('index'))
|
||||
#@todo repair
|
||||
else:
|
||||
output = render_template("import.html",
|
||||
messages=messages,
|
||||
remaining="\n".join(remaining_urls)
|
||||
)
|
||||
messages = []
|
||||
|
||||
return output
|
||||
|
||||
# Clear all statuses, so we do not see the 'unviewed' class
|
||||
|
|
@ -501,9 +510,27 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
# @todo handle ctrl break
|
||||
ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
|
||||
|
||||
# Check for new release version
|
||||
threading.Thread(target=check_for_new_version).start()
|
||||
return app
|
||||
|
||||
|
||||
# Check for new version and anonymous stats
|
||||
def check_for_new_version():
|
||||
import requests
|
||||
app.config['NEW_VERSION_AVAILABLE'] = True
|
||||
import urllib3
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
while not app.config.exit.is_set():
|
||||
r = requests.post("https://changedetection.io/check-ver.php",
|
||||
data={'version': datastore.data['version_tag'],
|
||||
'app_guid': datastore.data['app_guid']},
|
||||
|
||||
verify=False)
|
||||
|
||||
app.config.exit.wait(10)
|
||||
|
||||
# Requests for checking on the site use a pool of thread Workers managed by a Queue.
|
||||
class Worker(threading.Thread):
|
||||
current_uuid = None
|
||||
|
|
@ -517,16 +544,13 @@ class Worker(threading.Thread):
|
|||
|
||||
update_handler = fetch_site_status.perform_site_check(datastore=datastore)
|
||||
|
||||
while True:
|
||||
while not app.config.exit.is_set():
|
||||
|
||||
try:
|
||||
uuid = self.q.get(block=True, timeout=1)
|
||||
uuid = self.q.get(block=False)
|
||||
except queue.Empty:
|
||||
# We have a chance to kill this thread that needs to monitor for new jobs..
|
||||
# Delays here would be caused by a current response object pending
|
||||
# @todo switch to threaded response handler
|
||||
if app.config['STOP_THREADS']:
|
||||
return
|
||||
pass
|
||||
|
||||
else:
|
||||
self.current_uuid = uuid
|
||||
|
||||
|
|
@ -549,9 +573,13 @@ class Worker(threading.Thread):
|
|||
self.current_uuid = None # Done
|
||||
self.q.task_done()
|
||||
|
||||
app.config.exit.wait(1)
|
||||
|
||||
|
||||
|
||||
# Thread runner to check every minute, look for new watches to feed into the Queue.
|
||||
def ticker_thread_check_time_launch_checks():
|
||||
|
||||
# Spin up Workers.
|
||||
for _ in range(datastore.data['settings']['requests']['workers']):
|
||||
new_worker = Worker(update_q)
|
||||
|
|
@ -559,23 +587,19 @@ def ticker_thread_check_time_launch_checks():
|
|||
new_worker.start()
|
||||
|
||||
# Every minute check for new UUIDs to follow up on
|
||||
while True:
|
||||
|
||||
if app.config['STOP_THREADS']:
|
||||
return
|
||||
minutes = datastore.data['settings']['requests']['minutes_between_check']
|
||||
|
||||
while not app.config.exit.is_set():
|
||||
running_uuids = []
|
||||
for t in running_update_threads:
|
||||
running_uuids.append(t.current_uuid)
|
||||
|
||||
# Look at the dataset, find a stale watch to process
|
||||
minutes = datastore.data['settings']['requests']['minutes_between_check']
|
||||
threshold = time.time() - (minutes * 60)
|
||||
for uuid, watch in datastore.data['watching'].items():
|
||||
if watch['last_checked'] <= time.time() - (minutes * 60):
|
||||
|
||||
# @todo maybe update_q.queue is enough?
|
||||
if watch['last_checked'] <= threshold:
|
||||
if not uuid in running_uuids and uuid not in update_q.queue:
|
||||
update_q.put(uuid)
|
||||
|
||||
# Should be low so we can break this out in testing
|
||||
time.sleep(1)
|
||||
app.config.exit.wait(1)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ import time
|
|||
import requests
|
||||
import hashlib
|
||||
from inscriptis import get_text
|
||||
|
||||
import urllib3
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
# Some common stuff here that can be moved to a base class
|
||||
class perform_site_check():
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ class ChangeDetectionStore:
|
|||
self.datastore_path = datastore_path
|
||||
self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
|
||||
self.stop_thread = False
|
||||
|
||||
self.__data = {
|
||||
'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
|
||||
'watching': {},
|
||||
|
|
@ -79,8 +80,6 @@ class ChangeDetectionStore:
|
|||
if 'requests' in from_disk['settings']:
|
||||
self.__data['settings']['requests'].update(from_disk['settings']['requests'])
|
||||
|
||||
self.__data['tag'] = "0.27"
|
||||
|
||||
# Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
|
||||
# @todo pretty sure theres a python we todo this with an abstracted(?) object!
|
||||
|
||||
|
|
@ -101,6 +100,14 @@ class ChangeDetectionStore:
|
|||
self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
|
||||
self.add_watch(url='https://changedetection.io', tag='Tech news')
|
||||
|
||||
|
||||
self.__data['version_tag'] = "0.27"
|
||||
|
||||
if not 'app_guid' in self.__data:
|
||||
self.__data['app_guid'] = str(uuid_builder.uuid4())
|
||||
|
||||
self.needs_write = True
|
||||
|
||||
# Finally start the thread that will manage periodic data saves to JSON
|
||||
save_data_thread = threading.Thread(target=self.save_datastore).start()
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,9 @@
|
|||
{% if current_diff_url %}
|
||||
<a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</a>
|
||||
{% else %}
|
||||
<span id="version-text" class="pure-menu-heading">Version {{ version }}</span>
|
||||
{% if new_version_available %}
|
||||
<span id="version-text" class="pure-menu-heading">A new version is available</span>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
<ul class="pure-menu-list">
|
||||
|
|
|
|||
|
|
@ -30,10 +30,11 @@ def app(request):
|
|||
app_config = {'datastore_path': datastore_path}
|
||||
datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
|
||||
app = changedetection_app(app_config, datastore)
|
||||
app.config['STOP_THREADS'] = True
|
||||
|
||||
def teardown():
|
||||
datastore.stop_thread = True
|
||||
app.config['STOP_THREADS'] = True
|
||||
app.config.exit.set()
|
||||
try:
|
||||
os.unlink("{}/url-watches.json".format(datastore_path))
|
||||
except FileNotFoundError:
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue