Tidy up thread logic and version check

backend/__init__.py

@@ -17,6 +17,8 @@ import os
 import timeago
 
 import threading
+from threading import Event
+
 import queue
 
 from flask import Flask, render_template, request, send_file, send_from_directory,  abort, redirect, url_for
@@ -42,7 +44,9 @@ app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
 
-app.config['STOP_THREADS'] = False
+app.config.exit = Event()
+
+app.config['NEW_VERSION_AVAILABLE'] = False
 
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
@@ -79,10 +83,12 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
 def changedetection_app(config=None, datastore_o=None):
     global datastore
     datastore = datastore_o
-    # Hmm
+
     app.config.update(dict(DEBUG=True))
     app.config.update(config or {})
 
+
+
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
@@ -90,6 +96,7 @@ def changedetection_app(config=None, datastore_o=None):
     # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
     # You can divide up the stuff like this
 
+
     @app.route("/", methods=['GET'])
     def index():
         global messages
@@ -318,12 +325,14 @@ def changedetection_app(config=None, datastore_o=None):
 
             if len(remaining_urls) == 0:
                 return redirect(url_for('index'))
+            #@todo repair
         else:
             output = render_template("import.html",
                                      messages=messages,
                                      remaining="\n".join(remaining_urls)
                                      )
             messages = []
+
         return output
 
     # Clear all statuses, so we do not see the 'unviewed' class
@@ -501,9 +510,27 @@ def changedetection_app(config=None, datastore_o=None):
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
 
+    # Check for new release version
+    threading.Thread(target=check_for_new_version).start()
     return app
 
 
+# Check for new version and anonymous stats
+def check_for_new_version():
+    import requests
+    app.config['NEW_VERSION_AVAILABLE'] = True
+    import urllib3
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    while not app.config.exit.is_set():
+        r = requests.post("https://changedetection.io/check-ver.php",
+                          data={'version': datastore.data['version_tag'],
+                                'app_guid': datastore.data['app_guid']},
+
+                          verify=False)
+
+        app.config.exit.wait(10)
+
 # Requests for checking on the site use a pool of thread Workers managed by a Queue.
 class Worker(threading.Thread):
     current_uuid = None
@@ -517,16 +544,13 @@ class Worker(threading.Thread):
 
         update_handler = fetch_site_status.perform_site_check(datastore=datastore)
 
-        while True:
+        while not app.config.exit.is_set():
 
             try:
-                uuid = self.q.get(block=True, timeout=1)
+                uuid = self.q.get(block=False)
             except queue.Empty:
-                # We have a chance to kill this thread that needs to monitor for new jobs..
+                pass
-                # Delays here would be caused by a current response object pending
+
-                # @todo switch to threaded response handler
-                if app.config['STOP_THREADS']:
-                    return
             else:
                 self.current_uuid = uuid
 
@@ -549,9 +573,13 @@ class Worker(threading.Thread):
                 self.current_uuid = None  # Done
                 self.q.task_done()
 
+            app.config.exit.wait(1)
+
+
 
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
+
     # Spin up Workers.
     for _ in range(datastore.data['settings']['requests']['workers']):
         new_worker = Worker(update_q)
@@ -559,23 +587,19 @@ def ticker_thread_check_time_launch_checks():
         new_worker.start()
 
     # Every minute check for new UUIDs to follow up on
-    while True:
+    minutes = datastore.data['settings']['requests']['minutes_between_check']
-
-        if app.config['STOP_THREADS']:
-            return
 
+    while not app.config.exit.is_set():
         running_uuids = []
         for t in running_update_threads:
             running_uuids.append(t.current_uuid)
 
         # Look at the dataset, find a stale watch to process
-        minutes = datastore.data['settings']['requests']['minutes_between_check']
+        threshold = time.time() - (minutes * 60)
         for uuid, watch in datastore.data['watching'].items():
-            if watch['last_checked'] <= time.time() - (minutes * 60):
+            if watch['last_checked'] <= threshold:
-
-                # @todo maybe update_q.queue is enough?
                 if not uuid in running_uuids and uuid not in update_q.queue:
                     update_q.put(uuid)
 
         # Should be low so we can break this out in testing
-        time.sleep(1)
+        app.config.exit.wait(1)

backend/fetch_site_status.py

@@ -2,7 +2,8 @@ import time
 import requests
 import hashlib
 from inscriptis import get_text
-
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 # Some common stuff here that can be moved to a base class
 class perform_site_check():

backend/store.py

@@ -22,6 +22,7 @@ class ChangeDetectionStore:
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         self.stop_thread = False
+
         self.__data = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
             'watching': {},
@@ -79,8 +80,6 @@ class ChangeDetectionStore:
                     if 'requests' in from_disk['settings']:
                         self.__data['settings']['requests'].update(from_disk['settings']['requests'])
 
-                self.__data['tag'] = "0.27"
-
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
 
@@ -101,6 +100,14 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                 self.add_watch(url='https://changedetection.io', tag='Tech news')
 
+
+        self.__data['version_tag'] = "0.27"
+
+        if not 'app_guid' in self.__data:
+            self.__data['app_guid'] = str(uuid_builder.uuid4())
+
+        self.needs_write = True
+
         # Finally start the thread that will manage periodic data saves to JSON
         save_data_thread = threading.Thread(target=self.save_datastore).start()
 

backend/templates/base.html

@@ -21,7 +21,9 @@
         {% if current_diff_url %}
             <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</a>
         {% else %}
-            <span id="version-text" class="pure-menu-heading">Version {{ version }}</span>
+            {% if new_version_available %}
+              <span id="version-text" class="pure-menu-heading">A new version is available</span>
+            {% endif %}
         {% endif %}
 
         <ul class="pure-menu-list">

backend/tests/conftest.py

@@ -30,10 +30,11 @@ def app(request):
     app_config = {'datastore_path': datastore_path}
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
+    app.config['STOP_THREADS'] = True
 
     def teardown():
         datastore.stop_thread = True
-        app.config['STOP_THREADS'] = True
+        app.config.exit.set()
         try:
             os.unlink("{}/url-watches.json".format(datastore_path))
         except FileNotFoundError: