kopia lustrzana https://github.com/dgtlmoon/changedetection.io
Merge f3c6536fbc into 8df61f5eaa
dgtlmoon
GitHub
commit
b59e43bd07
|
|
@ -4,11 +4,14 @@
|
|||
|
||||
__version__ = '0.49.16'
|
||||
|
||||
from changedetectionio.strtobool import strtobool
|
||||
from json.decoder import JSONDecodeError
|
||||
# Set environment variables before importing other modules
|
||||
import os
|
||||
os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
|
||||
# Import eventlet for WSGI server - no monkey patching to avoid conflicts
|
||||
import eventlet
|
||||
|
||||
from changedetectionio.strtobool import strtobool
|
||||
from json.decoder import JSONDecodeError
|
||||
import eventlet.wsgi
|
||||
import getopt
|
||||
import platform
|
||||
|
|
@ -141,7 +144,27 @@ def main():
|
|||
logger.critical(str(e))
|
||||
return
|
||||
|
||||
# Get the Flask app
|
||||
app = changedetection_app(app_config, datastore)
|
||||
|
||||
# Now initialize Socket.IO after the app is fully set up
|
||||
try:
|
||||
from changedetectionio.realtime.socket_server import ChangeDetectionSocketIO
|
||||
from changedetectionio.flask_app import socketio_server
|
||||
import threading
|
||||
|
||||
# Create the Socket.IO server
|
||||
socketio_server = ChangeDetectionSocketIO(app, datastore)
|
||||
|
||||
# Run the Socket.IO server in a separate thread on port 5005
|
||||
socket_thread = threading.Thread(target=socketio_server.run,
|
||||
kwargs={'host': host, 'port': 5005})
|
||||
socket_thread.daemon = True
|
||||
socket_thread.start()
|
||||
|
||||
logger.info("Socket.IO server initialized successfully on port 5005")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to initialize Socket.IO server: {str(e)}")
|
||||
|
||||
signal.signal(signal.SIGTERM, sigshutdown_handler)
|
||||
signal.signal(signal.SIGINT, sigshutdown_handler)
|
||||
|
|
@ -204,5 +227,7 @@ def main():
|
|||
server_side=True), app)
|
||||
|
||||
else:
|
||||
# We'll integrate the Socket.IO server with the WSGI server
|
||||
# The Socket.IO server is already attached to the Flask app
|
||||
eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from changedetectionio.blueprint.ui.edit import construct_blueprint as construct
|
|||
from changedetectionio.blueprint.ui.notification import construct_blueprint as construct_notification_blueprint
|
||||
from changedetectionio.blueprint.ui.views import construct_blueprint as construct_views_blueprint
|
||||
|
||||
def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_update_threads, queuedWatchMetaData):
|
||||
def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_update_threads, queuedWatchMetaData, watch_check_completed):
|
||||
ui_blueprint = Blueprint('ui', __name__, template_folder="templates")
|
||||
|
||||
# Register the edit blueprint
|
||||
|
|
@ -20,7 +20,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
|
|||
ui_blueprint.register_blueprint(notification_blueprint)
|
||||
|
||||
# Register the views blueprint
|
||||
views_blueprint = construct_views_blueprint(datastore, update_q, queuedWatchMetaData)
|
||||
views_blueprint = construct_views_blueprint(datastore, update_q, queuedWatchMetaData, watch_check_completed)
|
||||
ui_blueprint.register_blueprint(views_blueprint)
|
||||
|
||||
# Import the login decorator
|
||||
|
|
@ -35,7 +35,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
|
|||
flash('Watch not found', 'error')
|
||||
else:
|
||||
flash("Cleared snapshot history for watch {}".format(uuid))
|
||||
|
||||
return redirect(url_for('watchlist.index'))
|
||||
|
||||
@ui_blueprint.route("/clear_history", methods=['GET', 'POST'])
|
||||
|
|
@ -47,7 +46,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
|
|||
if confirmtext == 'clear':
|
||||
for uuid in datastore.data['watching'].keys():
|
||||
datastore.clear_watch_history(uuid)
|
||||
|
||||
flash("Cleared snapshot history for all watches")
|
||||
else:
|
||||
flash('Incorrect confirmation text.', 'error')
|
||||
|
|
@ -248,6 +246,10 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
|
|||
|
||||
flash(f"{len(uuids)} watches were tagged")
|
||||
|
||||
if uuids:
|
||||
for uuid in uuids:
|
||||
watch_check_completed.send(watch_uuid=uuid)
|
||||
|
||||
return redirect(url_for('watchlist.index'))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from changedetectionio.store import ChangeDetectionStore
|
|||
from changedetectionio.auth_decorator import login_optionally_required
|
||||
from changedetectionio import html_tools
|
||||
|
||||
def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
|
||||
def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData, watch_check_completed):
|
||||
views_blueprint = Blueprint('ui_views', __name__, template_folder="../ui/templates")
|
||||
|
||||
@views_blueprint.route("/preview/<string:uuid>", methods=['GET'])
|
||||
|
|
|
|||
|
|
@ -84,7 +84,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
|
|||
has_proxies=datastore.proxy_list,
|
||||
has_unviewed=datastore.has_unviewed,
|
||||
hosted_sticky=os.getenv("SALTED_PASS", False) == False,
|
||||
now_time_server=time.time(),
|
||||
now_time_server=round(time.time()),
|
||||
pagination=pagination,
|
||||
queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
|
||||
search_q=request.args.get('q', '').strip(),
|
||||
|
|
|
|||
|
|
@ -100,14 +100,14 @@
|
|||
{% endif %}
|
||||
{% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
|
||||
|
||||
{% set is_unviewed = watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
|
||||
{% set checking_now = is_checking_now(watch) %}
|
||||
<tr id="{{ watch.uuid }}"
|
||||
<tr id="{{ watch.uuid }}" data-watch-uuid="{{ watch.uuid }}"
|
||||
class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
|
||||
{# socket.js also sets these vars on the row for update #}
|
||||
{% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
|
||||
{% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
|
||||
{% if watch.paused is defined and watch.paused != False %}paused{% endif %}
|
||||
{% if is_unviewed %}unviewed{% endif %}
|
||||
{% if watch.has_unviewed %}unviewed{% endif %}
|
||||
{% if watch.has_restock_info %} has-restock-info {% if watch['restock']['in_stock'] %}in-stock{% else %}not-in-stock{% endif %} {% else %}no-restock-info{% endif %}
|
||||
{% if watch.uuid in queued_uuids %}queued{% endif %}
|
||||
{% if checking_now %}checking-now{% endif %}
|
||||
|
|
@ -143,7 +143,7 @@
|
|||
<a href="{{ url_for('settings.settings_page', uuid=watch.uuid) }}#proxies">Try other proxies/location</a>
|
||||
{% endif %}
|
||||
<a href="{{ url_for('settings.settings_page', uuid=watch.uuid) }}#proxies">Try adding external proxies/locations</a>
|
||||
|
||||
|
||||
{% endif %}
|
||||
{% if 'empty result or contain only an image' in watch.last_error %}
|
||||
<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Detecting-changes-in-images">more help here</a>.
|
||||
|
|
@ -190,12 +190,13 @@
|
|||
</td>
|
||||
{% endif %}
|
||||
{#last_checked becomes fetch-start-time#}
|
||||
<td class="last-checked" data-timestamp="{{ watch.last_checked }}" {% if checking_now %} data-fetchduration={{ watch.fetch_time }} data-eta_complete="{{ watch.last_checked+watch.fetch_time }}" {% endif %} >
|
||||
{% if checking_now %}
|
||||
<span class="spinner"></span><span> Checking now</span>
|
||||
{% else %}
|
||||
{{watch|format_last_checked_time|safe}}</td>
|
||||
{% endif %}
|
||||
<td class="last-checked" data-timestamp="{{ watch.last_checked }}" data-fetchduration={{ watch.fetch_time }} data-eta_complete="{{ watch.last_checked+watch.fetch_time }}" >
|
||||
<div class="spinner-wrapper" style="display:none;" >
|
||||
<span class="spinner"></span><span> Checking now</span>
|
||||
</div>
|
||||
<span class="innertext">{{watch|format_last_checked_time|safe}}</span>
|
||||
</td>
|
||||
|
||||
|
||||
<td class="last-changed" data-timestamp="{{ watch.last_changed }}">{% if watch.history_n >=2 and watch.last_changed >0 %}
|
||||
{{watch.last_changed|format_timestamp_timeago}}
|
||||
|
|
@ -204,15 +205,17 @@
|
|||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
<a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('ui.form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
|
||||
class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
|
||||
<a href="" class="already-in-queue-button recheck pure-button pure-button-primary" style="display: none;" disabled="disabled">Queued</a>
|
||||
|
||||
<a href="{{ url_for('ui.form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}" class="recheck pure-button pure-button-primary">Recheck</a>
|
||||
<a href="{{ url_for('ui.ui_edit.edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
|
||||
|
||||
{% if watch.history_n >= 2 %}
|
||||
|
||||
{% set open_diff_in_new_tab = datastore.data['settings']['application']['ui'].get('open_diff_in_new_tab') %}
|
||||
{% set target_attr = ' target="' ~ watch.uuid ~ '"' if open_diff_in_new_tab else '' %}
|
||||
|
||||
{% if is_unviewed %}
|
||||
{% if watch.has_unviewed %}
|
||||
<a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid, from_version=watch.get_from_version_based_on_last_viewed) }}" {{target_attr}} class="pure-button pure-button-primary diff-link">History</a>
|
||||
{% else %}
|
||||
<a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary diff-link">History</a>
|
||||
|
|
@ -251,4 +254,4 @@
|
|||
</div>
|
||||
</form>
|
||||
</div>
|
||||
{% endblock %}
|
||||
{% endblock %}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
import queue
|
||||
from blinker import signal
|
||||
|
||||
class SignalPriorityQueue(queue.PriorityQueue):
|
||||
"""
|
||||
Extended PriorityQueue that sends a signal when items with a UUID are added.
|
||||
|
||||
This class extends the standard PriorityQueue and adds a signal emission
|
||||
after an item is put into the queue. If the item contains a UUID, the signal
|
||||
is sent with that UUID as a parameter.
|
||||
"""
|
||||
|
||||
def __init__(self, maxsize=0):
|
||||
super().__init__(maxsize)
|
||||
|
||||
def put(self, item, block=True, timeout=None):
|
||||
# Call the parent's put method first
|
||||
super().put(item, block, timeout)
|
||||
|
||||
# After putting the item in the queue, check if it has a UUID and emit signal
|
||||
if hasattr(item, 'item') and isinstance(item.item, dict) and 'uuid' in item.item:
|
||||
uuid = item.item['uuid']
|
||||
# Get the signal and send it if it exists
|
||||
watch_check_completed = signal('watch_check_completed')
|
||||
if watch_check_completed:
|
||||
# Send the watch_uuid parameter
|
||||
watch_check_completed.send(watch_uuid=uuid)
|
||||
|
|
@ -7,9 +7,11 @@ import queue
|
|||
import threading
|
||||
import time
|
||||
import timeago
|
||||
from blinker import signal
|
||||
|
||||
from changedetectionio.strtobool import strtobool
|
||||
from threading import Event
|
||||
from changedetectionio.custom_queue import SignalPriorityQueue
|
||||
|
||||
from flask import (
|
||||
Flask,
|
||||
|
|
@ -28,8 +30,13 @@ from flask_login import current_user
|
|||
from flask_paginate import Pagination, get_page_parameter
|
||||
from flask_restful import abort, Api
|
||||
from flask_cors import CORS
|
||||
|
||||
# Create specific signals for application events
|
||||
# Make this a global singleton to avoid multiple signal objects
|
||||
watch_check_completed = signal('watch_check_completed', doc='Signal sent when a watch check is completed')
|
||||
from flask_wtf import CSRFProtect
|
||||
from loguru import logger
|
||||
import eventlet
|
||||
|
||||
from changedetectionio import __version__
|
||||
from changedetectionio import queuedWatchMetaData
|
||||
|
|
@ -45,7 +52,7 @@ ticker_thread = None
|
|||
|
||||
extra_stylesheets = []
|
||||
|
||||
update_q = queue.PriorityQueue()
|
||||
update_q = SignalPriorityQueue()
|
||||
notification_q = queue.Queue()
|
||||
MAX_QUEUE_SIZE = 2000
|
||||
|
||||
|
|
@ -54,6 +61,9 @@ app = Flask(__name__,
|
|||
static_folder="static",
|
||||
template_folder="templates")
|
||||
|
||||
# Will be initialized in changedetection_app
|
||||
socketio_server = None
|
||||
|
||||
# Enable CORS, especially useful for the Chrome extension to operate from anywhere
|
||||
CORS(app)
|
||||
|
||||
|
|
@ -215,12 +225,15 @@ class User(flask_login.UserMixin):
|
|||
def changedetection_app(config=None, datastore_o=None):
|
||||
logger.trace("TRACE log is enabled")
|
||||
|
||||
global datastore
|
||||
global datastore, socketio_server
|
||||
datastore = datastore_o
|
||||
|
||||
# so far just for read-only via tests, but this will be moved eventually to be the main source
|
||||
# (instead of the global var)
|
||||
app.config['DATASTORE'] = datastore_o
|
||||
|
||||
# Store the signal in the app config to ensure it's accessible everywhere
|
||||
app.config['WATCH_CHECK_COMPLETED_SIGNAL'] = watch_check_completed
|
||||
|
||||
login_manager = flask_login.LoginManager(app)
|
||||
login_manager.login_view = 'login'
|
||||
|
|
@ -444,7 +457,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
|
||||
# watchlist UI buttons etc
|
||||
import changedetectionio.blueprint.ui as ui
|
||||
app.register_blueprint(ui.construct_blueprint(datastore, update_q, running_update_threads, queuedWatchMetaData))
|
||||
app.register_blueprint(ui.construct_blueprint(datastore, update_q, running_update_threads, queuedWatchMetaData, watch_check_completed))
|
||||
|
||||
import changedetectionio.blueprint.watchlist as watchlist
|
||||
app.register_blueprint(watchlist.construct_blueprint(datastore=datastore, update_q=update_q, queuedWatchMetaData=queuedWatchMetaData), url_prefix='')
|
||||
|
|
@ -467,6 +480,8 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
if not os.getenv("GITHUB_REF", False) and not strtobool(os.getenv('DISABLE_VERSION_CHECK', 'no')):
|
||||
threading.Thread(target=check_for_new_version).start()
|
||||
|
||||
# Return the Flask app - the Socket.IO will be attached to it but initialized separately
|
||||
# This avoids circular dependencies
|
||||
return app
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -436,55 +436,27 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False
|
|||
return re.sub(pattern, repl, html_content)
|
||||
|
||||
|
||||
def html_to_text_sub_worker(conn, html_content: str, render_anchor_tag_content=False, is_rss=False):
|
||||
# NOTE!! ANYTHING LIBXML, HTML5LIB ETC WILL CAUSE SOME SMALL MEMORY LEAK IN THE LOCAL "LIB" IMPLEMENTATION OUTSIDE PYTHON
|
||||
|
||||
|
||||
def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False, timeout=10) -> str:
|
||||
from inscriptis import get_text
|
||||
from inscriptis.model.config import ParserConfig
|
||||
|
||||
"""Converts html string to a string with just the text. If ignoring
|
||||
rendering anchor tag content is enable, anchor tag content are also
|
||||
included in the text
|
||||
|
||||
:param html_content: string with html content
|
||||
:param render_anchor_tag_content: boolean flag indicating whether to extract
|
||||
hyperlinks (the anchor tag content) together with text. This refers to the
|
||||
'href' inside 'a' tags.
|
||||
Anchor tag content is rendered in the following manner:
|
||||
'[ text ](anchor tag content)'
|
||||
:return: extracted text from the HTML
|
||||
"""
|
||||
# if anchor tag content flag is set to True define a config for
|
||||
# extracting this content
|
||||
if render_anchor_tag_content:
|
||||
parser_config = ParserConfig(
|
||||
annotation_rules={"a": ["hyperlink"]},
|
||||
display_links=True
|
||||
)
|
||||
# otherwise set config to None/default
|
||||
else:
|
||||
parser_config = None
|
||||
|
||||
# RSS Mode - Inscriptis will treat `title` as something else.
|
||||
# Make it as a regular block display element (//item/title)
|
||||
# This is a bit of a hack - the real way it to use XSLT to convert it to HTML #1874
|
||||
if is_rss:
|
||||
html_content = re.sub(r'<title([\s>])', r'<h1\1', html_content)
|
||||
html_content = re.sub(r'</title>', r'</h1>', html_content)
|
||||
|
||||
text_content = get_text(html_content, config=parser_config)
|
||||
conn.send(text_content)
|
||||
conn.close()
|
||||
|
||||
# NOTE!! ANYTHING LIBXML, HTML5LIB ETC WILL CAUSE SOME SMALL MEMORY LEAK IN THE LOCAL "LIB" IMPLEMENTATION OUTSIDE PYTHON
|
||||
def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False):
|
||||
from multiprocessing import Process, Pipe
|
||||
|
||||
parent_conn, child_conn = Pipe()
|
||||
p = Process(target=html_to_text_sub_worker, args=(child_conn, html_content, render_anchor_tag_content, is_rss))
|
||||
p.start()
|
||||
text = parent_conn.recv()
|
||||
p.join()
|
||||
return text
|
||||
return text_content
|
||||
|
||||
# Does LD+JSON exist with a @type=='product' and a .price set anywhere?
|
||||
def has_ldjson_product_info(content):
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from blinker import signal
|
||||
|
||||
from changedetectionio.strtobool import strtobool
|
||||
from changedetectionio.safe_jinja import render as jinja_render
|
||||
from . import watch_base
|
||||
|
|
@ -60,6 +62,10 @@ class model(watch_base):
|
|||
|
||||
return False
|
||||
|
||||
@property
|
||||
def has_unviewed(self):
|
||||
return int(self.newest_history_key) > int(self['last_viewed']) and self.__history_n >= 2
|
||||
|
||||
def ensure_data_dir_exists(self):
|
||||
if not os.path.isdir(self.watch_data_dir):
|
||||
logger.debug(f"> Creating data dir {self.watch_data_dir}")
|
||||
|
|
@ -120,6 +126,10 @@ class model(watch_base):
|
|||
'remote_server_reply': None,
|
||||
'track_ldjson_price_data': None
|
||||
})
|
||||
watch_check_completed = signal('watch_check_completed')
|
||||
if watch_check_completed:
|
||||
watch_check_completed.send(watch_uuid=self.get('uuid'))
|
||||
|
||||
return
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
"""
|
||||
Socket.IO realtime updates module for changedetection.io
|
||||
"""
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
import timeago
|
||||
from flask import Flask
|
||||
from flask_socketio import SocketIO
|
||||
import threading
|
||||
import json
|
||||
import time
|
||||
from loguru import logger
|
||||
import blinker
|
||||
|
||||
from changedetectionio.flask_app import _jinja2_filter_datetime, watch_check_completed
|
||||
|
||||
|
||||
class SignalHandler:
|
||||
"""A standalone class to receive signals"""
|
||||
def __init__(self, socketio_instance):
|
||||
self.socketio_instance = socketio_instance
|
||||
|
||||
# Get signal from app config
|
||||
app_signal = socketio_instance.main_app.config.get('WATCH_CHECK_COMPLETED_SIGNAL')
|
||||
if app_signal:
|
||||
app_signal.connect(self.handle_signal, weak=False)
|
||||
logger.info("SignalHandler: Connected to signal from app config")
|
||||
else:
|
||||
# Fallback if not in app config
|
||||
from changedetectionio.flask_app import watch_check_completed as wcc
|
||||
wcc.connect(self.handle_signal, weak=False)
|
||||
logger.info("SignalHandler: Connected to signal from direct import")
|
||||
|
||||
def handle_signal(self, *args, **kwargs):
|
||||
logger.info(f"SignalHandler: Signal received with {len(args)} args and {len(kwargs)} kwargs")
|
||||
# Safely extract the watch UUID from kwargs
|
||||
watch_uuid = kwargs.get('watch_uuid')
|
||||
if watch_uuid:
|
||||
# Get the datastore from the socket instance
|
||||
datastore = self.socketio_instance.datastore
|
||||
# Get the watch object from the datastore
|
||||
watch = datastore.data['watching'].get(watch_uuid)
|
||||
if watch:
|
||||
# Forward to the socket instance with the watch parameter
|
||||
self.socketio_instance.handle_watch_update(watch=watch)
|
||||
logger.info(f"Signal handler processed watch UUID {watch_uuid}")
|
||||
else:
|
||||
logger.warning(f"Watch UUID {watch_uuid} not found in datastore")
|
||||
|
||||
class ChangeDetectionSocketIO:
|
||||
def __init__(self, app, datastore):
|
||||
self.main_app = app
|
||||
self.datastore = datastore
|
||||
|
||||
# Use threading mode instead of eventlet
|
||||
self.socketio = SocketIO(self.main_app,
|
||||
async_mode='threading',
|
||||
cors_allowed_origins="*",
|
||||
logger=False,
|
||||
engineio_logger=False)
|
||||
|
||||
# Set up event handlers
|
||||
self.socketio.on_event('connect', self.handle_connect)
|
||||
self.socketio.on_event('disconnect', self.handle_disconnect)
|
||||
|
||||
# Don't patch the update_watch method - this was causing issues
|
||||
# Just start a background thread to periodically emit watch status
|
||||
self.thread = None
|
||||
self.thread_lock = threading.Lock()
|
||||
|
||||
# Create a dedicated signal handler
|
||||
self.signal_handler = SignalHandler(self)
|
||||
|
||||
def handle_connect(self):
|
||||
"""Handle client connection"""
|
||||
logger.info("Socket.IO: Client connected")
|
||||
|
||||
|
||||
def handle_disconnect(self):
|
||||
"""Handle client disconnection"""
|
||||
logger.info("Socket.IO: Client disconnected")
|
||||
|
||||
def handle_watch_update(self, **kwargs):
|
||||
"""Handle watch update signal from blinker"""
|
||||
try:
|
||||
watch = kwargs.get('watch')
|
||||
# Emit the watch update to all connected clients
|
||||
with self.main_app.app_context():
|
||||
from changedetectionio.flask_app import running_update_threads, update_q
|
||||
|
||||
# Get list of watches that are currently running
|
||||
running_uuids = []
|
||||
for t in running_update_threads:
|
||||
if hasattr(t, 'current_uuid') and t.current_uuid:
|
||||
running_uuids.append(t.current_uuid)
|
||||
|
||||
# Get list of watches in the queue
|
||||
queue_list = []
|
||||
for q_item in update_q.queue:
|
||||
if hasattr(q_item, 'item') and 'uuid' in q_item.item:
|
||||
queue_list.append(q_item.item['uuid'])
|
||||
|
||||
|
||||
# Create a simplified watch data object to send to clients
|
||||
watch_data = {
|
||||
'checking_now': True if watch.get('uuid') in running_uuids else False,
|
||||
'fetch_time': watch.get('fetch_time'),
|
||||
'has_error': watch.get('last_error') or watch.get('last_notification_error'),
|
||||
'last_changed': watch.get('last_changed'),
|
||||
'last_checked': watch.get('last_checked'),
|
||||
'last_checked_text': _jinja2_filter_datetime(watch),
|
||||
'last_changed_text': timeago.format(int(watch['last_changed']), time.time()) if watch.history_n >=2 and int(watch.get('last_changed',0)) >0 else 'Not yet',
|
||||
'queued': True if watch.get('uuid') in queue_list else False,
|
||||
'unviewed': watch.has_unviewed,
|
||||
'uuid': watch.get('uuid'),
|
||||
}
|
||||
self.socketio.emit("watch_update", watch_data)
|
||||
logger.debug(f"Socket.IO: Emitted update for watch {watch.get('uuid')}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Socket.IO error in handle_watch_update: {str(e)}")
|
||||
|
||||
|
||||
def run(self, host='0.0.0.0', port=5005):
|
||||
"""Run the Socket.IO server on a separate port"""
|
||||
# Start the background task when the server starts
|
||||
#self.start_background_task()
|
||||
|
||||
# Run the Socket.IO server
|
||||
# Use 0.0.0.0 to listen on all interfaces
|
||||
logger.info(f"Starting Socket.IO server on http://{host}:{port}")
|
||||
self.socketio.run(self.main_app, host=host, port=port, debug=False, use_reloader=False, allow_unsafe_werkzeug=True)
|
||||
|
|
@ -38,6 +38,9 @@ pytest tests/test_backend.py
|
|||
pytest tests/test_rss.py
|
||||
pytest tests/test_unique_lines.py
|
||||
|
||||
# Try high concurrency
|
||||
FETCH_WORKERS=130 pytest tests/test_history_consistency.py -v -l
|
||||
|
||||
# Check file:// will pickup a file when enabled
|
||||
echo "Hello world" > /tmp/test-file.txt
|
||||
ALLOW_FILE_URI=yes pytest tests/test_security.py
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
// Socket.IO client-side integration for changedetection.io
|
||||
|
||||
$(document).ready(function () {
|
||||
// Try to create the socket connection to port 5005 - if it fails, the site will still work normally
|
||||
try {
|
||||
// Connect to the dedicated Socket.IO server on port 5005
|
||||
const socket = io('http://127.0.0.1:5005');
|
||||
|
||||
// Connection status logging
|
||||
socket.on('connect', function () {
|
||||
console.log('Socket.IO connected');
|
||||
});
|
||||
|
||||
socket.on('disconnect', function () {
|
||||
console.log('Socket.IO disconnected');
|
||||
});
|
||||
|
||||
socket.on('checking_now', function (uuid_list) {
|
||||
console.log("Got checking now update");
|
||||
// Remove 'checking-now' class where it should no longer be
|
||||
$('.watch-table tbody tr.checking-now').each(function () {
|
||||
if (!uuid_list.includes($(this).data('watch-uuid'))) {
|
||||
$(this).removeClass('checking-now');
|
||||
}
|
||||
});
|
||||
|
||||
// Add the class on the rows where it should be
|
||||
uuid_list.forEach(function (uuid) {
|
||||
$('.watch-table tbody tr[data-watch-uuid="' + uuid + '"]').addClass('checking-now');
|
||||
});
|
||||
});
|
||||
|
||||
// Listen for periodically emitted watch data
|
||||
socket.on('watch_update', function (watch) {
|
||||
console.log(`Watch update ${watch.uuid}`);
|
||||
|
||||
|
||||
const $watchRow = $('tr[data-watch-uuid="' + watch.uuid + '"]');
|
||||
if ($watchRow.length) {
|
||||
$($watchRow).toggleClass('checking-now', watch.checking_now);
|
||||
$($watchRow).toggleClass('queued', watch.queued);
|
||||
$($watchRow).toggleClass('unviewed', watch.unviewed);
|
||||
$($watchRow).toggleClass('error', watch.has_error);
|
||||
$('td.last-changed', $watchRow).text(watch.last_checked_text)
|
||||
$('td.last-checked .innertext', $watchRow).text(watch.last_checked_text)
|
||||
$('td.last-checked', $watchRow).data('timestamp', watch.last_checked).data('fetchduration', watch.fetch_time);
|
||||
$('td.last-checked', $watchRow).data('eta_complete', watch.last_checked + watch.fetch_time);
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
} catch (e) {
|
||||
// If Socket.IO fails to initialize, just log it and continue
|
||||
console.log('Socket.IO initialization error:', e);
|
||||
}
|
||||
});
|
||||
|
|
@ -68,7 +68,7 @@ $(function () {
|
|||
if (eta_complete + 2 > nowtimeserver && fetch_duration > 3) {
|
||||
const remaining_seconds = Math.abs(eta_complete) - nowtimeserver - 1;
|
||||
|
||||
let r = (1.0 - (remaining_seconds / fetch_duration)) * 100;
|
||||
let r = Math.round((1.0 - (remaining_seconds / fetch_duration)) * 100);
|
||||
if (r < 10) {
|
||||
r = 10;
|
||||
}
|
||||
|
|
@ -76,7 +76,6 @@ $(function () {
|
|||
r = 100;
|
||||
}
|
||||
$(this).css('background-size', `${r}% 100%`);
|
||||
//$(this).text(`${r}% remain ${remaining_seconds}`);
|
||||
} else {
|
||||
$(this).css('background-size', `100% 100%`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
// Styles for Socket.IO real-time updates
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
|
||||
/* table related */
|
||||
.watch-table {
|
||||
width: 100%;
|
||||
font-size: 80%;
|
||||
|
||||
tr {
|
||||
&.unviewed {
|
||||
font-weight: bold;
|
||||
}
|
||||
&.error {
|
||||
color: var(--color-watch-table-error);
|
||||
}
|
||||
color: var(--color-watch-table-row-text);
|
||||
}
|
||||
|
||||
|
||||
td {
|
||||
white-space: nowrap;
|
||||
&.title-col {
|
||||
word-break: break-all;
|
||||
white-space: normal;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
th {
|
||||
white-space: nowrap;
|
||||
|
||||
a {
|
||||
font-weight: normal;
|
||||
|
||||
&.active {
|
||||
font-weight: bolder;
|
||||
}
|
||||
|
||||
&.inactive {
|
||||
.arrow {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.title-col a[target="_blank"]::after,
|
||||
.current-diff-url::after {
|
||||
content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
|
||||
margin: 0 3px 0 5px;
|
||||
}
|
||||
|
||||
|
||||
/* Row with 'checking-now' */
|
||||
tr.checking-now {
|
||||
td.last-checked {
|
||||
.spinner-wrapper {
|
||||
display: inline-block !important;
|
||||
}
|
||||
.innertext {
|
||||
display: none !important;
|
||||
}
|
||||
}
|
||||
}
|
||||
tr.queued {
|
||||
a.recheck {
|
||||
display: none !important;
|
||||
}
|
||||
a.already-in-queue-button {
|
||||
display: inline-block !important;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -13,6 +13,7 @@
|
|||
@import "parts/_menu";
|
||||
@import "parts/_love";
|
||||
@import "parts/preview_text_filter";
|
||||
@import "parts/_watch_table";
|
||||
@import "parts/_edit";
|
||||
@import "parts/_conditions_table";
|
||||
|
||||
|
|
@ -169,56 +170,6 @@ code {
|
|||
color: var(--color-text);
|
||||
}
|
||||
|
||||
/* table related */
|
||||
.watch-table {
|
||||
width: 100%;
|
||||
font-size: 80%;
|
||||
|
||||
tr {
|
||||
&.unviewed {
|
||||
font-weight: bold;
|
||||
}
|
||||
&.error {
|
||||
color: var(--color-watch-table-error);
|
||||
}
|
||||
color: var(--color-watch-table-row-text);
|
||||
}
|
||||
|
||||
|
||||
td {
|
||||
white-space: nowrap;
|
||||
&.title-col {
|
||||
word-break: break-all;
|
||||
white-space: normal;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
th {
|
||||
white-space: nowrap;
|
||||
|
||||
a {
|
||||
font-weight: normal;
|
||||
|
||||
&.active {
|
||||
font-weight: bolder;
|
||||
}
|
||||
|
||||
&.inactive {
|
||||
.arrow {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.title-col a[target="_blank"]::after,
|
||||
.current-diff-url::after {
|
||||
content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
|
||||
margin: 0 3px 0 5px;
|
||||
}
|
||||
}
|
||||
|
||||
.inline-tag {
|
||||
white-space: nowrap;
|
||||
border-radius: 5px;
|
||||
|
|
|
|||
|
|
@ -523,6 +523,43 @@ body.preview-text-enabled {
|
|||
z-index: 3;
|
||||
box-shadow: 1px 1px 4px var(--color-shadow-jump); }
|
||||
|
||||
/* table related */
|
||||
.watch-table {
|
||||
width: 100%;
|
||||
font-size: 80%;
|
||||
/* Row with 'checking-now' */ }
|
||||
.watch-table tr {
|
||||
color: var(--color-watch-table-row-text); }
|
||||
.watch-table tr.unviewed {
|
||||
font-weight: bold; }
|
||||
.watch-table tr.error {
|
||||
color: var(--color-watch-table-error); }
|
||||
.watch-table td {
|
||||
white-space: nowrap; }
|
||||
.watch-table td.title-col {
|
||||
word-break: break-all;
|
||||
white-space: normal; }
|
||||
.watch-table th {
|
||||
white-space: nowrap; }
|
||||
.watch-table th a {
|
||||
font-weight: normal; }
|
||||
.watch-table th a.active {
|
||||
font-weight: bolder; }
|
||||
.watch-table th a.inactive .arrow {
|
||||
display: none; }
|
||||
.watch-table .title-col a[target="_blank"]::after,
|
||||
.watch-table .current-diff-url::after {
|
||||
content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
|
||||
margin: 0 3px 0 5px; }
|
||||
.watch-table tr.checking-now td.last-checked .spinner-wrapper {
|
||||
display: inline-block !important; }
|
||||
.watch-table tr.checking-now td.last-checked .innertext {
|
||||
display: none !important; }
|
||||
.watch-table tr.queued a.recheck {
|
||||
display: none !important; }
|
||||
.watch-table tr.queued a.already-in-queue-button {
|
||||
display: inline-block !important; }
|
||||
|
||||
ul#conditions_match_logic {
|
||||
list-style: none; }
|
||||
ul#conditions_match_logic input, ul#conditions_match_logic label, ul#conditions_match_logic li {
|
||||
|
|
@ -735,34 +772,6 @@ code {
|
|||
background: var(--color-background-code);
|
||||
color: var(--color-text); }
|
||||
|
||||
/* table related */
|
||||
.watch-table {
|
||||
width: 100%;
|
||||
font-size: 80%; }
|
||||
.watch-table tr {
|
||||
color: var(--color-watch-table-row-text); }
|
||||
.watch-table tr.unviewed {
|
||||
font-weight: bold; }
|
||||
.watch-table tr.error {
|
||||
color: var(--color-watch-table-error); }
|
||||
.watch-table td {
|
||||
white-space: nowrap; }
|
||||
.watch-table td.title-col {
|
||||
word-break: break-all;
|
||||
white-space: normal; }
|
||||
.watch-table th {
|
||||
white-space: nowrap; }
|
||||
.watch-table th a {
|
||||
font-weight: normal; }
|
||||
.watch-table th a.active {
|
||||
font-weight: bolder; }
|
||||
.watch-table th a.inactive .arrow {
|
||||
display: none; }
|
||||
.watch-table .title-col a[target="_blank"]::after,
|
||||
.watch-table .current-diff-url::after {
|
||||
content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
|
||||
margin: 0 3px 0 5px; }
|
||||
|
||||
.inline-tag, .watch-tag-list, .tracking-ldjson-price-data, .restock-label {
|
||||
white-space: nowrap;
|
||||
border-radius: 5px;
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import threading
|
|||
import time
|
||||
import uuid as uuid_builder
|
||||
from loguru import logger
|
||||
from blinker import signal
|
||||
|
||||
from .processors import get_custom_watch_obj_for_processor
|
||||
from .processors.restock_diff import Restock
|
||||
|
|
@ -166,6 +167,10 @@ class ChangeDetectionStore:
|
|||
self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
|
||||
self.needs_write = True
|
||||
|
||||
watch_check_completed = signal('watch_check_completed')
|
||||
if watch_check_completed:
|
||||
watch_check_completed.send(watch_uuid=uuid)
|
||||
|
||||
def remove_password(self):
|
||||
self.__data['settings']['application']['password'] = False
|
||||
self.needs_write = True
|
||||
|
|
|
|||
|
|
@ -31,6 +31,10 @@
|
|||
</script>
|
||||
<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
|
||||
<script src="{{url_for('static_content', group='js', filename='csrf.js')}}" defer></script>
|
||||
<script src="https://cdn.socket.io/4.6.0/socket.io.min.js" integrity="sha384-c79GN5VsunZvi+Q/WObgk2in0CbZsHnjEqvFxC5DxHn9lTfNce2WW6h2pH6u/kF+" crossorigin="anonymous"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/timeago.js/4.0.2/timeago.min.js" integrity="sha512-SVDh1zH5N9ChofSlNAK43lcNS7lWze6DTVx1JCXH1Tmno+0/1jMpdbR8YDgDUfcUrPp1xyE53G42GFrcM0CMVg==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
|
||||
<script src="{{url_for('static_content', group='js', filename='socket.js')}}" defer></script>
|
||||
<script src="{{url_for('static_content', group='js', filename='timeago-init.js')}}" defer></script>
|
||||
</head>
|
||||
|
||||
<body class="">
|
||||
|
|
|
|||
|
|
@ -4,14 +4,14 @@ import time
|
|||
from flask import url_for
|
||||
from .util import live_server_setup, wait_for_all_checks
|
||||
|
||||
|
||||
# test pages with http://username@password:foobar.com/ work
|
||||
def test_basic_auth(client, live_server, measure_memory_usage):
|
||||
|
||||
live_server_setup(live_server)
|
||||
|
||||
# Add our URL to the import page
|
||||
test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
|
||||
|
||||
# This page will echo back any auth info
|
||||
test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
|
||||
time.sleep(1)
|
||||
res = client.post(
|
||||
url_for("imports.import_page"),
|
||||
data={"urls": test_url},
|
||||
|
|
@ -34,4 +34,4 @@ def test_basic_auth(client, live_server, measure_memory_usage):
|
|||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b'myuser mypass basic' in res.data
|
||||
assert b'myuser mypass basic' in res.data
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ from urllib.parse import urlparse, parse_qs
|
|||
|
||||
def test_consistent_history(client, live_server, measure_memory_usage):
|
||||
live_server_setup(live_server)
|
||||
|
||||
r = range(1, 30)
|
||||
workers = int(os.getenv("FETCH_WORKERS", 10))
|
||||
r = range(1, 10+workers)
|
||||
|
||||
for one in r:
|
||||
test_url = url_for('test_endpoint', content_type="text/html", content=str(one), _external=True)
|
||||
|
|
@ -35,8 +35,7 @@ def test_consistent_history(client, live_server, measure_memory_usage):
|
|||
)
|
||||
assert b"Settings updated." in res.data
|
||||
|
||||
|
||||
time.sleep(2)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
|
||||
|
||||
|
|
@ -46,11 +45,16 @@ def test_consistent_history(client, live_server, measure_memory_usage):
|
|||
|
||||
# assert the right amount of watches was found in the JSON
|
||||
assert len(json_obj['watching']) == len(r), "Correct number of watches was found in the JSON"
|
||||
|
||||
i=0
|
||||
# each one should have a history.txt containing just one line
|
||||
i=0
|
||||
for w in json_obj['watching'].keys():
|
||||
i+1
|
||||
# res = client.get(url_for("watchlist.index"))
|
||||
# with open('/tmp/debug.html', 'wb') as f:
|
||||
# f.write(res.data)
|
||||
history_txt_index_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, 'history.txt')
|
||||
assert os.path.isfile(history_txt_index_file), f"History.txt should exist where I expect it at {history_txt_index_file}"
|
||||
assert os.path.isfile(history_txt_index_file), f"history.txt for i: {i} should exist where I expect it at {history_txt_index_file}"
|
||||
|
||||
# Same like in model.Watch
|
||||
with open(history_txt_index_file, "r") as f:
|
||||
|
|
@ -58,8 +62,8 @@ def test_consistent_history(client, live_server, measure_memory_usage):
|
|||
assert len(tmp_history) == 1, "History.txt should contain 1 line"
|
||||
|
||||
# Should be two files,. the history.txt , and the snapshot.txt
|
||||
files_in_watch_dir = os.listdir(os.path.join(live_server.app.config['DATASTORE'].datastore_path,
|
||||
w))
|
||||
files_in_watch_dir = os.listdir(os.path.join(live_server.app.config['DATASTORE'].datastore_path, w))
|
||||
|
||||
# Find the snapshot one
|
||||
for fname in files_in_watch_dir:
|
||||
if fname != 'history.txt' and 'html' not in fname:
|
||||
|
|
@ -75,7 +79,6 @@ def test_consistent_history(client, live_server, measure_memory_usage):
|
|||
|
||||
assert len(files_in_watch_dir) == 3, "Should be just three files in the dir, html.br snapshot, history.txt and the extracted text snapshot"
|
||||
|
||||
|
||||
json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
|
||||
with open(json_db_file, 'r') as f:
|
||||
assert '"default"' not in f.read(), "'default' probably shouldnt be here, it came from when the 'default' Watch vars were accidently being saved"
|
||||
|
|
|
|||
|
|
@ -219,6 +219,15 @@ def test_rss_bad_chars_breaking(client, live_server):
|
|||
rss_token = extract_rss_token_from_UI(client)
|
||||
|
||||
uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
|
||||
i=0
|
||||
from loguru import logger
|
||||
# Because chardet could take a long time
|
||||
while i<=10:
|
||||
logger.debug(f"History was {live_server.app.config['DATASTORE'].data['watching'][uuid].history_n}..")
|
||||
if live_server.app.config['DATASTORE'].data['watching'][uuid].history_n ==2:
|
||||
break
|
||||
i+=1
|
||||
time.sleep(2)
|
||||
assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n == 2
|
||||
|
||||
# Check RSS feed is still working
|
||||
|
|
|
|||
|
|
@ -126,21 +126,56 @@ def extract_UUID_from_client(client):
|
|||
uuid = m.group(1)
|
||||
return uuid.strip()
|
||||
|
||||
def wait_for_all_checks(client):
|
||||
# actually this is not entirely true, it can still be 'processing' but not in the queue
|
||||
# Loop waiting until done..
|
||||
attempt=0
|
||||
# because sub-second rechecks are problematic in testing, use lots of delays
|
||||
time.sleep(1)
|
||||
while attempt < 60:
|
||||
res = client.get(url_for("watchlist.index"))
|
||||
if not b'Checking now' in res.data:
|
||||
break
|
||||
logging.getLogger().info(f"Waiting for watch-list to not say 'Checking now'.. {attempt}")
|
||||
time.sleep(1)
|
||||
def wait_for_all_checks(client=None):
|
||||
"""
|
||||
Waits until the queue is empty and remains empty for at least `required_empty_duration` seconds,
|
||||
and also ensures no running threads have `current_uuid` set.
|
||||
Retries for up to `max_attempts` times, sleeping `wait_between_attempts` seconds between checks.
|
||||
"""
|
||||
from changedetectionio.flask_app import update_q as global_update_q, running_update_threads
|
||||
|
||||
# Configuration
|
||||
attempt = 0
|
||||
|
||||
i=0
|
||||
|
||||
max_attempts = 60
|
||||
wait_between_attempts = 1
|
||||
required_empty_duration = 0.6
|
||||
|
||||
logger = logging.getLogger()
|
||||
time.sleep(0.5)
|
||||
|
||||
empty_since = None
|
||||
|
||||
while attempt < max_attempts:
|
||||
q_length = global_update_q.qsize()
|
||||
|
||||
# Check if any threads are still processing
|
||||
time.sleep(wait_between_attempts)
|
||||
any_threads_busy = any(t.current_uuid for t in running_update_threads)
|
||||
|
||||
|
||||
if q_length == 0 and not any_threads_busy:
|
||||
if empty_since is None:
|
||||
empty_since = time.time()
|
||||
logger.info(f"Queue empty and no active threads at attempt {attempt}, starting empty timer...")
|
||||
elif time.time() - empty_since >= required_empty_duration:
|
||||
logger.info(f"Queue has been empty and threads idle for {required_empty_duration} seconds. Done waiting.")
|
||||
break
|
||||
else:
|
||||
logger.info(f"Still waiting: queue empty and no active threads, but not yet {required_empty_duration} seconds...")
|
||||
else:
|
||||
if q_length != 0:
|
||||
logger.info(f"Queue not empty (size={q_length}), resetting timer.")
|
||||
if any_threads_busy:
|
||||
busy_threads = [t.name for t in running_update_threads if t.current_uuid]
|
||||
logger.info(f"Threads still busy: {busy_threads}, resetting timer.")
|
||||
empty_since = None
|
||||
|
||||
attempt += 1
|
||||
|
||||
time.sleep(1)
|
||||
time.sleep(wait_between_attempts)
|
||||
|
||||
def live_server_setup(live_server):
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from .processors.exceptions import ProcessorException
|
|||
import changedetectionio.content_fetchers.exceptions as content_fetchers_exceptions
|
||||
from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
|
||||
from changedetectionio import html_tools
|
||||
from changedetectionio.flask_app import watch_check_completed
|
||||
|
||||
import importlib
|
||||
import os
|
||||
|
|
@ -242,17 +243,16 @@ class update_worker(threading.Thread):
|
|||
os.unlink(full_path)
|
||||
|
||||
def run(self):
|
||||
|
||||
|
||||
while not self.app.config.exit.is_set():
|
||||
update_handler = None
|
||||
watch = None
|
||||
|
||||
try:
|
||||
queued_item_data = self.q.get(block=False)
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
else:
|
||||
|
||||
uuid = queued_item_data.item.get('uuid')
|
||||
fetch_start_time = round(time.time()) # Also used for a unique history key for now
|
||||
self.current_uuid = uuid
|
||||
|
|
@ -272,6 +272,8 @@ class update_worker(threading.Thread):
|
|||
logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
|
||||
|
||||
try:
|
||||
watch_check_completed.send(watch_uuid=watch['uuid'])
|
||||
|
||||
# Processor is what we are using for detecting the "Change"
|
||||
processor = watch.get('processor', 'text_json_diff')
|
||||
|
||||
|
|
@ -588,12 +590,18 @@ class update_worker(threading.Thread):
|
|||
'check_count': count
|
||||
})
|
||||
|
||||
|
||||
self.current_uuid = None # Done
|
||||
self.q.task_done()
|
||||
|
||||
# Send signal for watch check completion with the watch data
|
||||
if watch:
|
||||
logger.info(f"Sending watch_check_completed signal for UUID {watch['uuid']}")
|
||||
watch_check_completed.send(watch_uuid=watch['uuid'])
|
||||
|
||||
update_handler = None
|
||||
logger.debug(f"Watch {uuid} done in {time.time()-fetch_start_time:.2f}s")
|
||||
|
||||
|
||||
# Give the CPU time to interrupt
|
||||
time.sleep(0.1)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,9 @@ flask_restful
|
|||
flask_cors # For the Chrome extension to operate
|
||||
flask_wtf~=1.2
|
||||
flask~=2.3
|
||||
flask-socketio>=5.5.1
|
||||
python-socketio>=5.13.0
|
||||
python-engineio>=4.12.0
|
||||
inscriptis~=2.2
|
||||
pytz
|
||||
timeago~=1.0
|
||||
|
|
@ -115,3 +118,6 @@ psutil==7.0.0
|
|||
|
||||
ruff >= 0.11.2
|
||||
pre_commit >= 4.2.0
|
||||
|
||||
# For events between checking and socketio updates
|
||||
blinker
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue