kopia lustrzana https://github.com/dgtlmoon/changedetection.io
UI/Functionality - Ability to manage/apply filters and notifications across tags/groups
dgtlmoon
GitHub
rodzic
72311fb845
commit
52f2c00308
|
|
@ -317,25 +317,21 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
return "Access denied, bad token", 403
|
||||
|
||||
from . import diff
|
||||
limit_tag = request.args.get('tag')
|
||||
limit_tag = request.args.get('tag', '').lower().strip()
|
||||
# Be sure limit_tag is a uuid
|
||||
for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
|
||||
if limit_tag == tag.get('title', '').lower().strip():
|
||||
limit_tag = uuid
|
||||
|
||||
# Sort by last_changed and add the uuid which is usually the key..
|
||||
sorted_watches = []
|
||||
|
||||
# @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
|
||||
for uuid, watch in datastore.data['watching'].items():
|
||||
|
||||
if limit_tag != None:
|
||||
# Support for comma separated list of tags.
|
||||
for tag_in_watch in watch['tag'].split(','):
|
||||
tag_in_watch = tag_in_watch.strip()
|
||||
if tag_in_watch == limit_tag:
|
||||
watch['uuid'] = uuid
|
||||
sorted_watches.append(watch)
|
||||
|
||||
else:
|
||||
watch['uuid'] = uuid
|
||||
sorted_watches.append(watch)
|
||||
if limit_tag and not limit_tag in watch['tags']:
|
||||
continue
|
||||
watch['uuid'] = uuid
|
||||
sorted_watches.append(watch)
|
||||
|
||||
sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
|
||||
|
||||
|
|
@ -392,9 +388,17 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
@app.route("/", methods=['GET'])
|
||||
@login_optionally_required
|
||||
def index():
|
||||
global datastore
|
||||
from changedetectionio import forms
|
||||
|
||||
limit_tag = request.args.get('tag')
|
||||
limit_tag = request.args.get('tag', '').lower().strip()
|
||||
|
||||
# Be sure limit_tag is a uuid
|
||||
for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
|
||||
if limit_tag == tag.get('title', '').lower().strip():
|
||||
limit_tag = uuid
|
||||
|
||||
|
||||
# Redirect for the old rss path which used the /?rss=true
|
||||
if request.args.get('rss'):
|
||||
return redirect(url_for('rss', tag=limit_tag))
|
||||
|
|
@ -414,30 +418,15 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
sorted_watches = []
|
||||
search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
|
||||
for uuid, watch in datastore.data['watching'].items():
|
||||
|
||||
if limit_tag:
|
||||
# Support for comma separated list of tags.
|
||||
if not watch.get('tag'):
|
||||
if limit_tag and not limit_tag in watch['tags']:
|
||||
continue
|
||||
for tag_in_watch in watch.get('tag', '').split(','):
|
||||
tag_in_watch = tag_in_watch.strip()
|
||||
if tag_in_watch == limit_tag:
|
||||
watch['uuid'] = uuid
|
||||
if search_q:
|
||||
if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
|
||||
sorted_watches.append(watch)
|
||||
else:
|
||||
sorted_watches.append(watch)
|
||||
|
||||
else:
|
||||
#watch['uuid'] = uuid
|
||||
if search_q:
|
||||
if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
|
||||
sorted_watches.append(watch)
|
||||
else:
|
||||
if search_q:
|
||||
if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
|
||||
sorted_watches.append(watch)
|
||||
else:
|
||||
sorted_watches.append(watch)
|
||||
|
||||
existing_tags = datastore.get_all_tags()
|
||||
form = forms.quickWatchForm(request.form)
|
||||
page = request.args.get(get_page_parameter(), type=int, default=1)
|
||||
total_count = len(sorted_watches)
|
||||
|
|
@ -452,6 +441,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
# Don't link to hosting when we're on the hosting environment
|
||||
active_tag=limit_tag,
|
||||
app_rss_token=datastore.data['settings']['application']['rss_access_token'],
|
||||
datastore=datastore,
|
||||
form=form,
|
||||
guid=datastore.data['app_guid'],
|
||||
has_proxies=datastore.proxy_list,
|
||||
|
|
@ -463,7 +453,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
|
||||
sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
|
||||
system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
|
||||
tags=existing_tags,
|
||||
tags=datastore.data['settings']['application'].get('tags'),
|
||||
watches=sorted_watches
|
||||
)
|
||||
|
||||
|
|
@ -606,9 +596,13 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
|
||||
# proxy_override set to the json/text list of the items
|
||||
form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
|
||||
data=default,
|
||||
data=default
|
||||
)
|
||||
|
||||
# For the form widget tag uuid lookup
|
||||
form.tags.datastore = datastore # in _value
|
||||
|
||||
|
||||
form.fetch_backend.choices.append(("system", 'System settings default'))
|
||||
|
||||
# form.browser_steps[0] can be assumed that we 'goto url' first
|
||||
|
|
@ -659,6 +653,16 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
extra_update_obj['filter_text_replaced'] = True
|
||||
extra_update_obj['filter_text_removed'] = True
|
||||
|
||||
# Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
|
||||
tag_uuids = []
|
||||
if form.data.get('tags'):
|
||||
# Sometimes in testing this can be list, dont know why
|
||||
if type(form.data.get('tags')) == list:
|
||||
extra_update_obj['tags'] = form.data.get('tags')
|
||||
else:
|
||||
for t in form.data.get('tags').split(','):
|
||||
tag_uuids.append(datastore.add_tag(name=t))
|
||||
extra_update_obj['tags'] = tag_uuids
|
||||
|
||||
datastore.data['watching'][uuid].update(form.data)
|
||||
datastore.data['watching'][uuid].update(extra_update_obj)
|
||||
|
|
@ -713,7 +717,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
form=form,
|
||||
has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
|
||||
has_empty_checktime=using_default_check_time,
|
||||
has_extra_headers_file=watch.has_extra_headers_file or datastore.has_extra_headers_file,
|
||||
has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
|
||||
is_html_webdriver=is_html_webdriver,
|
||||
jq_support=jq_support,
|
||||
playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
|
||||
|
|
@ -1110,8 +1114,8 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
os.path.join(datastore_o.datastore_path, list_with_tags_file), "w"
|
||||
) as f:
|
||||
for uuid in datastore.data["watching"]:
|
||||
url = datastore.data["watching"][uuid]["url"]
|
||||
tag = datastore.data["watching"][uuid]["tag"]
|
||||
url = datastore.data["watching"][uuid].get('url')
|
||||
tag = datastore.data["watching"][uuid].get('tags', {})
|
||||
f.write("{} {}\r\n".format(url, tag))
|
||||
|
||||
# Add it to the Zip
|
||||
|
|
@ -1199,7 +1203,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
|
||||
add_paused = request.form.get('edit_and_watch_submit_button') != None
|
||||
processor = request.form.get('processor', 'text_json_diff')
|
||||
new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip(), extras={'paused': add_paused, 'processor': processor})
|
||||
new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
|
||||
|
||||
if new_uuid:
|
||||
if add_paused:
|
||||
|
|
@ -1267,9 +1271,11 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
elif tag != None:
|
||||
# Items that have this current tag
|
||||
for watch_uuid, watch in datastore.data['watching'].items():
|
||||
if (tag != None and tag in watch['tag']):
|
||||
if (tag != None and tag in watch.get('tags', {})):
|
||||
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
|
||||
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
|
||||
update_q.put(
|
||||
queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
|
||||
)
|
||||
i += 1
|
||||
|
||||
else:
|
||||
|
|
@ -1357,6 +1363,17 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
|
||||
flash("{} watches set to use default notification settings".format(len(uuids)))
|
||||
|
||||
elif (op == 'assign-tag'):
|
||||
op_extradata = request.form.get('op_extradata')
|
||||
tag_uuid = datastore.add_tag(name=op_extradata)
|
||||
if op_extradata and tag_uuid:
|
||||
for uuid in uuids:
|
||||
uuid = uuid.strip()
|
||||
if datastore.data['watching'].get(uuid):
|
||||
datastore.data['watching'][uuid]['tags'].append(tag_uuid)
|
||||
|
||||
flash("{} watches assigned tag".format(len(uuids)))
|
||||
|
||||
return redirect(url_for('index'))
|
||||
|
||||
@app.route("/api/share-url", methods=['GET'])
|
||||
|
|
@ -1366,7 +1383,6 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
the share-link can be imported/added"""
|
||||
import requests
|
||||
import json
|
||||
tag = request.args.get('tag')
|
||||
uuid = request.args.get('uuid')
|
||||
|
||||
# more for testing
|
||||
|
|
@ -1419,6 +1435,8 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
import changedetectionio.blueprint.price_data_follower as price_data_follower
|
||||
app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
|
||||
|
||||
import changedetectionio.blueprint.tags as tags
|
||||
app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
|
||||
|
||||
# @todo handle ctrl break
|
||||
ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
|
||||
|
|
|
|||
|
|
@ -218,6 +218,11 @@ class CreateWatch(Resource):
|
|||
return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
|
||||
|
||||
extras = copy.deepcopy(json_data)
|
||||
|
||||
# Because we renamed 'tag' to 'tags' but dont want to change the API (can do this in v2 of the API)
|
||||
if extras.get('tag'):
|
||||
extras['tags'] = extras.get('tag')
|
||||
|
||||
del extras['url']
|
||||
|
||||
new_uuid = self.datastore.add_watch(url=url, extras=extras)
|
||||
|
|
@ -259,13 +264,16 @@ class CreateWatch(Resource):
|
|||
"""
|
||||
list = {}
|
||||
|
||||
tag_limit = request.args.get('tag', None)
|
||||
for k, watch in self.datastore.data['watching'].items():
|
||||
if tag_limit:
|
||||
if not tag_limit.lower() in watch.all_tags:
|
||||
continue
|
||||
tag_limit = request.args.get('tag', '').lower()
|
||||
|
||||
list[k] = {'url': watch['url'],
|
||||
|
||||
for uuid, watch in self.datastore.data['watching'].items():
|
||||
# Watch tags by name (replace the other calls?)
|
||||
tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
|
||||
if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
|
||||
continue
|
||||
|
||||
list[uuid] = {'url': watch['url'],
|
||||
'title': watch['title'],
|
||||
'last_checked': watch['last_checked'],
|
||||
'last_changed': watch.last_changed,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
# Groups tags
|
||||
|
||||
## How it works
|
||||
|
||||
Watch has a list() of tag UUID's, which relate to a config under application.settings.tags
|
||||
|
||||
The 'tag' is actually a watch, because they basically will eventually share 90% of the same config.
|
||||
|
||||
So a tag is like an abstract of a watch
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
|
||||
from changedetectionio.store import ChangeDetectionStore
|
||||
from changedetectionio import login_optionally_required
|
||||
|
||||
|
||||
def construct_blueprint(datastore: ChangeDetectionStore):
|
||||
tags_blueprint = Blueprint('tags', __name__, template_folder="templates")
|
||||
|
||||
@tags_blueprint.route("/list", methods=['GET'])
|
||||
@login_optionally_required
|
||||
def tags_overview_page():
|
||||
from .form import SingleTag
|
||||
add_form = SingleTag(request.form)
|
||||
output = render_template("groups-overview.html",
|
||||
form=add_form,
|
||||
available_tags=datastore.data['settings']['application'].get('tags', {}),
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
@tags_blueprint.route("/add", methods=['POST'])
|
||||
@login_optionally_required
|
||||
def form_tag_add():
|
||||
from .form import SingleTag
|
||||
add_form = SingleTag(request.form)
|
||||
|
||||
if not add_form.validate():
|
||||
for widget, l in add_form.errors.items():
|
||||
flash(','.join(l), 'error')
|
||||
return redirect(url_for('tags.tags_overview_page'))
|
||||
|
||||
title = request.form.get('name').strip()
|
||||
|
||||
if datastore.tag_exists_by_name(title):
|
||||
flash(f'The tag "{title}" already exists', "error")
|
||||
return redirect(url_for('tags.tags_overview_page'))
|
||||
|
||||
datastore.add_tag(title)
|
||||
flash("Tag added")
|
||||
|
||||
|
||||
return redirect(url_for('tags.tags_overview_page'))
|
||||
|
||||
@tags_blueprint.route("/mute/<string:uuid>", methods=['GET'])
|
||||
@login_optionally_required
|
||||
def mute(uuid):
|
||||
if datastore.data['settings']['application']['tags'].get(uuid):
|
||||
datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
|
||||
return redirect(url_for('tags.tags_overview_page'))
|
||||
|
||||
@tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
|
||||
@login_optionally_required
|
||||
def delete(uuid):
|
||||
removed = 0
|
||||
# Delete the tag, and any tag reference
|
||||
if datastore.data['settings']['application']['tags'].get(uuid):
|
||||
del datastore.data['settings']['application']['tags'][uuid]
|
||||
|
||||
for watch_uuid, watch in datastore.data['watching'].items():
|
||||
if watch.get('tags') and uuid in watch['tags']:
|
||||
removed += 1
|
||||
watch['tags'].remove(uuid)
|
||||
|
||||
flash(f"Tag deleted and removed from {removed} watches")
|
||||
return redirect(url_for('tags.tags_overview_page'))
|
||||
|
||||
@tags_blueprint.route("/unlink/<string:uuid>", methods=['GET'])
|
||||
@login_optionally_required
|
||||
def unlink(uuid):
|
||||
unlinked = 0
|
||||
for watch_uuid, watch in datastore.data['watching'].items():
|
||||
if watch.get('tags') and uuid in watch['tags']:
|
||||
unlinked += 1
|
||||
watch['tags'].remove(uuid)
|
||||
|
||||
flash(f"Tag unlinked removed from {unlinked} watches")
|
||||
return redirect(url_for('tags.tags_overview_page'))
|
||||
|
||||
@tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
|
||||
@login_optionally_required
|
||||
def form_tag_edit(uuid):
|
||||
from changedetectionio import forms
|
||||
|
||||
if uuid == 'first':
|
||||
uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
|
||||
|
||||
default = datastore.data['settings']['application']['tags'].get(uuid)
|
||||
|
||||
form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
|
||||
data=default,
|
||||
)
|
||||
form.datastore=datastore # needed?
|
||||
|
||||
output = render_template("edit-tag.html",
|
||||
data=default,
|
||||
form=form,
|
||||
settings_application=datastore.data['settings']['application'],
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
@tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
|
||||
@login_optionally_required
|
||||
def form_tag_edit_submit(uuid):
|
||||
from changedetectionio import forms
|
||||
if uuid == 'first':
|
||||
uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
|
||||
|
||||
default = datastore.data['settings']['application']['tags'].get(uuid)
|
||||
|
||||
form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
|
||||
data=default,
|
||||
)
|
||||
# @todo subclass form so validation works
|
||||
#if not form.validate():
|
||||
# for widget, l in form.errors.items():
|
||||
# flash(','.join(l), 'error')
|
||||
# return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
|
||||
|
||||
datastore.data['settings']['application']['tags'][uuid].update(form.data)
|
||||
datastore.needs_write_urgent = True
|
||||
flash("Updated")
|
||||
|
||||
return redirect(url_for('tags.tags_overview_page'))
|
||||
|
||||
|
||||
@tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
|
||||
def form_tag_delete(uuid):
|
||||
return redirect(url_for('tags.tags_overview_page'))
|
||||
return tags_blueprint
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
from wtforms import (
|
||||
BooleanField,
|
||||
Form,
|
||||
IntegerField,
|
||||
RadioField,
|
||||
SelectField,
|
||||
StringField,
|
||||
SubmitField,
|
||||
TextAreaField,
|
||||
validators,
|
||||
)
|
||||
|
||||
|
||||
|
||||
class SingleTag(Form):
|
||||
|
||||
name = StringField('Tag name', [validators.InputRequired()], render_kw={"placeholder": "Name"})
|
||||
save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
{% extends 'base.html' %}
|
||||
{% block content %}
|
||||
{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
|
||||
{% from '_common_fields.jinja' import render_common_settings_form %}
|
||||
<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
|
||||
<script>
|
||||
|
||||
/*{% if emailprefix %}*/
|
||||
/*const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');*/
|
||||
/*{% endif %}*/
|
||||
|
||||
|
||||
</script>
|
||||
|
||||
<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
|
||||
<!--<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>-->
|
||||
<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
|
||||
|
||||
<div class="edit-form monospaced-textarea">
|
||||
|
||||
<div class="tabs collapsable">
|
||||
<ul>
|
||||
<li class="tab" id=""><a href="#general">General</a></li>
|
||||
<li class="tab"><a href="#filters-and-triggers">Filters & Triggers</a></li>
|
||||
<li class="tab"><a href="#notifications">Notifications</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="box-wrap inner">
|
||||
<form class="pure-form pure-form-stacked"
|
||||
action="{{ url_for('tags.form_tag_edit', uuid=data.uuid) }}" method="POST">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
|
||||
<div class="tab-pane-inner" id="general">
|
||||
<fieldset>
|
||||
<div class="pure-control-group">
|
||||
{{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }}
|
||||
</div>
|
||||
</fieldset>
|
||||
</div>
|
||||
|
||||
<div class="tab-pane-inner" id="filters-and-triggers">
|
||||
<div class="pure-control-group">
|
||||
{% set field = render_field(form.include_filters,
|
||||
rows=5,
|
||||
placeholder="#example
|
||||
xpath://body/div/span[contains(@class, 'example-class')]",
|
||||
class="m-d")
|
||||
%}
|
||||
{{ field }}
|
||||
{% if '/text()' in field %}
|
||||
<span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the <element> contains <![CDATA[]]></strong></span><br>
|
||||
{% endif %}
|
||||
<span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
|
||||
|
||||
<ul>
|
||||
<li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
|
||||
<li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
|
||||
<ul>
|
||||
<li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required, <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
|
||||
{% if jq_support %}
|
||||
<li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
|
||||
{% else %}
|
||||
<li>jq support not installed</li>
|
||||
{% endif %}
|
||||
</ul>
|
||||
</li>
|
||||
<li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
|
||||
<ul>
|
||||
<li>Example: <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
|
||||
href="http://xpather.com/" target="new">test your XPath here</a></li>
|
||||
<li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
|
||||
href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
|
||||
</span>
|
||||
</div>
|
||||
<fieldset class="pure-control-group">
|
||||
{{ render_field(form.subtractive_selectors, rows=5, placeholder="header
|
||||
footer
|
||||
nav
|
||||
.stockticker") }}
|
||||
<span class="pure-form-message-inline">
|
||||
<ul>
|
||||
<li> Remove HTML element(s) by CSS selector before text conversion. </li>
|
||||
<li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
|
||||
</ul>
|
||||
</span>
|
||||
</fieldset>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="tab-pane-inner" id="notifications">
|
||||
<fieldset>
|
||||
<div class="pure-control-group inline-radio">
|
||||
{{ render_checkbox_field(form.notification_muted) }}
|
||||
</div>
|
||||
{% if is_html_webdriver %}
|
||||
<div class="pure-control-group inline-radio">
|
||||
{{ render_checkbox_field(form.notification_screenshot) }}
|
||||
<span class="pure-form-message-inline">
|
||||
<strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
|
||||
</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="field-group" id="notification-field-group">
|
||||
{% if has_default_notification_urls %}
|
||||
<div class="inline-warning">
|
||||
<img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
|
||||
There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications.
|
||||
</div>
|
||||
{% endif %}
|
||||
<a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
|
||||
|
||||
{{ render_common_settings_form(form, emailprefix, settings_application) }}
|
||||
</div>
|
||||
</fieldset>
|
||||
</div>
|
||||
|
||||
<div id="actions">
|
||||
<div class="pure-control-group">
|
||||
{{ render_button(form.save_button) }}
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
{% extends 'base.html' %}
|
||||
{% block content %}
|
||||
{% from '_helpers.jinja' import render_simple_field, render_field %}
|
||||
<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
|
||||
|
||||
<div class="box">
|
||||
<form class="pure-form" action="{{ url_for('tags.form_tag_add') }}" method="POST" id="new-watch-form">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
|
||||
<fieldset>
|
||||
<legend>Add a new organisational tag</legend>
|
||||
<div id="watch-add-wrapper-zone">
|
||||
<div>
|
||||
{{ render_simple_field(form.name, placeholder="watch label / tag") }}
|
||||
</div>
|
||||
<div>
|
||||
{{ render_simple_field(form.save_button, title="Save" ) }}
|
||||
</div>
|
||||
</div>
|
||||
<br>
|
||||
<div style="color: #fff;">Groups allows you to manage filters and notifications for multiple watches under a single organisational tag.</div>
|
||||
</fieldset>
|
||||
</form>
|
||||
<!-- @todo maybe some overview matrix, 'tick' with which has notification, filter rules etc -->
|
||||
<div id="watch-table-wrapper">
|
||||
|
||||
<table class="pure-table pure-table-striped watch-table group-overview-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
<th>Tag / Label name</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<!--
|
||||
@Todo - connect Last checked, Last Changed, Number of Watches etc
|
||||
--->
|
||||
{% if not available_tags|length %}
|
||||
<tr>
|
||||
<td colspan="3">No website organisational tags/groups configured</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% for uuid, tag in available_tags.items() %}
|
||||
<tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
|
||||
<td class="watch-controls">
|
||||
<a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
|
||||
</td>
|
||||
<td class="title-col inline">{{tag.title}}</td>
|
||||
<td>
|
||||
<a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>
|
||||
<a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>
|
||||
<a class="pure-button pure-button-primary" href="{{ url_for('tags.unlink', uuid=uuid) }}" title="Keep the tag but unlink any watches">Unlink</a>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
|
@ -28,6 +28,8 @@ from changedetectionio.notification import (
|
|||
|
||||
from wtforms.fields import FormField
|
||||
|
||||
dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
|
||||
|
||||
valid_method = {
|
||||
'GET',
|
||||
'POST',
|
||||
|
|
@ -90,6 +92,29 @@ class SaltyPasswordField(StringField):
|
|||
else:
|
||||
self.data = False
|
||||
|
||||
class StringTagUUID(StringField):
|
||||
|
||||
# process_formdata(self, valuelist) handled manually in POST handler
|
||||
|
||||
# Is what is shown when field <input> is rendered
|
||||
def _value(self):
|
||||
# Tag UUID to name, on submit it will convert it back (in the submit handler of init.py)
|
||||
if self.data and type(self.data) is list:
|
||||
tag_titles = []
|
||||
for i in self.data:
|
||||
tag = self.datastore.data['settings']['application']['tags'].get(i)
|
||||
if tag:
|
||||
tag_title = tag.get('title')
|
||||
if tag_title:
|
||||
tag_titles.append(tag_title)
|
||||
|
||||
return ', '.join(tag_titles)
|
||||
|
||||
if not self.data:
|
||||
return ''
|
||||
|
||||
return 'error'
|
||||
|
||||
class TimeBetweenCheckForm(Form):
|
||||
weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
|
||||
days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
|
||||
|
|
@ -347,7 +372,7 @@ class quickWatchForm(Form):
|
|||
from . import processors
|
||||
|
||||
url = fields.URLField('URL', validators=[validateURL()])
|
||||
tag = StringField('Group tag', [validators.Optional()])
|
||||
tags = StringTagUUID('Group tag', [validators.Optional()])
|
||||
watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"})
|
||||
processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
|
||||
edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
|
||||
|
|
@ -355,6 +380,7 @@ class quickWatchForm(Form):
|
|||
|
||||
# Common to a single watch and the global settings
|
||||
class commonSettingsForm(Form):
|
||||
|
||||
notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
|
||||
notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
|
||||
notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
|
||||
|
|
@ -382,7 +408,7 @@ class SingleBrowserStep(Form):
|
|||
class watchForm(commonSettingsForm):
|
||||
|
||||
url = fields.URLField('URL', validators=[validateURL()])
|
||||
tag = StringField('Group tag', [validators.Optional()], default='')
|
||||
tags = StringTagUUID('Group tag', [validators.Optional()], default='')
|
||||
|
||||
time_between_check = FormField(TimeBetweenCheckForm)
|
||||
|
||||
|
|
|
|||
|
|
@ -120,9 +120,9 @@ class import_distill_io_json(Importer):
|
|||
except IndexError:
|
||||
pass
|
||||
|
||||
|
||||
# Does this need to be here anymore?
|
||||
if d.get('tags', False):
|
||||
extras['tag'] = ", ".join(d['tags'])
|
||||
extras['tags'] = ", ".join(d['tags'])
|
||||
|
||||
new_uuid = datastore.add_watch(url=d['uri'].strip(),
|
||||
extras=extras,
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ class model(dict):
|
|||
'schema_version' : 0,
|
||||
'shared_diff_access': False,
|
||||
'webdriver_delay': None , # Extra delay in seconds before extracting text
|
||||
'tags': {} #@todo use Tag.model initialisers
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
from .Watch import base_config
|
||||
import uuid
|
||||
|
||||
class model(dict):
|
||||
|
||||
def __init__(self, *arg, **kw):
|
||||
|
||||
self.update(base_config)
|
||||
|
||||
self['uuid'] = str(uuid.uuid4())
|
||||
|
||||
if kw.get('default'):
|
||||
self.update(kw['default'])
|
||||
del kw['default']
|
||||
|
||||
|
||||
# Goes at the end so we update the default object with the initialiser
|
||||
super(model, self).__init__(*arg, **kw)
|
||||
|
||||
|
|
@ -52,7 +52,8 @@ base_config = {
|
|||
'previous_md5_before_filters': False, # Used for skipping changedetection entirely
|
||||
'proxy': None, # Preferred proxy connection
|
||||
'subtractive_selectors': [],
|
||||
'tag': None,
|
||||
'tag': '', # Old system of text name for a tag, to be removed
|
||||
'tags': [], # list of UUIDs to App.Tags
|
||||
'text_should_not_be_present': [], # Text that should not present
|
||||
# Re #110, so then if this is set to None, we know to use the default value instead
|
||||
# Requires setting to None on submit if it's the same as the default
|
||||
|
|
@ -455,10 +456,6 @@ class model(dict):
|
|||
|
||||
return csv_output_filename
|
||||
|
||||
@property
|
||||
# Return list of tags, stripped and lowercase, used for searching
|
||||
def all_tags(self):
|
||||
return [s.strip().lower() for s in self.get('tag','').split(',')]
|
||||
|
||||
def has_special_diff_filter_options_set(self):
|
||||
|
||||
|
|
@ -473,40 +470,6 @@ class model(dict):
|
|||
# None is set
|
||||
return False
|
||||
|
||||
@property
|
||||
def has_extra_headers_file(self):
|
||||
if os.path.isfile(os.path.join(self.watch_data_dir, 'headers.txt')):
|
||||
return True
|
||||
|
||||
for f in self.all_tags:
|
||||
fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt"
|
||||
filepath = os.path.join(self.__datastore_path, fname)
|
||||
if os.path.isfile(filepath):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def get_all_headers(self):
|
||||
from .App import parse_headers_from_text_file
|
||||
headers = self.get('headers', {}).copy()
|
||||
# Available headers on the disk could 'headers.txt' in the watch data dir
|
||||
filepath = os.path.join(self.watch_data_dir, 'headers.txt')
|
||||
try:
|
||||
if os.path.isfile(filepath):
|
||||
headers.update(parse_headers_from_text_file(filepath))
|
||||
except Exception as e:
|
||||
print(f"ERROR reading headers.txt at {filepath}", str(e))
|
||||
|
||||
# Or each by tag, as tagname.txt in the main datadir
|
||||
for f in self.all_tags:
|
||||
fname = "headers-"+re.sub(r'[\W_]', '', f).lower().strip() + ".txt"
|
||||
filepath = os.path.join(self.__datastore_path, fname)
|
||||
try:
|
||||
if os.path.isfile(filepath):
|
||||
headers.update(parse_headers_from_text_file(filepath))
|
||||
except Exception as e:
|
||||
print(f"ERROR reading headers.txt at {filepath}", str(e))
|
||||
return headers
|
||||
|
||||
def get_last_fetched_before_filters(self):
|
||||
import brotli
|
||||
|
|
|
|||
|
|
@ -186,8 +186,13 @@ def create_notification_parameters(n_object, datastore):
|
|||
uuid = n_object['uuid'] if 'uuid' in n_object else ''
|
||||
|
||||
if uuid != '':
|
||||
watch_title = datastore.data['watching'][uuid]['title']
|
||||
watch_tag = datastore.data['watching'][uuid]['tag']
|
||||
watch_title = datastore.data['watching'][uuid].get('title', '')
|
||||
tag_list = []
|
||||
tags = datastore.get_all_tags_for_watch(uuid)
|
||||
if tags:
|
||||
for tag_uuid, tag in tags.items():
|
||||
tag_list.append(tag.get('title'))
|
||||
watch_tag = ', '.join(tag_list)
|
||||
else:
|
||||
watch_title = 'Change Detection'
|
||||
watch_tag = ''
|
||||
|
|
|
|||
|
|
@ -42,11 +42,10 @@ class perform_site_check(difference_detection_processor):
|
|||
|
||||
# Unset any existing notification error
|
||||
update_obj = {'last_notification_error': False, 'last_error': False}
|
||||
extra_headers = watch.get('headers', [])
|
||||
|
||||
# Tweak the base config with the per-watch ones
|
||||
request_headers = deepcopy(self.datastore.data['settings']['headers'])
|
||||
request_headers.update(extra_headers)
|
||||
request_headers = watch.get('headers', [])
|
||||
request_headers.update(self.datastore.get_all_base_headers())
|
||||
request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
|
||||
|
||||
# https://github.com/psf/requests/issues/4525
|
||||
# Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
|
||||
|
|
|
|||
|
|
@ -57,7 +57,6 @@ class perform_site_check(difference_detection_processor):
|
|||
|
||||
# DeepCopy so we can be sure we don't accidently change anything by reference
|
||||
watch = deepcopy(self.datastore.data['watching'].get(uuid))
|
||||
|
||||
if not watch:
|
||||
raise Exception("Watch no longer exists.")
|
||||
|
||||
|
|
@ -71,9 +70,9 @@ class perform_site_check(difference_detection_processor):
|
|||
update_obj = {'last_notification_error': False, 'last_error': False}
|
||||
|
||||
# Tweak the base config with the per-watch ones
|
||||
extra_headers = watch.get_all_headers()
|
||||
request_headers = self.datastore.get_all_headers()
|
||||
request_headers.update(extra_headers)
|
||||
request_headers = watch.get('headers', [])
|
||||
request_headers.update(self.datastore.get_all_base_headers())
|
||||
request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
|
||||
|
||||
# https://github.com/psf/requests/issues/4525
|
||||
# Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
|
||||
|
|
@ -191,21 +190,23 @@ class perform_site_check(difference_detection_processor):
|
|||
|
||||
fetcher.content = fetcher.content.replace('</body>', metadata + '</body>')
|
||||
|
||||
# Better would be if Watch.model could access the global data also
|
||||
# and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
|
||||
# https://realpython.com/inherit-python-dict/ instead of doing it procedurely
|
||||
include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
|
||||
include_filters_rule = [*watch.get('include_filters', []), *include_filters_from_tags]
|
||||
|
||||
include_filters_rule = deepcopy(watch.get('include_filters', []))
|
||||
# include_filters_rule = watch['include_filters']
|
||||
subtractive_selectors = watch.get(
|
||||
"subtractive_selectors", []
|
||||
) + self.datastore.data["settings"]["application"].get(
|
||||
"global_subtractive_selectors", []
|
||||
)
|
||||
subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
|
||||
*watch.get("subtractive_selectors", []),
|
||||
*self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
|
||||
]
|
||||
|
||||
# Inject a virtual LD+JSON price tracker rule
|
||||
if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
|
||||
include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
|
||||
|
||||
has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
|
||||
has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
|
||||
has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
|
||||
has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
|
||||
|
||||
if is_json and not has_filter_rule:
|
||||
include_filters_rule.append("json:$")
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ $(function () {
|
|||
$(this).closest('.unviewed').removeClass('unviewed');
|
||||
});
|
||||
|
||||
$("#checkbox-assign-tag").click(function (e) {
|
||||
$('#op_extradata').val(prompt("Enter a tag name"));
|
||||
});
|
||||
|
||||
$('.with-share-link > *').click(function () {
|
||||
$("#copied-clipboard").remove();
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ import threading
|
|||
import time
|
||||
import uuid as uuid_builder
|
||||
|
||||
dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
|
||||
|
||||
# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
|
||||
# Open a github issue if you know something :)
|
||||
# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
|
||||
|
|
@ -178,20 +180,6 @@ class ChangeDetectionStore:
|
|||
|
||||
return self.__data
|
||||
|
||||
def get_all_tags(self):
|
||||
tags = []
|
||||
for uuid, watch in self.data['watching'].items():
|
||||
if watch['tag'] is None:
|
||||
continue
|
||||
# Support for comma separated list of tags.
|
||||
for tag in watch['tag'].split(','):
|
||||
tag = tag.strip()
|
||||
if tag not in tags:
|
||||
tags.append(tag)
|
||||
|
||||
tags.sort()
|
||||
return tags
|
||||
|
||||
# Delete a single watch by UUID
|
||||
def delete(self, uuid):
|
||||
import pathlib
|
||||
|
|
@ -218,9 +206,9 @@ class ChangeDetectionStore:
|
|||
# Clone a watch by UUID
|
||||
def clone(self, uuid):
|
||||
url = self.data['watching'][uuid]['url']
|
||||
tag = self.data['watching'][uuid]['tag']
|
||||
tag = self.data['watching'][uuid].get('tags',[])
|
||||
extras = self.data['watching'][uuid]
|
||||
new_uuid = self.add_watch(url=url, tag=tag, extras=extras)
|
||||
new_uuid = self.add_watch(url=url, tag_uuids=tag, extras=extras)
|
||||
return new_uuid
|
||||
|
||||
def url_exists(self, url):
|
||||
|
|
@ -255,10 +243,11 @@ class ChangeDetectionStore:
|
|||
|
||||
self.needs_write_urgent = True
|
||||
|
||||
def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
|
||||
def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
|
||||
|
||||
if extras is None:
|
||||
extras = {}
|
||||
|
||||
# should always be str
|
||||
if tag is None or not tag:
|
||||
tag = ''
|
||||
|
|
@ -291,6 +280,7 @@ class ChangeDetectionStore:
|
|||
'processor',
|
||||
'subtractive_selectors',
|
||||
'tag',
|
||||
'tags',
|
||||
'text_should_not_be_present',
|
||||
'title',
|
||||
'trigger_text',
|
||||
|
|
@ -313,25 +303,34 @@ class ChangeDetectionStore:
|
|||
flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
|
||||
return None
|
||||
|
||||
with self.lock:
|
||||
# #Re 569
|
||||
new_watch = Watch.model(datastore_path=self.datastore_path, default={
|
||||
'url': url,
|
||||
'tag': tag,
|
||||
'date_created': int(time.time())
|
||||
})
|
||||
|
||||
new_uuid = new_watch['uuid']
|
||||
logging.debug("Added URL {} - {}".format(url, new_uuid))
|
||||
# #Re 569
|
||||
# Could be in 'tags', var or extras, smash them together and strip
|
||||
apply_extras['tags'] = []
|
||||
if tag or extras.get('tags'):
|
||||
tags = list(filter(None, list(set().union(tag.split(','), extras.get('tags', '').split(',')))))
|
||||
for t in list(map(str.strip, tags)):
|
||||
# for each stripped tag, add tag as UUID
|
||||
apply_extras['tags'].append(self.add_tag(t))
|
||||
|
||||
for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
|
||||
if k in apply_extras:
|
||||
del apply_extras[k]
|
||||
# Or if UUIDs given directly
|
||||
if tag_uuids:
|
||||
apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
|
||||
|
||||
new_watch.update(apply_extras)
|
||||
self.__data['watching'][new_uuid] = new_watch
|
||||
new_watch = Watch.model(datastore_path=self.datastore_path, url=url)
|
||||
|
||||
new_uuid = new_watch.get('uuid')
|
||||
|
||||
logging.debug("Added URL {} - {}".format(url, new_uuid))
|
||||
|
||||
for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
|
||||
if k in apply_extras:
|
||||
del apply_extras[k]
|
||||
|
||||
new_watch.update(apply_extras)
|
||||
new_watch.ensure_data_dir_exists()
|
||||
self.__data['watching'][new_uuid] = new_watch
|
||||
|
||||
self.__data['watching'][new_uuid].ensure_data_dir_exists()
|
||||
|
||||
if write_to_disk_now:
|
||||
self.sync_to_json()
|
||||
|
|
@ -511,10 +510,19 @@ class ChangeDetectionStore:
|
|||
filepath = os.path.join(self.datastore_path, 'headers.txt')
|
||||
return os.path.isfile(filepath)
|
||||
|
||||
def get_all_headers(self):
|
||||
def get_all_base_headers(self):
|
||||
from .model.App import parse_headers_from_text_file
|
||||
headers = copy(self.data['settings'].get('headers', {}))
|
||||
headers = {}
|
||||
# Global app settings
|
||||
headers.update(self.data['settings'].get('headers', {}))
|
||||
|
||||
return headers
|
||||
|
||||
def get_all_headers_in_textfile_for_watch(self, uuid):
|
||||
from .model.App import parse_headers_from_text_file
|
||||
headers = {}
|
||||
|
||||
# Global in /datastore/headers.txt
|
||||
filepath = os.path.join(self.datastore_path, 'headers.txt')
|
||||
try:
|
||||
if os.path.isfile(filepath):
|
||||
|
|
@ -522,8 +530,76 @@ class ChangeDetectionStore:
|
|||
except Exception as e:
|
||||
print(f"ERROR reading headers.txt at {filepath}", str(e))
|
||||
|
||||
watch = self.data['watching'].get(uuid)
|
||||
if watch:
|
||||
|
||||
# In /datastore/xyz-xyz/headers.txt
|
||||
filepath = os.path.join(watch.watch_data_dir, 'headers.txt')
|
||||
try:
|
||||
if os.path.isfile(filepath):
|
||||
headers.update(parse_headers_from_text_file(filepath))
|
||||
except Exception as e:
|
||||
print(f"ERROR reading headers.txt at {filepath}", str(e))
|
||||
|
||||
# In /datastore/tag-name.txt
|
||||
tags = self.get_all_tags_for_watch(uuid=uuid)
|
||||
for tag_uuid, tag in tags.items():
|
||||
fname = "headers-"+re.sub(r'[\W_]', '', tag.get('title')).lower().strip() + ".txt"
|
||||
filepath = os.path.join(self.datastore_path, fname)
|
||||
try:
|
||||
if os.path.isfile(filepath):
|
||||
headers.update(parse_headers_from_text_file(filepath))
|
||||
except Exception as e:
|
||||
print(f"ERROR reading headers.txt at {filepath}", str(e))
|
||||
|
||||
return headers
|
||||
|
||||
def get_tag_overrides_for_watch(self, uuid, attr):
|
||||
tags = self.get_all_tags_for_watch(uuid=uuid)
|
||||
ret = []
|
||||
|
||||
if tags:
|
||||
for tag_uuid, tag in tags.items():
|
||||
if attr in tag and tag[attr]:
|
||||
ret=[*ret, *tag[attr]]
|
||||
|
||||
return ret
|
||||
|
||||
def add_tag(self, name):
|
||||
print (">>> Adding new tag -", name)
|
||||
# If name exists, return that
|
||||
n = name.strip().lower()
|
||||
for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():
|
||||
if n == tag.get('title', '').lower().strip():
|
||||
print (f">>> Tag {name} already exists")
|
||||
return uuid
|
||||
|
||||
# Eventually almost everything todo with a watch will apply as a Tag
|
||||
# So we use the same model as a Watch
|
||||
with self.lock:
|
||||
new_tag = Watch.model(datastore_path=self.datastore_path, default={
|
||||
'title': name.strip(),
|
||||
'date_created': int(time.time())
|
||||
})
|
||||
|
||||
new_uuid = new_tag.get('uuid')
|
||||
|
||||
self.__data['settings']['application']['tags'][new_uuid] = new_tag
|
||||
|
||||
return new_uuid
|
||||
|
||||
def get_all_tags_for_watch(self, uuid):
|
||||
"""This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet"""
|
||||
watch = self.data['watching'].get(uuid)
|
||||
|
||||
# Should return a dict of full tag info linked by UUID
|
||||
if watch:
|
||||
return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', []))
|
||||
|
||||
return {}
|
||||
|
||||
def tag_exists_by_name(self, tag_name):
|
||||
return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
|
||||
|
||||
# Run all updates
|
||||
# IMPORTANT - Each update could be run even when they have a new install and the schema is correct
|
||||
|
|
@ -710,3 +786,16 @@ class ChangeDetectionStore:
|
|||
i+=1
|
||||
return
|
||||
|
||||
# Create tag objects and their references from existing tag text
|
||||
def update_12(self):
|
||||
i = 0
|
||||
for uuid, watch in self.data['watching'].items():
|
||||
# Split out and convert old tag string
|
||||
tag = watch.get('tag')
|
||||
if tag:
|
||||
tag_uuids = []
|
||||
for t in tag.split(','):
|
||||
tag_uuids.append(self.add_tag(name=t))
|
||||
|
||||
self.data['watching'][uuid]['tags'] = tag_uuids
|
||||
|
||||
|
|
|
|||
|
|
@ -58,6 +58,9 @@
|
|||
{% if current_user.is_authenticated or not has_password %}
|
||||
{% if not
|
||||
current_diff_url %}
|
||||
<li class="pure-menu-item">
|
||||
<a href="{{ url_for('tags.tags_overview_page')}}" class="pure-menu-link">GROUPS</a>
|
||||
</li>
|
||||
<li class="pure-menu-item">
|
||||
<a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
|
||||
</li>
|
||||
|
|
@ -86,7 +89,7 @@
|
|||
<!-- We use GET here so it offers people a chance to set bookmarks etc -->
|
||||
<form name="searchForm" action="" method="GET">
|
||||
<input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
|
||||
<input name="tag" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
|
||||
<input name="tags" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
|
||||
<button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
|
||||
{% include "svgs/search-icon.svg" %}
|
||||
</button>
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@
|
|||
{{ render_field(form.title, class="m-d") }}
|
||||
</div>
|
||||
<div class="pure-control-group">
|
||||
{{ render_field(form.tag) }}
|
||||
{{ render_field(form.tags) }}
|
||||
<span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
|
||||
</div>
|
||||
<div class="pure-control-group">
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
<div id="watch-add-wrapper-zone">
|
||||
<div>
|
||||
{{ render_simple_field(form.url, placeholder="https://...", required=true) }}
|
||||
{{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
|
||||
{{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
|
||||
</div>
|
||||
<div>
|
||||
{{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
|
||||
|
|
@ -30,12 +30,14 @@
|
|||
|
||||
<form class="pure-form" action="{{ url_for('form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
|
||||
<input type="hidden" id="op_extradata" name="op_extradata" value="" >
|
||||
<div id="checkbox-operations">
|
||||
<button class="pure-button button-secondary button-xsmall" name="op" value="pause">Pause</button>
|
||||
<button class="pure-button button-secondary button-xsmall" name="op" value="unpause">UnPause</button>
|
||||
<button class="pure-button button-secondary button-xsmall" name="op" value="mute">Mute</button>
|
||||
<button class="pure-button button-secondary button-xsmall" name="op" value="unmute">UnMute</button>
|
||||
<button class="pure-button button-secondary button-xsmall" name="op" value="recheck">Recheck</button>
|
||||
<button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag">Tag</button>
|
||||
<button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
|
||||
<button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
|
||||
<button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
|
||||
|
|
@ -47,9 +49,9 @@
|
|||
{% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
|
||||
<div>
|
||||
<a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
|
||||
{% for tag in tags %}
|
||||
{% for uuid, tag in tags.items() %}
|
||||
{% if tag != "" %}
|
||||
<a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
|
||||
<a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag == uuid }}">{{ tag.title }}</a>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
|
@ -143,9 +145,11 @@
|
|||
</span>
|
||||
{% endif %}
|
||||
|
||||
{% if not active_tag %}
|
||||
<span class="watch-tag-list">{{ watch.tag}}</span>
|
||||
{% endif %}
|
||||
|
||||
{% for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() %}
|
||||
<span class="watch-tag-list">{{ watch_tag.title }}</span>
|
||||
{% endfor %}
|
||||
|
||||
</td>
|
||||
<td class="last-checked">{{watch|format_last_checked_time|safe}}</td>
|
||||
<td class="last-changed">{% if watch.history_n >=2 and watch.last_changed >0 %}
|
||||
|
|
@ -178,7 +182,7 @@
|
|||
{% endif %}
|
||||
<li>
|
||||
<a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
|
||||
all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
|
||||
all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ def test_preferred_proxy(client, live_server):
|
|||
"fetch_backend": "html_requests",
|
||||
"headers": "",
|
||||
"proxy": "proxy-two",
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"url": url,
|
||||
},
|
||||
follow_redirects=True
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ def test_restock_detection(client, live_server):
|
|||
|
||||
client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": test_url, "tag": '', 'processor': 'restock_diff'},
|
||||
data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,15 @@ def test_check_access_control(app, client, live_server):
|
|||
res = client.get(url_for("diff_history_page", uuid="first"))
|
||||
assert b'Random content' in res.data
|
||||
|
||||
# Check wrong password does not let us in
|
||||
res = c.post(
|
||||
url_for("login"),
|
||||
data={"password": "WRONG PASSWORD"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"LOG OUT" not in res.data
|
||||
assert b"Incorrect password" in res.data
|
||||
|
||||
|
||||
# Menu should not be available yet
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import time
|
||||
from flask import url_for
|
||||
from .util import live_server_setup
|
||||
from .util import live_server_setup, wait_for_all_checks
|
||||
from changedetectionio import html_tools
|
||||
|
||||
|
||||
|
|
@ -39,7 +39,6 @@ def test_setup(client, live_server):
|
|||
live_server_setup(live_server)
|
||||
|
||||
def test_check_removed_line_contains_trigger(client, live_server):
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
# Give the endpoint time to spin up
|
||||
time.sleep(1)
|
||||
|
|
@ -54,7 +53,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
|
|||
assert b"1 Imported" in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Goto the edit page, add our ignore text
|
||||
# Add our URL to the import page
|
||||
|
|
@ -67,20 +66,20 @@ def test_check_removed_line_contains_trigger(client, live_server):
|
|||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
set_original(excluding='Something irrelevant')
|
||||
|
||||
# A line thats not the trigger should not trigger anything
|
||||
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
assert b'1 watches queued for rechecking.' in res.data
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' not in res.data
|
||||
|
||||
# The trigger line is REMOVED, this should trigger
|
||||
set_original(excluding='The golden line')
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' in res.data
|
||||
|
||||
|
|
@ -89,14 +88,14 @@ def test_check_removed_line_contains_trigger(client, live_server):
|
|||
client.get(url_for("mark_all_viewed"), follow_redirects=True)
|
||||
set_original(excluding=None)
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' not in res.data
|
||||
|
||||
# Remove it again, and we should get a trigger
|
||||
set_original(excluding='The golden line')
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' in res.data
|
||||
|
||||
|
|
@ -105,8 +104,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
|
|||
|
||||
|
||||
def test_check_add_line_contains_trigger(client, live_server):
|
||||
|
||||
sleep_time_for_fetch_thread = 3
|
||||
#live_server_setup(live_server)
|
||||
|
||||
# Give the endpoint time to spin up
|
||||
time.sleep(1)
|
||||
|
|
@ -136,8 +134,7 @@ def test_check_add_line_contains_trigger(client, live_server):
|
|||
assert b"1 Imported" in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
|
||||
wait_for_all_checks(client)
|
||||
# Goto the edit page, add our ignore text
|
||||
# Add our URL to the import page
|
||||
res = client.post(
|
||||
|
|
@ -150,23 +147,25 @@ def test_check_add_line_contains_trigger(client, live_server):
|
|||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
set_original(excluding='Something irrelevant')
|
||||
|
||||
# A line thats not the trigger should not trigger anything
|
||||
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
assert b'1 watches queued for rechecking.' in res.data
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' not in res.data
|
||||
|
||||
# The trigger line is ADDED, this should trigger
|
||||
set_original(add_line='<p>Oh yes please</p>')
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' in res.data
|
||||
|
||||
# Takes a moment for apprise to fire
|
||||
time.sleep(3)
|
||||
with open("test-datastore/notification.txt", 'r') as f:
|
||||
response= f.read()
|
||||
assert '-Oh yes please-' in response
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import time
|
||||
from flask import url_for
|
||||
from .util import live_server_setup, extract_api_key_from_UI
|
||||
from .util import live_server_setup, extract_api_key_from_UI, wait_for_all_checks
|
||||
|
||||
import json
|
||||
import uuid
|
||||
|
|
@ -57,6 +57,7 @@ def test_setup(client, live_server):
|
|||
live_server_setup(live_server)
|
||||
|
||||
def test_api_simple(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
|
||||
api_key = extract_api_key_from_UI(client)
|
||||
|
||||
|
|
@ -86,7 +87,7 @@ def test_api_simple(client, live_server):
|
|||
watch_uuid = res.json.get('uuid')
|
||||
assert res.status_code == 201
|
||||
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Verify its in the list and that recheck worked
|
||||
res = client.get(
|
||||
|
|
@ -107,7 +108,7 @@ def test_api_simple(client, live_server):
|
|||
)
|
||||
assert len(res.json) == 0
|
||||
|
||||
time.sleep(2)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
set_modified_response()
|
||||
# Trigger recheck of all ?recheck_all=1
|
||||
|
|
@ -115,7 +116,7 @@ def test_api_simple(client, live_server):
|
|||
url_for("createwatch", recheck_all='1'),
|
||||
headers={'x-api-key': api_key},
|
||||
)
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Did the recheck fire?
|
||||
res = client.get(
|
||||
|
|
@ -297,6 +298,8 @@ def test_api_watch_PUT_update(client, live_server):
|
|||
url_for("edit_page", uuid=watch_uuid),
|
||||
)
|
||||
assert b"cookie: yum" in res.data, "'cookie: yum' found in 'headers' section"
|
||||
assert b"One" in res.data, "Tag 'One' was found"
|
||||
assert b"Two" in res.data, "Tag 'Two' was found"
|
||||
|
||||
# HTTP PUT ( UPDATE an existing watch )
|
||||
res = client.put(
|
||||
|
|
@ -319,7 +322,8 @@ def test_api_watch_PUT_update(client, live_server):
|
|||
)
|
||||
assert b"new title" in res.data, "new title found in edit page"
|
||||
assert b"552" in res.data, "552 minutes found in edit page"
|
||||
assert b"One, Two" in res.data, "Tag 'One, Two' was found"
|
||||
assert b"One" in res.data, "Tag 'One' was found"
|
||||
assert b"Two" in res.data, "Tag 'Two' was found"
|
||||
assert b"cookie: all eaten" in res.data, "'cookie: all eaten' found in 'headers' section"
|
||||
|
||||
######################################################
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ def test_basic_auth(client, live_server):
|
|||
# Check form validation
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import time
|
||||
from flask import url_for
|
||||
from . util import live_server_setup
|
||||
from .util import live_server_setup, wait_for_all_checks
|
||||
from changedetectionio import html_tools
|
||||
|
||||
def set_original_ignore_response():
|
||||
|
|
@ -61,7 +61,7 @@ def set_modified_response_minus_block_text():
|
|||
|
||||
|
||||
def test_check_block_changedetection_text_NOT_present(client, live_server):
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
live_server_setup(live_server)
|
||||
# Use a mix of case in ZzZ to prove it works case-insensitive.
|
||||
ignore_text = "out of stoCk\r\nfoobar"
|
||||
|
|
@ -81,7 +81,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
|
|||
assert b"1 Imported" in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Goto the edit page, add our ignore text
|
||||
# Add our URL to the import page
|
||||
|
|
@ -96,7 +96,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
|
|||
assert b"Updated watch." in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
# Check it saved
|
||||
res = client.get(
|
||||
url_for("edit_page", uuid="first"),
|
||||
|
|
@ -107,7 +107,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
|
|||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should report nothing found (no new 'unviewed' class)
|
||||
res = client.get(url_for("index"))
|
||||
|
|
@ -120,7 +120,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
|
|||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should report nothing found (no new 'unviewed' class)
|
||||
res = client.get(url_for("index"))
|
||||
|
|
@ -131,7 +131,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
|
|||
# Now we set a change where the text is gone, it should now trigger
|
||||
set_modified_response_minus_block_text()
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' in res.data
|
||||
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
|
|||
# Add our URL to the import page
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
|
@ -157,7 +157,7 @@ def test_check_multiple_filters(client, live_server):
|
|||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": include_filters,
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ def test_element_removal_full(client, live_server):
|
|||
data={
|
||||
"subtractive_selectors": subtractive_selectors_data,
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
"fetch_backend": "html_requests",
|
||||
},
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ def test_check_filter_multiline(client, live_server):
|
|||
data={"include_filters": '',
|
||||
'extract_text': '/something.+?6 billion.+?lines/si',
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
'fetch_backend': "html_requests"
|
||||
},
|
||||
|
|
@ -146,7 +146,7 @@ def test_check_filter_and_regex_extract(client, live_server):
|
|||
data={"include_filters": include_filters,
|
||||
'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
'fetch_backend': "html_requests"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
|
|||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": test_url, "tag": 'cinema'},
|
||||
data={"url": test_url, "tags": 'cinema'},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Watch added" in res.data
|
||||
|
|
@ -89,7 +89,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
|
|||
|
||||
notification_form_data.update({
|
||||
"url": test_url,
|
||||
"tag": "my tag",
|
||||
"tags": "my tag",
|
||||
"title": "my title",
|
||||
"headers": "",
|
||||
"include_filters": '.ticket-available',
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import os
|
||||
import time
|
||||
from flask import url_for
|
||||
from .util import set_original_response, live_server_setup, extract_UUID_from_client
|
||||
from .util import set_original_response, live_server_setup, extract_UUID_from_client, wait_for_all_checks
|
||||
from changedetectionio.model import App
|
||||
|
||||
|
||||
|
|
@ -37,14 +37,14 @@ def run_filter_test(client, content_filter):
|
|||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": test_url, "tag": ''},
|
||||
data={"url": test_url, "tags": ''},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Watch added" in res.data
|
||||
|
||||
# Give the thread time to pick up the first version
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Goto the edit page, add our ignore text
|
||||
# Add our URL to the import page
|
||||
|
|
@ -71,8 +71,8 @@ def run_filter_test(client, content_filter):
|
|||
|
||||
notification_form_data.update({
|
||||
"url": test_url,
|
||||
"tag": "my tag",
|
||||
"title": "my title",
|
||||
"tags": "my tag",
|
||||
"title": "my title 123",
|
||||
"headers": "",
|
||||
"filter_failure_notification_send": 'y',
|
||||
"include_filters": content_filter,
|
||||
|
|
@ -85,43 +85,55 @@ def run_filter_test(client, content_filter):
|
|||
)
|
||||
|
||||
assert b"Updated watch." in res.data
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Now the notification should not exist, because we didnt reach the threshold
|
||||
assert not os.path.isfile("test-datastore/notification.txt")
|
||||
|
||||
for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
|
||||
# -2 because we would have checked twice above (on adding and on edit)
|
||||
for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2):
|
||||
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}"
|
||||
|
||||
# We should see something in the frontend
|
||||
assert b'Warning, no filters were found' in res.data
|
||||
|
||||
# One more check should trigger it (see -2 above)
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
wait_for_all_checks(client)
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
wait_for_all_checks(client)
|
||||
# Now it should exist and contain our "filter not found" alert
|
||||
assert os.path.isfile("test-datastore/notification.txt")
|
||||
notification = False
|
||||
|
||||
with open("test-datastore/notification.txt", 'r') as f:
|
||||
notification = f.read()
|
||||
|
||||
assert 'CSS/xPath filter was not present in the page' in notification
|
||||
assert content_filter.replace('"', '\\"') in notification
|
||||
|
||||
# Remove it and prove that it doesnt trigger when not expected
|
||||
# Remove it and prove that it doesn't trigger when not expected
|
||||
# It should register a change, but no 'filter not found'
|
||||
os.unlink("test-datastore/notification.txt")
|
||||
set_response_with_filter()
|
||||
|
||||
# Try several times, it should NOT have 'filter not found'
|
||||
for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should have sent a notification, but..
|
||||
assert os.path.isfile("test-datastore/notification.txt")
|
||||
# but it should not contain the info about the failed filter
|
||||
# but it should not contain the info about a failed filter (because there was none in this case)
|
||||
with open("test-datastore/notification.txt", 'r') as f:
|
||||
notification = f.read()
|
||||
assert not 'CSS/xPath filter was not present in the page' in notification
|
||||
|
||||
# Re #1247 - All tokens got replaced
|
||||
# Re #1247 - All tokens got replaced correctly in the notification
|
||||
res = client.get(url_for("index"))
|
||||
uuid = extract_UUID_from_client(client)
|
||||
# UUID is correct, but notification contains tag uuid as UUIID wtf
|
||||
assert uuid in notification
|
||||
|
||||
# cleanup for the next
|
||||
|
|
@ -137,7 +149,7 @@ def test_setup(live_server):
|
|||
|
||||
def test_check_include_filters_failure_notification(client, live_server):
|
||||
set_original_response()
|
||||
time.sleep(1)
|
||||
wait_for_all_checks(client)
|
||||
run_filter_test(client, '#nope-doesnt-exist')
|
||||
|
||||
def test_check_xpath_filter_failure_notification(client, live_server):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,262 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import time
|
||||
from flask import url_for
|
||||
from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name
|
||||
import os
|
||||
|
||||
|
||||
def test_setup(client, live_server):
|
||||
live_server_setup(live_server)
|
||||
|
||||
def set_original_response():
|
||||
test_return_data = """<html>
|
||||
<body>
|
||||
Some initial text<br>
|
||||
<p id="only-this">Should be only this</p>
|
||||
<br>
|
||||
<p id="not-this">And never this</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||
f.write(test_return_data)
|
||||
return None
|
||||
|
||||
def set_modified_response():
|
||||
test_return_data = """<html>
|
||||
<body>
|
||||
Some initial text<br>
|
||||
<p id="only-this">Should be REALLY only this</p>
|
||||
<br>
|
||||
<p id="not-this">And never this</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||
f.write(test_return_data)
|
||||
return None
|
||||
|
||||
def test_setup_group_tag(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
set_original_response()
|
||||
|
||||
# Add a tag with some config, import a tag and it should roughly work
|
||||
res = client.post(
|
||||
url_for("tags.form_tag_add"),
|
||||
data={"name": "test-tag"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Tag added" in res.data
|
||||
assert b"test-tag" in res.data
|
||||
|
||||
res = client.post(
|
||||
url_for("tags.form_tag_edit_submit", uuid="first"),
|
||||
data={"name": "test-tag",
|
||||
"include_filters": '#only-this',
|
||||
"subtractive_selectors": '#not-this'},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated" in res.data
|
||||
tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
|
||||
res = client.get(
|
||||
url_for("tags.form_tag_edit", uuid="first")
|
||||
)
|
||||
assert b"#only-this" in res.data
|
||||
assert b"#not-this" in res.data
|
||||
|
||||
# Tag should be setup and ready, now add a watch
|
||||
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url + "?first-imported=1 test-tag, extra-import-tag"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
res = client.get(url_for("index"))
|
||||
assert b'import-tag' in res.data
|
||||
assert b'extra-import-tag' in res.data
|
||||
|
||||
res = client.get(
|
||||
url_for("tags.tags_overview_page"),
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b'import-tag' in res.data
|
||||
assert b'extra-import-tag' in res.data
|
||||
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(url_for("index"))
|
||||
assert b'Warning, no filters were found' not in res.data
|
||||
|
||||
res = client.get(
|
||||
url_for("preview_page", uuid="first"),
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b'Should be only this' in res.data
|
||||
assert b'And never this' not in res.data
|
||||
|
||||
|
||||
# RSS Group tag filter
|
||||
# An extra one that should be excluded
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url + "?should-be-excluded=1 some-tag"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"1 Imported" in res.data
|
||||
wait_for_all_checks(client)
|
||||
set_modified_response()
|
||||
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
wait_for_all_checks(client)
|
||||
rss_token = extract_rss_token_from_UI(client)
|
||||
res = client.get(
|
||||
url_for("rss", token=rss_token, tag="extra-import-tag", _external=True),
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"should-be-excluded" not in res.data
|
||||
assert res.status_code == 200
|
||||
assert b"first-imported=1" in res.data
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
def test_tag_import_singular(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url + " test-tag, test-tag\r\n"+ test_url + "?x=1 test-tag, test-tag\r\n"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"2 Imported" in res.data
|
||||
|
||||
res = client.get(
|
||||
url_for("tags.tags_overview_page"),
|
||||
follow_redirects=True
|
||||
)
|
||||
# Should be only 1 tag because they both had the same
|
||||
assert res.data.count(b'test-tag') == 1
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
def test_tag_add_in_ui(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
#
|
||||
res = client.post(
|
||||
url_for("tags.form_tag_add"),
|
||||
data={"name": "new-test-tag"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Tag added" in res.data
|
||||
assert b"new-test-tag" in res.data
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
def test_group_tag_notification(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
set_original_response()
|
||||
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": test_url, "tags": 'test-tag, other-tag'},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Watch added" in res.data
|
||||
|
||||
notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
|
||||
notification_form_data = {"notification_urls": notification_url,
|
||||
"notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
|
||||
"notification_body": "BASE URL: {{base_url}}\n"
|
||||
"Watch URL: {{watch_url}}\n"
|
||||
"Watch UUID: {{watch_uuid}}\n"
|
||||
"Watch title: {{watch_title}}\n"
|
||||
"Watch tag: {{watch_tag}}\n"
|
||||
"Preview: {{preview_url}}\n"
|
||||
"Diff URL: {{diff_url}}\n"
|
||||
"Snapshot: {{current_snapshot}}\n"
|
||||
"Diff: {{diff}}\n"
|
||||
"Diff Added: {{diff_added}}\n"
|
||||
"Diff Removed: {{diff_removed}}\n"
|
||||
"Diff Full: {{diff_full}}\n"
|
||||
":-)",
|
||||
"notification_screenshot": True,
|
||||
"notification_format": "Text",
|
||||
"title": "test-tag"}
|
||||
|
||||
res = client.post(
|
||||
url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="test-tag")),
|
||||
data=notification_form_data,
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated" in res.data
|
||||
|
||||
wait_for_all_checks(client)
|
||||
|
||||
set_modified_response()
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(3)
|
||||
|
||||
assert os.path.isfile("test-datastore/notification.txt")
|
||||
|
||||
# Verify what was sent as a notification, this file should exist
|
||||
with open("test-datastore/notification.txt", "r") as f:
|
||||
notification_submission = f.read()
|
||||
os.unlink("test-datastore/notification.txt")
|
||||
|
||||
# Did we see the URL that had a change, in the notification?
|
||||
# Diff was correctly executed
|
||||
assert test_url in notification_submission
|
||||
assert ':-)' in notification_submission
|
||||
assert "Diff Full: Some initial text" in notification_submission
|
||||
assert "New GROUP TAG ChangeDetection.io" in notification_submission
|
||||
assert "test-tag" in notification_submission
|
||||
assert "other-tag" in notification_submission
|
||||
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
#@todo Test that multiple notifications fired
|
||||
#@todo Test that each of multiple notifications with different settings
|
||||
|
||||
|
||||
def test_limit_tag_ui(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
urls=[]
|
||||
|
||||
for i in range(20):
|
||||
urls.append(test_url+"?x="+str(i)+" test-tag")
|
||||
|
||||
for i in range(20):
|
||||
urls.append(test_url+"?non-grouped="+str(i))
|
||||
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": "\r\n".join(urls)},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"40 Imported" in res.data
|
||||
|
||||
res = client.get(url_for("index"))
|
||||
assert b'test-tag' in res.data
|
||||
|
||||
# All should be here
|
||||
assert res.data.count(b'processor-text_json_diff') == 40
|
||||
|
||||
tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
|
||||
|
||||
res = client.get(url_for("index", tag=tag_uuid))
|
||||
|
||||
# Just a subset should be here
|
||||
assert b'test-tag' in res.data
|
||||
assert res.data.count(b'processor-text_json_diff') == 20
|
||||
assert b"object at" not in res.data
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import time
|
||||
from flask import url_for
|
||||
from . util import live_server_setup
|
||||
from .util import live_server_setup, wait_for_all_checks
|
||||
from changedetectionio import html_tools
|
||||
|
||||
def test_setup(live_server):
|
||||
|
|
@ -84,7 +84,6 @@ def set_modified_ignore_response():
|
|||
|
||||
|
||||
def test_check_ignore_text_functionality(client, live_server):
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
# Use a mix of case in ZzZ to prove it works case-insensitive.
|
||||
ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
|
||||
|
|
@ -103,7 +102,7 @@ def test_check_ignore_text_functionality(client, live_server):
|
|||
assert b"1 Imported" in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Goto the edit page, add our ignore text
|
||||
# Add our URL to the import page
|
||||
|
|
@ -124,7 +123,7 @@ def test_check_ignore_text_functionality(client, live_server):
|
|||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should report nothing found (no new 'unviewed' class)
|
||||
res = client.get(url_for("index"))
|
||||
|
|
@ -137,7 +136,7 @@ def test_check_ignore_text_functionality(client, live_server):
|
|||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should report nothing found (no new 'unviewed' class)
|
||||
res = client.get(url_for("index"))
|
||||
|
|
@ -151,7 +150,7 @@ def test_check_ignore_text_functionality(client, live_server):
|
|||
# Just to be sure.. set a regular modified change..
|
||||
set_modified_original_ignore_response()
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' in res.data
|
||||
|
|
@ -167,7 +166,6 @@ def test_check_ignore_text_functionality(client, live_server):
|
|||
assert b'Deleted' in res.data
|
||||
|
||||
def test_check_global_ignore_text_functionality(client, live_server):
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
# Give the endpoint time to spin up
|
||||
time.sleep(1)
|
||||
|
|
@ -198,7 +196,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
|
|||
assert b"1 Imported" in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
|
||||
# Goto the edit page of the item, add our ignore text
|
||||
|
|
@ -220,7 +218,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
|
|||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# so that we are sure everything is viewed and in a known 'nothing changed' state
|
||||
res = client.get(url_for("diff_history_page", uuid="first"))
|
||||
|
|
@ -237,7 +235,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
|
|||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should report nothing found (no new 'unviewed' class)
|
||||
res = client.get(url_for("index"))
|
||||
|
|
@ -247,7 +245,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
|
|||
# Just to be sure.. set a regular modified change that will trigger it
|
||||
set_modified_original_ignore_response()
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' in res.data
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import time
|
||||
from flask import url_for
|
||||
from . util import live_server_setup
|
||||
from .util import live_server_setup, wait_for_all_checks
|
||||
|
||||
|
||||
def test_setup(live_server):
|
||||
|
|
@ -40,7 +40,7 @@ def set_some_changed_response():
|
|||
|
||||
|
||||
def test_normal_page_check_works_with_ignore_status_code(client, live_server):
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
|
||||
# Give the endpoint time to spin up
|
||||
time.sleep(1)
|
||||
|
|
@ -68,15 +68,15 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
|
|||
)
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
set_some_changed_response()
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should report nothing found (no new 'unviewed' class)
|
||||
res = client.get(url_for("index"))
|
||||
|
|
@ -109,13 +109,13 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
|
|||
# Add our URL to the import page
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"ignore_status_codes": "y", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Make a change
|
||||
set_some_changed_response()
|
||||
|
|
@ -123,7 +123,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
|
|||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should have 'unviewed' still
|
||||
# Because it should be looking at only that 'sametext' id
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ def test_jinja2_in_url_query(client, live_server):
|
|||
"date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}", )
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": full_url, "tag": "test"},
|
||||
data={"url": full_url, "tags": "test"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Watch added" in res.data
|
||||
|
|
|
|||
|
|
@ -208,7 +208,7 @@ def test_check_json_without_filter(client, live_server):
|
|||
)
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(
|
||||
url_for("preview_page", uuid="first"),
|
||||
|
|
@ -238,7 +238,7 @@ def check_json_filter(json_filter, client, live_server):
|
|||
assert b"1 Imported" in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Goto the edit page, add our ignore text
|
||||
# Add our URL to the import page
|
||||
|
|
@ -246,7 +246,7 @@ def check_json_filter(json_filter, client, live_server):
|
|||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": json_filter,
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
"fetch_backend": "html_requests"
|
||||
},
|
||||
|
|
@ -261,14 +261,14 @@ def check_json_filter(json_filter, client, live_server):
|
|||
assert bytes(escape(json_filter).encode('utf-8')) in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
# Make a change
|
||||
set_modified_response()
|
||||
|
||||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(4)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should have 'unviewed' still
|
||||
res = client.get(url_for("index"))
|
||||
|
|
@ -306,14 +306,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
|
|||
)
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
# Goto the edit page, add our ignore text
|
||||
# Add our URL to the import page
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": json_filter,
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
"fetch_backend": "html_requests"
|
||||
},
|
||||
|
|
@ -322,14 +322,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
|
|||
assert b"Updated watch." in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
# Make a change
|
||||
set_modified_response()
|
||||
|
||||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(url_for("diff_history_page", uuid="first"))
|
||||
# But the change should be there, tho its hard to test the change was detected because it will show old and new versions
|
||||
|
|
@ -366,7 +366,7 @@ def check_json_ext_filter(json_filter, client, live_server):
|
|||
assert b"1 Imported" in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Goto the edit page, add our ignore text
|
||||
# Add our URL to the import page
|
||||
|
|
@ -374,7 +374,7 @@ def check_json_ext_filter(json_filter, client, live_server):
|
|||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": json_filter,
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
"fetch_backend": "html_requests"
|
||||
},
|
||||
|
|
@ -389,14 +389,14 @@ def check_json_ext_filter(json_filter, client, live_server):
|
|||
assert bytes(escape(json_filter).encode('utf-8')) in res.data
|
||||
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
# Make a change
|
||||
set_modified_ext_response()
|
||||
|
||||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
# Give the thread time to pick it up
|
||||
time.sleep(4)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# It should have 'unviewed'
|
||||
res = client.get(url_for("index"))
|
||||
|
|
@ -428,14 +428,14 @@ def test_ignore_json_order(client, live_server):
|
|||
)
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
time.sleep(2)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||
f.write('{"world" : 123, "hello": 123}')
|
||||
|
||||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(2)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' not in res.data
|
||||
|
|
@ -446,7 +446,7 @@ def test_ignore_json_order(client, live_server):
|
|||
|
||||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(2)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(url_for("index"))
|
||||
assert b'unviewed' in res.data
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import os
|
|||
import time
|
||||
import re
|
||||
from flask import url_for
|
||||
from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup
|
||||
from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks
|
||||
from . util import extract_UUID_from_client
|
||||
import logging
|
||||
import base64
|
||||
|
|
@ -21,10 +21,11 @@ def test_setup(live_server):
|
|||
# Hard to just add more live server URLs when one test is already running (I think)
|
||||
# So we add our test here (was in a different file)
|
||||
def test_check_notification(client, live_server):
|
||||
#live_server_setup(live_server)
|
||||
set_original_response()
|
||||
|
||||
# Give the endpoint time to spin up
|
||||
time.sleep(3)
|
||||
time.sleep(1)
|
||||
|
||||
# Re 360 - new install should have defaults set
|
||||
res = client.get(url_for("settings_page"))
|
||||
|
|
@ -62,13 +63,13 @@ def test_check_notification(client, live_server):
|
|||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": test_url, "tag": ''},
|
||||
data={"url": test_url, "tags": ''},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Watch added" in res.data
|
||||
|
||||
# Give the thread time to pick up the first version
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# We write the PNG to disk, but a JPEG should appear in the notification
|
||||
# Write the last screenshot png
|
||||
|
|
@ -105,7 +106,7 @@ def test_check_notification(client, live_server):
|
|||
|
||||
notification_form_data.update({
|
||||
"url": test_url,
|
||||
"tag": "my tag",
|
||||
"tags": "my tag, my second tag",
|
||||
"title": "my title",
|
||||
"headers": "",
|
||||
"fetch_backend": "html_requests"})
|
||||
|
|
@ -128,7 +129,7 @@ def test_check_notification(client, live_server):
|
|||
|
||||
|
||||
## Now recheck, and it should have sent the notification
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
set_modified_response()
|
||||
|
||||
# Trigger a check
|
||||
|
|
@ -150,7 +151,7 @@ def test_check_notification(client, live_server):
|
|||
assert "b'" not in notification_submission
|
||||
assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
|
||||
assert "Watch title: my title" in notification_submission
|
||||
assert "Watch tag: my tag" in notification_submission
|
||||
assert "Watch tag: my tag, my second tag" in notification_submission
|
||||
assert "diff/" in notification_submission
|
||||
assert "preview/" in notification_submission
|
||||
assert ":-)" in notification_submission
|
||||
|
|
@ -193,11 +194,11 @@ def test_check_notification(client, live_server):
|
|||
|
||||
# Trigger a check
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(1)
|
||||
wait_for_all_checks(client)
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(1)
|
||||
wait_for_all_checks(client)
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
time.sleep(1)
|
||||
wait_for_all_checks(client)
|
||||
assert os.path.exists("test-datastore/notification.txt") == False
|
||||
|
||||
res = client.get(url_for("notification_logs"))
|
||||
|
|
@ -209,7 +210,7 @@ def test_check_notification(client, live_server):
|
|||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"url": test_url,
|
||||
"tag": "my tag",
|
||||
"tags": "my tag",
|
||||
"title": "my title",
|
||||
"notification_urls": '',
|
||||
"notification_title": '',
|
||||
|
|
@ -243,7 +244,7 @@ def test_notification_validation(client, live_server):
|
|||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": test_url, "tag": 'nice one'},
|
||||
data={"url": test_url, "tags": 'nice one'},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
|
|
@ -303,13 +304,13 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
|
|||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": test_url, "tag": 'nice one'},
|
||||
data={"url": test_url, "tags": 'nice one'},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Watch added" in res.data
|
||||
|
||||
time.sleep(2)
|
||||
wait_for_all_checks(client)
|
||||
set_modified_response()
|
||||
|
||||
client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ def test_check_notification_error_handling(client, live_server):
|
|||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": test_url, "tag": ''},
|
||||
data={"url": test_url, "tags": ''},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Watch added" in res.data
|
||||
|
|
@ -32,7 +32,7 @@ def test_check_notification_error_handling(client, live_server):
|
|||
"notification_body": "xxxxx",
|
||||
"notification_format": "Text",
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"title": "",
|
||||
"headers": "",
|
||||
"time_between_check-minutes": "180",
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ def test_headers_in_request(client, live_server):
|
|||
)
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
time.sleep(1)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
|
|
@ -43,7 +43,7 @@ def test_headers_in_request(client, live_server):
|
|||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
|
||||
"headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
|
||||
follow_redirects=True
|
||||
|
|
@ -95,14 +95,14 @@ def test_body_in_request(client, live_server):
|
|||
)
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# add the first 'version'
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"method": "POST",
|
||||
"fetch_backend": "html_requests",
|
||||
"body": "something something"},
|
||||
|
|
@ -110,7 +110,7 @@ def test_body_in_request(client, live_server):
|
|||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Now the change which should trigger a change
|
||||
body_value = 'Test Body Value'
|
||||
|
|
@ -118,7 +118,7 @@ def test_body_in_request(client, live_server):
|
|||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"method": "POST",
|
||||
"fetch_backend": "html_requests",
|
||||
"body": body_value},
|
||||
|
|
@ -126,7 +126,7 @@ def test_body_in_request(client, live_server):
|
|||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
||||
time.sleep(3)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# The service should echo back the body
|
||||
res = client.get(
|
||||
|
|
@ -163,7 +163,7 @@ def test_body_in_request(client, live_server):
|
|||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"method": "GET",
|
||||
"fetch_backend": "html_requests",
|
||||
"body": "invalid"},
|
||||
|
|
@ -187,7 +187,7 @@ def test_method_in_request(client, live_server):
|
|||
)
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
time.sleep(2)
|
||||
wait_for_all_checks(client)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url},
|
||||
|
|
@ -195,14 +195,14 @@ def test_method_in_request(client, live_server):
|
|||
)
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
time.sleep(2)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Attempt to add a method which is not valid
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"fetch_backend": "html_requests",
|
||||
"method": "invalid"},
|
||||
follow_redirects=True
|
||||
|
|
@ -214,7 +214,7 @@ def test_method_in_request(client, live_server):
|
|||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"fetch_backend": "html_requests",
|
||||
"method": "PATCH"},
|
||||
follow_redirects=True
|
||||
|
|
@ -222,7 +222,7 @@ def test_method_in_request(client, live_server):
|
|||
assert b"Updated watch." in res.data
|
||||
|
||||
# Give the thread time to pick up the first version
|
||||
time.sleep(2)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# The service should echo back the request verb
|
||||
res = client.get(
|
||||
|
|
@ -233,7 +233,7 @@ def test_method_in_request(client, live_server):
|
|||
# The test call service will return the verb as the body
|
||||
assert b"PATCH" in res.data
|
||||
|
||||
time.sleep(2)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
watches_with_method = 0
|
||||
with open('test-datastore/url-watches.json') as f:
|
||||
|
|
@ -265,7 +265,7 @@ def test_headers_textfile_in_request(client, live_server):
|
|||
)
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
time.sleep(1)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
|
||||
# Add some headers to a request
|
||||
|
|
@ -273,7 +273,7 @@ def test_headers_textfile_in_request(client, live_server):
|
|||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"url": test_url,
|
||||
"tag": "testtag",
|
||||
"tags": "testtag",
|
||||
"fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
|
||||
"headers": "xxx:ooo\ncool:yeah\r\n"},
|
||||
follow_redirects=True
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ def test_basic_search(client, live_server):
|
|||
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"title": "xxx-title", "url": urls[0], "tag": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"title": "xxx-title", "url": urls[0], "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
|
@ -62,7 +62,7 @@ def test_search_in_tag_limit(client, live_server):
|
|||
# By Title
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tag": urls[0].split(' ')[1], "headers": "",
|
||||
data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tags": urls[0].split(' ')[1], "headers": "",
|
||||
'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ def test_bad_access(client, live_server):
|
|||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"url": 'javascript:alert(document.domain)',
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"method": "GET",
|
||||
"fetch_backend": "html_requests",
|
||||
"body": ""},
|
||||
|
|
@ -29,7 +29,7 @@ def test_bad_access(client, live_server):
|
|||
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": ' javascript:alert(123)', "tag": ''},
|
||||
data={"url": ' javascript:alert(123)', "tags": ''},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
|
|
@ -37,7 +37,7 @@ def test_bad_access(client, live_server):
|
|||
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": '%20%20%20javascript:alert(123)%20%20', "tag": ''},
|
||||
data={"url": '%20%20%20javascript:alert(123)%20%20', "tags": ''},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
|
|
@ -46,7 +46,7 @@ def test_bad_access(client, live_server):
|
|||
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": ' source:javascript:alert(document.domain)', "tag": ''},
|
||||
data={"url": ' source:javascript:alert(document.domain)', "tags": ''},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
|
|
@ -56,7 +56,7 @@ def test_bad_access(client, live_server):
|
|||
|
||||
client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": 'file:///tasty/disk/drive', "tag": ''},
|
||||
data={"url": 'file:///tasty/disk/drive', "tags": ''},
|
||||
follow_redirects=True
|
||||
)
|
||||
time.sleep(1)
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ def test_share_watch(client, live_server):
|
|||
# Add our URL to the import page
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
import time
|
||||
from flask import url_for
|
||||
from urllib.request import urlopen
|
||||
from .util import set_original_response, set_modified_response, live_server_setup
|
||||
from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
|
||||
|
||||
sleep_time_for_fetch_thread = 3
|
||||
|
||||
|
|
@ -42,7 +42,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
|
|||
res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
|
||||
assert b'1 watches queued for rechecking.' in res.data
|
||||
|
||||
time.sleep(5)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
# Now something should be ready, indicated by having a 'unviewed' class
|
||||
res = client.get(url_for("index"))
|
||||
|
|
@ -60,7 +60,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
|
|||
# `subtractive_selectors` should still work in `source:` type requests
|
||||
def test_check_ignore_elements(client, live_server):
|
||||
set_original_response()
|
||||
time.sleep(2)
|
||||
time.sleep(1)
|
||||
test_url = 'source:'+url_for('test_endpoint', _external=True)
|
||||
# Add our URL to the import page
|
||||
res = client.post(
|
||||
|
|
@ -71,14 +71,14 @@ def test_check_ignore_elements(client, live_server):
|
|||
|
||||
assert b"1 Imported" in res.data
|
||||
|
||||
time.sleep(sleep_time_for_fetch_thread)
|
||||
wait_for_all_checks(client)
|
||||
|
||||
#####################
|
||||
# We want <span> and <p> ONLY, but ignore span with .foobar-detection
|
||||
|
||||
client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
|
||||
data={"include_filters": 'span,p', "url": test_url, "tags": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ def test_check_watch_field_storage(client, live_server):
|
|||
"title" : "My title",
|
||||
"ignore_text" : "ignore this",
|
||||
"url": test_url,
|
||||
"tag": "woohoo",
|
||||
"tags": "woohoo",
|
||||
"headers": "curl:foo",
|
||||
'fetch_backend': "html_requests"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server):
|
|||
time.sleep(1)
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
|
@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
|
|||
time.sleep(1)
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
|
@ -189,7 +189,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
|
|||
# Add our URL to the import page
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"include_filters": xpath_filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Updated watch." in res.data
|
||||
|
|
@ -231,7 +231,7 @@ def test_xpath_validation(client, live_server):
|
|||
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"include_filters": "/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"is not a valid XPath expression" in res.data
|
||||
|
|
@ -261,7 +261,7 @@ def test_check_with_prefix_include_filters(client, live_server):
|
|||
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -70,6 +70,16 @@ def extract_api_key_from_UI(client):
|
|||
api_key = m.group(1)
|
||||
return api_key.strip()
|
||||
|
||||
|
||||
# kinda funky, but works for now
|
||||
def get_UUID_for_tag_name(client, name):
|
||||
app_config = client.application.config.get('DATASTORE').data
|
||||
for uuid, tag in app_config['settings']['application'].get('tags', {}).items():
|
||||
if name == tag.get('title', '').lower().strip():
|
||||
return uuid
|
||||
return None
|
||||
|
||||
|
||||
# kinda funky, but works for now
|
||||
def extract_rss_token_from_UI(client):
|
||||
import re
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ def test_visual_selector_content_ready(client, live_server):
|
|||
|
||||
res = client.post(
|
||||
url_for("form_quick_watch_add"),
|
||||
data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
|
||||
data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"Watch added in Paused state, saving will unpause" in res.data
|
||||
|
|
@ -28,7 +28,7 @@ def test_visual_selector_content_ready(client, live_server):
|
|||
url_for("edit_page", uuid="first", unpause_on_save=1),
|
||||
data={
|
||||
"url": test_url,
|
||||
"tag": "",
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
'fetch_backend': "html_webdriver",
|
||||
'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();'
|
||||
|
|
|
|||
|
|
@ -26,10 +26,48 @@ class update_worker(threading.Thread):
|
|||
self.datastore = datastore
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def send_content_changed_notification(self, t, watch_uuid):
|
||||
def queue_notification_for_watch(self, n_object, watch):
|
||||
|
||||
from changedetectionio import diff
|
||||
|
||||
watch_history = watch.history
|
||||
dates = list(watch_history.keys())
|
||||
|
||||
# HTML needs linebreak, but MarkDown and Text can use a linefeed
|
||||
if n_object['notification_format'] == 'HTML':
|
||||
line_feed_sep = "<br>"
|
||||
else:
|
||||
line_feed_sep = "\n"
|
||||
|
||||
# Add text that was triggered
|
||||
snapshot_contents = watch.get_history_snapshot(dates[-1])
|
||||
trigger_text = watch.get('trigger_text', [])
|
||||
triggered_text = ''
|
||||
|
||||
if len(trigger_text):
|
||||
from . import html_tools
|
||||
triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
|
||||
if triggered_text:
|
||||
triggered_text = line_feed_sep.join(triggered_text)
|
||||
|
||||
|
||||
n_object.update({
|
||||
'current_snapshot': snapshot_contents,
|
||||
'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
|
||||
'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
|
||||
'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
|
||||
'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
|
||||
'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
|
||||
'triggered_text': triggered_text,
|
||||
'uuid': watch.get('uuid'),
|
||||
'watch_url': watch.get('url'),
|
||||
})
|
||||
logging.info (">> SENDING NOTIFICATION")
|
||||
self.notification_q.put(n_object)
|
||||
|
||||
|
||||
def send_content_changed_notification(self, watch_uuid):
|
||||
|
||||
from changedetectionio.notification import (
|
||||
default_notification_format_for_watch
|
||||
)
|
||||
|
|
@ -48,8 +86,9 @@ class update_worker(threading.Thread):
|
|||
"History index had 2 or more, but only 1 date loaded, timestamps were not unique? maybe two of the same timestamps got written, needs more delay?"
|
||||
)
|
||||
|
||||
n_object['notification_urls'] = watch['notification_urls'] if len(watch['notification_urls']) else \
|
||||
self.datastore.data['settings']['application']['notification_urls']
|
||||
# Should be a better parent getter in the model object
|
||||
# Prefer - Individual watch settings > Tag settings > Global settings (in that order)
|
||||
n_object['notification_urls'] = watch.get('notification_urls')
|
||||
|
||||
n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \
|
||||
self.datastore.data['settings']['application']['notification_title']
|
||||
|
|
@ -60,47 +99,51 @@ class update_worker(threading.Thread):
|
|||
n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \
|
||||
self.datastore.data['settings']['application']['notification_format']
|
||||
|
||||
|
||||
# Only prepare to notify if the rules above matched
|
||||
# (Individual watch) Only prepare to notify if the rules above matched
|
||||
sent = False
|
||||
if 'notification_urls' in n_object and n_object['notification_urls']:
|
||||
# HTML needs linebreak, but MarkDown and Text can use a linefeed
|
||||
if n_object['notification_format'] == 'HTML':
|
||||
line_feed_sep = "<br>"
|
||||
else:
|
||||
line_feed_sep = "\n"
|
||||
sent = True
|
||||
self.queue_notification_for_watch(n_object, watch)
|
||||
|
||||
# Add text that was triggered
|
||||
snapshot_contents = watch.get_history_snapshot(dates[-1])
|
||||
trigger_text = watch.get('trigger_text', [])
|
||||
triggered_text = ''
|
||||
# (Group tags) try by group tag
|
||||
if not sent:
|
||||
# Else, Try by tag, and use system default vars for format, body etc as fallback
|
||||
tags = self.datastore.get_all_tags_for_watch(uuid=watch_uuid)
|
||||
for tag_uuid, tag in tags.items():
|
||||
n_object = {}
|
||||
n_object['notification_urls'] = tag.get('notification_urls')
|
||||
|
||||
if len(trigger_text):
|
||||
from . import html_tools
|
||||
triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
|
||||
if triggered_text:
|
||||
triggered_text = line_feed_sep.join(triggered_text)
|
||||
n_object['notification_title'] = tag.get('notification_title') if tag.get('notification_title') else \
|
||||
self.datastore.data['settings']['application']['notification_title']
|
||||
|
||||
n_object['notification_body'] = tag.get('notification_body') if tag.get('notification_body') else \
|
||||
self.datastore.data['settings']['application']['notification_body']
|
||||
|
||||
n_object['notification_format'] = tag.get('notification_format') if tag.get('notification_format') != default_notification_format_for_watch else \
|
||||
self.datastore.data['settings']['application']['notification_format']
|
||||
|
||||
if 'notification_urls' in n_object and n_object.get('notification_urls') and not tag.get('notification_muted'):
|
||||
sent = True
|
||||
self.queue_notification_for_watch(n_object, watch)
|
||||
|
||||
# (Group tags) try by global
|
||||
if not sent:
|
||||
# leave this as is, but repeat in a loop for each tag also
|
||||
n_object['notification_urls'] = self.datastore.data['settings']['application'].get('notification_urls')
|
||||
n_object['notification_title'] = self.datastore.data['settings']['application'].get('notification_title')
|
||||
n_object['notification_body'] = self.datastore.data['settings']['application'].get('notification_body')
|
||||
n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
|
||||
if n_object.get('notification_urls') and n_object.get('notification_body') and n_object.get('notification_title'):
|
||||
sent = True
|
||||
self.queue_notification_for_watch(n_object, watch)
|
||||
|
||||
return sent
|
||||
|
||||
n_object.update({
|
||||
'current_snapshot': snapshot_contents,
|
||||
'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
|
||||
'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
|
||||
'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
|
||||
'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
|
||||
'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
|
||||
'triggered_text': triggered_text,
|
||||
'uuid': watch_uuid,
|
||||
'watch_url': watch['url'],
|
||||
})
|
||||
logging.info (">> SENDING NOTIFICATION")
|
||||
self.notification_q.put(n_object)
|
||||
else:
|
||||
logging.info (">> NO Notification sent, notification_url was empty in both watch and system")
|
||||
|
||||
def send_filter_failure_notification(self, watch_uuid):
|
||||
|
||||
threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
|
||||
watch = self.datastore.data['watching'].get(watch_uuid, False)
|
||||
watch = self.datastore.data['watching'].get(watch_uuid)
|
||||
if not watch:
|
||||
return
|
||||
|
||||
|
|
@ -177,7 +220,7 @@ class update_worker(threading.Thread):
|
|||
uuid = queued_item_data.item.get('uuid')
|
||||
self.current_uuid = uuid
|
||||
|
||||
if uuid in list(self.datastore.data['watching'].keys()):
|
||||
if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
|
||||
changed_detected = False
|
||||
contents = b''
|
||||
process_changedetection_results = True
|
||||
|
|
@ -360,7 +403,7 @@ class update_worker(threading.Thread):
|
|||
# Notifications should only trigger on the second time (first time, we gather the initial snapshot)
|
||||
if watch.history_n >= 2:
|
||||
if not self.datastore.data['watching'][uuid].get('notification_muted'):
|
||||
self.send_content_changed_notification(self, watch_uuid=uuid)
|
||||
self.send_content_changed_notification(watch_uuid=uuid)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue