kopia lustrzana https://github.com/dgtlmoon/changedetection.io
Send diff in notifications (#296)
dgtlmoon
GitHub
rodzic
a7c09c8990
commit
5a10acfd09
|
|
@ -14,6 +14,9 @@ jobs:
|
|||
with:
|
||||
python-version: 3.9
|
||||
|
||||
- name: Show env vars
|
||||
run: set
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
|
|
@ -27,12 +30,15 @@ jobs:
|
|||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
|
||||
- name: Unit tests
|
||||
run: |
|
||||
python3 -m unittest changedetectionio.tests.unit.test_notification_diff
|
||||
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
# Each test is totally isolated and performs its own cleanup/reset
|
||||
cd changedetectionio; ./run_all_tests.sh
|
||||
|
||||
|
||||
# https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
|
||||
# https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
|
||||
|
||||
|
|
|
|||
|
|
@ -464,7 +464,6 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
else:
|
||||
flash('No notification URLs set, cannot send test.', 'error')
|
||||
|
||||
|
||||
# Diff page [edit] link should go back to diff page
|
||||
if request.args.get("next") and request.args.get("next") == 'diff':
|
||||
return redirect(url_for('diff_history_page', uuid=uuid))
|
||||
|
|
@ -621,6 +620,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
|
||||
dates = list(watch['history'].keys())
|
||||
# Convert to int, sort and back to str again
|
||||
# @todo replace datastore getter that does this automatically
|
||||
dates = [int(i) for i in dates]
|
||||
dates.sort(reverse=True)
|
||||
dates = [str(i) for i in dates]
|
||||
|
|
@ -631,13 +631,11 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
|
||||
# Save the current newest history as the most recently viewed
|
||||
datastore.set_last_viewed(uuid, dates[0])
|
||||
|
||||
newest_file = watch['history'][dates[0]]
|
||||
with open(newest_file, 'r') as f:
|
||||
newest_version_file_contents = f.read()
|
||||
|
||||
previous_version = request.args.get('previous_version')
|
||||
|
||||
try:
|
||||
previous_file = watch['history'][previous_version]
|
||||
except KeyError:
|
||||
|
|
@ -843,8 +841,10 @@ def changedetection_app(config=None, datastore_o=None):
|
|||
|
||||
threading.Thread(target=notification_runner).start()
|
||||
|
||||
# Check for new release version
|
||||
threading.Thread(target=check_for_new_version).start()
|
||||
# Check for new release version, but not when running in test/build
|
||||
if not os.getenv("GITHUB_REF", False):
|
||||
threading.Thread(target=check_for_new_version).start()
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
|
@ -893,8 +893,6 @@ def notification_runner():
|
|||
except Exception as e:
|
||||
print("Watch URL: {} Error {}".format(n_object['watch_url'], e))
|
||||
|
||||
|
||||
|
||||
# Thread runner to check every minute, look for new watches to feed into the Queue.
|
||||
def ticker_thread_check_time_launch_checks():
|
||||
from changedetectionio import update_worker
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
# used for the notifications, the front-end is using a JS library
|
||||
|
||||
import difflib
|
||||
|
||||
# like .compare but a little different output
|
||||
def customSequenceMatcher(before, after, include_equal=False):
|
||||
cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
|
||||
|
||||
for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
|
||||
if include_equal and tag == 'equal':
|
||||
g = before[alo:ahi]
|
||||
yield g
|
||||
elif tag == 'delete':
|
||||
g = "(removed) {}".format(before[alo])
|
||||
yield g
|
||||
elif tag == 'replace':
|
||||
g = ["(changed) {}".format(before[alo]), "(-> into) {}".format(after[blo])]
|
||||
yield g
|
||||
elif tag == 'insert':
|
||||
g = "(added) {}".format(after[blo])
|
||||
yield g
|
||||
|
||||
# only_differences - only return info about the differences, no context
|
||||
# line_feed_sep could be "<br/>" or "<li>" or "\n" etc
|
||||
def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
|
||||
with open(newest_file, 'r') as f:
|
||||
newest_version_file_contents = f.read()
|
||||
newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
|
||||
|
||||
if previous_file:
|
||||
with open(previous_file, 'r') as f:
|
||||
previous_version_file_contents = f.read()
|
||||
previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
|
||||
else:
|
||||
previous_version_file_contents = ""
|
||||
|
||||
rendered_diff = customSequenceMatcher(previous_version_file_contents,
|
||||
newest_version_file_contents,
|
||||
include_equal)
|
||||
|
||||
# Recursively join lists
|
||||
f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
|
||||
return f(rendered_diff)
|
||||
|
|
@ -6,7 +6,7 @@ from wtforms.fields import html5
|
|||
from changedetectionio import content_fetcher
|
||||
import re
|
||||
|
||||
from changedetectionio.notification import default_notification_format, valid_notification_formats
|
||||
from changedetectionio.notification import default_notification_format, valid_notification_formats, default_notification_body, default_notification_title
|
||||
|
||||
class StringListField(StringField):
|
||||
widget = widgets.TextArea()
|
||||
|
|
@ -203,8 +203,8 @@ class quickWatchForm(Form):
|
|||
class commonSettingsForm(Form):
|
||||
|
||||
notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
|
||||
notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {watch_url}', validators=[validators.Optional(), ValidateTokensList()])
|
||||
notification_body = TextAreaField('Notification Body', default='{watch_url} had a change.', validators=[validators.Optional(), ValidateTokensList()])
|
||||
notification_title = StringField('Notification Title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
|
||||
notification_body = TextAreaField('Notification Body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
|
||||
notification_format = SelectField('Notification Format', choices=valid_notification_formats.keys(), default=default_notification_format)
|
||||
trigger_check = BooleanField('Send test notification on save')
|
||||
fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import os
|
||||
import apprise
|
||||
from apprise import NotifyFormat
|
||||
|
||||
|
|
@ -8,6 +7,8 @@ valid_tokens = {
|
|||
'watch_uuid': '',
|
||||
'watch_title': '',
|
||||
'watch_tag': '',
|
||||
'diff': '',
|
||||
'diff_full': '',
|
||||
'diff_url': '',
|
||||
'preview_url': '',
|
||||
'current_snapshot': ''
|
||||
|
|
@ -20,6 +21,8 @@ valid_notification_formats = {
|
|||
}
|
||||
|
||||
default_notification_format = 'Text'
|
||||
default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
|
||||
default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
|
||||
|
||||
def process_notification(n_object, datastore):
|
||||
import logging
|
||||
|
|
@ -33,8 +36,8 @@ def process_notification(n_object, datastore):
|
|||
apobj.add(url)
|
||||
|
||||
# Get the notification body from datastore
|
||||
n_body = n_object['notification_body']
|
||||
n_title = n_object['notification_title']
|
||||
n_body = n_object.get('notification_body', default_notification_body)
|
||||
n_title = n_object.get('notification_title', default_notification_title)
|
||||
n_format = valid_notification_formats.get(
|
||||
n_object['notification_format'],
|
||||
valid_notification_formats[default_notification_format],
|
||||
|
|
@ -88,15 +91,17 @@ def create_notification_parameters(n_object, datastore):
|
|||
|
||||
# Valid_tokens also used as a field validator
|
||||
tokens.update(
|
||||
{
|
||||
'base_url': base_url if base_url is not None else '',
|
||||
'watch_url': watch_url,
|
||||
'watch_uuid': uuid,
|
||||
'watch_title': watch_title if watch_title is not None else '',
|
||||
'watch_tag': watch_tag if watch_tag is not None else '',
|
||||
'diff_url': diff_url,
|
||||
'preview_url': preview_url,
|
||||
'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
|
||||
})
|
||||
{
|
||||
'base_url': base_url if base_url is not None else '',
|
||||
'watch_url': watch_url,
|
||||
'watch_uuid': uuid,
|
||||
'watch_title': watch_title if watch_title is not None else '',
|
||||
'watch_tag': watch_tag if watch_tag is not None else '',
|
||||
'diff_url': diff_url,
|
||||
'diff': n_object.get('diff', ''), # Null default in the case we use a test
|
||||
'diff_full': n_object.get('diff_full', ''), # Null default in the case we use a test
|
||||
'preview_url': preview_url,
|
||||
'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
|
||||
})
|
||||
|
||||
return tokens
|
||||
|
|
|
|||
|
|
@ -9,15 +9,16 @@
|
|||
# exit when any command fails
|
||||
set -e
|
||||
|
||||
|
||||
find tests/test_*py -type f|while read test_name
|
||||
do
|
||||
echo "TEST RUNNING $test_name"
|
||||
pytest $test_name
|
||||
done
|
||||
|
||||
echo "RUNNING WITH BASE_URL SET"
|
||||
|
||||
# Now re-run some tests with BASE_URL enabled
|
||||
# Re #65 - Ability to include a link back to the installation, in the notification.
|
||||
export BASE_URL="https://really-unique-domain.io"
|
||||
pytest tests/test_notification.py
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ import time
|
|||
import threading
|
||||
import os
|
||||
|
||||
from changedetectionio.notification import default_notification_format, default_notification_body, default_notification_title
|
||||
|
||||
# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
|
||||
# Open a github issue if you know something :)
|
||||
# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
|
||||
|
|
@ -157,6 +159,7 @@ class ChangeDetectionStore:
|
|||
|
||||
dates = list(self.__data['watching'][uuid]['history'].keys())
|
||||
# Convert to int, sort and back to str again
|
||||
# @todo replace datastore getter that does this automatically
|
||||
dates = [int(i) for i in dates]
|
||||
dates.sort(reverse=True)
|
||||
if len(dates):
|
||||
|
|
|
|||
|
|
@ -65,6 +65,14 @@
|
|||
<td><code>{preview_url}</code></td>
|
||||
<td>The URL of the preview page generated by changedetection.io.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>{diff}</code></td>
|
||||
<td>The diff output - differences only</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>{diff_full}</code></td>
|
||||
<td>The diff output - full difference output</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>{diff_url}</code></td>
|
||||
<td>The URL of the diff page generated by changedetection.io.</td>
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ def cleanup(datastore_path):
|
|||
for file in files:
|
||||
try:
|
||||
os.unlink("{}/{}".format(datastore_path, file))
|
||||
x = 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -55,6 +55,8 @@ def test_check_notification(client, live_server):
|
|||
"Preview: {preview_url}\n"
|
||||
"Diff URL: {diff_url}\n"
|
||||
"Snapshot: {current_snapshot}\n"
|
||||
"Diff: {diff}\n"
|
||||
"Diff Full: {diff_full}\n"
|
||||
":-)",
|
||||
"notification_format": "Text",
|
||||
"url": test_url,
|
||||
|
|
@ -114,6 +116,11 @@ def test_check_notification(client, live_server):
|
|||
|
||||
assert test_url in notification_submission
|
||||
|
||||
# Diff was correctly executed
|
||||
assert "Diff Full: (changed) Which is across multiple lines" in notification_submission
|
||||
assert "(-> into) which has this one new line" in notification_submission
|
||||
|
||||
|
||||
if env_base_url:
|
||||
# Re #65 - did we see our BASE_URl ?
|
||||
logging.debug (">>> BASE_URL checking in notification: %s", env_base_url)
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
"""Unit tests for the app."""
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
# What is this?
|
||||
This is test content for the python diff engine, we use the JS interface for the front end, because you can explore
|
||||
differences in words etc, but we use (at the moment) the python difflib engine.
|
||||
|
||||
This content `before.txt` and `after.txt` is for unit testing
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
After twenty years, as cursed as I may be
|
||||
for having learned computerese,
|
||||
I continue to examine bits, bytes and words
|
||||
xok
|
||||
and insure that I'm one of those computer nerds.
|
||||
and something new
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
After twenty years, as cursed as I may be
|
||||
for having learned computerese,
|
||||
I continue to examine bits, bytes and words
|
||||
ok
|
||||
and insure that I'm one of those computer nerds.
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
# run from dir above changedetectionio/ dir
|
||||
# python3 -m unittest changedetectionio.tests.unit.test_notification_diff
|
||||
|
||||
import unittest
|
||||
import os
|
||||
|
||||
from changedetectionio import diff
|
||||
|
||||
# mostly
|
||||
class TestDiffBuilder(unittest.TestCase):
|
||||
|
||||
def test_expected_diff_output(self):
|
||||
base_dir=os.path.dirname(__file__)
|
||||
output = diff.render_diff(base_dir+"/test-content/before.txt", base_dir+"/test-content/after.txt")
|
||||
output = output.split("\n")
|
||||
self.assertIn("(changed) ok", output)
|
||||
self.assertIn("(-> into) xok", output)
|
||||
self.assertIn("(added) and something new", output)
|
||||
|
||||
# @todo test blocks of changed, blocks of added, blocks of removed
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -56,9 +56,8 @@ class update_worker(threading.Thread):
|
|||
try:
|
||||
self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
|
||||
if changed_detected:
|
||||
|
||||
n_object = {}
|
||||
# A change was detected
|
||||
newest_version_file_contents = ""
|
||||
fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents)
|
||||
|
||||
# Update history with the stripped text for future reference, this will also mean we save the first
|
||||
|
|
@ -69,37 +68,55 @@ class update_worker(threading.Thread):
|
|||
|
||||
print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
|
||||
|
||||
# Get the newest snapshot data to be possibily used in a notification
|
||||
newest_key = self.datastore.get_newest_history_key(uuid)
|
||||
if newest_key:
|
||||
with open(watch['history'][newest_key], 'r') as f:
|
||||
newest_version_file_contents = f.read().strip()
|
||||
# Notifications should only trigger on the second time (first time, we gather the initial snapshot)
|
||||
if len(watch['history']) > 1:
|
||||
|
||||
n_object = {
|
||||
'watch_url': watch['url'],
|
||||
'uuid': uuid,
|
||||
'current_snapshot': newest_version_file_contents
|
||||
}
|
||||
dates = list(watch['history'].keys())
|
||||
# Convert to int, sort and back to str again
|
||||
# @todo replace datastore getter that does this automatically
|
||||
dates = [int(i) for i in dates]
|
||||
dates.sort(reverse=True)
|
||||
dates = [str(i) for i in dates]
|
||||
|
||||
# Did it have any notification alerts to hit?
|
||||
if len(watch['notification_urls']):
|
||||
print(">>> Notifications queued for UUID from watch {}".format(uuid))
|
||||
n_object['notification_urls'] = watch['notification_urls']
|
||||
n_object['notification_title'] = watch['notification_title']
|
||||
n_object['notification_body'] = watch['notification_body']
|
||||
n_object['notification_format'] = watch['notification_format']
|
||||
self.notification_q.put(n_object)
|
||||
prev_fname = watch['history'][dates[1]]
|
||||
|
||||
# No? maybe theres a global setting, queue them all
|
||||
elif len(self.datastore.data['settings']['application']['notification_urls']):
|
||||
print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
|
||||
n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
|
||||
n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
|
||||
n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
|
||||
n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format']
|
||||
self.notification_q.put(n_object)
|
||||
else:
|
||||
print(">>> NO notifications queued, watch and global notification URLs were empty.")
|
||||
|
||||
# Did it have any notification alerts to hit?
|
||||
if len(watch['notification_urls']):
|
||||
print(">>> Notifications queued for UUID from watch {}".format(uuid))
|
||||
n_object['notification_urls'] = watch['notification_urls']
|
||||
n_object['notification_title'] = watch['notification_title']
|
||||
n_object['notification_body'] = watch['notification_body']
|
||||
n_object['notification_format'] = watch['notification_format']
|
||||
|
||||
# No? maybe theres a global setting, queue them all
|
||||
elif len(self.datastore.data['settings']['application']['notification_urls']):
|
||||
print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
|
||||
n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
|
||||
n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
|
||||
n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
|
||||
n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format']
|
||||
else:
|
||||
print(">>> NO notifications queued, watch and global notification URLs were empty.")
|
||||
|
||||
# Only prepare to notify if the rules above matched
|
||||
if 'notification_urls' in n_object:
|
||||
# HTML needs linebreak, but MarkDown and Text can use a linefeed
|
||||
if n_object['notification_format'] == 'HTML':
|
||||
line_feed_sep = "</br>"
|
||||
else:
|
||||
line_feed_sep = "\n"
|
||||
|
||||
from changedetectionio import diff
|
||||
n_object.update({
|
||||
'watch_url': watch['url'],
|
||||
'uuid': uuid,
|
||||
'current_snapshot': str(contents),
|
||||
'diff_full': diff.render_diff(prev_fname, fname, line_feed_sep=line_feed_sep),
|
||||
'diff': diff.render_diff(prev_fname, fname, True, line_feed_sep=line_feed_sep)
|
||||
})
|
||||
|
||||
self.notification_q.put(n_object)
|
||||
|
||||
except Exception as e:
|
||||
print("!!!! Exception in update_worker !!!\n", e)
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue