From 9fe4f9599074b1bb2d597227134c26629e098733 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 2 Apr 2022 14:49:32 +0200 Subject: [PATCH] When fetching a snapshot via Chrome, make the most recent screenshot available on the Diff and Preview pages (#516) --- changedetectionio/__init__.py | 36 ++++++++++++-- changedetectionio/content_fetcher.py | 33 +++++++++---- changedetectionio/fetch_site_status.py | 7 ++- changedetectionio/forms.py | 4 +- changedetectionio/static/js/tabs.js | 6 ++- changedetectionio/static/styles/diff.css | 8 +++- changedetectionio/static/styles/diff.scss | 10 +++- changedetectionio/static/styles/styles.css | 15 +++--- changedetectionio/static/styles/styles.scss | 15 +++--- changedetectionio/store.py | 17 +++++++ changedetectionio/templates/diff.html | 53 +++++++++++++++------ changedetectionio/templates/edit.html | 2 +- changedetectionio/templates/preview.html | 40 ++++++++++++---- changedetectionio/templates/settings.html | 8 +++- changedetectionio/update_worker.py | 6 ++- 15 files changed, 201 insertions(+), 59 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 6412ebf0..353176a6 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -625,6 +625,7 @@ def changedetection_app(config=None, datastore_o=None): form.notification_body.data = datastore.data['settings']['application']['notification_body'] form.notification_format.data = datastore.data['settings']['application']['notification_format'] form.base_url.data = datastore.data['settings']['application']['base_url'] + form.real_browser_save_screenshot.data = datastore.data['settings']['application']['real_browser_save_screenshot'] if request.method == 'POST' and form.data.get('removepassword_button') == True: # Password unset is a GET, but we can lock the session to a salted env password to always need the password @@ -647,7 +648,8 @@ def changedetection_app(config=None, datastore_o=None): datastore.data['settings']['application']['global_subtractive_selectors'] = form.global_subtractive_selectors.data datastore.data['settings']['application']['global_ignore_text'] = form.global_ignore_text.data datastore.data['settings']['application']['ignore_whitespace'] = form.ignore_whitespace.data - + datastore.data['settings']['application']['real_browser_save_screenshot'] = form.real_browser_save_screenshot.data + if form.trigger_check.data: if len(form.notification_urls.data): n_object = {'watch_url': "Test from changedetection.io!", @@ -776,6 +778,9 @@ def changedetection_app(config=None, datastore_o=None): except Exception as e: previous_version_file_contents = "Unable to read {}.\n".format(previous_file) + + screenshot_url = datastore.get_screenshot(uuid) + output = render_template("diff.html", watch_a=watch, newest=newest_version_file_contents, previous=previous_version_file_contents, @@ -786,7 +791,8 @@ def changedetection_app(config=None, datastore_o=None): current_previous_version=str(previous_version), current_diff_url=watch['url'], extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']), - left_sticky=True) + left_sticky=True, + screenshot=screenshot_url) return output @@ -846,15 +852,17 @@ def changedetection_app(config=None, datastore_o=None): else: content.append({'line': "No history found", 'classes': ''}) - + screenshot_url = datastore.get_screenshot(uuid) output = render_template("preview.html", content=content, extra_stylesheets=extra_stylesheets, ignored_line_numbers=ignored_line_numbers, triggered_line_numbers=trigger_line_numbers, current_diff_url=watch['url'], + screenshot=screenshot_url, watch=watch, uuid=uuid) + return output @app.route("/settings/notification-logs", methods=['GET']) @@ -967,6 +975,28 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/static//", methods=['GET']) def static_content(group, filename): + if group == 'screenshot': + + from flask import make_response + + # Could be sensitive, follow password requirements + if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated: + abort(403) + + # These files should be in our subdirectory + try: + # set nocache, set content-type + watch_dir = datastore_o.datastore_path + "/" + filename + response = make_response(send_from_directory(filename="last-screenshot.png", directory=watch_dir, path=watch_dir + "/last-screenshot.png")) + response.headers['Content-type'] = 'image/png' + response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' + response.headers['Pragma'] = 'no-cache' + response.headers['Expires'] = 0 + return response + + except FileNotFoundError: + abort(404) + # These files should be in our subdirectory try: return send_from_directory("static/{}".format(group), path=filename) diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index 3d036774..1f40911e 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -42,6 +42,14 @@ class Fetcher(): # Should set self.error, self.status_code and self.content pass + @abstractmethod + def quit(self): + return + + @abstractmethod + def screenshot(self): + return + @abstractmethod def get_last_status_code(self): return self.status_code @@ -116,16 +124,16 @@ class html_webdriver(Fetcher): # request_body, request_method unused for now, until some magic in the future happens. # check env for WEBDRIVER_URL - driver = webdriver.Remote( + self.driver = webdriver.Remote( command_executor=self.command_executor, desired_capabilities=DesiredCapabilities.CHROME, proxy=self.proxy) try: - driver.get(url) + self.driver.get(url) except WebDriverException as e: # Be sure we close the session window - driver.quit() + self.quit() raise # @todo - how to check this? is it possible? @@ -135,26 +143,33 @@ class html_webdriver(Fetcher): # @todo - dom wait loaded? time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))) - self.content = driver.page_source + self.content = self.driver.page_source self.headers = {} - driver.quit() - + def screenshot(self): + return self.driver.get_screenshot_as_png() + # Does the connection to the webdriver work? run a test connection. def is_ready(self): from selenium import webdriver from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.common.exceptions import WebDriverException - driver = webdriver.Remote( + self.driver = webdriver.Remote( command_executor=self.command_executor, desired_capabilities=DesiredCapabilities.CHROME) # driver.quit() seems to cause better exceptions - driver.quit() - + self.quit() return True + def quit(self): + if self.driver: + try: + self.driver.quit() + except Exception as e: + print("Exception in chrome shutdown/quit" + str(e)) + # "html_requests" is listed as the default fetcher in store.py! class html_requests(Fetcher): fetcher_description = "Basic fast Plaintext/HTTP Client" diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 3c5fdc38..786e7a2f 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -21,6 +21,7 @@ class perform_site_check(): timestamp = int(time.time()) # used for storage etc too changed_detected = False + screenshot = False # as bytes stripped_text_from_html = "" watch = self.datastore.data['watching'][uuid] @@ -171,5 +172,9 @@ class perform_site_check(): if not watch['title'] or not len(watch['title']): update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content) + if self.datastore.data['settings']['application'].get('real_browser_save_screenshot', True): + screenshot = fetcher.screenshot() - return changed_detected, update_obj, text_content_before_ignored_filter + fetcher.quit() + + return changed_detected, update_obj, text_content_before_ignored_filter, screenshot diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 72e4b27b..ae73cb85 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -345,7 +345,6 @@ class watchForm(commonSettingsForm): return result class globalSettingsForm(commonSettingsForm): - password = SaltyPasswordField() minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck', [validators.NumberRange(min=1)]) @@ -355,4 +354,5 @@ class globalSettingsForm(commonSettingsForm): global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()]) ignore_whitespace = BooleanField('Ignore whitespace') save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"}) - removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"}) \ No newline at end of file + real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?') + removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"}) diff --git a/changedetectionio/static/js/tabs.js b/changedetectionio/static/js/tabs.js index 36fecf70..f600ef47 100644 --- a/changedetectionio/static/js/tabs.js +++ b/changedetectionio/static/js/tabs.js @@ -1,6 +1,11 @@ // Rewrite this is a plugin.. is all this JS really 'worth it?' +if(!window.location.hash) { + var tab=document.querySelectorAll("#default-tab a"); + tab[0].click(); +} + window.addEventListener('hashchange', function() { var tabs = document.getElementsByClassName('active'); while (tabs[0]) { @@ -21,7 +26,6 @@ if (!has_errors.length) { focus_error_tab(); } - function set_active_tab() { var tab=document.querySelectorAll("a[href='"+location.hash+"']"); if (tab.length) { diff --git a/changedetectionio/static/styles/diff.css b/changedetectionio/static/styles/diff.css index 6ab1009a..f655bbc7 100644 --- a/changedetectionio/static/styles/diff.css +++ b/changedetectionio/static/styles/diff.css @@ -1,7 +1,8 @@ #diff-ui { background: #fff; padding: 2em; - margin: 1em; + margin-left: 1em; + margin-right: 1em; border-radius: 5px; font-size: 11px; } #diff-ui table { @@ -70,3 +71,8 @@ td#diff-col div { /* ignored and triggered? make it obvious error */ .ignored.triggered { background-color: #ff0000; } + +.tab-pane-inner#screenshot { + text-align: center; } + .tab-pane-inner#screenshot img { + max-width: 99%; } diff --git a/changedetectionio/static/styles/diff.scss b/changedetectionio/static/styles/diff.scss index eaf06176..08ae4bdb 100644 --- a/changedetectionio/static/styles/diff.scss +++ b/changedetectionio/static/styles/diff.scss @@ -2,7 +2,8 @@ background: #fff; padding: 2em; - margin: 1em; + margin-left: 1em; + margin-right: 1em; border-radius: 5px; font-size: 11px; @@ -85,4 +86,11 @@ td#diff-col div { /* ignored and triggered? make it obvious error */ .ignored.triggered { background-color: #ff0000; +} + +.tab-pane-inner#screenshot { + text-align: center; + img { + max-width: 99%; + } } \ No newline at end of file diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index cdcd2487..4a6f2cb3 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -317,7 +317,7 @@ footer { right: auto; } section.content { padding-top: 110px; } - div.tabs ul li { + div.tabs.collapsable ul li { display: block; border-radius: 0px; } input[type='text'] { @@ -403,14 +403,15 @@ and also iPads specifically. padding: 20px; border-radius: 5px; } +.tab-pane-inner { + padding: 0px; } + .tab-pane-inner:not(:target) { + display: none; } + .tab-pane-inner:target { + display: block; } + .edit-form { min-width: 70%; } - .edit-form .tab-pane-inner { - padding: 0px; } - .edit-form .tab-pane-inner:not(:target) { - display: none; } - .edit-form .tab-pane-inner:target { - display: block; } .edit-form .box-wrap { position: relative; } .edit-form .inner { diff --git a/changedetectionio/static/styles/styles.scss b/changedetectionio/static/styles/styles.scss index 5ca7eb31..d3949165 100644 --- a/changedetectionio/static/styles/styles.scss +++ b/changedetectionio/static/styles/styles.scss @@ -35,7 +35,7 @@ a.github-link { section.content { padding-top: 5em; - padding-bottom: 5em; + padding-bottom: 1em; flex-direction: column; display: flex; align-items: center; @@ -437,7 +437,7 @@ footer { } // Make the tabs easier to hit, they will be all nice and horizontal - div.tabs ul li { + div.tabs.collapsable ul li { display: block; border-radius: 0px; } @@ -573,10 +573,7 @@ $form-edge-padding: 20px; } } - -.edit-form { - min-width: 70%; - .tab-pane-inner { +.tab-pane-inner { &:not(:target) { display: none; } @@ -585,7 +582,11 @@ $form-edge-padding: 20px; } // doesnt need padding because theres another row of buttons/activity padding: 0px; - } +} + +.edit-form { + min-width: 70%; + .box-wrap { position: relative; } diff --git a/changedetectionio/store.py b/changedetectionio/store.py index a3649501..e37710de 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -57,6 +57,7 @@ class ChangeDetectionStore: 'notification_title': default_notification_title, 'notification_body': default_notification_body, 'notification_format': default_notification_format, + 'real_browser_save_screenshot': True, } } } @@ -381,6 +382,22 @@ class ChangeDetectionStore: return fname + def get_screenshot(self, watch_uuid): + output_path = "{}/{}".format(self.datastore_path, watch_uuid) + fname = "{}/last-screenshot.png".format(output_path) + if path.isfile(fname): + return fname + + return False + + # Save as PNG, PNG is larger but better for doing visual diff in the future + def save_screenshot(self, watch_uuid, screenshot: bytes): + output_path = "{}/{}".format(self.datastore_path, watch_uuid) + fname = "{}/last-screenshot.png".format(output_path) + with open(fname, 'wb') as f: + f.write(screenshot) + f.close() + def sync_to_json(self): logging.info("Saving JSON..") diff --git a/changedetectionio/templates/diff.html b/changedetectionio/templates/diff.html index 9f697a5f..c5227e54 100644 --- a/changedetectionio/templates/diff.html +++ b/changedetectionio/templates/diff.html @@ -1,7 +1,6 @@ {% extends 'base.html' %} {% block content %} -

Differences

@@ -35,21 +34,45 @@ + +{% if screenshot %} + +
+ +
+{% endif %} +
-
Pro-tip: Use show current snapshot tab to visualise what will be ignored.
- - - - - - - - - -
- -
- Diff algorithm from the amazing github.com/kpdecker/jsdiff +
+
Pro-tip: Use show current snapshot tab to visualise what will be ignored. +
+ + + + + + + + + +
+ +
+ Diff algorithm from the amazing github.com/kpdecker/jsdiff +
+ +{% if screenshot %} +
+

+ For now, only the most recent screenshot is saved and displayed. +

+ + +
+{% endif %}
diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 445dcd68..b6e6e0d8 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -7,7 +7,7 @@
-
+
  • General
  • Request
    • diff --git a/changedetectionio/templates/preview.html b/changedetectionio/templates/preview.html index 2e66cbc2..f846b810 100644 --- a/changedetectionio/templates/preview.html +++ b/changedetectionio/templates/preview.html @@ -6,18 +6,40 @@

    Current - {{watch.last_checked|format_timestamp_timeago}}

+{% if screenshot %} + +
+ +
+{% endif %} +
- Grey lines are ignored Blue lines are triggers - - - - + + +
+
+ Grey lines are ignored Blue lines are triggers + + + + - - -
{% for row in content %}
{{row.line}}
{% endfor %} -
+
+
+ +{% if screenshot %} +
+

+ For now, only the most recent screenshot is saved and displayed. +

+ + +
+{% endif %}
{% endblock %} \ No newline at end of file diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 129e5353..e3145de3 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -8,7 +8,7 @@
-
+
  • General
  • Notifications
    • @@ -50,6 +50,12 @@ {{ render_field(form.extract_title_as_title) }} Note: This will automatically apply to all existing watches.
+ +
+ {{ render_field(form.real_browser_save_screenshot) }} + When using a Chrome browser, a screenshot from the last check will be available on the Diff page +
+
diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 601eef45..51e65424 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -38,11 +38,12 @@ class update_worker(threading.Thread): changed_detected = False contents = "" + screenshot = False update_obj= {} now = time.time() try: - changed_detected, update_obj, contents = update_handler.run(uuid) + changed_detected, update_obj, contents, screenshot = update_handler.run(uuid) # Re #342 # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes. @@ -140,6 +141,9 @@ class update_worker(threading.Thread): # Always record that we atleast tried self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3), 'last_checked': round(time.time())}) + # Always save the screenshot if it's available + if screenshot: + self.datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot) self.current_uuid = None # Done self.q.task_done()