Adding jinja2/browsersteps test (#2915)

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -34,7 +34,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -48,7 +48,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(
         url_for("edit_page", uuid=uuid, unpause_on_save=1))
     assert b'No proxy' in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -81,7 +81,7 @@ def test_socks5(client, live_server, measure_memory_usage):
     assert "Awesome, you made it".encode('utf-8') in res.data
 
     # PROXY CHECKER WIDGET CHECK - this needs more checking
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     res = client.get(
         url_for("check_proxies.start_check", uuid=uuid),

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -99,7 +99,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     assert b'ldjson-price-track-offer' in res.data
 
     # Accept it
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     #time.sleep(1)
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     client.get(url_for("form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_backend.py

@@ -68,7 +68,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     # Check the 'get latest snapshot works'
     res = client.get(url_for("watch_get_latest_html", uuid=uuid))

changedetectionio/tests/test_encoding.py

@@ -40,7 +40,7 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
 
 
     # Content type recording worked
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"
 
     res = client.get(

changedetectionio/tests/test_filter_failure_notification.py

@@ -51,7 +51,7 @@ def run_filter_test(client, live_server, content_filter):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"
 

changedetectionio/tests/test_group.py

@@ -288,7 +288,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = extract_UUID_from_client(client)
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data
@@ -315,7 +315,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = extract_UUID_from_client(client)
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data

changedetectionio/tests/test_ignore.py

@@ -36,7 +36,7 @@ def test_ignore(client, live_server, measure_memory_usage):
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     # use the highlighter endpoint
     res = client.post(
         url_for("highlight_submit_ignore_url", uuid=uuid),

changedetectionio/tests/test_live_preview.py

@@ -29,7 +29,7 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage):
         data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(
         url_for("edit_page", uuid=uuid),
         data={

changedetectionio/tests/test_notification.py

@@ -6,7 +6,7 @@ from flask import url_for
 from loguru import logger
 
 from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
-    set_longer_modified_response
+    set_longer_modified_response, get_index
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -76,7 +76,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
     testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
 
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     datastore = 'test-datastore'
     with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
         f.write(base64.b64decode(testimage_png))
@@ -328,7 +328,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
 
     # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK
-    res = client.get(url_for("index"))
+    res = get_index(client)
     assert b'notification-error' not in res.data
 
     with open("test-datastore/notification.txt", 'r') as f:

changedetectionio/tests/test_request.py

@@ -378,7 +378,8 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     with open('test-datastore/headers.txt', 'w') as f:
         f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")
 
-    with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    with open(f'test-datastore/{uuid}/headers.txt', 'w') as f:
         f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")
 
     wait_for_all_checks(client)

changedetectionio/tests/test_restock_itemprop.py

@@ -380,7 +380,7 @@ def test_change_with_notification_values(client, live_server):
 
     ## Now test the "SEND TEST NOTIFICATION" is working
     os.unlink("test-datastore/notification.txt")
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
     time.sleep(5)
     assert os.path.isfile("test-datastore/notification.txt"), "Notification received"

changedetectionio/tests/test_rss.py

@@ -132,7 +132,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={

changedetectionio/tests/test_scheduler.py

@@ -39,7 +39,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
 
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
 
@@ -104,7 +104,7 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure
 
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
 

changedetectionio/tests/util.py

@@ -285,15 +285,43 @@ def live_server_setup(live_server):
             <p id="remove">This text should be removed</p>
               <form onsubmit="event.preventDefault();">
             <!-- obfuscated text so that we dont accidentally get a false positive due to conversion of the source :) --->
-                <button name="test-button" onclick="getElementById('remove').remove();getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=')">Click here</button>
-                <div id=some-content></div>
+                <button name="test-button" onclick="
+                getElementById('remove').remove();
+                getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=');
+                getElementById('reflect-text').innerHTML = getElementById('test-input-text').value;
+                ">Click here</button>
+                
+                <div id="some-content"></div>
+                
                 <pre>
                 {header_text.lower()}
                 </pre>
-              </body>
+                
+                <br>
+                <!-- used for testing that the jinja2 compiled here --->
+                <input type="text" value="" id="test-input-text" /><br>
+                <div id="reflect-text">Waiting to reflect text from #test-input-text here</div>
+              </form>
+                
+           </body>
          </html>""", 200)
         resp.headers['Content-Type'] = 'text/html'
         return resp
 
     live_server.start()
 
+def get_index(client):
+    import inspect
+    # Get the caller's frame (parent function)
+    frame = inspect.currentframe()
+    caller_frame = frame.f_back  # Go back to the caller's frame
+    caller_name = caller_frame.f_code.co_name
+    caller_line = caller_frame.f_lineno
+
+    print(f"Called by: {caller_name}, Line: {caller_line}")
+
+    res = client.get(url_for("index"))
+    with open(f"test-datastore/index-{caller_name}-{caller_line}.html", 'wb') as f:
+        f.write(res.data)
+
+    return res

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -2,14 +2,16 @@
 
 import os
 from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from ..util import live_server_setup, wait_for_all_checks, get_index
 
-def test_setup(client, live_server, measure_memory_usage):
+def test_setup(client, live_server):
     live_server_setup(live_server)
 
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
+    live_server.stop()
+    live_server.start()
 
     import os
     import json
@@ -27,7 +29,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={
@@ -87,7 +89,9 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
 
 def test_basic_browserstep(client, live_server, measure_memory_usage):
 
-    #live_server_setup(live_server)
+    live_server.stop()
+    live_server.start()
+
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
     test_url = url_for('test_interactive_html_endpoint', _external=True)
@@ -108,9 +112,13 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
             "url": test_url,
             "tags": "",
             'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Click element',
-            'browser_steps-0-selector': 'button[name=test-button]',
-            'browser_steps-0-optional_value': '',
+            'browser_steps-0-operation': 'Enter text in field',
+            'browser_steps-0-selector': '#test-input-text',
+            # Should get set to the actual text (jinja2 rendered)
+            'browser_steps-0-optional_value': "Hello-Jinja2-{% now  'Europe/Berlin', '%Y-%m-%d' %}",
+            'browser_steps-1-operation': 'Click element',
+            'browser_steps-1-selector': 'button[name=test-button]',
+            'browser_steps-1-optional_value': '',
             # For now, cookies doesnt work in headers because it must be a full cookiejar object
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
         },
@@ -119,7 +127,7 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data
@@ -132,13 +140,32 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"This text should be removed" not in res.data
     assert b"I smell JavaScript because the button was pressed" in res.data
 
+    assert b'Hello-Jinja2-20' in res.data
+
     assert b"testheader: yes" in res.data
     assert b"user-agent: mycustomagent" in res.data
+    live_server.stop()
+
+def test_non_200_errors_report_browsersteps(client, live_server):
+
+    live_server.stop()
+    live_server.start()
 
     four_o_four_url =  url_for('test_endpoint', status_code=404, _external=True)
     four_o_four_url = four_o_four_url.replace('localhost.localdomain', 'cdio')
     four_o_four_url = four_o_four_url.replace('localhost', 'cdio')
 
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": four_o_four_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+
     # now test for 404 errors
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
@@ -153,12 +180,14 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"unpaused" in res.data
+
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = get_index(client)
+
     assert b'Error - 404' in res.data
 
     client.get(
         url_for("form_delete", uuid="all"),
         follow_redirects=True
-    )
+    )