kopia lustrzana https://github.com/dgtlmoon/changedetection.io
pull/608/head
rodzic
31fea55ee4
commit
67c833d2bc
|
@ -605,12 +605,12 @@ def changedetection_app(config=None, datastore_o=None):
|
||||||
if request.method == 'POST' and not form.validate():
|
if request.method == 'POST' and not form.validate():
|
||||||
flash("An error occurred, please see below.", "error")
|
flash("An error occurred, please see below.", "error")
|
||||||
|
|
||||||
|
|
||||||
output = render_template("edit.html",
|
output = render_template("edit.html",
|
||||||
uuid=uuid,
|
uuid=uuid,
|
||||||
watch=datastore.data['watching'][uuid],
|
watch=datastore.data['watching'][uuid],
|
||||||
form=form,
|
form=form,
|
||||||
has_empty_checktime=using_default_check_time,
|
has_empty_checktime=using_default_check_time,
|
||||||
|
using_global_webdriver_wait=default['webdriver_delay'] is None,
|
||||||
current_base_url=datastore.data['settings']['application']['base_url'],
|
current_base_url=datastore.data['settings']['application']['base_url'],
|
||||||
emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
|
emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
|
||||||
)
|
)
|
||||||
|
|
|
@ -28,6 +28,9 @@ class Fetcher():
|
||||||
system_http_proxy = os.getenv('HTTP_PROXY')
|
system_http_proxy = os.getenv('HTTP_PROXY')
|
||||||
system_https_proxy = os.getenv('HTTPS_PROXY')
|
system_https_proxy = os.getenv('HTTPS_PROXY')
|
||||||
|
|
||||||
|
# Time ONTOP of the system defined env minimum time
|
||||||
|
render_extract_delay=0
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_error(self):
|
def get_error(self):
|
||||||
return self.error
|
return self.error
|
||||||
|
@ -147,7 +150,7 @@ class base_html_playwright(Fetcher):
|
||||||
# - `'commit'` - consider operation to be finished when network response is received and the document started loading.
|
# - `'commit'` - consider operation to be finished when network response is received and the document started loading.
|
||||||
# Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
|
# Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
|
||||||
# This seemed to solve nearly all 'TimeoutErrors'
|
# This seemed to solve nearly all 'TimeoutErrors'
|
||||||
extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
|
extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
|
||||||
page.wait_for_timeout(extra_wait * 1000)
|
page.wait_for_timeout(extra_wait * 1000)
|
||||||
except playwright._impl._api_types.TimeoutError as e:
|
except playwright._impl._api_types.TimeoutError as e:
|
||||||
raise EmptyReply(url=url, status_code=None)
|
raise EmptyReply(url=url, status_code=None)
|
||||||
|
@ -240,7 +243,7 @@ class base_html_webdriver(Fetcher):
|
||||||
# raise EmptyReply(url=url, status_code=r.status_code)
|
# raise EmptyReply(url=url, status_code=r.status_code)
|
||||||
|
|
||||||
# @todo - dom wait loaded?
|
# @todo - dom wait loaded?
|
||||||
time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
|
time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
|
||||||
self.content = self.driver.page_source
|
self.content = self.driver.page_source
|
||||||
self.headers = {}
|
self.headers = {}
|
||||||
self.screenshot = self.driver.get_screenshot_as_png()
|
self.screenshot = self.driver.get_screenshot_as_png()
|
||||||
|
|
|
@ -97,7 +97,13 @@ class perform_site_check():
|
||||||
proxy_args = self.set_proxy_from_list(watch)
|
proxy_args = self.set_proxy_from_list(watch)
|
||||||
fetcher = klass(proxy_override=proxy_args)
|
fetcher = klass(proxy_override=proxy_args)
|
||||||
|
|
||||||
# Proxy List support
|
# Configurable per-watch or global extra delay before extracting text (for webDriver types)
|
||||||
|
system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
|
||||||
|
if watch['webdriver_delay'] is not None:
|
||||||
|
fetcher.render_extract_delay = watch['webdriver_delay']
|
||||||
|
elif system_webdriver_delay is not None:
|
||||||
|
fetcher.render_extract_delay = system_webdriver_delay
|
||||||
|
|
||||||
fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
|
fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
|
||||||
|
|
||||||
# Fetching complete, now filters
|
# Fetching complete, now filters
|
||||||
|
|
|
@ -318,6 +318,7 @@ class commonSettingsForm(Form):
|
||||||
notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format)
|
notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format)
|
||||||
fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
|
fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
|
||||||
extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
|
extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
|
||||||
|
webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")] )
|
||||||
|
|
||||||
class watchForm(commonSettingsForm):
|
class watchForm(commonSettingsForm):
|
||||||
|
|
||||||
|
|
|
@ -41,7 +41,8 @@ class model(dict):
|
||||||
'notification_body': default_notification_body,
|
'notification_body': default_notification_body,
|
||||||
'notification_format': default_notification_format,
|
'notification_format': default_notification_format,
|
||||||
'real_browser_save_screenshot': True,
|
'real_browser_save_screenshot': True,
|
||||||
'schema_version' : 0
|
'schema_version' : 0,
|
||||||
|
'webdriver_delay': None # Extra delay in seconds before extracting text
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -43,7 +43,8 @@ class model(dict):
|
||||||
# Re #110, so then if this is set to None, we know to use the default value instead
|
# Re #110, so then if this is set to None, we know to use the default value instead
|
||||||
# Requires setting to None on submit if it's the same as the default
|
# Requires setting to None on submit if it's the same as the default
|
||||||
# Should be all None by default, so we use the system default in this case.
|
# Should be all None by default, so we use the system default in this case.
|
||||||
'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}
|
'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
|
||||||
|
'webdriver_delay': None
|
||||||
}
|
}
|
||||||
|
|
||||||
def __init__(self, *arg, **kw):
|
def __init__(self, *arg, **kw):
|
||||||
|
|
|
@ -0,0 +1,16 @@
|
||||||
|
$(document).ready(function() {
|
||||||
|
function toggle() {
|
||||||
|
if ($('input[name="application-fetch_backend"]:checked').val() != 'html_requests') {
|
||||||
|
$('#requests-override-options').hide();
|
||||||
|
$('#webdriver-override-options').show();
|
||||||
|
} else {
|
||||||
|
$('#requests-override-options').show();
|
||||||
|
$('#webdriver-override-options').hide();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$('input[name="application-fetch_backend"]').click(function (e) {
|
||||||
|
toggle();
|
||||||
|
});
|
||||||
|
toggle();
|
||||||
|
|
||||||
|
});
|
|
@ -2,8 +2,10 @@ $(document).ready(function() {
|
||||||
function toggle() {
|
function toggle() {
|
||||||
if ($('input[name="fetch_backend"]:checked').val() != 'html_requests') {
|
if ($('input[name="fetch_backend"]:checked').val() != 'html_requests') {
|
||||||
$('#requests-override-options').hide();
|
$('#requests-override-options').hide();
|
||||||
|
$('#webdriver-override-options').show();
|
||||||
} else {
|
} else {
|
||||||
$('#requests-override-options').show();
|
$('#requests-override-options').show();
|
||||||
|
$('#webdriver-override-options').hide();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$('input[name="fetch_backend"]').click(function (e) {
|
$('input[name="fetch_backend"]').click(function (e) {
|
||||||
|
|
|
@ -73,6 +73,21 @@
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
<fieldset class="pure-group" id="webdriver-override-options">
|
||||||
|
<div class="pure-form-message-inline">
|
||||||
|
<strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
|
||||||
|
<br/>
|
||||||
|
This will wait <i>n</i> seconds before extracting the text.
|
||||||
|
</div>
|
||||||
|
<div class="pure-control-group">
|
||||||
|
{{ render_field(form.webdriver_delay) }}
|
||||||
|
</div>
|
||||||
|
{% if using_global_webdriver_wait %}
|
||||||
|
<div class="pure-form-message-inline">
|
||||||
|
<strong>Using the current global default settings</strong>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
</fieldset>
|
||||||
<fieldset class="pure-group" id="requests-override-options">
|
<fieldset class="pure-group" id="requests-override-options">
|
||||||
<div class="pure-form-message-inline">
|
<div class="pure-form-message-inline">
|
||||||
<strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
|
<strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
|
||||||
|
|
|
@ -12,6 +12,7 @@
|
||||||
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
|
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
|
||||||
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
|
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
|
||||||
|
|
||||||
|
<script type="text/javascript" src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
|
||||||
<div class="edit-form">
|
<div class="edit-form">
|
||||||
<div class="tabs collapsable">
|
<div class="tabs collapsable">
|
||||||
<ul>
|
<ul>
|
||||||
|
@ -87,6 +88,16 @@
|
||||||
<p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
|
<p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
<fieldset class="pure-group" id="webdriver-override-options">
|
||||||
|
<div class="pure-form-message-inline">
|
||||||
|
<strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
|
||||||
|
<br/>
|
||||||
|
This will wait <i>n</i> seconds before extracting the text.
|
||||||
|
</div>
|
||||||
|
<div class="pure-control-group">
|
||||||
|
{{ render_field(form.application.form.webdriver_delay) }}
|
||||||
|
</div>
|
||||||
|
</fieldset>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
Ładowanie…
Reference in New Issue