kopia lustrzana https://github.com/dgtlmoon/changedetection.io
feature: Support XPath2.0 to 3.1 (#1774)
Constantin Hong
GitHub
rodzic
5229094e44
commit
26931e0167
|
|
@ -268,3 +268,7 @@ I offer commercial support, this software is depended on by network security, ae
|
|||
[license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
|
||||
[release-link]: https://github.com/dgtlmoon/changedetection.io/releases
|
||||
[docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io
|
||||
|
||||
## Third-party licenses
|
||||
|
||||
changedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE)
|
||||
|
|
|
|||
|
|
@ -69,11 +69,12 @@ xpath://body/div/span[contains(@class, 'example-class')]",
|
|||
{% endif %}
|
||||
</ul>
|
||||
</li>
|
||||
<li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
|
||||
<li>XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with <code>xpath:</code>
|
||||
<ul>
|
||||
<li>Example: <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
|
||||
<li>Example: <code>//*[contains(@class, 'sametext')]</code> or <code>xpath:count(//*[contains(@class, 'sametext')])</code>, <a
|
||||
href="http://xpather.com/" target="new">test your XPath here</a></li>
|
||||
<li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
|
||||
<li>To use XPath1.0: Prefix with <code>xpath1:</code></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -328,11 +328,30 @@ class ValidateCSSJSONXPATHInput(object):
|
|||
return
|
||||
|
||||
# Does it look like XPath?
|
||||
if line.strip()[0] == '/':
|
||||
if line.strip()[0] == '/' or line.strip().startswith('xpath:'):
|
||||
if not self.allow_xpath:
|
||||
raise ValidationError("XPath not permitted in this field!")
|
||||
from lxml import etree, html
|
||||
import elementpath
|
||||
# xpath 2.0-3.1
|
||||
from elementpath.xpath3 import XPath3Parser
|
||||
tree = html.fromstring("<html></html>")
|
||||
line = line.replace('xpath:', '')
|
||||
|
||||
try:
|
||||
elementpath.select(tree, line.strip(), parser=XPath3Parser)
|
||||
except elementpath.ElementPathError as e:
|
||||
message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
|
||||
raise ValidationError(message % (line, str(e)))
|
||||
except:
|
||||
raise ValidationError("A system-error occurred when validating your XPath expression")
|
||||
|
||||
if line.strip().startswith('xpath1:'):
|
||||
if not self.allow_xpath:
|
||||
raise ValidationError("XPath not permitted in this field!")
|
||||
from lxml import etree, html
|
||||
tree = html.fromstring("<html></html>")
|
||||
line = line.replace('xpath1:', '')
|
||||
|
||||
try:
|
||||
tree.xpath(line.strip())
|
||||
|
|
|
|||
|
|
@ -69,10 +69,89 @@ def element_removal(selectors: List[str], html_content):
|
|||
selector = ",".join(selectors)
|
||||
return subtractive_css_selector(selector, html_content)
|
||||
|
||||
def elementpath_tostring(obj):
|
||||
"""
|
||||
change elementpath.select results to string type
|
||||
# The MIT License (MIT), Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati)
|
||||
# https://github.com/sissaschool/elementpath/blob/dfcc2fd3d6011b16e02bf30459a7924f547b47d0/elementpath/xpath_tokens.py#L1038
|
||||
"""
|
||||
|
||||
import elementpath
|
||||
from decimal import Decimal
|
||||
import math
|
||||
|
||||
if obj is None:
|
||||
return ''
|
||||
# https://elementpath.readthedocs.io/en/latest/xpath_api.html#elementpath.select
|
||||
elif isinstance(obj, elementpath.XPathNode):
|
||||
return obj.string_value
|
||||
elif isinstance(obj, bool):
|
||||
return 'true' if obj else 'false'
|
||||
elif isinstance(obj, Decimal):
|
||||
value = format(obj, 'f')
|
||||
if '.' in value:
|
||||
return value.rstrip('0').rstrip('.')
|
||||
return value
|
||||
|
||||
elif isinstance(obj, float):
|
||||
if math.isnan(obj):
|
||||
return 'NaN'
|
||||
elif math.isinf(obj):
|
||||
return str(obj).upper()
|
||||
|
||||
value = str(obj)
|
||||
if '.' in value:
|
||||
value = value.rstrip('0').rstrip('.')
|
||||
if '+' in value:
|
||||
value = value.replace('+', '')
|
||||
if 'e' in value:
|
||||
return value.upper()
|
||||
return value
|
||||
|
||||
return str(obj)
|
||||
|
||||
# Return str Utf-8 of matched rules
|
||||
def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False):
|
||||
from lxml import etree, html
|
||||
import elementpath
|
||||
# xpath 2.0-3.1
|
||||
from elementpath.xpath3 import XPath3Parser
|
||||
|
||||
parser = etree.HTMLParser()
|
||||
if is_rss:
|
||||
# So that we can keep CDATA for cdata_in_document_to_text() to process
|
||||
parser = etree.XMLParser(strip_cdata=False)
|
||||
|
||||
tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
|
||||
html_block = ""
|
||||
|
||||
r = elementpath.select(tree, xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'}, parser=XPath3Parser)
|
||||
#@note: //title/text() wont work where <title>CDATA..
|
||||
|
||||
if type(r) != list:
|
||||
r = [r]
|
||||
|
||||
for element in r:
|
||||
# When there's more than 1 match, then add the suffix to separate each line
|
||||
# And where the matched result doesn't include something that will cause Inscriptis to add a newline
|
||||
# (This way each 'match' reliably has a new-line in the diff)
|
||||
# Divs are converted to 4 whitespaces by inscriptis
|
||||
if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
|
||||
html_block += TEXT_FILTER_LIST_LINE_SUFFIX
|
||||
|
||||
if type(element) == str:
|
||||
html_block += element
|
||||
elif issubclass(type(element), etree._Element) or issubclass(type(element), etree._ElementTree):
|
||||
html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
|
||||
else:
|
||||
html_block += elementpath_tostring(element)
|
||||
|
||||
return html_block
|
||||
|
||||
# Return str Utf-8 of matched rules
|
||||
# 'xpath1:'
|
||||
def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False):
|
||||
from lxml import etree, html
|
||||
|
||||
parser = None
|
||||
if is_rss:
|
||||
|
|
|
|||
|
|
@ -173,6 +173,11 @@ class perform_site_check(difference_detection_processor):
|
|||
html_content=self.fetcher.content,
|
||||
append_pretty_line_formatting=not watch.is_source_type_url,
|
||||
is_rss=is_rss)
|
||||
elif filter_rule.startswith('xpath1:'):
|
||||
html_content += html_tools.xpath1_filter(xpath_filter=filter_rule.replace('xpath1:', ''),
|
||||
html_content=fetcher.content,
|
||||
append_pretty_line_formatting=not is_source,
|
||||
is_rss=is_rss)
|
||||
else:
|
||||
# CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
|
||||
html_content += html_tools.include_filters(include_filters=filter_rule,
|
||||
|
|
|
|||
|
|
@ -847,4 +847,14 @@ class ChangeDetectionStore:
|
|||
if not watch.get('date_created'):
|
||||
self.data['watching'][uuid]['date_created'] = i
|
||||
i+=1
|
||||
return
|
||||
return
|
||||
|
||||
# #1774 - protect xpath1 against migration
|
||||
def update_14(self):
|
||||
for awatch in self.__data["watching"]:
|
||||
if self.__data["watching"][awatch]['include_filters']:
|
||||
for num, selector in enumerate(self.__data["watching"][awatch]['include_filters']):
|
||||
if selector.startswith('/'):
|
||||
self.__data["watching"][awatch]['include_filters'][num] = 'xpath1:' + selector
|
||||
if selector.startswith('xpath:'):
|
||||
self.__data["watching"][awatch]['include_filters'][num] = selector.replace('xpath:', 'xpath1:', 1)
|
||||
|
|
|
|||
|
|
@ -290,11 +290,12 @@ xpath://body/div/span[contains(@class, 'example-class')]",
|
|||
{% endif %}
|
||||
</ul>
|
||||
</li>
|
||||
<li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
|
||||
<li>XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with <code>xpath:</code>
|
||||
<ul>
|
||||
<li>Example: <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
|
||||
<li>Example: <code>//*[contains(@class, 'sametext')]</code> or <code>xpath:count(//*[contains(@class, 'sametext')])</code>, <a
|
||||
href="http://xpather.com/" target="new">test your XPath here</a></li>
|
||||
<li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
|
||||
<li>To use XPath1.0: Prefix with <code>xpath1:</code></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -6,9 +6,11 @@ from .util import live_server_setup, wait_for_all_checks
|
|||
|
||||
from ..html_tools import *
|
||||
|
||||
|
||||
def test_setup(live_server):
|
||||
live_server_setup(live_server)
|
||||
|
||||
|
||||
def set_original_response():
|
||||
test_return_data = """<html>
|
||||
<body>
|
||||
|
|
@ -26,6 +28,7 @@ def set_original_response():
|
|||
f.write(test_return_data)
|
||||
return None
|
||||
|
||||
|
||||
def set_modified_response():
|
||||
test_return_data = """<html>
|
||||
<body>
|
||||
|
|
@ -44,11 +47,12 @@ def set_modified_response():
|
|||
|
||||
return None
|
||||
|
||||
|
||||
# Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
|
||||
def test_check_xpath_filter_utf8(client, live_server):
|
||||
filter='//item/*[self::description]'
|
||||
filter = '//item/*[self::description]'
|
||||
|
||||
d='''<?xml version="1.0" encoding="UTF-8"?>
|
||||
d = '''<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
|
||||
<channel>
|
||||
<title>rpilocator.com</title>
|
||||
|
|
@ -102,9 +106,9 @@ def test_check_xpath_filter_utf8(client, live_server):
|
|||
|
||||
# Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
|
||||
def test_check_xpath_text_function_utf8(client, live_server):
|
||||
filter='//item/title/text()'
|
||||
filter = '//item/title/text()'
|
||||
|
||||
d='''<?xml version="1.0" encoding="UTF-8"?>
|
||||
d = '''<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
|
||||
<channel>
|
||||
<title>rpilocator.com</title>
|
||||
|
|
@ -163,15 +167,12 @@ def test_check_xpath_text_function_utf8(client, live_server):
|
|||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
def test_check_markup_xpath_filter_restriction(client, live_server):
|
||||
|
||||
def test_check_markup_xpath_filter_restriction(client, live_server):
|
||||
xpath_filter = "//*[contains(@class, 'sametext')]"
|
||||
|
||||
set_original_response()
|
||||
|
||||
# Give the endpoint time to spin up
|
||||
time.sleep(1)
|
||||
|
||||
# Add our URL to the import page
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
|
|
@ -214,7 +215,6 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
|
|||
|
||||
|
||||
def test_xpath_validation(client, live_server):
|
||||
|
||||
# Add our URL to the import page
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
|
|
@ -235,6 +235,48 @@ def test_xpath_validation(client, live_server):
|
|||
assert b'Deleted' in res.data
|
||||
|
||||
|
||||
def test_xpath23_prefix_validation(client, live_server):
|
||||
# Add our URL to the import page
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"1 Imported" in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": "xpath:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"is not a valid XPath expression" in res.data
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
|
||||
def test_xpath1_validation(client, live_server):
|
||||
# Add our URL to the import page
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"1 Imported" in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": "xpath1:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"is not a valid XPath expression" in res.data
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
|
||||
# actually only really used by the distll.io importer, but could be handy too
|
||||
def test_check_with_prefix_include_filters(client, live_server):
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
|
|
@ -254,7 +296,8 @@ def test_check_with_prefix_include_filters(client, live_server):
|
|||
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
|
||||
data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "",
|
||||
'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
|
|
@ -266,13 +309,15 @@ def test_check_with_prefix_include_filters(client, live_server):
|
|||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Some text thats the same" in res.data #in selector
|
||||
assert b"Some text that will change" not in res.data #not in selector
|
||||
assert b"Some text thats the same" in res.data # in selector
|
||||
assert b"Some text that will change" not in res.data # not in selector
|
||||
|
||||
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
|
||||
|
||||
def test_various_rules(client, live_server):
|
||||
# Just check these don't error
|
||||
#live_server_setup(live_server)
|
||||
# live_server_setup(live_server)
|
||||
with open("test-datastore/endpoint-content.txt", "w") as f:
|
||||
f.write("""<html>
|
||||
<body>
|
||||
|
|
@ -285,10 +330,11 @@ def test_various_rules(client, live_server):
|
|||
<a href=''>some linky </a>
|
||||
<a href=''>another some linky </a>
|
||||
<!-- related to https://github.com/dgtlmoon/changedetection.io/pull/1774 -->
|
||||
<input type="email" id="email" />
|
||||
<input type="email" id="email" />
|
||||
</body>
|
||||
</html>
|
||||
""")
|
||||
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
|
|
@ -298,7 +344,6 @@ def test_various_rules(client, live_server):
|
|||
assert b"1 Imported" in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
|
||||
for r in ['//div', '//a', 'xpath://div', 'xpath://a']:
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
|
|
@ -313,3 +358,153 @@ def test_various_rules(client, live_server):
|
|||
assert b"Updated watch." in res.data
|
||||
res = client.get(url_for("index"))
|
||||
assert b'fetch-error' not in res.data, f"Should not see errors after '{r} filter"
|
||||
|
||||
res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
assert b'Deleted' in res.data
|
||||
|
||||
|
||||
def test_xpath_20(client, live_server):
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"1 Imported" in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
set_original_response()
|
||||
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": "//*[contains(@class, 'sametext')]|//*[contains(@class, 'changetext')]",
|
||||
"url": test_url,
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Updated watch." in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(
|
||||
url_for("preview_page", uuid="first"),
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Some text thats the same" in res.data # in selector
|
||||
assert b"Some text that will change" in res.data # in selector
|
||||
|
||||
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
|
||||
|
||||
def test_xpath_20_function_count(client, live_server):
|
||||
set_original_response()
|
||||
|
||||
# Add our URL to the import page
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"1 Imported" in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": "xpath:count(//div) * 123456789987654321",
|
||||
"url": test_url,
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Updated watch." in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(
|
||||
url_for("preview_page", uuid="first"),
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"246913579975308642" in res.data # in selector
|
||||
|
||||
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
|
||||
|
||||
def test_xpath_20_function_count2(client, live_server):
|
||||
set_original_response()
|
||||
|
||||
# Add our URL to the import page
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"1 Imported" in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={"include_filters": "/html/body/count(div) * 123456789987654321",
|
||||
"url": test_url,
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Updated watch." in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(
|
||||
url_for("preview_page", uuid="first"),
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"246913579975308642" in res.data # in selector
|
||||
|
||||
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
|
||||
|
||||
def test_xpath_20_function_string_join_matches(client, live_server):
|
||||
set_original_response()
|
||||
|
||||
# Add our URL to the import page
|
||||
test_url = url_for('test_endpoint', _external=True)
|
||||
res = client.post(
|
||||
url_for("import_page"),
|
||||
data={"urls": test_url},
|
||||
follow_redirects=True
|
||||
)
|
||||
assert b"1 Imported" in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.post(
|
||||
url_for("edit_page", uuid="first"),
|
||||
data={
|
||||
"include_filters": "xpath:string-join(//*[contains(@class, 'sametext')]|//*[matches(@class, 'changetext')], 'specialconjunction')",
|
||||
"url": test_url,
|
||||
"tags": "",
|
||||
"headers": "",
|
||||
'fetch_backend': "html_requests"},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Updated watch." in res.data
|
||||
wait_for_all_checks(client)
|
||||
|
||||
res = client.get(
|
||||
url_for("preview_page", uuid="first"),
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
assert b"Some text thats the samespecialconjunctionSome text that will change" in res.data # in selector
|
||||
|
||||
client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,203 @@
|
|||
import sys
|
||||
import os
|
||||
import pytest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
import html_tools
|
||||
|
||||
# test generation guide.
|
||||
# 1. Do not include encoding in the xml declaration if the test object is a str type.
|
||||
# 2. Always paraphrase test.
|
||||
|
||||
hotels = """
|
||||
<hotel>
|
||||
<branch location="California">
|
||||
<staff>
|
||||
<given_name>Christopher</given_name>
|
||||
<surname>Anderson</surname>
|
||||
<age>25</age>
|
||||
</staff>
|
||||
<staff>
|
||||
<given_name>Christopher</given_name>
|
||||
<surname>Carter</surname>
|
||||
<age>30</age>
|
||||
</staff>
|
||||
</branch>
|
||||
<branch location="Las Vegas">
|
||||
<staff>
|
||||
<given_name>Lisa</given_name>
|
||||
<surname>Walker</surname>
|
||||
<age>60</age>
|
||||
</staff>
|
||||
<staff>
|
||||
<given_name>Jessica</given_name>
|
||||
<surname>Walker</surname>
|
||||
<age>32</age>
|
||||
</staff>
|
||||
<staff>
|
||||
<given_name>Jennifer</given_name>
|
||||
<surname>Roberts</surname>
|
||||
<age>50</age>
|
||||
</staff>
|
||||
</branch>
|
||||
</hotel>"""
|
||||
|
||||
@pytest.mark.parametrize("html_content", [hotels])
|
||||
@pytest.mark.parametrize("xpath, answer", [('(//staff/given_name, //staff/age)', '25'),
|
||||
("xs:date('2023-10-10')", '2023-10-10'),
|
||||
("if (/hotel/branch[@location = 'California']/staff[1]/age = 25) then 'is 25' else 'is not 25'", 'is 25'),
|
||||
("if (//hotel/branch[@location = 'California']/staff[1]/age = 25) then 'is 25' else 'is not 25'", 'is 25'),
|
||||
("if (count(/hotel/branch/staff) = 5) then true() else false()", 'true'),
|
||||
("if (count(//hotel/branch/staff) = 5) then true() else false()", 'true'),
|
||||
("for $i in /hotel/branch/staff return if ($i/age >= 40) then upper-case($i/surname) else lower-case($i/surname)", 'anderson'),
|
||||
("given_name = 'Christopher' and age = 40", 'false'),
|
||||
("//given_name = 'Christopher' and //age = 40", 'false'),
|
||||
#("(staff/given_name, staff/age)", 'Lisa'),
|
||||
("(//staff/given_name, //staff/age)", 'Lisa'),
|
||||
#("hotel/branch[@location = 'California']/staff/age union hotel/branch[@location = 'Las Vegas']/staff/age", ''),
|
||||
("(//hotel/branch[@location = 'California']/staff/age union //hotel/branch[@location = 'Las Vegas']/staff/age)", '60'),
|
||||
("(200 to 210)", "205"),
|
||||
("(//hotel/branch[@location = 'California']/staff/age union //hotel/branch[@location = 'Las Vegas']/staff/age)", "50"),
|
||||
("(1, 9, 9, 5)", "5"),
|
||||
("(3, (), (14, 15), 92, 653)", "653"),
|
||||
("for $i in /hotel/branch/staff return $i/given_name", "Christopher"),
|
||||
("for $i in //hotel/branch/staff return $i/given_name", "Christopher"),
|
||||
("distinct-values(for $i in /hotel/branch/staff return $i/given_name)", "Jessica"),
|
||||
("distinct-values(for $i in //hotel/branch/staff return $i/given_name)", "Jessica"),
|
||||
("for $i in (7 to 15) return $i*10", "130"),
|
||||
("some $i in /hotel/branch/staff satisfies $i/age < 20", "false"),
|
||||
("some $i in //hotel/branch/staff satisfies $i/age < 20", "false"),
|
||||
("every $i in /hotel/branch/staff satisfies $i/age > 20", "true"),
|
||||
("every $i in //hotel/branch/staff satisfies $i/age > 20 ", "true"),
|
||||
("let $x := branch[@location = 'California'], $y := branch[@location = 'Las Vegas'] return (avg($x/staff/age), avg($y/staff/age))", "27.5"),
|
||||
("let $x := //branch[@location = 'California'], $y := //branch[@location = 'Las Vegas'] return (avg($x/staff/age), avg($y/staff/age))", "27.5"),
|
||||
("let $nu := 1, $de := 1000 return 'probability = ' || $nu div $de * 100 || '%'", "0.1%"),
|
||||
("let $nu := 2, $probability := function ($argument) { 'probability = ' || $nu div $argument * 100 || '%'}, $de := 5 return $probability($de)", "40%"),
|
||||
("'XPATH2.0-3.1 dissemination' instance of xs:string ", "true"),
|
||||
("'new stackoverflow question incoming' instance of xs:integer ", "false"),
|
||||
("'50000' cast as xs:integer", "50000"),
|
||||
("//branch[@location = 'California']/staff[1]/surname eq 'Anderson'", "true"),
|
||||
("fn:false()", "false")])
|
||||
def test_hotels(html_content, xpath, answer):
|
||||
html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)
|
||||
assert type(html_content) == str
|
||||
assert answer in html_content
|
||||
|
||||
|
||||
|
||||
branches_to_visit = """<?xml version="1.0" ?>
|
||||
<branches_to_visit>
|
||||
<manager name="Godot" room_no="501">
|
||||
<branch>Area 51</branch>
|
||||
<branch>A place with no name</branch>
|
||||
<branch>Stalsk12</branch>
|
||||
</manager>
|
||||
<manager name="Freya" room_no="305">
|
||||
<branch>Stalsk12</branch>
|
||||
<branch>Barcelona</branch>
|
||||
<branch>Paris</branch>
|
||||
</manager>
|
||||
</branches_to_visit>"""
|
||||
@pytest.mark.parametrize("html_content", [branches_to_visit])
|
||||
@pytest.mark.parametrize("xpath, answer", [
|
||||
("manager[@name = 'Godot']/branch union manager[@name = 'Freya']/branch", "Area 51"),
|
||||
("//manager[@name = 'Godot']/branch union //manager[@name = 'Freya']/branch", "Stalsk12"),
|
||||
("manager[@name = 'Godot']/branch | manager[@name = 'Freya']/branch", "Stalsk12"),
|
||||
("//manager[@name = 'Godot']/branch | //manager[@name = 'Freya']/branch", "Stalsk12"),
|
||||
("manager/branch intersect manager[@name = 'Godot']/branch", "A place with no name"),
|
||||
("//manager/branch intersect //manager[@name = 'Godot']/branch", "A place with no name"),
|
||||
("manager[@name = 'Godot']/branch intersect manager[@name = 'Freya']/branch", ""),
|
||||
("manager/branch except manager[@name = 'Godot']/branch", "Barcelona"),
|
||||
("manager[@name = 'Godot']/branch[1] eq 'Area 51'", "true"),
|
||||
("//manager[@name = 'Godot']/branch[1] eq 'Area 51'", "true"),
|
||||
("manager[@name = 'Godot']/branch[1] eq 'Seoul'", "false"),
|
||||
("//manager[@name = 'Godot']/branch[1] eq 'Seoul'", "false"),
|
||||
("manager[@name = 'Godot']/branch[2] eq manager[@name = 'Freya']/branch[2]", "false"),
|
||||
("//manager[@name = 'Godot']/branch[2] eq //manager[@name = 'Freya']/branch[2]", "false"),
|
||||
("manager[1]/@room_no lt manager[2]/@room_no", "false"),
|
||||
("//manager[1]/@room_no lt //manager[2]/@room_no", "false"),
|
||||
("manager[1]/@room_no gt manager[2]/@room_no", "true"),
|
||||
("//manager[1]/@room_no gt //manager[2]/@room_no", "true"),
|
||||
("manager[@name = 'Godot']/branch[1] = 'Area 51'", "true"),
|
||||
("//manager[@name = 'Godot']/branch[1] = 'Area 51'", "true"),
|
||||
("manager[@name = 'Godot']/branch[1] = 'Seoul'", "false"),
|
||||
("//manager[@name = 'Godot']/branch[1] = 'Seoul'", "false"),
|
||||
("manager[@name = 'Godot']/branch = 'Area 51'", "true"),
|
||||
("//manager[@name = 'Godot']/branch = 'Area 51'", "true"),
|
||||
("manager[@name = 'Godot']/branch = 'Barcelona'", "false"),
|
||||
("//manager[@name = 'Godot']/branch = 'Barcelona'", "false"),
|
||||
("manager[1]/@room_no > manager[2]/@room_no", "true"),
|
||||
("//manager[1]/@room_no > //manager[2]/@room_no", "true"),
|
||||
("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[1]/branch[1]", "false"),
|
||||
("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[1]/branch[1]", "false"),
|
||||
("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[1]/branch[3]", "true"),
|
||||
("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[1]/branch[3]", "true"),
|
||||
("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] << manager[1]/branch[1]", "false"),
|
||||
("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] << //manager[1]/branch[1]", "false"),
|
||||
("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] >> manager[1]/branch[1]", "true"),
|
||||
("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] >> //manager[1]/branch[1]", "true"),
|
||||
("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[@name = 'Freya']/branch[ . = 'Stalsk12']", "false"),
|
||||
("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[@name = 'Freya']/branch[ . = 'Stalsk12']", "false"),
|
||||
("manager[1]/@name || manager[2]/@name", "GodotFreya"),
|
||||
("//manager[1]/@name || //manager[2]/@name", "GodotFreya"),
|
||||
])
|
||||
def test_branches_to_visit(html_content, xpath, answer):
|
||||
html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)
|
||||
assert type(html_content) == str
|
||||
assert answer in html_content
|
||||
|
||||
trips = """
|
||||
<trips>
|
||||
<trip reservation_number="10">
|
||||
<depart>2023-10-06</depart>
|
||||
<arrive>2023-10-10</arrive>
|
||||
<traveler name="Christopher Anderson">
|
||||
<duration>4</duration>
|
||||
<price>2000.00</price>
|
||||
</traveler>
|
||||
</trip>
|
||||
<trip reservation_number="12">
|
||||
<depart>2023-10-06</depart>
|
||||
<arrive>2023-10-12</arrive>
|
||||
<traveler name="Frank Carter">
|
||||
<duration>6</duration>
|
||||
<price>3500.34</price>
|
||||
</traveler>
|
||||
</trip>
|
||||
</trips>"""
|
||||
@pytest.mark.parametrize("html_content", [trips])
|
||||
@pytest.mark.parametrize("xpath, answer", [
|
||||
("1 + 9 * 9 + 5 div 5", "83"),
|
||||
("(1 + 9 * 9 + 5) div 6", "14.5"),
|
||||
("23 idiv 3", "7"),
|
||||
("23 div 3", "7.66666666"),
|
||||
("for $i in ./trip return $i/traveler/duration * $i/traveler/price", "21002.04"),
|
||||
("for $i in ./trip return $i/traveler/duration ", "4"),
|
||||
("for $i in .//trip return $i/traveler/duration * $i/traveler/price", "21002.04"),
|
||||
("sum(for $i in ./trip return $i/traveler/duration * $i/traveler/price)", "29002.04"),
|
||||
("sum(for $i in .//trip return $i/traveler/duration * $i/traveler/price)", "29002.04"),
|
||||
#("trip[1]/depart - trip[1]/arrive", "fail_to_get_answer"),
|
||||
#("//trip[1]/depart - //trip[1]/arrive", "fail_to_get_answer"),
|
||||
#("trip[1]/depart + trip[1]/arrive", "fail_to_get_answer"),
|
||||
#("xs:date(trip[1]/depart) + xs:date(trip[1]/arrive)", "fail_to_get_answer"),
|
||||
("(//trip[1]/arrive cast as xs:date) - (//trip[1]/depart cast as xs:date)", "P4D"),
|
||||
("(//trip[1]/depart cast as xs:date) - (//trip[1]/arrive cast as xs:date)", "-P4D"),
|
||||
("(//trip[1]/depart cast as xs:date) + xs:dayTimeDuration('P3D')", "2023-10-09"),
|
||||
("(//trip[1]/depart cast as xs:date) - xs:dayTimeDuration('P3D')", "2023-10-03"),
|
||||
("(456, 623) instance of xs:integer", "false"),
|
||||
("(456, 623) instance of xs:integer*", "true"),
|
||||
("/trips/trip instance of element()", "false"),
|
||||
("/trips/trip instance of element()*", "true"),
|
||||
("/trips/trip[1]/arrive instance of xs:date", "false"),
|
||||
("date(/trips/trip[1]/arrive) instance of xs:date", "true"),
|
||||
("'8' cast as xs:integer", "8"),
|
||||
("'11.1E3' cast as xs:double", "11100"),
|
||||
("6.5 cast as xs:integer", "6"),
|
||||
#("/trips/trip[1]/arrive cast as xs:dateTime", "fail_to_get_answer"),
|
||||
("/trips/trip[1]/arrive cast as xs:date", "2023-10-10"),
|
||||
("('2023-10-12') cast as xs:date", "2023-10-12"),
|
||||
("for $i in //trip return concat($i/depart, ' ', $i/arrive)", "2023-10-06 2023-10-10"),
|
||||
])
|
||||
def test_trips(html_content, xpath, answer):
|
||||
html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)
|
||||
assert type(html_content) == str
|
||||
assert answer in html_content
|
||||
|
|
@ -46,6 +46,9 @@ beautifulsoup4
|
|||
# XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
|
||||
lxml
|
||||
|
||||
# XPath 2.0-3.1 support
|
||||
elementpath
|
||||
|
||||
selenium~=4.14.0
|
||||
|
||||
werkzeug~=3.0
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue