diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 19c83b94..668f5ca0 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -436,46 +436,61 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False return re.sub(pattern, repl, html_content) - - # NOTE!! ANYTHING LIBXML, HTML5LIB ETC WILL CAUSE SOME SMALL MEMORY LEAK IN THE LOCAL "LIB" IMPLEMENTATION OUTSIDE PYTHON -def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False): +import os +def html_to_text_sub_worker(temp_file_path, html_content, render_anchor_tag_content=False, is_rss=False): from inscriptis import get_text from inscriptis.model.config import ParserConfig + try: + if render_anchor_tag_content: + parser_config = ParserConfig( + annotation_rules={"a": ["hyperlink"]}, + display_links=True + ) + else: + parser_config = None - """Converts html string to a string with just the text. If ignoring - rendering anchor tag content is enable, anchor tag content are also - included in the text + if is_rss: + html_content = re.sub(r'])', r'', r'', html_content) - :param html_content: string with html content - :param render_anchor_tag_content: boolean flag indicating whether to extract - hyperlinks (the anchor tag content) together with text. This refers to the - 'href' inside 'a' tags. - Anchor tag content is rendered in the following manner: - '[ text ](anchor tag content)' - :return: extracted text from the HTML - """ - # if anchor tag content flag is set to True define a config for - # extracting this content - if render_anchor_tag_content: - parser_config = ParserConfig( - annotation_rules={"a": ["hyperlink"]}, - display_links=True - ) - # otherwise set config to None/default - else: - parser_config = None + text_content = get_text(html_content, config=parser_config) - # RSS Mode - Inscriptis will treat `title` as something else. - # Make it as a regular block display element (//item/title) - # This is a bit of a hack - the real way it to use XSLT to convert it to HTML #1874 - if is_rss: - html_content = re.sub(r'])', r'', r'', html_content) + with open(temp_file_path, "w", encoding="utf-8") as f: + f.write(text_content) - text_content = get_text(html_content, config=parser_config) - return text_content + except Exception as e: + # Write error to file so the parent can read it + with open(temp_file_path, "w", encoding="utf-8") as f: + f.write(f"[ERROR] {e}") + +import tempfile +from multiprocessing import Process +def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False, timeout=10) -> str: + + + with tempfile.NamedTemporaryFile(delete=False, mode="w+", encoding="utf-8") as tmp_file: + temp_file_path = tmp_file.name + + p = Process( + target=html_to_text_sub_worker, + args=(temp_file_path, html_content, render_anchor_tag_content, is_rss) + ) + p.start() + p.join(timeout) + + if p.is_alive(): + p.terminate() + p.join() + + try: + with open(temp_file_path, "r", encoding="utf-8") as f: + result = f.read() + finally: + os.remove(temp_file_path) + + return result # Does LD+JSON exist with a @type=='product' and a .price set anywhere? def has_ldjson_product_info(content):