Files
rss-to-opds/tests/test_xhtml.py
2026-11-15 18:15:03 -05:00

400 lines
16 KiB
Python

"""Tests for the HTML to XHTML normaliser.
The bar every case is held to: the output must parse with a strict XML parser,
because that is what an EPUB reader does with it.
"""
from __future__ import annotations
import unittest
from app.xhtml import parse_article
from tests.support import assert_well_formed, body, collapse, parse_xml
BASE = "https://example.com/posts/one?utm=x"
def render(raw: str, image_map=None, base: str = BASE, max_input_bytes: int = 0) -> str:
"""Parse and serialise, asserting well-formedness on the way out."""
article = parse_article(raw, base, max_input_bytes=max_input_bytes)
fragment = article.to_xhtml(image_map)
assert_well_formed(unittest.TestCase(), fragment)
return fragment
class MalformedInputTests(unittest.TestCase):
"""Feed HTML is broken in predictable ways; none of it may reach the output."""
def test_unclosed_paragraphs_are_balanced(self):
out = render("<p>one<p>two<p>three")
self.assertEqual(body(out).count("<p>"), 3)
self.assertEqual(body(out).count("</p>"), 3)
self.assertIn("three", out)
def test_block_element_closes_open_paragraph(self):
# A naive stack would nest the div inside the paragraph.
out = render("<p>lead<div>block</div>trailing")
self.assertIn("<p>lead</p>", out)
self.assertIn("<div>block</div>", out)
self.assertNotIn("<p>lead<div>", out)
def test_unclosed_list_items(self):
out = render("<ul><li>a<li>b<li>c</ul>")
self.assertEqual(body(out).count("<li>"), 3)
self.assertEqual(body(out).count("</li>"), 3)
def test_stray_end_tag_is_ignored(self):
out = render("<p>text</span></div></p>")
self.assertIn("<p>text</p>", out)
def test_unclosed_table_row_and_cells(self):
out = render("<table><tr><td>a<td>b<tr><td>c</table>")
self.assertEqual(body(out).count("<tr>"), 2)
self.assertEqual(body(out).count("<td>"), 3)
self.assertIn("<td>c</td>", out)
def test_self_closing_non_void_element_swallows_nothing(self):
# `<div/>` must not open a subtree that eats the rest of the document, and
# the empty element it leaves behind is noise worth dropping.
out = render("<div/>after")
self.assertIn("after", out)
self.assertEqual(body(out), "after")
def test_void_elements_are_self_closed(self):
out = render("a<br>b<hr>c")
self.assertIn("<br/>", out)
self.assertIn("<hr/>", out)
self.assertNotIn("<br>", out)
def test_control_characters_are_removed(self):
# NUL and vertical tab cannot be represented in XML even as entities.
out = render("<p>be\x00fore\x0baft\x0cft</p>")
self.assertNotIn("\x00", out)
self.assertNotIn("\x0b", out)
self.assertNotIn("\x0c", out)
self.assertIn("before", out)
def test_unpaired_surrogate_is_removed(self):
out = render("<p>ok\ud800tail</p>")
self.assertNotIn("\ud800", out)
self.assertIn("oktail", out)
def test_zero_width_characters_are_removed(self):
out = render("<p>a\u200bb\ufeffc</p>")
self.assertEqual(collapse(parse_xml(out).all_text()), "abc")
def test_comments_are_dropped(self):
out = render("<p>keep<!-- drop me --> this</p>")
self.assertNotIn("drop me", out)
def test_processing_instruction_and_doctype_are_dropped(self):
out = render("<?xml version='1.0'?><!DOCTYPE html><p>body</p>")
self.assertNotIn("DOCTYPE", out)
self.assertIn("body", out)
def test_deeply_nested_wrappers_keep_their_text(self):
out = render("<section><article><aside>deep</aside></article></section>")
self.assertIn("deep", out)
self.assertNotIn("<section>", out)
self.assertNotIn("<aside>", out)
class EscapingTests(unittest.TestCase):
"""Nothing in the output may be interpretable as markup or as an entity."""
def test_bare_ampersand_is_escaped(self):
out = render("<p>Tom & Jerry</p>")
self.assertIn("Tom &amp; Jerry", out)
def test_named_and_numeric_entities_are_decoded_then_reescaped(self):
out = render("<p>&nbsp;&#233;&amp;</p>")
# &nbsp; must not survive as an entity, and must not break the parse.
self.assertNotIn("&nbsp;", out)
self.assertIn("\u00a0", out)
self.assertIn("\u00e9", out)
def test_text_that_looks_like_xml_is_escaped(self):
out = render("<p>&lt;script&gt;alert(1)&lt;/script&gt;</p>")
self.assertIn("&lt;script&gt;", out)
self.assertEqual(body(out).count("<script>"), 0)
def test_quotes_in_attributes_are_escaped(self):
out = render('<p><a href="https://e.com/?q=&quot;x&quot;" title="a&quot;b">t</a></p>')
self.assertIn("&quot;", out)
def test_cdata_section_is_unwrapped_to_text(self):
out = render("<p><![CDATA[raw text]]></p>")
self.assertIn("raw text", out)
def test_gets_escaped_inside_attributes_only_once(self):
out = render('<p><a href="https://e.com/a?b=1&amp;c=2">t</a></p>')
self.assertIn('href="https://e.com/a?b=1&amp;c=2"', out)
class DangerousContentTests(unittest.TestCase):
"""Scriptable and resource-fetching constructs must not survive."""
def test_script_element_and_body_are_removed(self):
out = render("<p>before</p><script>alert('xss')</script><p>after</p>")
self.assertNotIn("script", out.lower())
self.assertIn("before", out)
self.assertIn("after", out)
def test_unclosed_script_does_not_swallow_forever(self):
out = render("<script>alert(1)</script><p>visible</p>")
self.assertIn("visible", out)
def test_style_element_is_removed(self):
out = render("<style>p{color:red}</style><p>text</p>")
self.assertNotIn("color:red", out)
def test_iframe_and_object_are_removed_with_contents(self):
out = render('<iframe src="https://e.com/x"></iframe><p>t</p>')
self.assertNotIn("iframe", out)
def test_style_attribute_is_dropped(self):
out = render('<p style="background:url(javascript:1)">t</p>')
self.assertNotIn("style", out)
def test_event_handler_attributes_are_dropped(self):
out = render('<p onclick="evil()" onmouseover="evil()">t</p>')
self.assertNotIn("onclick", out)
self.assertNotIn("onmouseover", out)
def test_class_attribute_is_dropped(self):
# class fights the book's own stylesheet and means nothing offline.
out = render('<p class="x">t</p>')
self.assertNotIn("class", body(out))
def test_id_is_kept_for_anchor_targets(self):
out = render('<p id="section-2">t</p>')
self.assertIn('id="section-2"', out)
def test_invalid_id_is_dropped(self):
# An id has to be a usable XML name; anything else is not an anchor. The
# quote case needs single-quoted markup, or the parser would read a
# shorter, valid value and the case would prove nothing.
cases = {
"leading digit": '<p id="9lives">t</p>',
"space": '<p id="has space">t</p>',
"slash": '<p id="sla/sh">t</p>',
"quote": "<p id='quote\"d'>t</p>",
"empty": '<p id="">t</p>',
"too long": f'<p id="{"a" * 100}">t</p>',
}
for label, markup in cases.items():
self.assertNotIn("id=", body(render(markup)), label)
def test_javascript_url_is_dropped(self):
out = render('<a href="javascript:alert(1)">click</a>')
self.assertNotIn("javascript", out)
self.assertIn("click", out)
def test_data_url_link_is_dropped(self):
out = render('<a href="data:text/html,<script>x</script>">t</a>')
self.assertNotIn("data:", out)
def test_data_url_image_is_dropped(self):
out = render('<img src="data:image/png;base64,AAAA"><p>t</p>')
self.assertNotIn("data:", out)
self.assertNotIn("<img", out)
def test_img_without_source_is_dropped(self):
out = render('<p><img alt="no source"></p>')
self.assertNotIn("<img", out)
def test_only_href_survives_on_links(self):
out = render('<a href="https://e.com/a" target="_blank" rel="noopener" download>x</a>')
self.assertIn('href="https://e.com/a"', out)
self.assertNotIn("target", out)
self.assertNotIn("rel=", out)
class UrlResolutionTests(unittest.TestCase):
"""Bodies are re-based into the book, so relative URLs must be resolved."""
def test_root_relative_link_uses_article_origin(self):
out = render('<a href="/about">about</a>')
self.assertIn('href="https://example.com/about"', out)
def test_document_relative_link_uses_article_directory(self):
out = render('<a href="sibling.html">s</a>')
self.assertIn('href="https://example.com/posts/sibling.html"', out)
def test_protocol_relative_link_is_resolved(self):
out = render('<a href="//cdn.example.com/x">x</a>')
self.assertIn('href="https://cdn.example.com/x"', out)
def test_mailto_link_is_kept(self):
out = render('<a href="mailto:someone@example.com">mail</a>')
self.assertIn('href="mailto:someone@example.com"', out)
def test_anchor_only_link_stays_in_document(self):
# A same-document link must not be rewritten to the source web page: the
# matching id is kept, so it jumps within the book instead.
out = render('<p id="section">target</p><a href="#section">jump</a>')
self.assertIn('href="#section"', out)
self.assertIn('id="section"', out)
def test_relative_image_is_absolutised(self):
article = parse_article('<img src="images/a.png">', BASE)
self.assertEqual(article.image_urls, ["https://example.com/posts/images/a.png"])
def test_blockquote_cite_is_validated(self):
out = render('<blockquote cite="javascript:x">q</blockquote>')
self.assertNotIn("cite", out)
out = render('<blockquote cite="https://e.com/src">q</blockquote>')
self.assertIn('cite="https://e.com/src"', out)
class AnchorIdTests(unittest.TestCase):
"""Ids survive so an article's own table of contents works offline."""
def test_duplicate_ids_are_renamed_not_dropped(self):
out = render('<p id="a">1</p><p id="a">2</p><p id="a">3</p>')
self.assertIn('id="a"', out)
self.assertIn('id="a-2"', out)
self.assertIn('id="a-3"', out)
def test_id_on_void_element_is_dropped(self):
out = render('<img id="x" src="a.png">')
self.assertNotIn('id="x"', out)
def test_renamed_ids_do_not_collide_with_existing_ones(self):
out = render('<p id="a">1</p><p id="a-2">2</p><p id="a">3</p>')
self.assertEqual(body(out).count('id="a-2"'), 1)
self.assertIn('id="a-3"', out)
def test_unique_ids_are_left_alone(self):
out = render('<p id="one">1</p><p id="two">2</p>')
self.assertIn('id="one"', out)
self.assertIn('id="two"', out)
def test_an_element_keeping_only_an_id_is_not_pruned(self):
out = render('<div id="anchor"></div><p>text</p>')
self.assertIn('id="anchor"', out)
class ImageHandlingTests(unittest.TestCase):
"""Images are collected for the fetcher and rewritten by the caller."""
def test_image_urls_are_collected_in_order_and_deduplicated(self):
article = parse_article('<p><img src="a.png"><img src="a.png"><img src="b.png"></p>', BASE)
self.assertEqual(
article.image_urls,
["https://example.com/posts/a.png", "https://example.com/posts/b.png"],
)
def test_remote_urls_are_kept_when_no_map_is_given(self):
out = render('<img src="a.png" alt="alt text">')
self.assertIn('src="https://example.com/posts/a.png"', out)
self.assertIn('alt="alt text"', out)
def test_map_rewrites_src_to_local_path(self):
url = "https://example.com/posts/a.png"
out = render('<img src="a.png">', image_map={url: "images/img-1.png"})
self.assertIn('src="images/img-1.png"', out)
def test_none_in_map_drops_image_and_empty_wrapper(self):
url = "https://example.com/posts/a.png"
out = render('<div><img src="a.png"></div><p>text</p>', image_map={url: None})
self.assertNotIn("<img", out)
self.assertNotIn("<div>", out)
self.assertIn("<p>text</p>", out)
def test_alt_and_title_are_preserved_for_localised_images(self):
url = "https://example.com/posts/a.png"
out = render('<img src="a.png" alt="A" title="T">', image_map={url: "images/i.png"})
self.assertIn('alt="A"', out)
self.assertIn('title="T"', out)
class WhitespaceAndStructureTests(unittest.TestCase):
def test_whitespace_is_collapsed_outside_pre(self):
out = render("<p>one\n\n two\t\tthree</p>")
self.assertIn("<p>one two three</p>", out)
def test_whitespace_is_preserved_inside_pre(self):
out = render("<pre>def f():\n return 1</pre>")
self.assertIn("def f():\n return 1", out)
def test_empty_paragraphs_are_pruned(self):
out = render("<p> </p><p>&nbsp;</p><p>real</p>")
self.assertEqual(body(out).count("<p>"), 1)
self.assertIn("real", out)
def test_empty_table_cells_are_kept(self):
# An empty cell is layout, not noise, so pruning must leave it alone.
out = render("<table><tr><td></td><td>b</td></tr></table>")
self.assertEqual(body(out).count("<td></td>"), 1)
def test_plain_text_body_is_wrapped_in_a_paragraph(self):
out = render("just words with no markup")
self.assertEqual(body(out), "<p>just words with no markup</p>")
def test_whitespace_only_body_renders_empty(self):
article = parse_article(" \n ", BASE)
self.assertEqual(article.to_xhtml(), "")
def test_empty_input_renders_empty(self):
self.assertEqual(parse_article("", BASE).to_xhtml(), "")
def test_heading_levels_survive(self):
out = render("<h1>a</h1><h2>b</h2><h3>c</h3>")
for tag in ("h1", "h2", "h3"):
self.assertIn(f"<{tag}>", out)
def test_colspan_is_clamped(self):
out = render('<td colspan="99999">x</td>')
self.assertNotIn("colspan", out)
out = render('<td colspan="2">x</td>')
self.assertIn('colspan="2"', out)
def test_element_structure_is_preserved(self):
out = render("<ul><li>a<ul><li>b</li></ul></li></ul>")
tree = parse_xml(out)
# The fragment's own root is the article wrapper.
self.assertEqual(tree.must_attr("class"), "article-body")
outer = tree.must_find("ul")
items = outer.must_find_all("li")
self.assertEqual(len(items), 2)
# The nested list belongs to the first item, not to the outer list.
self.assertTrue(items[0].find_all("ul"))
class ArticleApiTests(unittest.TestCase):
def test_text_extraction_flattens_and_collapses(self):
article = parse_article("<p>One</p> <p>Two &amp; three</p>", BASE)
self.assertEqual(article.text(), "One Two & three")
def test_text_extraction_is_limited_at_a_word_boundary(self):
article = parse_article("<p>alpha beta gamma delta</p>", BASE)
text = article.text(limit=12)
self.assertTrue(text.endswith("\u2026"))
self.assertLessEqual(len(text), 13)
self.assertIn("alpha beta", text)
def test_truncation_flag_and_output_validity(self):
article = parse_article("<p>" + "word " * 5000 + "</p>", BASE, max_input_bytes=2000)
self.assertTrue(article.truncated)
assert_well_formed(self, article.to_xhtml())
def test_no_truncation_when_under_limit(self):
article = parse_article("<p>short</p>", BASE, max_input_bytes=10_000)
self.assertFalse(article.truncated)
def test_output_is_deterministic(self):
raw = '<p>a &amp; b</p><div><img src="i.png"><ul><li>x</ul></div>'
self.assertEqual(render(raw), render(raw))
def test_image_urls_are_cached_between_calls(self):
article = parse_article('<img src="a.png">', BASE)
self.assertIs(article.image_urls, article.image_urls)
def test_output_wrapper_has_the_expected_class(self):
self.assertTrue(render("<p>x</p>").startswith('<div class="article-body">'))
if __name__ == "__main__":
unittest.main()