400 lines
16 KiB
Python
400 lines
16 KiB
Python
"""Tests for the HTML to XHTML normaliser.
|
|
|
|
The bar every case is held to: the output must parse with a strict XML parser,
|
|
because that is what an EPUB reader does with it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import unittest
|
|
|
|
from app.xhtml import parse_article
|
|
from tests.support import assert_well_formed, body, collapse, parse_xml
|
|
|
|
BASE = "https://example.com/posts/one?utm=x"
|
|
|
|
|
|
def render(raw: str, image_map=None, base: str = BASE, max_input_bytes: int = 0) -> str:
|
|
"""Parse and serialise, asserting well-formedness on the way out."""
|
|
article = parse_article(raw, base, max_input_bytes=max_input_bytes)
|
|
fragment = article.to_xhtml(image_map)
|
|
assert_well_formed(unittest.TestCase(), fragment)
|
|
return fragment
|
|
|
|
|
|
class MalformedInputTests(unittest.TestCase):
|
|
"""Feed HTML is broken in predictable ways; none of it may reach the output."""
|
|
|
|
def test_unclosed_paragraphs_are_balanced(self):
|
|
out = render("<p>one<p>two<p>three")
|
|
self.assertEqual(body(out).count("<p>"), 3)
|
|
self.assertEqual(body(out).count("</p>"), 3)
|
|
self.assertIn("three", out)
|
|
|
|
def test_block_element_closes_open_paragraph(self):
|
|
# A naive stack would nest the div inside the paragraph.
|
|
out = render("<p>lead<div>block</div>trailing")
|
|
self.assertIn("<p>lead</p>", out)
|
|
self.assertIn("<div>block</div>", out)
|
|
self.assertNotIn("<p>lead<div>", out)
|
|
|
|
def test_unclosed_list_items(self):
|
|
out = render("<ul><li>a<li>b<li>c</ul>")
|
|
self.assertEqual(body(out).count("<li>"), 3)
|
|
self.assertEqual(body(out).count("</li>"), 3)
|
|
|
|
def test_stray_end_tag_is_ignored(self):
|
|
out = render("<p>text</span></div></p>")
|
|
self.assertIn("<p>text</p>", out)
|
|
|
|
def test_unclosed_table_row_and_cells(self):
|
|
out = render("<table><tr><td>a<td>b<tr><td>c</table>")
|
|
self.assertEqual(body(out).count("<tr>"), 2)
|
|
self.assertEqual(body(out).count("<td>"), 3)
|
|
self.assertIn("<td>c</td>", out)
|
|
|
|
def test_self_closing_non_void_element_swallows_nothing(self):
|
|
# `<div/>` must not open a subtree that eats the rest of the document, and
|
|
# the empty element it leaves behind is noise worth dropping.
|
|
out = render("<div/>after")
|
|
self.assertIn("after", out)
|
|
self.assertEqual(body(out), "after")
|
|
|
|
def test_void_elements_are_self_closed(self):
|
|
out = render("a<br>b<hr>c")
|
|
self.assertIn("<br/>", out)
|
|
self.assertIn("<hr/>", out)
|
|
self.assertNotIn("<br>", out)
|
|
|
|
def test_control_characters_are_removed(self):
|
|
# NUL and vertical tab cannot be represented in XML even as entities.
|
|
out = render("<p>be\x00fore\x0baft\x0cft</p>")
|
|
self.assertNotIn("\x00", out)
|
|
self.assertNotIn("\x0b", out)
|
|
self.assertNotIn("\x0c", out)
|
|
self.assertIn("before", out)
|
|
|
|
def test_unpaired_surrogate_is_removed(self):
|
|
out = render("<p>ok\ud800tail</p>")
|
|
self.assertNotIn("\ud800", out)
|
|
self.assertIn("oktail", out)
|
|
|
|
def test_zero_width_characters_are_removed(self):
|
|
out = render("<p>a\u200bb\ufeffc</p>")
|
|
self.assertEqual(collapse(parse_xml(out).all_text()), "abc")
|
|
|
|
def test_comments_are_dropped(self):
|
|
out = render("<p>keep<!-- drop me --> this</p>")
|
|
self.assertNotIn("drop me", out)
|
|
|
|
def test_processing_instruction_and_doctype_are_dropped(self):
|
|
out = render("<?xml version='1.0'?><!DOCTYPE html><p>body</p>")
|
|
self.assertNotIn("DOCTYPE", out)
|
|
self.assertIn("body", out)
|
|
|
|
def test_deeply_nested_wrappers_keep_their_text(self):
|
|
out = render("<section><article><aside>deep</aside></article></section>")
|
|
self.assertIn("deep", out)
|
|
self.assertNotIn("<section>", out)
|
|
self.assertNotIn("<aside>", out)
|
|
|
|
|
|
class EscapingTests(unittest.TestCase):
|
|
"""Nothing in the output may be interpretable as markup or as an entity."""
|
|
|
|
def test_bare_ampersand_is_escaped(self):
|
|
out = render("<p>Tom & Jerry</p>")
|
|
self.assertIn("Tom & Jerry", out)
|
|
|
|
def test_named_and_numeric_entities_are_decoded_then_reescaped(self):
|
|
out = render("<p> é&</p>")
|
|
# must not survive as an entity, and must not break the parse.
|
|
self.assertNotIn(" ", out)
|
|
self.assertIn("\u00a0", out)
|
|
self.assertIn("\u00e9", out)
|
|
|
|
def test_text_that_looks_like_xml_is_escaped(self):
|
|
out = render("<p><script>alert(1)</script></p>")
|
|
self.assertIn("<script>", out)
|
|
self.assertEqual(body(out).count("<script>"), 0)
|
|
|
|
def test_quotes_in_attributes_are_escaped(self):
|
|
out = render('<p><a href="https://e.com/?q="x"" title="a"b">t</a></p>')
|
|
self.assertIn(""", out)
|
|
|
|
def test_cdata_section_is_unwrapped_to_text(self):
|
|
out = render("<p><![CDATA[raw text]]></p>")
|
|
self.assertIn("raw text", out)
|
|
|
|
def test_gets_escaped_inside_attributes_only_once(self):
|
|
out = render('<p><a href="https://e.com/a?b=1&c=2">t</a></p>')
|
|
self.assertIn('href="https://e.com/a?b=1&c=2"', out)
|
|
|
|
|
|
class DangerousContentTests(unittest.TestCase):
|
|
"""Scriptable and resource-fetching constructs must not survive."""
|
|
|
|
def test_script_element_and_body_are_removed(self):
|
|
out = render("<p>before</p><script>alert('xss')</script><p>after</p>")
|
|
self.assertNotIn("script", out.lower())
|
|
self.assertIn("before", out)
|
|
self.assertIn("after", out)
|
|
|
|
def test_unclosed_script_does_not_swallow_forever(self):
|
|
out = render("<script>alert(1)</script><p>visible</p>")
|
|
self.assertIn("visible", out)
|
|
|
|
def test_style_element_is_removed(self):
|
|
out = render("<style>p{color:red}</style><p>text</p>")
|
|
self.assertNotIn("color:red", out)
|
|
|
|
def test_iframe_and_object_are_removed_with_contents(self):
|
|
out = render('<iframe src="https://e.com/x"></iframe><p>t</p>')
|
|
self.assertNotIn("iframe", out)
|
|
|
|
def test_style_attribute_is_dropped(self):
|
|
out = render('<p style="background:url(javascript:1)">t</p>')
|
|
self.assertNotIn("style", out)
|
|
|
|
def test_event_handler_attributes_are_dropped(self):
|
|
out = render('<p onclick="evil()" onmouseover="evil()">t</p>')
|
|
self.assertNotIn("onclick", out)
|
|
self.assertNotIn("onmouseover", out)
|
|
|
|
def test_class_attribute_is_dropped(self):
|
|
# class fights the book's own stylesheet and means nothing offline.
|
|
out = render('<p class="x">t</p>')
|
|
self.assertNotIn("class", body(out))
|
|
|
|
def test_id_is_kept_for_anchor_targets(self):
|
|
out = render('<p id="section-2">t</p>')
|
|
self.assertIn('id="section-2"', out)
|
|
|
|
def test_invalid_id_is_dropped(self):
|
|
# An id has to be a usable XML name; anything else is not an anchor. The
|
|
# quote case needs single-quoted markup, or the parser would read a
|
|
# shorter, valid value and the case would prove nothing.
|
|
cases = {
|
|
"leading digit": '<p id="9lives">t</p>',
|
|
"space": '<p id="has space">t</p>',
|
|
"slash": '<p id="sla/sh">t</p>',
|
|
"quote": "<p id='quote\"d'>t</p>",
|
|
"empty": '<p id="">t</p>',
|
|
"too long": f'<p id="{"a" * 100}">t</p>',
|
|
}
|
|
for label, markup in cases.items():
|
|
self.assertNotIn("id=", body(render(markup)), label)
|
|
|
|
def test_javascript_url_is_dropped(self):
|
|
out = render('<a href="javascript:alert(1)">click</a>')
|
|
self.assertNotIn("javascript", out)
|
|
self.assertIn("click", out)
|
|
|
|
def test_data_url_link_is_dropped(self):
|
|
out = render('<a href="data:text/html,<script>x</script>">t</a>')
|
|
self.assertNotIn("data:", out)
|
|
|
|
def test_data_url_image_is_dropped(self):
|
|
out = render('<img src="data:image/png;base64,AAAA"><p>t</p>')
|
|
self.assertNotIn("data:", out)
|
|
self.assertNotIn("<img", out)
|
|
|
|
def test_img_without_source_is_dropped(self):
|
|
out = render('<p><img alt="no source"></p>')
|
|
self.assertNotIn("<img", out)
|
|
|
|
def test_only_href_survives_on_links(self):
|
|
out = render('<a href="https://e.com/a" target="_blank" rel="noopener" download>x</a>')
|
|
self.assertIn('href="https://e.com/a"', out)
|
|
self.assertNotIn("target", out)
|
|
self.assertNotIn("rel=", out)
|
|
|
|
|
|
class UrlResolutionTests(unittest.TestCase):
|
|
"""Bodies are re-based into the book, so relative URLs must be resolved."""
|
|
|
|
def test_root_relative_link_uses_article_origin(self):
|
|
out = render('<a href="/about">about</a>')
|
|
self.assertIn('href="https://example.com/about"', out)
|
|
|
|
def test_document_relative_link_uses_article_directory(self):
|
|
out = render('<a href="sibling.html">s</a>')
|
|
self.assertIn('href="https://example.com/posts/sibling.html"', out)
|
|
|
|
def test_protocol_relative_link_is_resolved(self):
|
|
out = render('<a href="//cdn.example.com/x">x</a>')
|
|
self.assertIn('href="https://cdn.example.com/x"', out)
|
|
|
|
def test_mailto_link_is_kept(self):
|
|
out = render('<a href="mailto:someone@example.com">mail</a>')
|
|
self.assertIn('href="mailto:someone@example.com"', out)
|
|
|
|
def test_anchor_only_link_stays_in_document(self):
|
|
# A same-document link must not be rewritten to the source web page: the
|
|
# matching id is kept, so it jumps within the book instead.
|
|
out = render('<p id="section">target</p><a href="#section">jump</a>')
|
|
self.assertIn('href="#section"', out)
|
|
self.assertIn('id="section"', out)
|
|
|
|
def test_relative_image_is_absolutised(self):
|
|
article = parse_article('<img src="images/a.png">', BASE)
|
|
self.assertEqual(article.image_urls, ["https://example.com/posts/images/a.png"])
|
|
|
|
def test_blockquote_cite_is_validated(self):
|
|
out = render('<blockquote cite="javascript:x">q</blockquote>')
|
|
self.assertNotIn("cite", out)
|
|
out = render('<blockquote cite="https://e.com/src">q</blockquote>')
|
|
self.assertIn('cite="https://e.com/src"', out)
|
|
|
|
|
|
class AnchorIdTests(unittest.TestCase):
|
|
"""Ids survive so an article's own table of contents works offline."""
|
|
|
|
def test_duplicate_ids_are_renamed_not_dropped(self):
|
|
out = render('<p id="a">1</p><p id="a">2</p><p id="a">3</p>')
|
|
self.assertIn('id="a"', out)
|
|
self.assertIn('id="a-2"', out)
|
|
self.assertIn('id="a-3"', out)
|
|
|
|
def test_id_on_void_element_is_dropped(self):
|
|
out = render('<img id="x" src="a.png">')
|
|
self.assertNotIn('id="x"', out)
|
|
|
|
def test_renamed_ids_do_not_collide_with_existing_ones(self):
|
|
out = render('<p id="a">1</p><p id="a-2">2</p><p id="a">3</p>')
|
|
self.assertEqual(body(out).count('id="a-2"'), 1)
|
|
self.assertIn('id="a-3"', out)
|
|
|
|
def test_unique_ids_are_left_alone(self):
|
|
out = render('<p id="one">1</p><p id="two">2</p>')
|
|
self.assertIn('id="one"', out)
|
|
self.assertIn('id="two"', out)
|
|
|
|
def test_an_element_keeping_only_an_id_is_not_pruned(self):
|
|
out = render('<div id="anchor"></div><p>text</p>')
|
|
self.assertIn('id="anchor"', out)
|
|
|
|
|
|
class ImageHandlingTests(unittest.TestCase):
|
|
"""Images are collected for the fetcher and rewritten by the caller."""
|
|
|
|
def test_image_urls_are_collected_in_order_and_deduplicated(self):
|
|
article = parse_article('<p><img src="a.png"><img src="a.png"><img src="b.png"></p>', BASE)
|
|
self.assertEqual(
|
|
article.image_urls,
|
|
["https://example.com/posts/a.png", "https://example.com/posts/b.png"],
|
|
)
|
|
|
|
def test_remote_urls_are_kept_when_no_map_is_given(self):
|
|
out = render('<img src="a.png" alt="alt text">')
|
|
self.assertIn('src="https://example.com/posts/a.png"', out)
|
|
self.assertIn('alt="alt text"', out)
|
|
|
|
def test_map_rewrites_src_to_local_path(self):
|
|
url = "https://example.com/posts/a.png"
|
|
out = render('<img src="a.png">', image_map={url: "images/img-1.png"})
|
|
self.assertIn('src="images/img-1.png"', out)
|
|
|
|
def test_none_in_map_drops_image_and_empty_wrapper(self):
|
|
url = "https://example.com/posts/a.png"
|
|
out = render('<div><img src="a.png"></div><p>text</p>', image_map={url: None})
|
|
self.assertNotIn("<img", out)
|
|
self.assertNotIn("<div>", out)
|
|
self.assertIn("<p>text</p>", out)
|
|
|
|
def test_alt_and_title_are_preserved_for_localised_images(self):
|
|
url = "https://example.com/posts/a.png"
|
|
out = render('<img src="a.png" alt="A" title="T">', image_map={url: "images/i.png"})
|
|
self.assertIn('alt="A"', out)
|
|
self.assertIn('title="T"', out)
|
|
|
|
|
|
class WhitespaceAndStructureTests(unittest.TestCase):
|
|
def test_whitespace_is_collapsed_outside_pre(self):
|
|
out = render("<p>one\n\n two\t\tthree</p>")
|
|
self.assertIn("<p>one two three</p>", out)
|
|
|
|
def test_whitespace_is_preserved_inside_pre(self):
|
|
out = render("<pre>def f():\n return 1</pre>")
|
|
self.assertIn("def f():\n return 1", out)
|
|
|
|
def test_empty_paragraphs_are_pruned(self):
|
|
out = render("<p> </p><p> </p><p>real</p>")
|
|
self.assertEqual(body(out).count("<p>"), 1)
|
|
self.assertIn("real", out)
|
|
|
|
def test_empty_table_cells_are_kept(self):
|
|
# An empty cell is layout, not noise, so pruning must leave it alone.
|
|
out = render("<table><tr><td></td><td>b</td></tr></table>")
|
|
self.assertEqual(body(out).count("<td></td>"), 1)
|
|
|
|
def test_plain_text_body_is_wrapped_in_a_paragraph(self):
|
|
out = render("just words with no markup")
|
|
self.assertEqual(body(out), "<p>just words with no markup</p>")
|
|
|
|
def test_whitespace_only_body_renders_empty(self):
|
|
article = parse_article(" \n ", BASE)
|
|
self.assertEqual(article.to_xhtml(), "")
|
|
|
|
def test_empty_input_renders_empty(self):
|
|
self.assertEqual(parse_article("", BASE).to_xhtml(), "")
|
|
|
|
def test_heading_levels_survive(self):
|
|
out = render("<h1>a</h1><h2>b</h2><h3>c</h3>")
|
|
for tag in ("h1", "h2", "h3"):
|
|
self.assertIn(f"<{tag}>", out)
|
|
|
|
def test_colspan_is_clamped(self):
|
|
out = render('<td colspan="99999">x</td>')
|
|
self.assertNotIn("colspan", out)
|
|
out = render('<td colspan="2">x</td>')
|
|
self.assertIn('colspan="2"', out)
|
|
|
|
def test_element_structure_is_preserved(self):
|
|
out = render("<ul><li>a<ul><li>b</li></ul></li></ul>")
|
|
tree = parse_xml(out)
|
|
# The fragment's own root is the article wrapper.
|
|
self.assertEqual(tree.must_attr("class"), "article-body")
|
|
outer = tree.must_find("ul")
|
|
items = outer.must_find_all("li")
|
|
self.assertEqual(len(items), 2)
|
|
# The nested list belongs to the first item, not to the outer list.
|
|
self.assertTrue(items[0].find_all("ul"))
|
|
|
|
|
|
class ArticleApiTests(unittest.TestCase):
|
|
def test_text_extraction_flattens_and_collapses(self):
|
|
article = parse_article("<p>One</p> <p>Two & three</p>", BASE)
|
|
self.assertEqual(article.text(), "One Two & three")
|
|
|
|
def test_text_extraction_is_limited_at_a_word_boundary(self):
|
|
article = parse_article("<p>alpha beta gamma delta</p>", BASE)
|
|
text = article.text(limit=12)
|
|
self.assertTrue(text.endswith("\u2026"))
|
|
self.assertLessEqual(len(text), 13)
|
|
self.assertIn("alpha beta", text)
|
|
|
|
def test_truncation_flag_and_output_validity(self):
|
|
article = parse_article("<p>" + "word " * 5000 + "</p>", BASE, max_input_bytes=2000)
|
|
self.assertTrue(article.truncated)
|
|
assert_well_formed(self, article.to_xhtml())
|
|
|
|
def test_no_truncation_when_under_limit(self):
|
|
article = parse_article("<p>short</p>", BASE, max_input_bytes=10_000)
|
|
self.assertFalse(article.truncated)
|
|
|
|
def test_output_is_deterministic(self):
|
|
raw = '<p>a & b</p><div><img src="i.png"><ul><li>x</ul></div>'
|
|
self.assertEqual(render(raw), render(raw))
|
|
|
|
def test_image_urls_are_cached_between_calls(self):
|
|
article = parse_article('<img src="a.png">', BASE)
|
|
self.assertIs(article.image_urls, article.image_urls)
|
|
|
|
def test_output_wrapper_has_the_expected_class(self):
|
|
self.assertTrue(render("<p>x</p>").startswith('<div class="article-body">'))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|