"""Tests for the HTML to XHTML normaliser.
The bar every case is held to: the output must parse with a strict XML parser,
because that is what an EPUB reader does with it.
"""
from __future__ import annotations
import unittest
from app.xhtml import parse_article
from tests.support import assert_well_formed, body, collapse, parse_xml
BASE = "https://example.com/posts/one?utm=x"
def render(raw: str, image_map=None, base: str = BASE, max_input_bytes: int = 0) -> str:
"""Parse and serialise, asserting well-formedness on the way out."""
article = parse_article(raw, base, max_input_bytes=max_input_bytes)
fragment = article.to_xhtml(image_map)
assert_well_formed(unittest.TestCase(), fragment)
return fragment
class MalformedInputTests(unittest.TestCase):
"""Feed HTML is broken in predictable ways; none of it may reach the output."""
def test_unclosed_paragraphs_are_balanced(self):
out = render("
one
two
three")
self.assertEqual(body(out).count("
"), 3)
self.assertEqual(body(out).count("
"), 3)
self.assertIn("three", out)
def test_block_element_closes_open_paragraph(self):
# A naive stack would nest the div inside the paragraph.
out = render("lead
block
trailing")
self.assertIn("lead
", out)
self.assertIn("block
", out)
self.assertNotIn("lead
", out)
def test_unclosed_list_items(self):
out = render("
")
self.assertEqual(body(out).count("
"), 3)
self.assertEqual(body(out).count(" "), 3)
def test_stray_end_tag_is_ignored(self):
out = render("
text
")
self.assertIn("text
", out)
def test_unclosed_table_row_and_cells(self):
out = render("")
self.assertEqual(body(out).count(""), 2)
self.assertEqual(body(out).count(""), 3)
self.assertIn(" c ", out)
def test_self_closing_non_void_element_swallows_nothing(self):
# `
` must not open a subtree that eats the rest of the document, and
# the empty element it leaves behind is noise worth dropping.
out = render("
after")
self.assertIn("after", out)
self.assertEqual(body(out), "after")
def test_void_elements_are_self_closed(self):
out = render("a b c")
self.assertIn(" ", out)
self.assertIn(" ", out)
self.assertNotIn(" ", out)
def test_control_characters_are_removed(self):
# NUL and vertical tab cannot be represented in XML even as entities.
out = render("be\x00fore\x0baft\x0cft
")
self.assertNotIn("\x00", out)
self.assertNotIn("\x0b", out)
self.assertNotIn("\x0c", out)
self.assertIn("before", out)
def test_unpaired_surrogate_is_removed(self):
out = render("ok\ud800tail
")
self.assertNotIn("\ud800", out)
self.assertIn("oktail", out)
def test_zero_width_characters_are_removed(self):
out = render("a\u200bb\ufeffc
")
self.assertEqual(collapse(parse_xml(out).all_text()), "abc")
def test_comments_are_dropped(self):
out = render("keep this
")
self.assertNotIn("drop me", out)
def test_processing_instruction_and_doctype_are_dropped(self):
out = render("body
")
self.assertNotIn("DOCTYPE", out)
self.assertIn("body", out)
def test_deeply_nested_wrappers_keep_their_text(self):
out = render("")
self.assertIn("deep", out)
self.assertNotIn("", out)
self.assertNotIn("", out)
class EscapingTests(unittest.TestCase):
"""Nothing in the output may be interpretable as markup or as an entity."""
def test_bare_ampersand_is_escaped(self):
out = render("Tom & Jerry
")
self.assertIn("Tom & Jerry", out)
def test_named_and_numeric_entities_are_decoded_then_reescaped(self):
out = render(" é&
")
# must not survive as an entity, and must not break the parse.
self.assertNotIn(" ", out)
self.assertIn("\u00a0", out)
self.assertIn("\u00e9", out)
def test_text_that_looks_like_xml_is_escaped(self):
out = render("<script>alert(1)</script>
")
self.assertIn("<script>", out)
self.assertEqual(body(out).count("after
")
self.assertNotIn("script", out.lower())
self.assertIn("before", out)
self.assertIn("after", out)
def test_unclosed_script_does_not_swallow_forever(self):
out = render("visible
")
self.assertIn("visible", out)
def test_style_element_is_removed(self):
out = render("text
")
self.assertNotIn("color:red", out)
def test_iframe_and_object_are_removed_with_contents(self):
out = render('t
')
self.assertNotIn("iframe", out)
def test_style_attribute_is_dropped(self):
out = render('t
')
self.assertNotIn("style", out)
def test_event_handler_attributes_are_dropped(self):
out = render('t
')
self.assertNotIn("onclick", out)
self.assertNotIn("onmouseover", out)
def test_class_attribute_is_dropped(self):
# class fights the book's own stylesheet and means nothing offline.
out = render('t
')
self.assertNotIn("class", body(out))
def test_id_is_kept_for_anchor_targets(self):
out = render('t
')
self.assertIn('id="section-2"', out)
def test_invalid_id_is_dropped(self):
# An id has to be a usable XML name; anything else is not an anchor. The
# quote case needs single-quoted markup, or the parser would read a
# shorter, valid value and the case would prove nothing.
cases = {
"leading digit": 't
',
"space": 't
',
"slash": 't
',
"quote": "t
",
"empty": 't
',
"too long": f't
',
}
for label, markup in cases.items():
self.assertNotIn("id=", body(render(markup)), label)
def test_javascript_url_is_dropped(self):
out = render('click ')
self.assertNotIn("javascript", out)
self.assertIn("click", out)
def test_data_url_link_is_dropped(self):
out = render('t ')
self.assertNotIn("data:", out)
def test_data_url_image_is_dropped(self):
out = render('t
')
self.assertNotIn("data:", out)
self.assertNotIn(" ')
self.assertNotIn(" x')
self.assertIn('href="https://e.com/a"', out)
self.assertNotIn("target", out)
self.assertNotIn("rel=", out)
class UrlResolutionTests(unittest.TestCase):
"""Bodies are re-based into the book, so relative URLs must be resolved."""
def test_root_relative_link_uses_article_origin(self):
out = render('about ')
self.assertIn('href="https://example.com/about"', out)
def test_document_relative_link_uses_article_directory(self):
out = render('s ')
self.assertIn('href="https://example.com/posts/sibling.html"', out)
def test_protocol_relative_link_is_resolved(self):
out = render('x ')
self.assertIn('href="https://cdn.example.com/x"', out)
def test_mailto_link_is_kept(self):
out = render('mail ')
self.assertIn('href="mailto:someone@example.com"', out)
def test_anchor_only_link_stays_in_document(self):
# A same-document link must not be rewritten to the source web page: the
# matching id is kept, so it jumps within the book instead.
out = render('target
jump ')
self.assertIn('href="#section"', out)
self.assertIn('id="section"', out)
def test_relative_image_is_absolutised(self):
article = parse_article(' ', BASE)
self.assertEqual(article.image_urls, ["https://example.com/posts/images/a.png"])
def test_blockquote_cite_is_validated(self):
out = render('q ')
self.assertNotIn("cite", out)
out = render('q ')
self.assertIn('cite="https://e.com/src"', out)
class AnchorIdTests(unittest.TestCase):
"""Ids survive so an article's own table of contents works offline."""
def test_duplicate_ids_are_renamed_not_dropped(self):
out = render('1
2
3
')
self.assertIn('id="a"', out)
self.assertIn('id="a-2"', out)
self.assertIn('id="a-3"', out)
def test_id_on_void_element_is_dropped(self):
out = render(' ')
self.assertNotIn('id="x"', out)
def test_renamed_ids_do_not_collide_with_existing_ones(self):
out = render('1
2
3
')
self.assertEqual(body(out).count('id="a-2"'), 1)
self.assertIn('id="a-3"', out)
def test_unique_ids_are_left_alone(self):
out = render('1
2
')
self.assertIn('id="one"', out)
self.assertIn('id="two"', out)
def test_an_element_keeping_only_an_id_is_not_pruned(self):
out = render('
text
')
self.assertIn('id="anchor"', out)
class ImageHandlingTests(unittest.TestCase):
"""Images are collected for the fetcher and rewritten by the caller."""
def test_image_urls_are_collected_in_order_and_deduplicated(self):
article = parse_article('
', BASE)
self.assertEqual(
article.image_urls,
["https://example.com/posts/a.png", "https://example.com/posts/b.png"],
)
def test_remote_urls_are_kept_when_no_map_is_given(self):
out = render(' ')
self.assertIn('src="https://example.com/posts/a.png"', out)
self.assertIn('alt="alt text"', out)
def test_map_rewrites_src_to_local_path(self):
url = "https://example.com/posts/a.png"
out = render(' ', image_map={url: "images/img-1.png"})
self.assertIn('src="images/img-1.png"', out)
def test_none_in_map_drops_image_and_empty_wrapper(self):
url = "https://example.com/posts/a.png"
out = render('text
', image_map={url: None})
self.assertNotIn(" ", out)
self.assertIn("text
", out)
def test_alt_and_title_are_preserved_for_localised_images(self):
url = "https://example.com/posts/a.png"
out = render(' ', image_map={url: "images/i.png"})
self.assertIn('alt="A"', out)
self.assertIn('title="T"', out)
class WhitespaceAndStructureTests(unittest.TestCase):
def test_whitespace_is_collapsed_outside_pre(self):
out = render("one\n\n two\t\tthree
")
self.assertIn("one two three
", out)
def test_whitespace_is_preserved_inside_pre(self):
out = render("def f():\n return 1 ")
self.assertIn("def f():\n return 1", out)
def test_empty_paragraphs_are_pruned(self):
out = render("
real
")
self.assertEqual(body(out).count(""), 1)
self.assertIn("real", out)
def test_empty_table_cells_are_kept(self):
# An empty cell is layout, not noise, so pruning must leave it alone.
out = render("
")
self.assertEqual(body(out).count(" "), 1)
def test_plain_text_body_is_wrapped_in_a_paragraph(self):
out = render("just words with no markup")
self.assertEqual(body(out), "just words with no markup
")
def test_whitespace_only_body_renders_empty(self):
article = parse_article(" \n ", BASE)
self.assertEqual(article.to_xhtml(), "")
def test_empty_input_renders_empty(self):
self.assertEqual(parse_article("", BASE).to_xhtml(), "")
def test_heading_levels_survive(self):
out = render("a b c ")
for tag in ("h1", "h2", "h3"):
self.assertIn(f"<{tag}>", out)
def test_colspan_is_clamped(self):
out = render('x ')
self.assertNotIn("colspan", out)
out = render('x ')
self.assertIn('colspan="2"', out)
def test_element_structure_is_preserved(self):
out = render("")
tree = parse_xml(out)
# The fragment's own root is the article wrapper.
self.assertEqual(tree.must_attr("class"), "article-body")
outer = tree.must_find("ul")
items = outer.must_find_all("li")
self.assertEqual(len(items), 2)
# The nested list belongs to the first item, not to the outer list.
self.assertTrue(items[0].find_all("ul"))
class ArticleApiTests(unittest.TestCase):
def test_text_extraction_flattens_and_collapses(self):
article = parse_article("One
Two & three
", BASE)
self.assertEqual(article.text(), "One Two & three")
def test_text_extraction_is_limited_at_a_word_boundary(self):
article = parse_article("alpha beta gamma delta
", BASE)
text = article.text(limit=12)
self.assertTrue(text.endswith("\u2026"))
self.assertLessEqual(len(text), 13)
self.assertIn("alpha beta", text)
def test_truncation_flag_and_output_validity(self):
article = parse_article("" + "word " * 5000 + "
", BASE, max_input_bytes=2000)
self.assertTrue(article.truncated)
assert_well_formed(self, article.to_xhtml())
def test_no_truncation_when_under_limit(self):
article = parse_article("short
", BASE, max_input_bytes=10_000)
self.assertFalse(article.truncated)
def test_output_is_deterministic(self):
raw = 'a & b
'
self.assertEqual(render(raw), render(raw))
def test_image_urls_are_cached_between_calls(self):
article = parse_article(' ', BASE)
self.assertIs(article.image_urls, article.image_urls)
def test_output_wrapper_has_the_expected_class(self):
self.assertTrue(render("x
").startswith(''))
if __name__ == "__main__":
unittest.main()