Initial commit

This commit is contained in:
2026-11-15 18:15:03 -05:00
commit b76aea2899
36 changed files with 9424 additions and 0 deletions

747
app/xhtml.py Normal file
View File

@@ -0,0 +1,747 @@
"""Turn arbitrary feed HTML into well-formed XHTML for the EPUB pipeline.
EPUB is XML end to end: the OPF, the NCX, the navigation document and every
content document are parsed by a strict XML parser (the reader firmware this
server was written alongside uses expat). Feed article bodies are the opposite
of strict -- unclosed ``<li>``, stray ``&``, ``<br>`` instead of ``<br/>``,
control characters, embedded scripts and tracking pixels. Handing that to the
EPUB writer would produce a book that some readers refuse to open, so everything
goes through this normaliser first.
How it stays safe, in order of importance:
* The input is parsed into an in-memory tree and re-serialised from scratch.
Any malformedness in the source therefore cannot reach the output: whatever the
parser tolerated is rebuilt with balanced tags, quoted attributes and escaped
text. There is no pass-through of raw markup.
* Characters that XML 1.0 forbids outright (most C0 controls, unpaired
surrogates) are removed, because they are not expressible even when escaped.
* Entities are decoded to real Unicode by the parser and re-escaped on output, so
a stray ``&`` in the source cannot produce a document that will not parse.
* Scriptable and metadata elements are dropped with their contents; every other
unknown element is unwrapped, keeping its text.
* Only URLs that survive scheme validation reach an attribute, which keeps
``javascript:`` and ``data:`` payloads out of the book.
The tree is kept around after parsing so a caller can collect image URLs, fetch
them, and then serialise with the fetched images rewritten to local paths.
"""
from __future__ import annotations
import html
import logging
import re
from collections.abc import Iterator
from html.parser import HTMLParser
from urllib.parse import urljoin, urlsplit
LOGGER = logging.getLogger(__name__)
# Elements kept in the output. Mirrors the firmware reader's sanitiser
# (lib/FreshRss/src/FreshRssArticle.cpp) with the additions that make sense on a
# device with a real layout engine: images, and the small semantic set EPUB
# readers style (q, cite, time, abbr, del, ins, small, mark).
KEPT_TAGS = frozenset(
{
"p",
"div",
"span",
"br",
"hr",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"ul",
"ol",
"li",
"dl",
"dt",
"dd",
"blockquote",
"pre",
"code",
"em",
"i",
"strong",
"b",
"u",
"s",
"sub",
"sup",
"a",
"img",
"figure",
"figcaption",
"table",
"thead",
"tbody",
"tfoot",
"tr",
"td",
"th",
"caption",
"small",
"mark",
"cite",
"time",
"abbr",
"q",
"ins",
"del",
}
)
# Dropped along with everything inside them. These are either scriptable,
# fetch extra resources, or are document-level metadata that has no place inside
# an EPUB content document.
DROP_WITH_CONTENT = frozenset(
{
"script",
"style",
"iframe",
"frame",
"frameset",
"object",
"embed",
"applet",
"svg",
"math",
"form",
"input",
"button",
"select",
"option",
"optgroup",
"textarea",
"label",
"fieldset",
"legend",
"noscript",
"template",
"canvas",
"video",
"audio",
"source",
"track",
"map",
"area",
"head",
"title",
"meta",
"link",
"base",
"dialog",
"slot",
}
)
# HTML void elements: no end tag exists, so the parser must not expect one.
VOID_TAGS = frozenset({"br", "hr", "img"})
_HTML_VOID = frozenset(
{
"area",
"base",
"br",
"col",
"embed",
"hr",
"img",
"input",
"link",
"meta",
"param",
"source",
"track",
"wbr",
}
)
# Elements that silently end an open element in HTML. Feeds are full of
# unclosed <p> and <li>, and the browser-correct fix is to close the open
# element rather than to nest one block inside another (which is what a naive
# stack would do, producing <p>text<div>...</div></p>).
_IMPLICIT_CLOSE: dict[str, frozenset[str]] = {
"p": frozenset(
{
"address",
"article",
"aside",
"blockquote",
"details",
"div",
"dl",
"fieldset",
"figcaption",
"figure",
"footer",
"form",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"header",
"hr",
"main",
"nav",
"ol",
"p",
"pre",
"section",
"table",
"ul",
}
),
"li": frozenset({"li"}),
"dt": frozenset({"dt", "dd"}),
"dd": frozenset({"dt", "dd"}),
"td": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
"th": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
"tr": frozenset({"tr", "tbody", "tfoot", "thead"}),
"option": frozenset({"option", "optgroup"}),
"thead": frozenset({"tbody", "tfoot"}),
"tbody": frozenset({"tbody", "tfoot"}),
}
# Attributes kept, per element. Everything else is discarded: inline styles and
# layout attributes fight the EPUB stylesheet, and event handlers, tracking
# attributes and class names have no meaning in a book.
_KEPT_ATTRS: dict[str, frozenset[str]] = {
"a": frozenset({"href", "title"}),
"img": frozenset({"src", "alt", "title"}),
"blockquote": frozenset({"cite"}),
"q": frozenset({"cite"}),
"time": frozenset({"datetime"}),
"ins": frozenset({"datetime"}),
"del": frozenset({"datetime"}),
"abbr": frozenset({"title"}),
"td": frozenset({"colspan", "rowspan"}),
"th": frozenset({"colspan", "rowspan"}),
"ol": frozenset({"start"}),
}
# Empty structural elements worth removing once their contents are gone (for
# example a wrapper whose only child was a rejected image). Table cells are
# deliberately absent: an empty cell is layout, not noise.
_DROPPABLE_WHEN_EMPTY = frozenset(
{
"p",
"div",
"span",
"li",
"ul",
"ol",
"dl",
"blockquote",
"figure",
"figcaption",
"section",
"article",
}
)
_LINK_SCHEMES = ("http", "https", "mailto")
_IMAGE_SCHEMES = ("http", "https")
# XML 1.0 permits #x9, #xA, #xD, #x20-#xD7FF, #xE000-#xFFFD, #x10000-#x10FFFF.
# Everything else -- NUL bytes, vertical tabs from mangled feeds, form feeds,
# unpaired surrogates -- cannot be represented even as a character reference and
# must be deleted before serialisation.
_ILLEGAL_XML_RE = re.compile(
"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]"
)
# Collapses runs of layout whitespace. A non-breaking space is deliberately not
# matched: feeds use it for intentional spacing, and reflowable text keeps it.
_WHITESPACE_RE = re.compile(r"[^\S\xa0]+")
# Trims layout whitespace from the edges of a block. Python's str.strip() treats
# a non-breaking space as whitespace and would eat one the author typed on
# purpose, so the pattern excludes it here too.
_BLOCK_TRIM_RE = re.compile(r"\A[^\S\xa0]+|[^\S\xa0]+\Z")
_ZERO_WIDTH_RE = re.compile("[\u200b\u200c\u200d\u2060\ufeff]")
# XML name rules for an id: starts with a letter or underscore, then a bounded run
# of name characters. Anything else is not a usable anchor target.
_ID_RE = re.compile(r"\A[A-Za-z_][A-Za-z0-9_.-]{0,63}\Z")
_ID_ATTRS = frozenset({"id"})
# Wrappers that need the attribute filter relaxed for integer attributes.
_INT_ATTR_LIMITS = {"colspan": (1, 100), "rowspan": (1, 100), "start": (-10000, 10000)}
class _Element:
"""One element node in the parsed tree."""
__slots__ = ("attrs", "children", "tag")
def __init__(self, tag: str) -> None:
self.tag = tag
self.attrs: dict[str, str] = {}
self.children: list[_Element | _Text] = []
class _Text:
"""One text node in the parsed tree."""
__slots__ = ("text",)
def __init__(self, text: str) -> None:
self.text = text
class _TreeBuilder(HTMLParser):
"""Build a normalised element tree from feed HTML.
The parser is tolerant by construction -- it never raises on malformed input,
it only decides what to keep -- so this class is the single place where all
of the "this feed is broken" handling lives.
"""
def __init__(self) -> None:
# convert_charrefs decodes &amp;/&#233;/&nbsp; to Unicode for us, so the
# serialiser only ever has to escape what it writes itself.
super().__init__(convert_charrefs=True)
self.root = _Element("#root")
self._stack: list[_Element] = [self.root]
# Names of open drop-with-content elements; while non-empty, all input is
# discarded except the end tag that closes the innermost one.
self._suppressed: list[str] = []
# -- parser callbacks -------------------------------------------------
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if self._suppressed:
if tag in DROP_WITH_CONTENT and tag not in _HTML_VOID:
self._suppressed.append(tag)
return
if tag in DROP_WITH_CONTENT:
if tag not in _HTML_VOID:
self._suppressed.append(tag)
return
self._close_implied(tag)
if tag in _HTML_VOID and tag not in VOID_TAGS:
# Void but not kept (<col>, <wbr>, <param>): it cannot have content,
# so it must never open a subtree that would swallow the rest.
# br/hr/img are void *and* kept, so they fall through to be emitted.
return
element = _Element(tag)
element.attrs = self._filter_attrs(tag, attrs)
self._stack[-1].children.append(element)
if tag not in VOID_TAGS:
self._stack.append(element)
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
# <br/> lands here. A self-closing element is complete: it is appended
# and never pushed, so there is nothing left open. A self-closing
# drop-with-content element has no content to discard and must not open a
# suppression region, or the rest of the document would be swallowed.
if self._suppressed or tag in DROP_WITH_CONTENT or tag in _HTML_VOID:
return
self._close_implied(tag)
element = _Element(tag)
element.attrs = self._filter_attrs(tag, attrs)
self._stack[-1].children.append(element)
def handle_endtag(self, tag: str) -> None:
if self._suppressed:
if tag == self._suppressed[-1]:
self._suppressed.pop()
return
if tag in _HTML_VOID:
return
# Close down to the matching open element, tolerating unclosed tags in
# between (this is exactly what browsers do).
for index in range(len(self._stack) - 1, 0, -1):
if self._stack[index].tag == tag:
del self._stack[index:]
return
# Stray end tag with no matching open element: ignore it.
def handle_data(self, data: str) -> None:
if self._suppressed or not data:
return
self._stack[-1].children.append(_Text(data))
def handle_comment(self, data: str) -> None:
return
def handle_decl(self, decl: str) -> None:
return
def handle_pi(self, data: str) -> None:
return
def unknown_decl(self, data: str) -> None:
if self._suppressed:
return
# An HTML parser treats <![CDATA[...]]> as a bogus comment and discards
# it, but a feed body that passed through an XML parser upstream can
# legitimately contain one, and its payload is real content. The parser
# hands over the section body with the closing ]] already stripped.
marker = "CDATA["
if data.startswith(marker):
inner = data[len(marker) :]
if inner:
self._stack[-1].children.append(_Text(inner))
# -- helpers ---------------------------------------------------------
def _close_implied(self, tag: str) -> None:
"""Close open elements that HTML would close implicitly."""
while len(self._stack) > 1:
conditions = _IMPLICIT_CLOSE.get(self._stack[-1].tag)
if conditions is None or tag not in conditions:
return
self._stack.pop()
def _filter_attrs(self, tag: str, attrs: list[tuple[str, str | None]]) -> dict[str, str]:
if tag not in KEPT_TAGS:
# The element is unwrapped later, so its attributes are never used.
return {}
allowed = _KEPT_ATTRS.get(tag, frozenset())
if tag not in VOID_TAGS:
# Anchor targets survive so an in-book table of contents still jumps
# to its section when the article is read offline.
allowed = allowed | _ID_ATTRS
kept: dict[str, str] = {}
for raw_name, raw_value in attrs:
name = raw_name.lower()
if name not in allowed:
continue
value = html.unescape(raw_value or "") if raw_value is not None else ""
value = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", value)).strip()
if name == "id":
if not _ID_RE.match(value):
continue
kept[name] = value
continue
if name in _INT_ATTR_LIMITS:
bounds = _INT_ATTR_LIMITS[name]
try:
number = int(value)
except ValueError:
continue
if not bounds[0] <= number <= bounds[1]:
continue
value = str(number)
if not value:
continue
kept[name] = value
return kept
class Article:
"""A parsed, normalised article body ready for EPUB serialisation."""
def __init__(self, tree: _Element, base_url: str, truncated: bool) -> None:
self._tree = tree
self._base_url = base_url
self.truncated = truncated
self._image_urls: list[str] | None = None
@property
def image_urls(self) -> list[str]:
"""Absolute http(s) image URLs, in document order, de-duplicated."""
if self._image_urls is None:
seen: list[str] = []
known: set[str] = set()
for element in _iter_elements(self._tree):
if element.tag != "img":
continue
src = element.attrs.get("src", "")
if src and src not in known:
known.add(src)
seen.append(src)
self._image_urls = seen
return self._image_urls
def text(self, limit: int = 600) -> str:
"""Flattened plain text, for summaries and the search index."""
parts: list[str] = []
for element in _iter_text(self._tree):
parts.append(element.text)
collapsed = _WHITESPACE_RE.sub(" ", _ILLEGAL_XML_RE.sub("", "".join(parts))).strip()
if len(collapsed) <= limit:
return collapsed
# Cut at a word boundary so a summary never ends mid-word.
cut = collapsed[:limit].rsplit(" ", 1)[0]
return (cut or collapsed[:limit]).rstrip() + "\u2026"
def to_xhtml(self, image_map: dict[str, str | None] | None = None) -> str:
"""Serialise the body as an XHTML fragment.
``image_map`` maps an original absolute image URL to the local href to
use in its place; a value of None drops that image (and its wrapper, if it
becomes empty). Pass None to leave remote image URLs untouched -- the
caller must then declare remote resources in the OPF manifest.
Applying a map rewrites the tree in place, so an Article is rendered
once. The server builds a book and serialises it exactly once.
"""
if image_map is not None:
_apply_image_map(self._tree, image_map)
# A dropped image can leave an empty wrapper behind.
_prune_empty(self._tree)
body = _serialise_children(self._tree, inside_pre=False)
if not body.strip():
return ""
return f'<div class="article-body">\n{body}\n</div>'
class _Resolve:
"""Decides which URLs survive normalisation, and where they point."""
def __init__(self, base_url: str) -> None:
self.base_url = base_url
def link(self, raw: str) -> str | None:
# A same-document reference is left alone so it resolves against the book
# rather than against the web page the article came from. The matching id
# was kept by the attribute filter.
if raw.startswith("#"):
return raw
resolved = _absolutise(raw, self.base_url)
if resolved is None:
return None
if urlsplit(resolved).scheme.lower() not in _LINK_SCHEMES:
return None
return resolved
def image(self, raw: str) -> str | None:
resolved = _absolutise(raw, self.base_url)
if resolved is None:
return None
if urlsplit(resolved).scheme.lower() not in _IMAGE_SCHEMES:
return None
return resolved
def parse_article(raw_html: str, base_url: str = "", max_input_bytes: int = 0) -> Article:
"""Parse a feed article body into a normalised tree.
``base_url`` is the article's own URL; relative links and images inside the
body are resolved against it, which is the only way they keep working once
the body is inside a book with a different base.
"""
text = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", raw_html or ""))
truncated = False
if max_input_bytes > 0 and len(text.encode("utf-8", "ignore")) > max_input_bytes:
# Cut on a character boundary, not a byte boundary, and do not try to be
# clever about tags: the parser tolerates a truncated tail.
while len(text.encode("utf-8", "ignore")) > max_input_bytes:
text = text[: int(len(text) * 0.98) or 1]
truncated = True
LOGGER.warning("article body truncated to %d bytes before parsing", max_input_bytes)
builder = _TreeBuilder()
builder.feed(text)
builder.close()
tree = builder.root
if not _has_elements(tree):
# A body with no markup at all (plain text feed) still needs block
# structure, or the EPUB would contain a bare text node.
paragraph = _Element("p")
paragraph.children = list(tree.children)
tree.children = [paragraph]
_resolve_urls(tree, _Resolve(base_url), set())
_prune_empty(tree)
return Article(tree, base_url, truncated)
def _absolutise(raw: str, base_url: str) -> str | None:
candidate = (raw or "").strip()
if not candidate:
return None
# Control characters and raw whitespace inside a URL are always corruption.
candidate = _ILLEGAL_XML_RE.sub("", candidate)
if any(char.isspace() for char in candidate):
candidate = candidate.split()[0]
try:
if base_url:
return urljoin(base_url, candidate)
return candidate
except ValueError:
return None
def _resolve_urls(element: _Element, resolve: _Resolve, seen_ids: set[str]) -> None:
"""Validate URLs and de-duplicate anchor ids, in document order.
Runs once, at parse time, so everything downstream (the image collector, the
fetcher, the OPF manifest) sees final absolute URLs and there is no second
chance for a bad scheme to slip through.
"""
for child in element.children:
if isinstance(child, _Text):
continue
_dedupe_id(child, seen_ids)
if child.tag == "a":
href = child.attrs.get("href")
if href is not None:
target = resolve.link(href)
if target is None:
child.attrs.pop("href", None)
else:
child.attrs["href"] = target
elif child.tag == "img":
src = child.attrs.get("src")
target = resolve.image(src) if src else None
if target is None:
# Mark it for pruning: an image with no usable source is noise.
child.tag = "#dropped-image"
else:
child.attrs["src"] = target
elif child.tag in ("blockquote", "q"):
cite = child.attrs.get("cite")
if cite is not None:
target = resolve.link(cite)
if target is None:
child.attrs.pop("cite", None)
else:
child.attrs["cite"] = target
_resolve_urls(child, resolve, seen_ids)
def _dedupe_id(element: _Element, seen_ids: set[str]) -> None:
"""Make an element's id unique within the document.
Duplicate ids are invalid and would make the first anchor win, so a repeat is
renamed rather than dropped -- dropping it would break the link pointing at
it. One book holds one article, so a document-wide set is enough.
"""
anchor = element.attrs.get("id")
if anchor is None:
return
if anchor not in seen_ids:
seen_ids.add(anchor)
return
index = 2
while f"{anchor}-{index}" in seen_ids:
index += 1
renamed = f"{anchor}-{index}"
seen_ids.add(renamed)
element.attrs["id"] = renamed
def _apply_image_map(element: _Element, image_map: dict[str, str | None]) -> None:
"""Point images at their local copies inside the book.
A URL mapped to None means the fetch failed or the image was refused, so the
element is marked for pruning instead of being left pointing at the network.
"""
for child in element.children:
if isinstance(child, _Text):
continue
if child.tag == "img":
src = child.attrs.get("src")
if src is not None and src in image_map:
target = image_map[src]
if target is None:
child.tag = "#dropped-image"
else:
child.attrs["src"] = target
_apply_image_map(child, image_map)
def _prune_empty(node: _Element) -> None:
"""Remove rejected images and the empty wrappers they leave behind."""
kept: list[_Element | _Text] = []
for child in node.children:
if isinstance(child, _Element):
if child.tag == "#dropped-image":
continue
_prune_empty(child)
if _is_noise(child):
continue
kept.append(child)
node.children = kept
def _is_noise(element: _Element) -> bool:
"""True for a droppable wrapper with no content left inside it.
Whitespace-only text does not count as content (``<p>&nbsp;</p>`` is noise),
but a retained ``id`` does: it is the target of a link.
"""
if element.tag not in _DROPPABLE_WHEN_EMPTY or element.attrs:
return False
for child in element.children:
if isinstance(child, _Element):
return False
if child.text.strip():
return False
return True
def _has_elements(element: _Element) -> bool:
"""True when the tree contains at least one element node."""
return any(True for _ in _iter_elements(element))
def _serialise_children(node: _Element, inside_pre: bool) -> str:
return "".join(_serialise(child, inside_pre) for child in node.children)
def _serialise(node: _Element | _Text, inside_pre: bool) -> str:
if isinstance(node, _Text):
if inside_pre:
return escape_text(node.text)
# Collapse whitespace outside <pre>: feed HTML arrives indented, and that
# indentation is pure bytes in the book.
return escape_text(_WHITESPACE_RE.sub(" ", node.text))
if node.tag not in KEPT_TAGS:
# Unwrapped elements keep their children but lose their own tag, so no
# unknown element can appear in the output.
return _serialise_children(node, inside_pre)
child_pre = inside_pre or node.tag == "pre"
inner = _serialise_children(node, child_pre)
if node.tag in ("p", "div", "li", "td", "th", "blockquote", "figcaption", "caption"):
inner = _BLOCK_TRIM_RE.sub("", inner)
attributes = "".join(
f' {name}="{escape_attribute(value)}"' for name, value in _ordered_attrs(node)
)
if node.tag in VOID_TAGS:
return f"<{node.tag}{attributes}/>"
return f"<{node.tag}{attributes}>{inner}</{node.tag}>"
def _ordered_attrs(element: _Element) -> list[tuple[str, str]]:
"""Stable attribute order, with the most meaningful attribute first."""
priority = {"id": 0, "href": 1, "src": 2, "alt": 3, "title": 4}
return sorted(element.attrs.items(), key=lambda item: (priority.get(item[0], 9), item[0]))
def escape_text(text: str) -> str:
return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
def escape_attribute(value: str) -> str:
# Attribute values are normalised by XML parsers anyway (tab/newline become
# spaces), so do it explicitly rather than leaving it implicit.
text = value.replace("\t", " ").replace("\n", " ").replace("\r", " ")
return (
text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
)
def _iter_elements(node: _Element) -> Iterator[_Element]:
"""Depth-first traversal over element nodes only."""
for child in node.children:
if isinstance(child, _Element):
yield child
yield from _iter_elements(child)
def _iter_text(node: _Element) -> Iterator[_Text]:
"""Depth-first traversal over text nodes only."""
for child in node.children:
if isinstance(child, _Text):
yield child
else:
yield from _iter_text(child)