Initial commit
This commit is contained in:
747
app/xhtml.py
Normal file
747
app/xhtml.py
Normal file
@@ -0,0 +1,747 @@
|
||||
"""Turn arbitrary feed HTML into well-formed XHTML for the EPUB pipeline.
|
||||
|
||||
EPUB is XML end to end: the OPF, the NCX, the navigation document and every
|
||||
content document are parsed by a strict XML parser (the reader firmware this
|
||||
server was written alongside uses expat). Feed article bodies are the opposite
|
||||
of strict -- unclosed ``<li>``, stray ``&``, ``<br>`` instead of ``<br/>``,
|
||||
control characters, embedded scripts and tracking pixels. Handing that to the
|
||||
EPUB writer would produce a book that some readers refuse to open, so everything
|
||||
goes through this normaliser first.
|
||||
|
||||
How it stays safe, in order of importance:
|
||||
|
||||
* The input is parsed into an in-memory tree and re-serialised from scratch.
|
||||
Any malformedness in the source therefore cannot reach the output: whatever the
|
||||
parser tolerated is rebuilt with balanced tags, quoted attributes and escaped
|
||||
text. There is no pass-through of raw markup.
|
||||
* Characters that XML 1.0 forbids outright (most C0 controls, unpaired
|
||||
surrogates) are removed, because they are not expressible even when escaped.
|
||||
* Entities are decoded to real Unicode by the parser and re-escaped on output, so
|
||||
a stray ``&`` in the source cannot produce a document that will not parse.
|
||||
* Scriptable and metadata elements are dropped with their contents; every other
|
||||
unknown element is unwrapped, keeping its text.
|
||||
* Only URLs that survive scheme validation reach an attribute, which keeps
|
||||
``javascript:`` and ``data:`` payloads out of the book.
|
||||
|
||||
The tree is kept around after parsing so a caller can collect image URLs, fetch
|
||||
them, and then serialise with the fetched images rewritten to local paths.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# Elements kept in the output. Mirrors the firmware reader's sanitiser
|
||||
# (lib/FreshRss/src/FreshRssArticle.cpp) with the additions that make sense on a
|
||||
# device with a real layout engine: images, and the small semantic set EPUB
|
||||
# readers style (q, cite, time, abbr, del, ins, small, mark).
|
||||
KEPT_TAGS = frozenset(
|
||||
{
|
||||
"p",
|
||||
"div",
|
||||
"span",
|
||||
"br",
|
||||
"hr",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"ul",
|
||||
"ol",
|
||||
"li",
|
||||
"dl",
|
||||
"dt",
|
||||
"dd",
|
||||
"blockquote",
|
||||
"pre",
|
||||
"code",
|
||||
"em",
|
||||
"i",
|
||||
"strong",
|
||||
"b",
|
||||
"u",
|
||||
"s",
|
||||
"sub",
|
||||
"sup",
|
||||
"a",
|
||||
"img",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"table",
|
||||
"thead",
|
||||
"tbody",
|
||||
"tfoot",
|
||||
"tr",
|
||||
"td",
|
||||
"th",
|
||||
"caption",
|
||||
"small",
|
||||
"mark",
|
||||
"cite",
|
||||
"time",
|
||||
"abbr",
|
||||
"q",
|
||||
"ins",
|
||||
"del",
|
||||
}
|
||||
)
|
||||
|
||||
# Dropped along with everything inside them. These are either scriptable,
|
||||
# fetch extra resources, or are document-level metadata that has no place inside
|
||||
# an EPUB content document.
|
||||
DROP_WITH_CONTENT = frozenset(
|
||||
{
|
||||
"script",
|
||||
"style",
|
||||
"iframe",
|
||||
"frame",
|
||||
"frameset",
|
||||
"object",
|
||||
"embed",
|
||||
"applet",
|
||||
"svg",
|
||||
"math",
|
||||
"form",
|
||||
"input",
|
||||
"button",
|
||||
"select",
|
||||
"option",
|
||||
"optgroup",
|
||||
"textarea",
|
||||
"label",
|
||||
"fieldset",
|
||||
"legend",
|
||||
"noscript",
|
||||
"template",
|
||||
"canvas",
|
||||
"video",
|
||||
"audio",
|
||||
"source",
|
||||
"track",
|
||||
"map",
|
||||
"area",
|
||||
"head",
|
||||
"title",
|
||||
"meta",
|
||||
"link",
|
||||
"base",
|
||||
"dialog",
|
||||
"slot",
|
||||
}
|
||||
)
|
||||
|
||||
# HTML void elements: no end tag exists, so the parser must not expect one.
|
||||
VOID_TAGS = frozenset({"br", "hr", "img"})
|
||||
_HTML_VOID = frozenset(
|
||||
{
|
||||
"area",
|
||||
"base",
|
||||
"br",
|
||||
"col",
|
||||
"embed",
|
||||
"hr",
|
||||
"img",
|
||||
"input",
|
||||
"link",
|
||||
"meta",
|
||||
"param",
|
||||
"source",
|
||||
"track",
|
||||
"wbr",
|
||||
}
|
||||
)
|
||||
|
||||
# Elements that silently end an open element in HTML. Feeds are full of
|
||||
# unclosed <p> and <li>, and the browser-correct fix is to close the open
|
||||
# element rather than to nest one block inside another (which is what a naive
|
||||
# stack would do, producing <p>text<div>...</div></p>).
|
||||
_IMPLICIT_CLOSE: dict[str, frozenset[str]] = {
|
||||
"p": frozenset(
|
||||
{
|
||||
"address",
|
||||
"article",
|
||||
"aside",
|
||||
"blockquote",
|
||||
"details",
|
||||
"div",
|
||||
"dl",
|
||||
"fieldset",
|
||||
"figcaption",
|
||||
"figure",
|
||||
"footer",
|
||||
"form",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"header",
|
||||
"hr",
|
||||
"main",
|
||||
"nav",
|
||||
"ol",
|
||||
"p",
|
||||
"pre",
|
||||
"section",
|
||||
"table",
|
||||
"ul",
|
||||
}
|
||||
),
|
||||
"li": frozenset({"li"}),
|
||||
"dt": frozenset({"dt", "dd"}),
|
||||
"dd": frozenset({"dt", "dd"}),
|
||||
"td": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
|
||||
"th": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
|
||||
"tr": frozenset({"tr", "tbody", "tfoot", "thead"}),
|
||||
"option": frozenset({"option", "optgroup"}),
|
||||
"thead": frozenset({"tbody", "tfoot"}),
|
||||
"tbody": frozenset({"tbody", "tfoot"}),
|
||||
}
|
||||
|
||||
# Attributes kept, per element. Everything else is discarded: inline styles and
|
||||
# layout attributes fight the EPUB stylesheet, and event handlers, tracking
|
||||
# attributes and class names have no meaning in a book.
|
||||
_KEPT_ATTRS: dict[str, frozenset[str]] = {
|
||||
"a": frozenset({"href", "title"}),
|
||||
"img": frozenset({"src", "alt", "title"}),
|
||||
"blockquote": frozenset({"cite"}),
|
||||
"q": frozenset({"cite"}),
|
||||
"time": frozenset({"datetime"}),
|
||||
"ins": frozenset({"datetime"}),
|
||||
"del": frozenset({"datetime"}),
|
||||
"abbr": frozenset({"title"}),
|
||||
"td": frozenset({"colspan", "rowspan"}),
|
||||
"th": frozenset({"colspan", "rowspan"}),
|
||||
"ol": frozenset({"start"}),
|
||||
}
|
||||
|
||||
# Empty structural elements worth removing once their contents are gone (for
|
||||
# example a wrapper whose only child was a rejected image). Table cells are
|
||||
# deliberately absent: an empty cell is layout, not noise.
|
||||
_DROPPABLE_WHEN_EMPTY = frozenset(
|
||||
{
|
||||
"p",
|
||||
"div",
|
||||
"span",
|
||||
"li",
|
||||
"ul",
|
||||
"ol",
|
||||
"dl",
|
||||
"blockquote",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"section",
|
||||
"article",
|
||||
}
|
||||
)
|
||||
|
||||
_LINK_SCHEMES = ("http", "https", "mailto")
|
||||
_IMAGE_SCHEMES = ("http", "https")
|
||||
|
||||
# XML 1.0 permits #x9, #xA, #xD, #x20-#xD7FF, #xE000-#xFFFD, #x10000-#x10FFFF.
|
||||
# Everything else -- NUL bytes, vertical tabs from mangled feeds, form feeds,
|
||||
# unpaired surrogates -- cannot be represented even as a character reference and
|
||||
# must be deleted before serialisation.
|
||||
_ILLEGAL_XML_RE = re.compile(
|
||||
"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]"
|
||||
)
|
||||
# Collapses runs of layout whitespace. A non-breaking space is deliberately not
|
||||
# matched: feeds use it for intentional spacing, and reflowable text keeps it.
|
||||
_WHITESPACE_RE = re.compile(r"[^\S\xa0]+")
|
||||
# Trims layout whitespace from the edges of a block. Python's str.strip() treats
|
||||
# a non-breaking space as whitespace and would eat one the author typed on
|
||||
# purpose, so the pattern excludes it here too.
|
||||
_BLOCK_TRIM_RE = re.compile(r"\A[^\S\xa0]+|[^\S\xa0]+\Z")
|
||||
_ZERO_WIDTH_RE = re.compile("[\u200b\u200c\u200d\u2060\ufeff]")
|
||||
# XML name rules for an id: starts with a letter or underscore, then a bounded run
|
||||
# of name characters. Anything else is not a usable anchor target.
|
||||
_ID_RE = re.compile(r"\A[A-Za-z_][A-Za-z0-9_.-]{0,63}\Z")
|
||||
_ID_ATTRS = frozenset({"id"})
|
||||
|
||||
# Wrappers that need the attribute filter relaxed for integer attributes.
|
||||
_INT_ATTR_LIMITS = {"colspan": (1, 100), "rowspan": (1, 100), "start": (-10000, 10000)}
|
||||
|
||||
|
||||
class _Element:
|
||||
"""One element node in the parsed tree."""
|
||||
|
||||
__slots__ = ("attrs", "children", "tag")
|
||||
|
||||
def __init__(self, tag: str) -> None:
|
||||
self.tag = tag
|
||||
self.attrs: dict[str, str] = {}
|
||||
self.children: list[_Element | _Text] = []
|
||||
|
||||
|
||||
class _Text:
|
||||
"""One text node in the parsed tree."""
|
||||
|
||||
__slots__ = ("text",)
|
||||
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = text
|
||||
|
||||
|
||||
class _TreeBuilder(HTMLParser):
|
||||
"""Build a normalised element tree from feed HTML.
|
||||
|
||||
The parser is tolerant by construction -- it never raises on malformed input,
|
||||
it only decides what to keep -- so this class is the single place where all
|
||||
of the "this feed is broken" handling lives.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# convert_charrefs decodes &/é/ to Unicode for us, so the
|
||||
# serialiser only ever has to escape what it writes itself.
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.root = _Element("#root")
|
||||
self._stack: list[_Element] = [self.root]
|
||||
# Names of open drop-with-content elements; while non-empty, all input is
|
||||
# discarded except the end tag that closes the innermost one.
|
||||
self._suppressed: list[str] = []
|
||||
|
||||
# -- parser callbacks -------------------------------------------------
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if self._suppressed:
|
||||
if tag in DROP_WITH_CONTENT and tag not in _HTML_VOID:
|
||||
self._suppressed.append(tag)
|
||||
return
|
||||
if tag in DROP_WITH_CONTENT:
|
||||
if tag not in _HTML_VOID:
|
||||
self._suppressed.append(tag)
|
||||
return
|
||||
self._close_implied(tag)
|
||||
if tag in _HTML_VOID and tag not in VOID_TAGS:
|
||||
# Void but not kept (<col>, <wbr>, <param>): it cannot have content,
|
||||
# so it must never open a subtree that would swallow the rest.
|
||||
# br/hr/img are void *and* kept, so they fall through to be emitted.
|
||||
return
|
||||
element = _Element(tag)
|
||||
element.attrs = self._filter_attrs(tag, attrs)
|
||||
self._stack[-1].children.append(element)
|
||||
if tag not in VOID_TAGS:
|
||||
self._stack.append(element)
|
||||
|
||||
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
# <br/> lands here. A self-closing element is complete: it is appended
|
||||
# and never pushed, so there is nothing left open. A self-closing
|
||||
# drop-with-content element has no content to discard and must not open a
|
||||
# suppression region, or the rest of the document would be swallowed.
|
||||
if self._suppressed or tag in DROP_WITH_CONTENT or tag in _HTML_VOID:
|
||||
return
|
||||
self._close_implied(tag)
|
||||
element = _Element(tag)
|
||||
element.attrs = self._filter_attrs(tag, attrs)
|
||||
self._stack[-1].children.append(element)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if self._suppressed:
|
||||
if tag == self._suppressed[-1]:
|
||||
self._suppressed.pop()
|
||||
return
|
||||
if tag in _HTML_VOID:
|
||||
return
|
||||
# Close down to the matching open element, tolerating unclosed tags in
|
||||
# between (this is exactly what browsers do).
|
||||
for index in range(len(self._stack) - 1, 0, -1):
|
||||
if self._stack[index].tag == tag:
|
||||
del self._stack[index:]
|
||||
return
|
||||
# Stray end tag with no matching open element: ignore it.
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._suppressed or not data:
|
||||
return
|
||||
self._stack[-1].children.append(_Text(data))
|
||||
|
||||
def handle_comment(self, data: str) -> None:
|
||||
return
|
||||
|
||||
def handle_decl(self, decl: str) -> None:
|
||||
return
|
||||
|
||||
def handle_pi(self, data: str) -> None:
|
||||
return
|
||||
|
||||
def unknown_decl(self, data: str) -> None:
|
||||
if self._suppressed:
|
||||
return
|
||||
# An HTML parser treats <![CDATA[...]]> as a bogus comment and discards
|
||||
# it, but a feed body that passed through an XML parser upstream can
|
||||
# legitimately contain one, and its payload is real content. The parser
|
||||
# hands over the section body with the closing ]] already stripped.
|
||||
marker = "CDATA["
|
||||
if data.startswith(marker):
|
||||
inner = data[len(marker) :]
|
||||
if inner:
|
||||
self._stack[-1].children.append(_Text(inner))
|
||||
|
||||
# -- helpers ---------------------------------------------------------
|
||||
|
||||
def _close_implied(self, tag: str) -> None:
|
||||
"""Close open elements that HTML would close implicitly."""
|
||||
while len(self._stack) > 1:
|
||||
conditions = _IMPLICIT_CLOSE.get(self._stack[-1].tag)
|
||||
if conditions is None or tag not in conditions:
|
||||
return
|
||||
self._stack.pop()
|
||||
|
||||
def _filter_attrs(self, tag: str, attrs: list[tuple[str, str | None]]) -> dict[str, str]:
|
||||
if tag not in KEPT_TAGS:
|
||||
# The element is unwrapped later, so its attributes are never used.
|
||||
return {}
|
||||
allowed = _KEPT_ATTRS.get(tag, frozenset())
|
||||
if tag not in VOID_TAGS:
|
||||
# Anchor targets survive so an in-book table of contents still jumps
|
||||
# to its section when the article is read offline.
|
||||
allowed = allowed | _ID_ATTRS
|
||||
kept: dict[str, str] = {}
|
||||
for raw_name, raw_value in attrs:
|
||||
name = raw_name.lower()
|
||||
if name not in allowed:
|
||||
continue
|
||||
value = html.unescape(raw_value or "") if raw_value is not None else ""
|
||||
value = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", value)).strip()
|
||||
if name == "id":
|
||||
if not _ID_RE.match(value):
|
||||
continue
|
||||
kept[name] = value
|
||||
continue
|
||||
if name in _INT_ATTR_LIMITS:
|
||||
bounds = _INT_ATTR_LIMITS[name]
|
||||
try:
|
||||
number = int(value)
|
||||
except ValueError:
|
||||
continue
|
||||
if not bounds[0] <= number <= bounds[1]:
|
||||
continue
|
||||
value = str(number)
|
||||
if not value:
|
||||
continue
|
||||
kept[name] = value
|
||||
return kept
|
||||
|
||||
|
||||
class Article:
|
||||
"""A parsed, normalised article body ready for EPUB serialisation."""
|
||||
|
||||
def __init__(self, tree: _Element, base_url: str, truncated: bool) -> None:
|
||||
self._tree = tree
|
||||
self._base_url = base_url
|
||||
self.truncated = truncated
|
||||
self._image_urls: list[str] | None = None
|
||||
|
||||
@property
|
||||
def image_urls(self) -> list[str]:
|
||||
"""Absolute http(s) image URLs, in document order, de-duplicated."""
|
||||
if self._image_urls is None:
|
||||
seen: list[str] = []
|
||||
known: set[str] = set()
|
||||
for element in _iter_elements(self._tree):
|
||||
if element.tag != "img":
|
||||
continue
|
||||
src = element.attrs.get("src", "")
|
||||
if src and src not in known:
|
||||
known.add(src)
|
||||
seen.append(src)
|
||||
self._image_urls = seen
|
||||
return self._image_urls
|
||||
|
||||
def text(self, limit: int = 600) -> str:
|
||||
"""Flattened plain text, for summaries and the search index."""
|
||||
parts: list[str] = []
|
||||
for element in _iter_text(self._tree):
|
||||
parts.append(element.text)
|
||||
collapsed = _WHITESPACE_RE.sub(" ", _ILLEGAL_XML_RE.sub("", "".join(parts))).strip()
|
||||
if len(collapsed) <= limit:
|
||||
return collapsed
|
||||
# Cut at a word boundary so a summary never ends mid-word.
|
||||
cut = collapsed[:limit].rsplit(" ", 1)[0]
|
||||
return (cut or collapsed[:limit]).rstrip() + "\u2026"
|
||||
|
||||
def to_xhtml(self, image_map: dict[str, str | None] | None = None) -> str:
|
||||
"""Serialise the body as an XHTML fragment.
|
||||
|
||||
``image_map`` maps an original absolute image URL to the local href to
|
||||
use in its place; a value of None drops that image (and its wrapper, if it
|
||||
becomes empty). Pass None to leave remote image URLs untouched -- the
|
||||
caller must then declare remote resources in the OPF manifest.
|
||||
|
||||
Applying a map rewrites the tree in place, so an Article is rendered
|
||||
once. The server builds a book and serialises it exactly once.
|
||||
"""
|
||||
if image_map is not None:
|
||||
_apply_image_map(self._tree, image_map)
|
||||
# A dropped image can leave an empty wrapper behind.
|
||||
_prune_empty(self._tree)
|
||||
body = _serialise_children(self._tree, inside_pre=False)
|
||||
if not body.strip():
|
||||
return ""
|
||||
return f'<div class="article-body">\n{body}\n</div>'
|
||||
|
||||
|
||||
class _Resolve:
|
||||
"""Decides which URLs survive normalisation, and where they point."""
|
||||
|
||||
def __init__(self, base_url: str) -> None:
|
||||
self.base_url = base_url
|
||||
|
||||
def link(self, raw: str) -> str | None:
|
||||
# A same-document reference is left alone so it resolves against the book
|
||||
# rather than against the web page the article came from. The matching id
|
||||
# was kept by the attribute filter.
|
||||
if raw.startswith("#"):
|
||||
return raw
|
||||
resolved = _absolutise(raw, self.base_url)
|
||||
if resolved is None:
|
||||
return None
|
||||
if urlsplit(resolved).scheme.lower() not in _LINK_SCHEMES:
|
||||
return None
|
||||
return resolved
|
||||
|
||||
def image(self, raw: str) -> str | None:
|
||||
resolved = _absolutise(raw, self.base_url)
|
||||
if resolved is None:
|
||||
return None
|
||||
if urlsplit(resolved).scheme.lower() not in _IMAGE_SCHEMES:
|
||||
return None
|
||||
return resolved
|
||||
|
||||
|
||||
def parse_article(raw_html: str, base_url: str = "", max_input_bytes: int = 0) -> Article:
|
||||
"""Parse a feed article body into a normalised tree.
|
||||
|
||||
``base_url`` is the article's own URL; relative links and images inside the
|
||||
body are resolved against it, which is the only way they keep working once
|
||||
the body is inside a book with a different base.
|
||||
"""
|
||||
text = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", raw_html or ""))
|
||||
truncated = False
|
||||
if max_input_bytes > 0 and len(text.encode("utf-8", "ignore")) > max_input_bytes:
|
||||
# Cut on a character boundary, not a byte boundary, and do not try to be
|
||||
# clever about tags: the parser tolerates a truncated tail.
|
||||
while len(text.encode("utf-8", "ignore")) > max_input_bytes:
|
||||
text = text[: int(len(text) * 0.98) or 1]
|
||||
truncated = True
|
||||
LOGGER.warning("article body truncated to %d bytes before parsing", max_input_bytes)
|
||||
|
||||
builder = _TreeBuilder()
|
||||
builder.feed(text)
|
||||
builder.close()
|
||||
tree = builder.root
|
||||
|
||||
if not _has_elements(tree):
|
||||
# A body with no markup at all (plain text feed) still needs block
|
||||
# structure, or the EPUB would contain a bare text node.
|
||||
paragraph = _Element("p")
|
||||
paragraph.children = list(tree.children)
|
||||
tree.children = [paragraph]
|
||||
|
||||
_resolve_urls(tree, _Resolve(base_url), set())
|
||||
_prune_empty(tree)
|
||||
return Article(tree, base_url, truncated)
|
||||
|
||||
|
||||
def _absolutise(raw: str, base_url: str) -> str | None:
|
||||
candidate = (raw or "").strip()
|
||||
if not candidate:
|
||||
return None
|
||||
# Control characters and raw whitespace inside a URL are always corruption.
|
||||
candidate = _ILLEGAL_XML_RE.sub("", candidate)
|
||||
if any(char.isspace() for char in candidate):
|
||||
candidate = candidate.split()[0]
|
||||
try:
|
||||
if base_url:
|
||||
return urljoin(base_url, candidate)
|
||||
return candidate
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_urls(element: _Element, resolve: _Resolve, seen_ids: set[str]) -> None:
|
||||
"""Validate URLs and de-duplicate anchor ids, in document order.
|
||||
|
||||
Runs once, at parse time, so everything downstream (the image collector, the
|
||||
fetcher, the OPF manifest) sees final absolute URLs and there is no second
|
||||
chance for a bad scheme to slip through.
|
||||
"""
|
||||
for child in element.children:
|
||||
if isinstance(child, _Text):
|
||||
continue
|
||||
_dedupe_id(child, seen_ids)
|
||||
if child.tag == "a":
|
||||
href = child.attrs.get("href")
|
||||
if href is not None:
|
||||
target = resolve.link(href)
|
||||
if target is None:
|
||||
child.attrs.pop("href", None)
|
||||
else:
|
||||
child.attrs["href"] = target
|
||||
elif child.tag == "img":
|
||||
src = child.attrs.get("src")
|
||||
target = resolve.image(src) if src else None
|
||||
if target is None:
|
||||
# Mark it for pruning: an image with no usable source is noise.
|
||||
child.tag = "#dropped-image"
|
||||
else:
|
||||
child.attrs["src"] = target
|
||||
elif child.tag in ("blockquote", "q"):
|
||||
cite = child.attrs.get("cite")
|
||||
if cite is not None:
|
||||
target = resolve.link(cite)
|
||||
if target is None:
|
||||
child.attrs.pop("cite", None)
|
||||
else:
|
||||
child.attrs["cite"] = target
|
||||
_resolve_urls(child, resolve, seen_ids)
|
||||
|
||||
|
||||
def _dedupe_id(element: _Element, seen_ids: set[str]) -> None:
|
||||
"""Make an element's id unique within the document.
|
||||
|
||||
Duplicate ids are invalid and would make the first anchor win, so a repeat is
|
||||
renamed rather than dropped -- dropping it would break the link pointing at
|
||||
it. One book holds one article, so a document-wide set is enough.
|
||||
"""
|
||||
anchor = element.attrs.get("id")
|
||||
if anchor is None:
|
||||
return
|
||||
if anchor not in seen_ids:
|
||||
seen_ids.add(anchor)
|
||||
return
|
||||
index = 2
|
||||
while f"{anchor}-{index}" in seen_ids:
|
||||
index += 1
|
||||
renamed = f"{anchor}-{index}"
|
||||
seen_ids.add(renamed)
|
||||
element.attrs["id"] = renamed
|
||||
|
||||
|
||||
def _apply_image_map(element: _Element, image_map: dict[str, str | None]) -> None:
|
||||
"""Point images at their local copies inside the book.
|
||||
|
||||
A URL mapped to None means the fetch failed or the image was refused, so the
|
||||
element is marked for pruning instead of being left pointing at the network.
|
||||
"""
|
||||
for child in element.children:
|
||||
if isinstance(child, _Text):
|
||||
continue
|
||||
if child.tag == "img":
|
||||
src = child.attrs.get("src")
|
||||
if src is not None and src in image_map:
|
||||
target = image_map[src]
|
||||
if target is None:
|
||||
child.tag = "#dropped-image"
|
||||
else:
|
||||
child.attrs["src"] = target
|
||||
_apply_image_map(child, image_map)
|
||||
|
||||
|
||||
def _prune_empty(node: _Element) -> None:
|
||||
"""Remove rejected images and the empty wrappers they leave behind."""
|
||||
kept: list[_Element | _Text] = []
|
||||
for child in node.children:
|
||||
if isinstance(child, _Element):
|
||||
if child.tag == "#dropped-image":
|
||||
continue
|
||||
_prune_empty(child)
|
||||
if _is_noise(child):
|
||||
continue
|
||||
kept.append(child)
|
||||
node.children = kept
|
||||
|
||||
|
||||
def _is_noise(element: _Element) -> bool:
|
||||
"""True for a droppable wrapper with no content left inside it.
|
||||
|
||||
Whitespace-only text does not count as content (``<p> </p>`` is noise),
|
||||
but a retained ``id`` does: it is the target of a link.
|
||||
"""
|
||||
if element.tag not in _DROPPABLE_WHEN_EMPTY or element.attrs:
|
||||
return False
|
||||
for child in element.children:
|
||||
if isinstance(child, _Element):
|
||||
return False
|
||||
if child.text.strip():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _has_elements(element: _Element) -> bool:
|
||||
"""True when the tree contains at least one element node."""
|
||||
return any(True for _ in _iter_elements(element))
|
||||
|
||||
|
||||
def _serialise_children(node: _Element, inside_pre: bool) -> str:
|
||||
return "".join(_serialise(child, inside_pre) for child in node.children)
|
||||
|
||||
|
||||
def _serialise(node: _Element | _Text, inside_pre: bool) -> str:
|
||||
if isinstance(node, _Text):
|
||||
if inside_pre:
|
||||
return escape_text(node.text)
|
||||
# Collapse whitespace outside <pre>: feed HTML arrives indented, and that
|
||||
# indentation is pure bytes in the book.
|
||||
return escape_text(_WHITESPACE_RE.sub(" ", node.text))
|
||||
if node.tag not in KEPT_TAGS:
|
||||
# Unwrapped elements keep their children but lose their own tag, so no
|
||||
# unknown element can appear in the output.
|
||||
return _serialise_children(node, inside_pre)
|
||||
|
||||
child_pre = inside_pre or node.tag == "pre"
|
||||
inner = _serialise_children(node, child_pre)
|
||||
if node.tag in ("p", "div", "li", "td", "th", "blockquote", "figcaption", "caption"):
|
||||
inner = _BLOCK_TRIM_RE.sub("", inner)
|
||||
attributes = "".join(
|
||||
f' {name}="{escape_attribute(value)}"' for name, value in _ordered_attrs(node)
|
||||
)
|
||||
if node.tag in VOID_TAGS:
|
||||
return f"<{node.tag}{attributes}/>"
|
||||
return f"<{node.tag}{attributes}>{inner}</{node.tag}>"
|
||||
|
||||
|
||||
def _ordered_attrs(element: _Element) -> list[tuple[str, str]]:
|
||||
"""Stable attribute order, with the most meaningful attribute first."""
|
||||
priority = {"id": 0, "href": 1, "src": 2, "alt": 3, "title": 4}
|
||||
return sorted(element.attrs.items(), key=lambda item: (priority.get(item[0], 9), item[0]))
|
||||
|
||||
|
||||
def escape_text(text: str) -> str:
|
||||
return text.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
|
||||
|
||||
def escape_attribute(value: str) -> str:
|
||||
# Attribute values are normalised by XML parsers anyway (tab/newline become
|
||||
# spaces), so do it explicitly rather than leaving it implicit.
|
||||
text = value.replace("\t", " ").replace("\n", " ").replace("\r", " ")
|
||||
return (
|
||||
text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)
|
||||
)
|
||||
|
||||
|
||||
def _iter_elements(node: _Element) -> Iterator[_Element]:
|
||||
"""Depth-first traversal over element nodes only."""
|
||||
for child in node.children:
|
||||
if isinstance(child, _Element):
|
||||
yield child
|
||||
yield from _iter_elements(child)
|
||||
|
||||
|
||||
def _iter_text(node: _Element) -> Iterator[_Text]:
|
||||
"""Depth-first traversal over text nodes only."""
|
||||
for child in node.children:
|
||||
if isinstance(child, _Text):
|
||||
yield child
|
||||
else:
|
||||
yield from _iter_text(child)
|
||||
Reference in New Issue
Block a user