Initial commit

This commit is contained in:
2026-11-15 18:15:03 -05:00
commit b76aea2899
36 changed files with 9424 additions and 0 deletions

492
app/epub.py Normal file
View File

@@ -0,0 +1,492 @@
"""EPUB 3 generation, entirely in memory.
An EPUB is a ZIP with a strict shape, and a reader that rejects a book usually
does so because of a shape violation rather than because of its content. The
rules that matter here, and that the tests pin down:
* ``mimetype`` is the first entry, stored uncompressed, with no extra field. A
ZIP where it is deflated or not first is not an EPUB.
* ``META-INF/container.xml`` names the OPF, which is the package document.
* XML everywhere: the OPF, the navigation document, the NCX fallback and every
content document have to parse with a strict XML parser. Article bodies are
normalised by :mod:`app.xhtml` before they come anywhere near this module.
* A publication needs ``dc:identifier``, ``dc:title``, ``dc:language`` and a
``dcterms:modified`` timestamp; a navigation document marked ``properties="nav"``
is mandatory, and the NCX is kept as the EPUB 2 fallback because older readers
still look for it.
* Every manifest href resolves to a real entry, and every spine idref resolves to
a manifest item. Both are checked in the tests.
Nothing touches the filesystem: the ZIP is assembled in a ``BytesIO`` and handed
back as ``bytes``.
"""
from __future__ import annotations
import io
import logging
import zipfile
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from app import images, urls, xhtml
LOGGER = logging.getLogger(__name__)
MIMETYPE = "application/epub+zip"
# Where each kind of part lives inside the package.
OPF_PATH = "OEBPS/content.opf"
NAV_PATH = "OEBPS/nav.xhtml"
NCX_PATH = "OEBPS/toc.ncx"
CSS_PATH = "OEBPS/styles/main.css"
COVER_PATH = "OEBPS/text/cover.xhtml"
ARTICLE_PATH = "OEBPS/text/article.xhtml"
IMAGE_DIR = "OEBPS/images"
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
DC_NAMESPACE = "http://purl.org/dc/elements/1.1/"
NCX_NAMESPACE = "http://www.daisy.org/z3986/2005/ncx/"
OPS_NAMESPACE = "http://www.idpf.org/2007/ops"
# A deliberately plain stylesheet. It sets a readable measure and gives code,
# quotes and images somewhere sane to sit, and it never fights the reader's own
# typography: no colours, no font families, no absolute sizes.
DEFAULT_STYLESHEET = """\
body { margin: 5%; line-height: 1.5; }
h1, h2, h3 { line-height: 1.25; page-break-after: avoid; }
.article-title { margin-bottom: 0.2em; }
.byline { margin-top: 0; font-size: 0.9em; opacity: 0.8; }
img { max-width: 100%; height: auto; }
figure { margin: 1em 0; }
figcaption { font-size: 0.9em; font-style: italic; }
blockquote { margin: 1em 1.5em; }
pre { white-space: pre-wrap; word-wrap: break-word; }
code, pre { font-family: monospace; }
table { border-collapse: collapse; max-width: 100%; }
td, th { border: 1px solid currentColor; padding: 0.3em 0.5em; }
hr { border: none; border-top: 1px solid currentColor; }
.source { margin-top: 2em; font-size: 0.85em; opacity: 0.8; }
.cover { text-align: center; }
.cover img { max-height: 90%; }
"""
# Content documents sit one directory below the images.
_TEXT_PREFIX = "../"
# A literal middle dot rather than the HTML entity ·: XML predefines only
# amp, lt, gt, quot and apos, and this package ships no DOCTYPE that could define
# any others, so an HTML entity here would make the document unparseable.
BYLINE_SEPARATOR = " \u00b7 "
class EpubTooLargeError(Exception):
"""The assembled book exceeded the configured size cap."""
@dataclass(frozen=True)
class BookImage:
"""An image embedded in the book."""
href: str # relative to the OPF, e.g. "images/img-1.png"
media_type: str
data: bytes
is_cover: bool = False
@property
def document_href(self) -> str:
"""The same image, as referenced from a document in ``text/``."""
return _TEXT_PREFIX + self.href
@dataclass
class Book:
"""Everything needed to write one EPUB."""
identifier: str
title: str
language: str
modified: datetime
content_xhtml: str
published: datetime | None = None
author: str | None = None
publisher: str | None = None
source_url: str | None = None
feed_title: str | None = None
images: list[BookImage] = field(default_factory=list)
stylesheet: str = DEFAULT_STYLESHEET
include_source_link: bool = True
@property
def cover_image(self) -> BookImage | None:
for image in self.images:
if image.is_cover:
return image
return None
def build_epub(book: Book) -> bytes:
"""Assemble a book into EPUB bytes."""
buffer = io.BytesIO()
# ZIP entry timestamps come from the book so that identical input produces
# identical bytes; a book that changes only because it was rebuilt would
# otherwise defeat every cache between here and the reader.
stamp = _zip_timestamp(book.modified)
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
# The mimetype entry must come first and must not be compressed, which is
# why it is written through an explicit ZipInfo rather than writestr.
info = zipfile.ZipInfo("mimetype", date_time=stamp)
info.compress_type = zipfile.ZIP_STORED
info.external_attr = 0o644 << 16
archive.writestr(info, MIMETYPE)
_write(archive, "META-INF/container.xml", _container_xml(), stamp)
_write(archive, OPF_PATH, _opf_document(book), stamp)
_write(archive, NAV_PATH, _nav_document(book), stamp)
_write(archive, NCX_PATH, _ncx_document(book), stamp)
_write(archive, CSS_PATH, book.stylesheet, stamp)
_write(archive, COVER_PATH, _cover_document(book), stamp)
_write(archive, ARTICLE_PATH, _article_document(book), stamp)
for image in book.images:
package_dir = OPF_PATH.rsplit("/", 1)[0]
_write(archive, f"{package_dir}/{image.href}", image.data, stamp)
return buffer.getvalue()
def build_article_epub(
item,
*,
epub_config,
catalog_config,
fetcher=None,
max_content_bytes: int = 1_048_576,
) -> bytes:
"""Turn one FreshRSS item into an EPUB.
The article body comes from FreshRSS, normalised by :mod:`app.xhtml`. Images
are embedded when a fetcher is supplied, and any that cannot be fetched are
dropped from the body rather than left pointing at the network -- a book that
claims to work offline should.
"""
article = xhtml.parse_article(
item.content_html, base_url=item.url or "", max_input_bytes=max_content_bytes
)
book_images: list[BookImage] = []
image_map: dict[str, str | None] | None = None
if epub_config.include_images and fetcher is not None:
wanted = article.image_urls[: epub_config.max_images_per_article]
fetched = fetcher.fetch_all(wanted)
image_map = {}
for url in wanted:
image = fetched.get(url)
if image is None:
image_map[url] = None
continue
index = len(book_images) + 1
href = f"images/img-{index}{images.MEDIA_TYPES[image.media_type]}"
book_images.append(BookImage(href=href, media_type=image.media_type, data=image.data))
image_map[url] = _TEXT_PREFIX + href
elif article.image_urls:
# Keeping remote URLs means the OPF has to declare remote resources, which
# is handled by the caller not setting include_images.
image_map = None
book = Book(
identifier=urls.stable_urn("book", item.id),
title=item.title,
language=catalog_config.language,
modified=item.updated_at,
published=item.published_at,
content_xhtml=article.to_xhtml(image_map),
author=item.author or catalog_config.author,
publisher=catalog_config.publisher,
source_url=item.url,
feed_title=item.feed_title,
images=book_images,
include_source_link=epub_config.include_source_link,
stylesheet=_load_stylesheet(epub_config.stylesheet),
)
if epub_config.cover_image:
_choose_cover(book)
data = build_epub(book)
if len(data) > epub_config.max_epub_bytes:
raise EpubTooLargeError(
f"book is {len(data)} bytes, over the {epub_config.max_epub_bytes} byte cap"
)
return data
def _load_stylesheet(path: str | None) -> str:
"""Use the configured stylesheet, falling back to the built-in one."""
if not path:
return DEFAULT_STYLESHEET
try:
return Path(path).read_text(encoding="utf-8")
except OSError as exc:
LOGGER.warning("cannot read stylesheet %s (%s); using the built-in one", path, exc)
return DEFAULT_STYLESHEET
def _choose_cover(book: Book) -> None:
"""Promote the first suitably large image to be the book's cover.
A reader's library view shows a cover or a blank tile, and a feed's lead image
is almost always the right one. Images whose size cannot be read are skipped:
promoting an unknown image risks a cover that is a tracking pixel.
"""
for index, image in enumerate(book.images):
size = images.dimensions(image.data)
if size is None:
continue
width, height = size
if images.looks_like_a_cover(width, height):
book.images[index] = BookImage(
href=image.href,
media_type=image.media_type,
data=image.data,
is_cover=True,
)
return
def _write(
archive: zipfile.ZipFile,
path: str,
content: str | bytes,
stamp: tuple[int, int, int, int, int, int],
) -> None:
"""Write one entry, encoding text parts and passing binary parts through."""
info = zipfile.ZipInfo(path, date_time=stamp)
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o644 << 16
data = content if isinstance(content, bytes) else content.encode("utf-8")
archive.writestr(info, data)
def _zip_timestamp(moment: datetime) -> tuple[int, int, int, int, int, int]:
"""A ZIP timestamp, clamped to the range the format can express (from 1980)."""
year = max(1980, min(2107, moment.year))
return (year, moment.month, moment.day, moment.hour, moment.minute, moment.second)
# -- documents -----------------------------------------------------------
def _container_xml() -> str:
return (
'<?xml version="1.0" encoding="utf-8"?>\n'
'<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">\n'
" <rootfiles>\n"
f' <rootfile full-path="{OPF_PATH}" media-type="application/oebps-package+xml"/>\n'
" </rootfiles>\n"
"</container>\n"
)
def _opf_document(book: Book) -> str:
metadata = [
f' <dc:identifier id="pub-id">{_escape(book.identifier)}</dc:identifier>',
f" <dc:title>{_escape(book.title)}</dc:title>",
f" <dc:language>{_escape(book.language)}</dc:language>",
]
if book.author:
metadata.append(f" <dc:creator>{_escape(book.author)}</dc:creator>")
if book.publisher:
metadata.append(f" <dc:publisher>{_escape(book.publisher)}</dc:publisher>")
if book.published:
metadata.append(f" <dc:date>{_timestamp(book.published)}</dc:date>")
if book.source_url:
metadata.append(f" <dc:source>{_escape(book.source_url)}</dc:source>")
metadata.append(f' <meta property="dcterms:modified">{_timestamp(book.modified)}</meta>')
if book.cover_image is not None:
# The EPUB 2 spelling of the same statement, for readers that predate the
# cover-image property below.
metadata.append(' <meta name="cover" content="cover-image"/>')
manifest = [
' <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>',
' <item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>',
' <item id="css" href="styles/main.css" media-type="text/css"/>',
' <item id="cover" href="text/cover.xhtml" media-type="application/xhtml+xml"/>',
' <item id="article" href="text/article.xhtml" media-type="application/xhtml+xml"/>',
]
cover = book.cover_image
for index, image in enumerate(book.images, start=1):
properties = ' properties="cover-image"' if image.is_cover else ""
identifier = "cover-image" if image.is_cover else f"image-{index}"
manifest.append(
f' <item id="{identifier}" href="{_escape(image.href)}" '
f'media-type="{image.media_type}"{properties}/>'
)
if cover is None and book.images:
# Images are embedded rather than linked, so nothing to declare here; the
# branch exists to keep the intent obvious if remote images are ever kept.
pass
spine = [
' <itemref idref="cover"/>',
' <itemref idref="article"/>',
]
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<package xmlns="{OPF_NAMESPACE}" version="3.0" '
f'unique-identifier="pub-id" xml:lang="{_escape(book.language)}">',
f' <metadata xmlns:dc="{DC_NAMESPACE}">',
*metadata,
" </metadata>",
" <manifest>",
*manifest,
" </manifest>",
' <spine toc="ncx">',
*spine,
" </spine>",
" <guide>",
' <reference type="cover" title="Cover" href="text/cover.xhtml"/>',
" </guide>",
"</package>",
"",
]
)
def _nav_document(book: Book) -> str:
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<html xmlns="{XHTML_NAMESPACE}" xmlns:epub="{OPS_NAMESPACE}" '
f'xml:lang="{_escape(book.language)}" lang="{_escape(book.language)}">',
" <head>",
" <title>Contents</title>",
' <meta charset="utf-8"/>',
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
" </head>",
" <body>",
' <nav epub:type="toc" id="toc">',
" <h1>Contents</h1>",
" <ol>",
f' <li><a href="text/cover.xhtml">{_escape(book.title)}</a></li>',
' <li><a href="text/article.xhtml">Article</a></li>',
" </ol>",
" </nav>",
" </body>",
"</html>",
"",
]
)
def _ncx_document(book: Book) -> str:
points = [
("navpoint-1", 1, "Cover", "text/cover.xhtml"),
("navpoint-2", 2, "Article", "text/article.xhtml"),
]
body = []
for identifier, order, label, href in points:
body.extend(
[
f' <navPoint id="{identifier}" playOrder="{order}">',
f" <navLabel><text>{_escape(label)}</text></navLabel>",
f' <content src="{href}"/>',
" </navPoint>",
]
)
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<ncx xmlns="{NCX_NAMESPACE}" version="2005-1">',
" <head>",
f' <meta name="dtb:uid" content="{_escape(book.identifier)}"/>',
' <meta name="dtb:depth" content="1"/>',
' <meta name="dtb:totalPageCount" content="0"/>',
' <meta name="dtb:maxPageNumber" content="0"/>',
" </head>",
f" <docTitle><text>{_escape(book.title)}</text></docTitle>",
" <navMap>",
*body,
" </navMap>",
"</ncx>",
"",
]
)
def _cover_document(book: Book) -> str:
lines = [
'<?xml version="1.0" encoding="utf-8"?>',
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
f'lang="{_escape(book.language)}">',
" <head>",
f" <title>{_escape(book.title)}</title>",
' <meta charset="utf-8"/>',
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
" </head>",
' <body class="cover">',
]
cover = book.cover_image
if cover is not None:
lines.append(f' <img src="{_escape(cover.document_href)}" alt="{_escape(book.title)}"/>')
lines.append(f' <h1 class="article-title">{_escape(book.title)}</h1>')
if book.author:
lines.append(f' <p class="byline">{_escape(book.author)}</p>')
if book.feed_title:
lines.append(f' <p class="byline">{_escape(book.feed_title)}</p>')
if book.published:
lines.append(f' <p class="byline">{_date_only(book.published)}</p>')
lines.extend([" </body>", "</html>", ""])
return "\n".join(lines)
def _article_document(book: Book) -> str:
byline_bits = []
if book.author:
byline_bits.append(_escape(book.author))
if book.published:
byline_bits.append(_date_only(book.published))
lines = [
'<?xml version="1.0" encoding="utf-8"?>',
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
f'lang="{_escape(book.language)}">',
" <head>",
f" <title>{_escape(book.title)}</title>",
' <meta charset="utf-8"/>',
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
" </head>",
" <body>",
f' <h1 class="article-title">{_escape(book.title)}</h1>',
]
if byline_bits:
lines.append(f' <p class="byline">{BYLINE_SEPARATOR.join(byline_bits)}</p>')
if book.content_xhtml:
lines.append(book.content_xhtml)
else:
# Never ship an empty content document: a reader that opens a blank page
# looks broken, and the original is the most useful thing to show instead.
lines.append(" <p>This article had no content to download.</p>")
if book.include_source_link and book.source_url:
lines.append(
' <p class="source">Originally published at '
f'<a href="{_escape(book.source_url)}">{_escape(book.source_url)}</a>.</p>'
)
lines.extend([" </body>", "</html>", ""])
return "\n".join(lines)
# -- formatting ----------------------------------------------------------
def _escape(text: str) -> str:
"""XML-escape text for a document body or an attribute value."""
return xhtml.escape_text(text).replace('"', "&quot;")
def _timestamp(moment: datetime) -> str:
"""An EPUB date: UTC, seconds precision, ``Z`` suffix."""
return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def _date_only(moment: datetime) -> str:
return moment.astimezone(UTC).strftime("%Y-%m-%d")