Initial commit
This commit is contained in:
492
app/epub.py
Normal file
492
app/epub.py
Normal file
@@ -0,0 +1,492 @@
|
||||
"""EPUB 3 generation, entirely in memory.
|
||||
|
||||
An EPUB is a ZIP with a strict shape, and a reader that rejects a book usually
|
||||
does so because of a shape violation rather than because of its content. The
|
||||
rules that matter here, and that the tests pin down:
|
||||
|
||||
* ``mimetype`` is the first entry, stored uncompressed, with no extra field. A
|
||||
ZIP where it is deflated or not first is not an EPUB.
|
||||
* ``META-INF/container.xml`` names the OPF, which is the package document.
|
||||
* XML everywhere: the OPF, the navigation document, the NCX fallback and every
|
||||
content document have to parse with a strict XML parser. Article bodies are
|
||||
normalised by :mod:`app.xhtml` before they come anywhere near this module.
|
||||
* A publication needs ``dc:identifier``, ``dc:title``, ``dc:language`` and a
|
||||
``dcterms:modified`` timestamp; a navigation document marked ``properties="nav"``
|
||||
is mandatory, and the NCX is kept as the EPUB 2 fallback because older readers
|
||||
still look for it.
|
||||
* Every manifest href resolves to a real entry, and every spine idref resolves to
|
||||
a manifest item. Both are checked in the tests.
|
||||
|
||||
Nothing touches the filesystem: the ZIP is assembled in a ``BytesIO`` and handed
|
||||
back as ``bytes``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app import images, urls, xhtml
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
MIMETYPE = "application/epub+zip"
|
||||
|
||||
# Where each kind of part lives inside the package.
|
||||
OPF_PATH = "OEBPS/content.opf"
|
||||
NAV_PATH = "OEBPS/nav.xhtml"
|
||||
NCX_PATH = "OEBPS/toc.ncx"
|
||||
CSS_PATH = "OEBPS/styles/main.css"
|
||||
COVER_PATH = "OEBPS/text/cover.xhtml"
|
||||
ARTICLE_PATH = "OEBPS/text/article.xhtml"
|
||||
IMAGE_DIR = "OEBPS/images"
|
||||
|
||||
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
|
||||
OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
|
||||
DC_NAMESPACE = "http://purl.org/dc/elements/1.1/"
|
||||
NCX_NAMESPACE = "http://www.daisy.org/z3986/2005/ncx/"
|
||||
OPS_NAMESPACE = "http://www.idpf.org/2007/ops"
|
||||
|
||||
# A deliberately plain stylesheet. It sets a readable measure and gives code,
|
||||
# quotes and images somewhere sane to sit, and it never fights the reader's own
|
||||
# typography: no colours, no font families, no absolute sizes.
|
||||
DEFAULT_STYLESHEET = """\
|
||||
body { margin: 5%; line-height: 1.5; }
|
||||
h1, h2, h3 { line-height: 1.25; page-break-after: avoid; }
|
||||
.article-title { margin-bottom: 0.2em; }
|
||||
.byline { margin-top: 0; font-size: 0.9em; opacity: 0.8; }
|
||||
img { max-width: 100%; height: auto; }
|
||||
figure { margin: 1em 0; }
|
||||
figcaption { font-size: 0.9em; font-style: italic; }
|
||||
blockquote { margin: 1em 1.5em; }
|
||||
pre { white-space: pre-wrap; word-wrap: break-word; }
|
||||
code, pre { font-family: monospace; }
|
||||
table { border-collapse: collapse; max-width: 100%; }
|
||||
td, th { border: 1px solid currentColor; padding: 0.3em 0.5em; }
|
||||
hr { border: none; border-top: 1px solid currentColor; }
|
||||
.source { margin-top: 2em; font-size: 0.85em; opacity: 0.8; }
|
||||
.cover { text-align: center; }
|
||||
.cover img { max-height: 90%; }
|
||||
"""
|
||||
|
||||
# Content documents sit one directory below the images.
|
||||
_TEXT_PREFIX = "../"
|
||||
|
||||
# A literal middle dot rather than the HTML entity ·: XML predefines only
|
||||
# amp, lt, gt, quot and apos, and this package ships no DOCTYPE that could define
|
||||
# any others, so an HTML entity here would make the document unparseable.
|
||||
BYLINE_SEPARATOR = " \u00b7 "
|
||||
|
||||
|
||||
class EpubTooLargeError(Exception):
|
||||
"""The assembled book exceeded the configured size cap."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BookImage:
|
||||
"""An image embedded in the book."""
|
||||
|
||||
href: str # relative to the OPF, e.g. "images/img-1.png"
|
||||
media_type: str
|
||||
data: bytes
|
||||
is_cover: bool = False
|
||||
|
||||
@property
|
||||
def document_href(self) -> str:
|
||||
"""The same image, as referenced from a document in ``text/``."""
|
||||
return _TEXT_PREFIX + self.href
|
||||
|
||||
|
||||
@dataclass
|
||||
class Book:
|
||||
"""Everything needed to write one EPUB."""
|
||||
|
||||
identifier: str
|
||||
title: str
|
||||
language: str
|
||||
modified: datetime
|
||||
content_xhtml: str
|
||||
published: datetime | None = None
|
||||
author: str | None = None
|
||||
publisher: str | None = None
|
||||
source_url: str | None = None
|
||||
feed_title: str | None = None
|
||||
images: list[BookImage] = field(default_factory=list)
|
||||
stylesheet: str = DEFAULT_STYLESHEET
|
||||
include_source_link: bool = True
|
||||
|
||||
@property
|
||||
def cover_image(self) -> BookImage | None:
|
||||
for image in self.images:
|
||||
if image.is_cover:
|
||||
return image
|
||||
return None
|
||||
|
||||
|
||||
def build_epub(book: Book) -> bytes:
|
||||
"""Assemble a book into EPUB bytes."""
|
||||
buffer = io.BytesIO()
|
||||
# ZIP entry timestamps come from the book so that identical input produces
|
||||
# identical bytes; a book that changes only because it was rebuilt would
|
||||
# otherwise defeat every cache between here and the reader.
|
||||
stamp = _zip_timestamp(book.modified)
|
||||
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
# The mimetype entry must come first and must not be compressed, which is
|
||||
# why it is written through an explicit ZipInfo rather than writestr.
|
||||
info = zipfile.ZipInfo("mimetype", date_time=stamp)
|
||||
info.compress_type = zipfile.ZIP_STORED
|
||||
info.external_attr = 0o644 << 16
|
||||
archive.writestr(info, MIMETYPE)
|
||||
|
||||
_write(archive, "META-INF/container.xml", _container_xml(), stamp)
|
||||
_write(archive, OPF_PATH, _opf_document(book), stamp)
|
||||
_write(archive, NAV_PATH, _nav_document(book), stamp)
|
||||
_write(archive, NCX_PATH, _ncx_document(book), stamp)
|
||||
_write(archive, CSS_PATH, book.stylesheet, stamp)
|
||||
_write(archive, COVER_PATH, _cover_document(book), stamp)
|
||||
_write(archive, ARTICLE_PATH, _article_document(book), stamp)
|
||||
for image in book.images:
|
||||
package_dir = OPF_PATH.rsplit("/", 1)[0]
|
||||
_write(archive, f"{package_dir}/{image.href}", image.data, stamp)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def build_article_epub(
|
||||
item,
|
||||
*,
|
||||
epub_config,
|
||||
catalog_config,
|
||||
fetcher=None,
|
||||
max_content_bytes: int = 1_048_576,
|
||||
) -> bytes:
|
||||
"""Turn one FreshRSS item into an EPUB.
|
||||
|
||||
The article body comes from FreshRSS, normalised by :mod:`app.xhtml`. Images
|
||||
are embedded when a fetcher is supplied, and any that cannot be fetched are
|
||||
dropped from the body rather than left pointing at the network -- a book that
|
||||
claims to work offline should.
|
||||
"""
|
||||
article = xhtml.parse_article(
|
||||
item.content_html, base_url=item.url or "", max_input_bytes=max_content_bytes
|
||||
)
|
||||
book_images: list[BookImage] = []
|
||||
image_map: dict[str, str | None] | None = None
|
||||
|
||||
if epub_config.include_images and fetcher is not None:
|
||||
wanted = article.image_urls[: epub_config.max_images_per_article]
|
||||
fetched = fetcher.fetch_all(wanted)
|
||||
image_map = {}
|
||||
for url in wanted:
|
||||
image = fetched.get(url)
|
||||
if image is None:
|
||||
image_map[url] = None
|
||||
continue
|
||||
index = len(book_images) + 1
|
||||
href = f"images/img-{index}{images.MEDIA_TYPES[image.media_type]}"
|
||||
book_images.append(BookImage(href=href, media_type=image.media_type, data=image.data))
|
||||
image_map[url] = _TEXT_PREFIX + href
|
||||
elif article.image_urls:
|
||||
# Keeping remote URLs means the OPF has to declare remote resources, which
|
||||
# is handled by the caller not setting include_images.
|
||||
image_map = None
|
||||
|
||||
book = Book(
|
||||
identifier=urls.stable_urn("book", item.id),
|
||||
title=item.title,
|
||||
language=catalog_config.language,
|
||||
modified=item.updated_at,
|
||||
published=item.published_at,
|
||||
content_xhtml=article.to_xhtml(image_map),
|
||||
author=item.author or catalog_config.author,
|
||||
publisher=catalog_config.publisher,
|
||||
source_url=item.url,
|
||||
feed_title=item.feed_title,
|
||||
images=book_images,
|
||||
include_source_link=epub_config.include_source_link,
|
||||
stylesheet=_load_stylesheet(epub_config.stylesheet),
|
||||
)
|
||||
if epub_config.cover_image:
|
||||
_choose_cover(book)
|
||||
data = build_epub(book)
|
||||
if len(data) > epub_config.max_epub_bytes:
|
||||
raise EpubTooLargeError(
|
||||
f"book is {len(data)} bytes, over the {epub_config.max_epub_bytes} byte cap"
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def _load_stylesheet(path: str | None) -> str:
|
||||
"""Use the configured stylesheet, falling back to the built-in one."""
|
||||
if not path:
|
||||
return DEFAULT_STYLESHEET
|
||||
try:
|
||||
return Path(path).read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
LOGGER.warning("cannot read stylesheet %s (%s); using the built-in one", path, exc)
|
||||
return DEFAULT_STYLESHEET
|
||||
|
||||
|
||||
def _choose_cover(book: Book) -> None:
|
||||
"""Promote the first suitably large image to be the book's cover.
|
||||
|
||||
A reader's library view shows a cover or a blank tile, and a feed's lead image
|
||||
is almost always the right one. Images whose size cannot be read are skipped:
|
||||
promoting an unknown image risks a cover that is a tracking pixel.
|
||||
"""
|
||||
for index, image in enumerate(book.images):
|
||||
size = images.dimensions(image.data)
|
||||
if size is None:
|
||||
continue
|
||||
width, height = size
|
||||
if images.looks_like_a_cover(width, height):
|
||||
book.images[index] = BookImage(
|
||||
href=image.href,
|
||||
media_type=image.media_type,
|
||||
data=image.data,
|
||||
is_cover=True,
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
def _write(
|
||||
archive: zipfile.ZipFile,
|
||||
path: str,
|
||||
content: str | bytes,
|
||||
stamp: tuple[int, int, int, int, int, int],
|
||||
) -> None:
|
||||
"""Write one entry, encoding text parts and passing binary parts through."""
|
||||
info = zipfile.ZipInfo(path, date_time=stamp)
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.external_attr = 0o644 << 16
|
||||
data = content if isinstance(content, bytes) else content.encode("utf-8")
|
||||
archive.writestr(info, data)
|
||||
|
||||
|
||||
def _zip_timestamp(moment: datetime) -> tuple[int, int, int, int, int, int]:
|
||||
"""A ZIP timestamp, clamped to the range the format can express (from 1980)."""
|
||||
year = max(1980, min(2107, moment.year))
|
||||
return (year, moment.month, moment.day, moment.hour, moment.minute, moment.second)
|
||||
|
||||
|
||||
# -- documents -----------------------------------------------------------
|
||||
|
||||
|
||||
def _container_xml() -> str:
|
||||
return (
|
||||
'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
'<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">\n'
|
||||
" <rootfiles>\n"
|
||||
f' <rootfile full-path="{OPF_PATH}" media-type="application/oebps-package+xml"/>\n'
|
||||
" </rootfiles>\n"
|
||||
"</container>\n"
|
||||
)
|
||||
|
||||
|
||||
def _opf_document(book: Book) -> str:
|
||||
metadata = [
|
||||
f' <dc:identifier id="pub-id">{_escape(book.identifier)}</dc:identifier>',
|
||||
f" <dc:title>{_escape(book.title)}</dc:title>",
|
||||
f" <dc:language>{_escape(book.language)}</dc:language>",
|
||||
]
|
||||
if book.author:
|
||||
metadata.append(f" <dc:creator>{_escape(book.author)}</dc:creator>")
|
||||
if book.publisher:
|
||||
metadata.append(f" <dc:publisher>{_escape(book.publisher)}</dc:publisher>")
|
||||
if book.published:
|
||||
metadata.append(f" <dc:date>{_timestamp(book.published)}</dc:date>")
|
||||
if book.source_url:
|
||||
metadata.append(f" <dc:source>{_escape(book.source_url)}</dc:source>")
|
||||
metadata.append(f' <meta property="dcterms:modified">{_timestamp(book.modified)}</meta>')
|
||||
if book.cover_image is not None:
|
||||
# The EPUB 2 spelling of the same statement, for readers that predate the
|
||||
# cover-image property below.
|
||||
metadata.append(' <meta name="cover" content="cover-image"/>')
|
||||
|
||||
manifest = [
|
||||
' <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>',
|
||||
' <item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>',
|
||||
' <item id="css" href="styles/main.css" media-type="text/css"/>',
|
||||
' <item id="cover" href="text/cover.xhtml" media-type="application/xhtml+xml"/>',
|
||||
' <item id="article" href="text/article.xhtml" media-type="application/xhtml+xml"/>',
|
||||
]
|
||||
cover = book.cover_image
|
||||
for index, image in enumerate(book.images, start=1):
|
||||
properties = ' properties="cover-image"' if image.is_cover else ""
|
||||
identifier = "cover-image" if image.is_cover else f"image-{index}"
|
||||
manifest.append(
|
||||
f' <item id="{identifier}" href="{_escape(image.href)}" '
|
||||
f'media-type="{image.media_type}"{properties}/>'
|
||||
)
|
||||
if cover is None and book.images:
|
||||
# Images are embedded rather than linked, so nothing to declare here; the
|
||||
# branch exists to keep the intent obvious if remote images are ever kept.
|
||||
pass
|
||||
|
||||
spine = [
|
||||
' <itemref idref="cover"/>',
|
||||
' <itemref idref="article"/>',
|
||||
]
|
||||
return "\n".join(
|
||||
[
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<package xmlns="{OPF_NAMESPACE}" version="3.0" '
|
||||
f'unique-identifier="pub-id" xml:lang="{_escape(book.language)}">',
|
||||
f' <metadata xmlns:dc="{DC_NAMESPACE}">',
|
||||
*metadata,
|
||||
" </metadata>",
|
||||
" <manifest>",
|
||||
*manifest,
|
||||
" </manifest>",
|
||||
' <spine toc="ncx">',
|
||||
*spine,
|
||||
" </spine>",
|
||||
" <guide>",
|
||||
' <reference type="cover" title="Cover" href="text/cover.xhtml"/>',
|
||||
" </guide>",
|
||||
"</package>",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _nav_document(book: Book) -> str:
|
||||
return "\n".join(
|
||||
[
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<html xmlns="{XHTML_NAMESPACE}" xmlns:epub="{OPS_NAMESPACE}" '
|
||||
f'xml:lang="{_escape(book.language)}" lang="{_escape(book.language)}">',
|
||||
" <head>",
|
||||
" <title>Contents</title>",
|
||||
' <meta charset="utf-8"/>',
|
||||
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
|
||||
" </head>",
|
||||
" <body>",
|
||||
' <nav epub:type="toc" id="toc">',
|
||||
" <h1>Contents</h1>",
|
||||
" <ol>",
|
||||
f' <li><a href="text/cover.xhtml">{_escape(book.title)}</a></li>',
|
||||
' <li><a href="text/article.xhtml">Article</a></li>',
|
||||
" </ol>",
|
||||
" </nav>",
|
||||
" </body>",
|
||||
"</html>",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _ncx_document(book: Book) -> str:
|
||||
points = [
|
||||
("navpoint-1", 1, "Cover", "text/cover.xhtml"),
|
||||
("navpoint-2", 2, "Article", "text/article.xhtml"),
|
||||
]
|
||||
body = []
|
||||
for identifier, order, label, href in points:
|
||||
body.extend(
|
||||
[
|
||||
f' <navPoint id="{identifier}" playOrder="{order}">',
|
||||
f" <navLabel><text>{_escape(label)}</text></navLabel>",
|
||||
f' <content src="{href}"/>',
|
||||
" </navPoint>",
|
||||
]
|
||||
)
|
||||
return "\n".join(
|
||||
[
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<ncx xmlns="{NCX_NAMESPACE}" version="2005-1">',
|
||||
" <head>",
|
||||
f' <meta name="dtb:uid" content="{_escape(book.identifier)}"/>',
|
||||
' <meta name="dtb:depth" content="1"/>',
|
||||
' <meta name="dtb:totalPageCount" content="0"/>',
|
||||
' <meta name="dtb:maxPageNumber" content="0"/>',
|
||||
" </head>",
|
||||
f" <docTitle><text>{_escape(book.title)}</text></docTitle>",
|
||||
" <navMap>",
|
||||
*body,
|
||||
" </navMap>",
|
||||
"</ncx>",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _cover_document(book: Book) -> str:
|
||||
lines = [
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
|
||||
f'lang="{_escape(book.language)}">',
|
||||
" <head>",
|
||||
f" <title>{_escape(book.title)}</title>",
|
||||
' <meta charset="utf-8"/>',
|
||||
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
|
||||
" </head>",
|
||||
' <body class="cover">',
|
||||
]
|
||||
cover = book.cover_image
|
||||
if cover is not None:
|
||||
lines.append(f' <img src="{_escape(cover.document_href)}" alt="{_escape(book.title)}"/>')
|
||||
lines.append(f' <h1 class="article-title">{_escape(book.title)}</h1>')
|
||||
if book.author:
|
||||
lines.append(f' <p class="byline">{_escape(book.author)}</p>')
|
||||
if book.feed_title:
|
||||
lines.append(f' <p class="byline">{_escape(book.feed_title)}</p>')
|
||||
if book.published:
|
||||
lines.append(f' <p class="byline">{_date_only(book.published)}</p>')
|
||||
lines.extend([" </body>", "</html>", ""])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _article_document(book: Book) -> str:
|
||||
byline_bits = []
|
||||
if book.author:
|
||||
byline_bits.append(_escape(book.author))
|
||||
if book.published:
|
||||
byline_bits.append(_date_only(book.published))
|
||||
lines = [
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
|
||||
f'lang="{_escape(book.language)}">',
|
||||
" <head>",
|
||||
f" <title>{_escape(book.title)}</title>",
|
||||
' <meta charset="utf-8"/>',
|
||||
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
|
||||
" </head>",
|
||||
" <body>",
|
||||
f' <h1 class="article-title">{_escape(book.title)}</h1>',
|
||||
]
|
||||
if byline_bits:
|
||||
lines.append(f' <p class="byline">{BYLINE_SEPARATOR.join(byline_bits)}</p>')
|
||||
if book.content_xhtml:
|
||||
lines.append(book.content_xhtml)
|
||||
else:
|
||||
# Never ship an empty content document: a reader that opens a blank page
|
||||
# looks broken, and the original is the most useful thing to show instead.
|
||||
lines.append(" <p>This article had no content to download.</p>")
|
||||
if book.include_source_link and book.source_url:
|
||||
lines.append(
|
||||
' <p class="source">Originally published at '
|
||||
f'<a href="{_escape(book.source_url)}">{_escape(book.source_url)}</a>.</p>'
|
||||
)
|
||||
lines.extend([" </body>", "</html>", ""])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# -- formatting ----------------------------------------------------------
|
||||
|
||||
|
||||
def _escape(text: str) -> str:
|
||||
"""XML-escape text for a document body or an attribute value."""
|
||||
return xhtml.escape_text(text).replace('"', """)
|
||||
|
||||
|
||||
def _timestamp(moment: datetime) -> str:
|
||||
"""An EPUB date: UTC, seconds precision, ``Z`` suffix."""
|
||||
return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _date_only(moment: datetime) -> str:
|
||||
return moment.astimezone(UTC).strftime("%Y-%m-%d")
|
||||
Reference in New Issue
Block a user