Files
rss-to-opds/tests/test_epub.py
2026-11-15 18:15:03 -05:00

534 lines
22 KiB
Python

"""Tests for image metadata and EPUB generation.
The EPUB checks are structural on purpose. A book is rejected by a reader for
shape reasons -- a deflated mimetype, a manifest href with no entry, a document
that will not parse -- far more often than for content reasons, so those are the
properties asserted here.
"""
from __future__ import annotations
import unittest
import zipfile
from datetime import UTC, datetime
from io import BytesIO
from app import epub, images
from app.fetch import FetchedImage
from app.xhtml import parse_article
from tests.support import (
gif_bytes,
jpeg_bytes,
parse_xml,
png_bytes,
webp_bytes,
)
MODIFIED = datetime(2026, 11, 15, 12, 30, 45, tzinfo=UTC)
PUBLISHED = datetime(2026, 11, 14, 9, 0, 0, tzinfo=UTC)
class ImageMetadataTests(unittest.TestCase):
"""Sniffing and sizing, against structurally real image bytes."""
def test_sniffing_each_supported_format(self):
cases = {
"image/png": png_bytes(),
"image/jpeg": jpeg_bytes(),
"image/gif": gif_bytes(),
"image/webp": webp_bytes(),
}
for expected, data in cases.items():
self.assertEqual(images.sniff_media_type(data), expected)
def test_dimensions_are_read_for_each_format(self):
cases = [
(png_bytes(800, 400), (800, 400)),
(jpeg_bytes(640, 480), (640, 480)),
(gif_bytes(320, 200), (320, 200)),
(webp_bytes(1024, 512), (1024, 512)),
]
for data, expected in cases:
self.assertEqual(images.dimensions(data), expected)
def test_unknown_bytes_have_no_media_type(self):
self.assertIsNone(images.sniff_media_type(b"not an image at all"))
self.assertIsNone(images.sniff_media_type(b""))
self.assertIsNone(images.sniff_media_type(b"\x89PNG"))
def test_truncated_images_do_not_crash_the_sizer(self):
for data in (png_bytes()[:12], jpeg_bytes()[:6], gif_bytes()[:8], webp_bytes()[:20]):
self.assertIsNone(images.dimensions(data))
def test_svg_is_not_an_embeddable_type(self):
# SVG can carry script and external references; it is not on the list.
self.assertIsNone(images.normalise_media_type("image/svg+xml"))
def test_content_type_parameters_are_ignored(self):
self.assertEqual(images.normalise_media_type("image/jpeg; charset=binary"), "image/jpeg")
def test_aliases_are_normalised(self):
self.assertEqual(images.normalise_media_type("IMAGE/JPG"), "image/jpeg")
self.assertEqual(images.normalise_media_type("image/x-png"), "image/png")
def test_bytes_win_over_a_wrong_declared_type(self):
# A mislabelled payload must not produce an invalid manifest entry.
self.assertEqual(images.identify(png_bytes(), "text/html"), "image/png")
self.assertEqual(images.identify(b"unknown", "image/png"), "image/png")
self.assertIsNone(images.identify(b"unknown", "text/html"))
def test_cover_heuristic(self):
self.assertTrue(images.looks_like_a_cover(800, 400))
self.assertTrue(images.looks_like_a_cover(400, 200))
self.assertFalse(images.looks_like_a_cover(399, 200))
self.assertFalse(images.looks_like_a_cover(1, 1))
self.assertFalse(images.looks_like_a_cover(800, 100))
def make_book(**overrides) -> epub.Book:
values = {
"identifier": "urn:uuid:12345678-1234-1234-1234-123456789abc",
"title": "A Test Article & Its Friends",
"language": "en",
"modified": MODIFIED,
"published": PUBLISHED,
"content_xhtml": '<div class="article-body">\n<p>Body text.</p>\n</div>',
"author": "A. Writer",
"publisher": "Example Press",
"source_url": "https://example.com/articles/1",
"feed_title": "Example Tech",
}
values.update(overrides)
return epub.Book(**values)
class EpubContainerTests(unittest.TestCase):
"""The ZIP-level rules that make a file an EPUB at all."""
def setUp(self):
self.data = epub.build_epub(make_book())
self.archive = zipfile.ZipFile(BytesIO(self.data))
self.addCleanup(self.archive.close)
def test_mimetype_is_the_first_entry(self):
self.assertEqual(self.archive.namelist()[0], "mimetype")
def test_mimetype_is_stored_not_deflated(self):
info = self.archive.getinfo("mimetype")
self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
self.assertEqual(self.archive.read("mimetype"), b"application/epub+zip")
def test_mimetype_has_no_extra_field(self):
# An extra field in the local header is a known cause of readers refusing
# a book, so the raw bytes are inspected rather than the parsed record.
offset = self.archive.getinfo("mimetype").header_offset
local_header = self.data[offset : offset + 30]
extra_length = int.from_bytes(local_header[28:30], "little")
self.assertEqual(extra_length, 0)
def test_container_names_the_package_document(self):
xml = self.archive.read("META-INF/container.xml").decode("utf-8")
root = parse_xml(xml)
rootfile = root.must_find("rootfiles").must_find("rootfile")
self.assertEqual(rootfile.must_attr("full-path"), epub.OPF_PATH)
self.assertEqual(rootfile.must_attr("media-type"), "application/oebps-package+xml")
# Whatever it points at must actually exist.
self.assertIn(rootfile.must_attr("full-path"), self.archive.namelist())
def test_expected_parts_are_present(self):
names = set(self.archive.namelist())
for path in (
"META-INF/container.xml",
epub.OPF_PATH,
epub.NAV_PATH,
epub.NCX_PATH,
epub.CSS_PATH,
epub.COVER_PATH,
epub.ARTICLE_PATH,
):
self.assertIn(path, names)
class EpubDocumentTests(unittest.TestCase):
"""Every text part has to parse with a strict XML parser."""
def setUp(self):
self.data = epub.build_epub(make_book())
self.archive = zipfile.ZipFile(BytesIO(self.data))
self.addCleanup(self.archive.close)
def test_every_xml_part_is_well_formed(self):
for path in (
epub.OPF_PATH,
epub.NAV_PATH,
epub.NCX_PATH,
epub.COVER_PATH,
epub.ARTICLE_PATH,
"META-INF/container.xml",
):
with self.subTest(path=path):
parse_xml(self.archive.read(path))
def test_all_xml_starts_with_a_declaration(self):
for path in (
epub.OPF_PATH,
epub.NAV_PATH,
epub.NCX_PATH,
epub.COVER_PATH,
epub.ARTICLE_PATH,
):
with self.subTest(path=path):
text = self.archive.read(path).decode("utf-8")
self.assertTrue(text.startswith('<?xml version="1.0" encoding="utf-8"?>'))
def test_xhtml_documents_declare_the_namespace(self):
for path in (epub.NAV_PATH, epub.COVER_PATH, epub.ARTICLE_PATH):
with self.subTest(path=path):
root = parse_xml(self.archive.read(path))
self.assertEqual(root.must_attr("xmlns"), epub.XHTML_NAMESPACE)
def test_no_doctype_is_emitted(self):
# A DOCTYPE would have to be resolved by the reader's XML parser; EPUB 3
# makes it optional, so it is left out entirely.
for path in (epub.OPF_PATH, epub.NAV_PATH, epub.ARTICLE_PATH):
with self.subTest(path=path):
self.assertNotIn("<!DOCTYPE", self.archive.read(path).decode("utf-8"))
class EpubPackageTests(unittest.TestCase):
"""OPF metadata, manifest and spine."""
def setUp(self):
self.data = epub.build_epub(make_book())
self.archive = zipfile.ZipFile(BytesIO(self.data))
self.addCleanup(self.archive.close)
self.opf = parse_xml(self.archive.read(epub.OPF_PATH))
def manifest(self) -> dict:
items = self.opf.must_find("manifest").must_find_all("item")
return {item.must_attr("id"): item for item in items}
def test_package_declares_version_and_unique_identifier(self):
self.assertEqual(self.opf.must_attr("version"), "3.0")
self.assertEqual(self.opf.must_attr("unique-identifier"), "pub-id")
def test_required_dublin_core_metadata_is_present(self):
metadata = self.opf.must_find("metadata")
identifier = metadata.must_find("identifier")
self.assertEqual(identifier.must_attr("id"), "pub-id")
self.assertEqual(identifier.all_text(), make_book().identifier)
self.assertEqual(metadata.must_find("title").all_text(), make_book().title)
self.assertEqual(metadata.must_find("language").all_text(), "en")
def test_optional_metadata_is_present_when_known(self):
metadata = self.opf.must_find("metadata")
self.assertEqual(metadata.must_find("creator").all_text(), "A. Writer")
self.assertEqual(metadata.must_find("publisher").all_text(), "Example Press")
self.assertEqual(metadata.must_find("source").all_text(), "https://example.com/articles/1")
self.assertEqual(metadata.must_find("date").all_text(), "2026-11-14T09:00:00Z")
def test_dcterms_modified_has_the_required_format(self):
import re
meta = [
item
for item in self.opf.must_find("metadata").must_find_all("meta")
if item.attr("property") == "dcterms:modified"
]
self.assertEqual(len(meta), 1)
value = meta[0].all_text()
self.assertRegex(value, r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
self.assertEqual(value, "2026-11-15T12:30:45Z")
def test_absent_metadata_is_omitted_rather_than_emptied(self):
data = epub.build_epub(make_book(author=None, publisher=None, source_url=None))
opf = parse_xml(zipfile.ZipFile(BytesIO(data)).read(epub.OPF_PATH))
metadata = opf.must_find("metadata")
self.assertIsNone(metadata.find("creator"))
self.assertIsNone(metadata.find("publisher"))
self.assertIsNone(metadata.find("source"))
def test_navigation_document_is_marked(self):
nav = self.manifest()["nav"]
self.assertEqual(nav.must_attr("href"), "nav.xhtml")
self.assertEqual(nav.must_attr("properties"), "nav")
def test_spine_toc_points_at_the_ncx(self):
self.assertEqual(self.opf.must_find("spine").must_attr("toc"), "ncx")
def test_every_manifest_href_exists_in_the_zip(self):
for identifier, item in self.manifest().items():
with self.subTest(item=identifier):
self.assertIn("OEBPS/" + item.must_attr("href"), self.archive.namelist())
def test_every_spine_idref_resolves(self):
manifest = self.manifest()
for itemref in self.opf.must_find("spine").must_find_all("itemref"):
self.assertIn(itemref.must_attr("idref"), manifest)
def test_every_manifest_item_has_a_media_type(self):
for identifier, item in self.manifest().items():
with self.subTest(item=identifier):
self.assertTrue(item.must_attr("media-type"))
def test_guide_points_at_the_cover_page(self):
reference = self.opf.must_find("guide").must_find("reference")
self.assertEqual(reference.must_attr("type"), "cover")
self.assertEqual(reference.must_attr("href"), "text/cover.xhtml")
def test_ncx_has_matching_uid_and_navpoints(self):
ncx = parse_xml(self.archive.read(epub.NCX_PATH))
self.assertEqual(ncx.must_attr("version"), "2005-1")
uid = [
m for m in ncx.must_find("head").must_find_all("meta") if m.attr("name") == "dtb:uid"
]
self.assertEqual(uid[0].must_attr("content"), make_book().identifier)
self.assertEqual(len(ncx.must_find("navMap").must_find_all("navPoint")), 2)
class EpubContentTests(unittest.TestCase):
def build(self, **overrides) -> tuple[zipfile.ZipFile, bytes]:
data = epub.build_epub(make_book(**overrides))
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
return archive, data
def test_article_document_contains_the_body_and_title(self):
archive, _ = self.build()
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("Body text.", text)
self.assertIn("A Test Article &amp; Its Friends", text)
def test_article_document_carries_a_byline_and_date(self):
archive, _ = self.build()
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("A. Writer", text)
self.assertIn("2026-11-14", text)
def test_source_link_can_be_suppressed(self):
archive, _ = self.build(include_source_link=False)
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertNotIn("Originally published", text)
def test_empty_body_still_produces_a_readable_document(self):
archive, _ = self.build(content_xhtml="")
parse_xml(archive.read(epub.ARTICLE_PATH))
self.assertIn("no content", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
def test_title_with_markup_is_escaped(self):
archive, _ = self.build(title='<script>alert("x")</script>')
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertNotIn("<script>", text)
parse_xml(archive.read(epub.ARTICLE_PATH))
def test_stylesheet_is_embedded(self):
archive, _ = self.build(stylesheet="body { color: red; }")
self.assertEqual(archive.read(epub.CSS_PATH).decode("utf-8"), "body { color: red; }")
def test_default_stylesheet_is_not_empty(self):
archive, _ = self.build()
self.assertIn("line-height", archive.read(epub.CSS_PATH).decode("utf-8"))
class EpubImageTests(unittest.TestCase):
def test_images_are_embedded_and_declared(self):
cover = epub.BookImage(
href="images/img-1.png", media_type="image/png", data=png_bytes(800, 400)
)
data = epub.build_epub(make_book(images=[cover]))
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
self.assertIn("OEBPS/images/img-1.png", archive.namelist())
opf = parse_xml(archive.read(epub.OPF_PATH))
item = opf.must_find("manifest").must_find("item")
self.assertIsNotNone(item)
def test_cover_image_is_marked_and_referenced(self):
cover = epub.BookImage(
href="images/img-1.png",
media_type="image/png",
data=png_bytes(800, 400),
is_cover=True,
)
data = epub.build_epub(make_book(images=[cover]))
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
opf = parse_xml(archive.read(epub.OPF_PATH))
covers = [
item
for item in opf.must_find("manifest").must_find_all("item")
if item.attr("properties") == "cover-image"
]
self.assertEqual(len(covers), 1)
self.assertEqual(covers[0].must_attr("id"), "cover-image")
# The EPUB 2 spelling for older readers.
metas = [
m for m in opf.must_find("metadata").must_find_all("meta") if m.attr("name") == "cover"
]
self.assertEqual(metas[0].must_attr("content"), "cover-image")
# And the cover page shows it, with a path relative to text/.
self.assertIn("../images/img-1.png", archive.read(epub.COVER_PATH).decode("utf-8"))
def test_cover_image_is_not_promoted_without_a_real_cover(self):
small = epub.BookImage(
href="images/img-1.png", media_type="image/png", data=png_bytes(16, 16)
)
book = make_book(images=[small])
_choose = getattr(epub, "_choose_cover")
_choose(book)
self.assertIsNone(book.cover_image)
def test_deterministic_output_for_identical_input(self):
left = epub.build_epub(make_book())
right = epub.build_epub(make_book())
self.assertEqual(left, right)
class BuildArticleEpubTests(unittest.TestCase):
"""The end-to-end assembly step, with and without an image fetcher."""
class Item:
def __init__(self, content: str, title: str = "An Article") -> None:
self.id = "tag:google.com,2005:reader/item/0000000000000001"
self.title = title
self.content_html = content
self.author = "A. Writer"
self.url = "https://example.com/a/1"
self.feed_title = "Example Tech"
self.published = 1_700_000_000
self.updated = 1_700_000_000
self.published_at = datetime.fromtimestamp(1_700_000_000, tz=UTC)
self.updated_at = datetime.fromtimestamp(1_700_000_000, tz=UTC)
class Config:
include_images = True
max_images_per_article = 20
cover_image = True
include_source_link = True
stylesheet = None
max_epub_bytes = 33_554_432
class Catalog:
language = "en"
author = "FreshRSS"
publisher = "Example Press"
class FakeFetcher:
"""Returns one PNG for every URL, recording what it was asked for."""
def __init__(self, fail: bool = False) -> None:
self.asked: list[str] = []
self.fail = fail
def fetch_all(self, urls):
self.asked.extend(urls)
if self.fail:
return {url: None for url in urls}
return {
url: FetchedImage(url=url, media_type="image/png", data=png_bytes(800, 400))
for url in urls
}
def build(self, **overrides) -> tuple[zipfile.ZipFile, bytes]:
item = overrides.pop("item", self.Item("<p>Hello</p>"))
config = overrides.pop("config", self.Config())
fetcher = overrides.pop("fetcher", None)
data = epub.build_article_epub(
item, epub_config=config, catalog_config=self.Catalog(), fetcher=fetcher
)
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
return archive, data
def test_book_is_valid_without_a_fetcher(self):
archive, _ = self.build()
parse_xml(archive.read(epub.OPF_PATH))
self.assertIn("Hello", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
def test_article_identifier_is_stable(self):
_, first = self.build()
_, second = self.build()
self.assertEqual(first, second)
def test_remote_images_are_embedded_when_a_fetcher_is_supplied(self):
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
fetcher = self.FakeFetcher()
archive, _ = self.build(item=item, fetcher=fetcher)
self.assertEqual(fetcher.asked, ["https://cdn.example.com/a.png"])
self.assertIn("OEBPS/images/img-1.png", archive.namelist())
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("../images/img-1.png", article)
self.assertNotIn("https://cdn.example.com", article)
def test_unfetchable_images_are_dropped_from_the_body(self):
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
archive, _ = self.build(item=item, fetcher=self.FakeFetcher(fail=True))
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertNotIn("<img", article)
self.assertNotIn("cdn.example.com", article)
def test_lead_image_becomes_the_cover(self):
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
archive, _ = self.build(item=item, fetcher=self.FakeFetcher())
opf = parse_xml(archive.read(epub.OPF_PATH))
covers = [
item
for item in opf.must_find("manifest").must_find_all("item")
if item.attr("properties") == "cover-image"
]
self.assertEqual(len(covers), 1)
def test_images_are_left_remote_when_inlining_is_off(self):
class NoImages(self.Config):
include_images = False
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
archive, _ = self.build(item=item, config=NoImages(), fetcher=self.FakeFetcher())
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("https://cdn.example.com/a.png", article)
def test_image_count_is_capped(self):
class OneImage(self.Config):
max_images_per_article = 1
item = self.Item(
'<img src="https://cdn.example.com/a.png"><img src="https://cdn.example.com/b.png">'
)
fetcher = self.FakeFetcher()
archive, _ = self.build(item=item, config=OneImage(), fetcher=fetcher)
self.assertEqual(len(fetcher.asked), 1)
self.assertEqual(len(archive.namelist()), 9)
def test_oversized_book_is_refused(self):
class TinyLimit(self.Config):
max_epub_bytes = 1024
with self.assertRaises(epub.EpubTooLargeError):
self.build(config=TinyLimit(), fetcher=self.FakeFetcher())
def test_stylesheet_path_falls_back_when_unreadable(self):
class BadStylesheet(self.Config):
stylesheet = "/nonexistent/stylesheet.css"
archive, _ = self.build(config=BadStylesheet())
self.assertIn("line-height", archive.read(epub.CSS_PATH).decode("utf-8"))
def test_sanitised_body_has_no_scripts(self):
item = self.Item("<p>ok</p><script>alert(1)</script>")
archive, _ = self.build(item=item)
self.assertNotIn("alert", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
def test_parsed_content_respects_the_byte_cap(self):
item = self.Item("<p>" + "word " * 20_000 + "</p>")
data = epub.build_article_epub(
item,
epub_config=self.Config(),
catalog_config=self.Catalog(),
max_content_bytes=4096,
)
self.assertLess(len(data), 100_000)
if __name__ == "__main__":
unittest.main()