Initial commit

This commit is contained in:
2026-11-15 18:15:03 -05:00
commit b76aea2899
36 changed files with 9424 additions and 0 deletions

12
tests/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
"""Test package.
Logging is quietened here so that expected warnings from exercised code paths
(for example the clear-text password notice) do not drown out test results. A
test that asserts on a log message can raise the level itself.
"""
from __future__ import annotations
import logging
logging.getLogger("app").setLevel(logging.CRITICAL)

605
tests/stub_freshrss.py Normal file
View File

@@ -0,0 +1,605 @@
"""An in-process FreshRSS stand-in.
This exists so the client can be tested against the wire format a real FreshRSS
produces, rather than against a mock that would happily accept anything. The
response shapes here mirror the Google Reader API responses the firmware client
consumes (``lib/FreshRss/src/FreshRssSync.cpp``), including the parts that are
easy to get wrong: ``Auth=`` embedded in a multi-line ClientLogin body, category
labels nested inside each subscription, item payloads that put the body under
``content``/``summary``, and one ``i=`` parameter per id on ``edit-tag``.
It also records every request, so tests can assert on the exact query parameters
the client sent -- which is how the unread filter and the page size are verified.
The same class backs ``tools/fake-freshrss.py``, so the server can be smoke
tested by hand without pointing it at a real account.
"""
from __future__ import annotations
import json
import threading
import urllib.parse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Any
API_PREFIX = "/api/greader.php"
READ_TAG = "user/-/state/com.google/read"
STARRED_TAG = "user/-/state/com.google/starred"
READING_LIST = "user/-/state/com.google/reading-list"
def _tags(entry: Any) -> list[str]:
"""Category tags on an entry, tolerating a malformed payload.
Tests deliberately inject non-object entries to check that the client skips
them, so the stub's own filtering must survive them too.
"""
if not isinstance(entry, dict):
return []
tags = entry.get("categories")
if not isinstance(tags, list):
return []
return [tag for tag in tags if isinstance(tag, str)]
def _published(entry: Any) -> int:
"""Sort key that tolerates a malformed entry."""
if not isinstance(entry, dict):
return 0
value = entry.get("published")
return value if isinstance(value, int) else 0
def _with_read_state(entry: Any, read_ids: set[str]) -> Any:
"""The entry as the server would send it, with the read tag applied."""
if not isinstance(entry, dict) or entry.get("id") not in read_ids:
return entry
if READ_TAG in _tags(entry):
return entry
projected = dict(entry)
projected["categories"] = [*_tags(entry), READ_TAG]
return projected
def _is_tagged(entry: Any, tag: str, read_ids: set[str]) -> bool:
"""Whether an entry carries a tag, including tracked read state.
Marking an item read adds the read tag on the server, so an ``xt=read`` query
stops returning it. Without modelling that, a test could not tell whether
mark-read actually took effect.
"""
if tag in _tags(entry):
return True
return tag == READ_TAG and isinstance(entry, dict) and entry.get("id") in read_ids
def item_id(index: int) -> str:
"""A FreshRSS-shaped item id: the exact form that needs URL-safe encoding."""
return f"tag:google.com,2005:reader/item/{index:016x}"
class StubFreshRSS:
"""A scriptable FreshRSS with just enough behaviour to trust the client."""
def __init__(
self,
*,
username: str = "tyler",
password: str = "api-password",
page_limit: int = 1000,
bind_host: str = "127.0.0.1",
bind_port: int = 0,
public_host: str | None = None,
) -> None:
self.username = username
self.password = password
self.page_limit = page_limit
self._bind_host = bind_host
self._bind_port = bind_port
# The name other containers should use to reach this one. It defaults to
# the bind address, which is right for a local test and wrong inside a
# compose network, where the service name is the address.
self._public_host = public_host or bind_host
self.auth_token = "stub-auth-token"
self.write_token = "stub-write-token"
self.subscriptions: list[dict[str, Any]] = []
# Items are typed loosely on purpose: this class stands in for an upstream
# that can send anything, and tests rely on being able to inject malformed
# payloads to check that the client skips them.
self.items: dict[str, list[Any]] = {}
self.read: set[str] = set()
self.requests: list[dict[str, Any]] = []
self.supports_unread_counts = True
# When set, subscription list replies with 200 and a body that is not JSON,
# so the client's response handling can be tested against a proxy or an
# error page rather than against well-formed output.
self.corrupt_json = False
# Injected failures as (status, path fragment or None) pairs.
self._failures: list[tuple[int, str | None]] = []
self._lock = threading.Lock()
self._server: ThreadingHTTPServer | None = None
self._thread: threading.Thread | None = None
self.port = 0
# -- scripting -------------------------------------------------------
def add_feed(
self,
stream_id: str,
title: str,
categories: list[str] | None = None,
*,
html_url: str | None = None,
) -> str:
"""Register a feed under one or more category labels."""
self.subscriptions.append(
{
"id": stream_id,
"title": title,
"categories": [
{"id": f"user/-/label/{label}", "label": label} for label in (categories or [])
],
"htmlUrl": html_url or stream_id.removeprefix("feed/"),
"url": stream_id.removeprefix("feed/"),
"sortid": f"{len(self.subscriptions):08x}",
}
)
self.items.setdefault(stream_id, [])
return stream_id
def add_item(
self,
stream_id: str,
index: int,
title: str,
*,
content: str = "<p>body</p>",
summary: str | None = None,
published: int = 1_700_000_000,
author: str | None = "A. Writer",
url: str | None = None,
read: bool = False,
starred: bool = False,
categories: list[str] | None = None,
) -> str:
"""Add an article to a feed, returning its id."""
identifier = item_id(index)
tags = [READING_LIST, *(categories or [])]
if read:
tags.append(READ_TAG)
if starred:
tags.append(STARRED_TAG)
entry: dict[str, Any] = {
"id": identifier,
"title": title,
"published": published,
"updated": published,
"timestampUsec": str(published * 1_000_000),
"crawlTimeMsec": str(published * 1000),
"categories": tags,
"alternate": [
{
"href": url or f"https://example.com/articles/{index}",
"type": "text/html",
}
],
"origin": {
"streamId": stream_id,
"title": self._feed_title(stream_id),
"htmlUrl": stream_id.removeprefix("feed/"),
},
}
if author is not None:
entry["author"] = author
entry["content"] = {"content": content, "direction": "ltr"}
entry["summary"] = {
"content": summary if summary is not None else content,
"direction": "ltr",
}
self.items.setdefault(stream_id, []).append(entry)
if read:
self.read.add(identifier)
return identifier
def inject_raw_item(self, stream_id: str, payload: Any) -> None:
"""Push an arbitrary payload into a stream.
Upstream is not guaranteed to answer with well-formed objects, so tests
need a way to send junk through the real client path rather than trusting
that it would be handled. Guarded because handler threads read this.
"""
with self._lock:
self.items.setdefault(stream_id, []).append(payload)
def fail_next(self, status: int, count: int = 1, *, path: str | None = None) -> None:
"""Make the next ``count`` requests fail with ``status``.
``path`` narrows it to requests whose path contains that fragment, which
matters because the client logs in before anything else: an untargeted
injected failure would be spent on the login and never reach the call the
test is about. Guarded because it is called between requests while
handler threads are running.
"""
with self._lock:
self._failures.extend([(status, path)] * count)
def _feed_title(self, stream_id: str) -> str:
for feed in self.subscriptions:
if feed["id"] == stream_id:
return feed["title"]
return stream_id
# -- lifecycle -------------------------------------------------------
@property
def base_url(self) -> str:
return f"http://{self._public_host}:{self.port}"
@property
def api_url(self) -> str:
return self.base_url + API_PREFIX
def start(self) -> StubFreshRSS:
handler = _make_handler(self)
server = ThreadingHTTPServer((self._bind_host, self._bind_port), handler)
self._server = server
# Read the port back, because a request for port 0 means "any free port"
# and only the operating system knows which one that turned out to be.
self.port = server.server_address[1]
# A short poll interval keeps teardown cheap: shutdown() only returns once
# the serve loop notices the flag, and the default half second would be
# paid by every test that starts a stub.
self._thread = threading.Thread(
target=server.serve_forever,
kwargs={"poll_interval": 0.02},
daemon=True,
)
self._thread.start()
return self
def stop(self) -> None:
if self._server is not None:
self._server.shutdown()
self._server.server_close()
self._server = None
if self._thread is not None:
self._thread.join(timeout=5)
self._thread = None
def __enter__(self) -> StubFreshRSS:
return self.start()
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: object,
) -> None:
self.stop()
# -- request handling ------------------------------------------------
def record(self, entry: dict[str, Any]) -> None:
with self._lock:
self.requests.append(entry)
def next_failure(self, path: str) -> int | None:
"""The next injected failure that applies to ``path``, if any."""
with self._lock:
for index, (status, target) in enumerate(self._failures):
if target is None or target in path:
del self._failures[index]
return status
return None
def find_requests(self, fragment: str) -> list[dict[str, Any]]:
"""Every recorded request whose path contains ``fragment``.
A substring match rather than a suffix, because a stream path is followed
by the encoded stream id (``/stream/contents/feed%2F...``) and so never
ends where the interesting part of the route does.
"""
with self._lock:
return [entry for entry in self.requests if fragment in entry["path"]]
def mark_read(self, identifiers: list[str]) -> None:
with self._lock:
self.read.update(identifiers)
def clear_read(self) -> None:
"""Forget every read mark, for tests that re-run a marking step."""
with self._lock:
self.read.clear()
def read_snapshot(self) -> set[str]:
"""A copy of the read marks, taken under the lock."""
with self._lock:
return set(self.read)
def subscription_payload(self) -> dict[str, Any]:
return {"subscriptions": self.subscriptions}
def unread_count_payload(self) -> dict[str, Any]:
counts = []
for feed in self.subscriptions:
unread = sum(
1
for entry in self.items.get(feed["id"], [])
if isinstance(entry, dict) and entry.get("id") not in self.read
)
counts.append(
{
"id": feed["id"],
"count": unread,
"newestItemTimestampUsec": "1700000000000000",
}
)
return {"max": self.page_limit, "unreadcounts": counts}
def stream_payload(self, stream_id: str, query: dict[str, list[str]]) -> dict[str, Any]:
"""Items for a stream, honouring ``xt``, ``ot``, ``n`` and ``r``."""
read_ids = self.read_snapshot()
if stream_id == READING_LIST:
entries = [entry for group in self.items.values() for entry in group]
elif stream_id == STARRED_TAG:
entries = [
entry
for group in self.items.values()
for entry in group
if _is_tagged(entry, STARRED_TAG, read_ids)
]
elif stream_id.startswith("user/-/label/"):
label = stream_id.rsplit("/", 1)[-1]
labelled = {
feed["id"]
for feed in self.subscriptions
if any(category["label"] == label for category in feed["categories"])
}
entries = [
entry
for feed_id, group in self.items.items()
if feed_id in labelled
for entry in group
]
else:
entries = list(self.items.get(stream_id, []))
excluded = set(query.get("xt", []))
if excluded:
entries = [
entry
for entry in entries
if not any(_is_tagged(entry, tag, read_ids) for tag in excluded)
]
older_than = query.get("ot", [None])[0]
if older_than and older_than.isdigit():
entries = [
entry
for entry in entries
if not isinstance(entry, dict) or _published(entry) >= int(older_than)
]
descending = query.get("r", ["d"])[0] != "o"
entries.sort(key=_published, reverse=descending)
# Marking an item read adds the tag to the item itself upstream, so the
# returned payload has to carry it too -- otherwise a client could never
# observe a read mark it just made.
entries = [_with_read_state(entry, read_ids) for entry in entries]
limit = query.get("n", [None])[0]
if limit and limit.isdigit():
entries = entries[: int(limit)]
return {"id": stream_id, "updated": 1_700_000_000, "items": entries}
def _make_handler(stub: StubFreshRSS):
"""Build the request handler class bound to one stub instance."""
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "StubFreshRSS/1.0"
# The parameter must keep the base class's name for keyword compatibility,
# even though it shadows the builtin.
def log_message(self, format: str, *args: object) -> None:
"""Silence the default access log; tests assert on recorded requests."""
# -- plumbing ----------------------------------------------------
def _respond(self, status: int, body: bytes, content_type: str) -> None:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
if body and self.command != "HEAD":
self.wfile.write(body)
def _respond_json(self, payload: Any, status: int = 200) -> None:
self._respond(
status,
json.dumps(payload).encode("utf-8"),
"application/json; charset=utf-8",
)
def _record(self, body: str) -> tuple[str, dict[str, list[str]]]:
parsed = urllib.parse.urlsplit(self.path)
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
route = parsed.path
if route.startswith(API_PREFIX):
route = route[len(API_PREFIX) :]
stub.record(
{
"method": self.command,
"path": route,
"raw_path": self.path,
"query": query,
"body": body,
"authorization": self.headers.get("Authorization", ""),
}
)
return route, query
def _authorised(self) -> bool:
expected = f"GoogleLogin auth={stub.auth_token}"
return self.headers.get("Authorization", "") == expected
def _drain_body(self) -> str:
length = int(self.headers.get("Content-Length") or 0)
if not length:
return ""
return self.rfile.read(length).decode("utf-8", "replace")
# -- routes ------------------------------------------------------
def do_HEAD(self) -> None:
self.do_GET()
def do_GET(self) -> None:
route, query = self._record("")
failure = stub.next_failure(route)
if failure is not None:
self._respond_json({"error": "injected failure"}, status=failure)
return
if route == "/healthz":
# Unauthenticated on purpose: it only says the server answered, and
# it is what the demo compose stack waits on before starting the
# catalogue. Real FreshRSS has no such route.
self._respond_json({"status": "ok"})
return
if route == "/accounts/ClientLogin":
self._respond_json({"error": "ClientLogin is a POST"}, status=405)
return
if route == "/reader/api/0/subscription/list":
if not self._require_auth():
return
if stub.corrupt_json:
self._respond(200, b"<html>login page</html>", "text/html")
return
self._respond_json(stub.subscription_payload())
return
if route == "/reader/api/0/unread-count":
if not self._require_auth():
return
if not stub.supports_unread_counts:
self._respond_json({"error": "unsupported"}, status=404)
return
self._respond_json(stub.unread_count_payload())
return
if route == "/reader/api/0/token":
if not self._require_auth():
return
self._respond(200, stub.write_token.encode("utf-8"), "text/plain")
return
if route.startswith("/reader/api/0/stream/contents/"):
if not self._require_auth():
return
stream_id = urllib.parse.unquote(route[len("/reader/api/0/stream/contents/") :])
self._respond_json(stub.stream_payload(stream_id, query))
return
self._respond_json({"error": "not found"}, status=404)
def do_POST(self) -> None:
body = self._drain_body()
route, _query = self._record(body)
failure = stub.next_failure(route)
if failure is not None:
self._respond_json({"error": "injected failure"}, status=failure)
return
if route == "/accounts/ClientLogin":
self._handle_login(body)
return
if route == "/reader/api/0/edit-tag":
if not self._require_auth():
return
self._handle_edit_tag(body)
return
self._respond_json({"error": "not found"}, status=404)
def _require_auth(self) -> bool:
if self._authorised():
return True
self._respond_json({"error": "unauthorised"}, status=401)
return False
def _handle_login(self, body: str) -> None:
form = urllib.parse.parse_qs(body)
username = form.get("Email", [""])[0]
password = form.get("Passwd", [""])[0]
if username != stub.username or password != stub.password:
self._respond_json({"error": "BadAuthentication"}, status=403)
return
# The real endpoint returns three lines; only Auth matters to us, and a
# parser that grabs the first "=" would get SID instead.
payload = f"SID=stub-sid\nLSID=stub-lsid\nAuth={stub.auth_token}\n"
self._respond(200, payload.encode("utf-8"), "text/plain; charset=utf-8")
def _handle_edit_tag(self, body: str) -> None:
form = urllib.parse.parse_qs(body, keep_blank_values=True)
token = form.get("T", [""])[0]
if token != stub.write_token:
self._respond_json({"error": "Invalid token"}, status=401)
return
if form.get("a", [""])[0] != READ_TAG:
self._respond_json({"error": "unexpected tag"}, status=400)
return
stub.mark_read(form.get("i", []))
self._respond(200, b"OK", "text/plain; charset=utf-8")
return Handler
def build_demo_catalog(**stub_options: Any) -> StubFreshRSS:
"""A small but realistic account, used by tests and the manual smoke tool.
Three feeds across two categories, with a read item, a starred item, an item
in a feed that has no category, and content that exercises the sanitiser
(relative links, a script tag, an unclosed list, an entity).
Keyword options are forwarded to :class:`StubFreshRSS`, which is how
``tools/fake-freshrss.py`` serves this same account to a container --
``bind_host``, ``bind_port`` and ``public_host``, since a compose network needs
a wildcard bind, a fixed port and a service name in place of loopback. The
defaults are the loopback bind and ephemeral port the tests rely on.
"""
stub = StubFreshRSS(**stub_options)
tech = stub.add_feed("feed/https://example.com/tech/atom.xml", "Example Tech", ["Tech"])
news = stub.add_feed(
"feed/https://news.example.org/rss?source=daily",
"Daily News",
["News"],
)
lonely = stub.add_feed("feed/https://solo.example.net/feed.xml", "Solo Blog")
stub.add_item(
tech,
1,
"A Tech Post & Its Friends",
content=(
'<p>Intro with an <a href="/relative-link">in-page link</a>.</p>'
"<script>alert('nope')</script>"
"<ul><li>first<li>second</ul>"
'<p><img src="/images/diagram.png" alt="A diagram"></p>'
"<p>Entities: &amp; &lt;tag&gt; &nbsp; done.</p>"
),
published=1_700_000_100,
)
stub.add_item(tech, 2, "An Older Tech Post", published=1_699_000_000)
stub.add_item(
tech,
3,
"Already Read Post",
published=1_698_000_000,
read=True,
)
stub.add_item(news, 4, "Markets Dip Slightly", published=1_700_000_200, author=None)
stub.add_item(
news,
5,
"Starred Investigation",
published=1_700_000_300,
starred=True,
)
stub.add_item(lonely, 6, "Uncategorised Musings", published=1_700_000_050)
return stub

235
tests/support.py Normal file
View File

@@ -0,0 +1,235 @@
"""Shared test helpers.
The XML helper here is deliberately not ``xml.etree``: that API is documented as
unsafe for untrusted documents because it will expand entities and honour a DTD.
Even though every document these tests parse is one this project generated, the
parser is configured to refuse DTDs, entity declarations and external entity
resolution outright. Refusing them is stronger than trusting the parser's
amplification limits, and it keeps the test suite free of the pattern that makes
reviewers (and linters) rightly nervous.
"""
from __future__ import annotations
import re
from xml.parsers import expat
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
_WS_RE = re.compile(r"\s+")
def local_name(tag: str) -> str:
"""The part of a tag after any namespace prefix.
Expat is used without namespace processing here, so a prefixed element arrives
as ``dc:title``. Structural assertions care about the element, not the
prefix, so lookups compare the local name and read naturally.
"""
return tag.rsplit(":", 1)[-1]
class XmlNode:
"""A minimal element tree node, enough for structural assertions."""
__slots__ = ("attrs", "children", "tag", "text")
def __init__(self, tag: str, attrs: dict[str, str] | None = None) -> None:
self.tag = tag
self.attrs: dict[str, str] = attrs or {}
self.children: list[XmlNode] = []
self.text = ""
def attr(self, name: str, default: str | None = None) -> str | None:
"""Return an attribute value (None when absent)."""
return self.attrs.get(name, default)
def must_find(self, tag: str) -> XmlNode:
"""The first descendant with this tag, failing loudly when absent.
Structural assertions want to say "the OPF has no <manifest>" rather than
hand back a None that only explodes on the next attribute access.
"""
found = self.find(tag)
if found is None:
raise AssertionError(f"no <{tag}> element in <{self.tag}>")
return found
def must_find_all(self, tag: str) -> list[XmlNode]:
"""Every descendant with this tag, failing loudly when there are none."""
found = self.find_all(tag)
if not found:
raise AssertionError(f"no <{tag}> elements in <{self.tag}>")
return found
def must_attr(self, name: str) -> str:
"""An attribute value that must be present."""
value = self.attrs.get(name)
if value is None:
raise AssertionError(f"no {name!r} attribute on <{self.tag}>")
return value
def find_all(self, tag: str) -> list[XmlNode]:
"""All descendants matching a tag name, in document order.
Matches the local name, so ``find("title")`` finds ``dc:title`` while
``find("dc:title")`` still works.
"""
wanted = local_name(tag)
found: list[XmlNode] = []
for child in self.children:
if child.tag == tag or local_name(child.tag) == wanted:
found.append(child)
found.extend(child.find_all(tag))
return found
def find(self, tag: str) -> XmlNode | None:
"""First descendant with a matching tag name, or None."""
matches = self.find_all(tag)
return matches[0] if matches else None
def all_text(self) -> str:
"""Concatenated text of this node and its descendants."""
return self.text + "".join(child.all_text() for child in self.children)
def direct_text(self) -> str:
"""Text belonging to this node only, ignoring descendants."""
return self.text
def parse_xml(data: str | bytes) -> XmlNode:
"""Parse XML into an :class:`XmlNode` tree, refusing any DTD or entity.
Raises ``expat.ExpatError`` for malformed input and ``ValueError`` for a
document that tries to declare a doctype or an entity.
"""
parser = expat.ParserCreate()
parser.buffer_text = True
# Parameter entities come from a DTD, which we never accept.
parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_NEVER)
def reject_doctype(doctype_name, system_id, public_id, has_internal_subset):
raise ValueError("doctype declarations are not allowed in generated XML")
def reject_entity(*_args):
raise ValueError("entity declarations are not allowed in generated XML")
def reject_external(*_args):
# Returning 0 makes expat fail the parse instead of resolving the entity.
return 0
parser.StartDoctypeDeclHandler = reject_doctype
parser.EntityDeclHandler = reject_entity
parser.ExternalEntityRefHandler = reject_external
root = XmlNode("#document")
stack = [root]
def start_element(tag: str, attrs: dict[str, str]) -> None:
stack.append(XmlNode(tag, dict(attrs)))
def end_element(_tag: str) -> None:
node = stack.pop()
stack[-1].children.append(node)
def characters(data: str) -> None:
stack[-1].text += data
parser.StartElementHandler = start_element
parser.EndElementHandler = end_element
parser.CharacterDataHandler = characters
parser.Parse(data, True)
if not root.children:
raise ValueError("XML document has no root element")
return root.children[0]
def assert_well_formed(testcase, fragment: str) -> None:
"""Assert that ``fragment`` parses as XML inside a single wrapper root."""
markup = f'<root xmlns="{XHTML_NAMESPACE}">{fragment}</root>'
try:
parse_xml(markup)
except (expat.ExpatError, ValueError) as exc: # pragma: no cover - on failure
testcase.fail(f"output is not well-formed XML: {exc}\n{fragment}")
def collapse(text: str) -> str:
"""Collapse all whitespace, for assertions that do not care about layout."""
return _WS_RE.sub(" ", text).strip()
def body(fragment: str) -> str:
"""Return the inner markup of the article wrapper the sanitiser emits."""
prefix = '<div class="article-body">'
if fragment.startswith(prefix):
fragment = fragment[len(prefix) :]
suffix = "</div>"
if fragment.endswith(suffix):
fragment = fragment[: -len(suffix)]
return fragment.strip()
# -- image builders ------------------------------------------------------
#
# These produce bytes that are structurally real -- correct magic numbers and
# container headers -- so sniffing and dimension parsing are exercised against the
# formats rather than against hand-written expectations. Pixel data is minimal
# because nothing here ever decodes an image.
def png_bytes(width: int = 800, height: int = 400) -> bytes:
"""A structurally valid PNG of the given size."""
import struct
import zlib
def chunk(tag: bytes, payload: bytes) -> bytes:
crc = zlib.crc32(tag + payload) & 0xFFFFFFFF
return struct.pack(">I", len(payload)) + tag + payload + struct.pack(">I", crc)
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
rows = b"".join(b"\x00" + b"\xff\x00\x00" * width for _ in range(height))
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(rows, 9))
+ chunk(b"IEND", b"")
)
def jpeg_bytes(width: int = 800, height: int = 400) -> bytes:
"""SOI + APP0 + SOF0 + EOI: a JPEG the dimension walker can read."""
import struct
app0 = b"\xff\xe0" + struct.pack(">H", 16) + b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00"
# The frame header carries height and width, in that order.
sof0 = (
b"\xff\xc0"
+ struct.pack(">H", 17)
+ b"\x08"
+ struct.pack(">HH", height, width)
+ b"\x03\x01\x11\x00\x02\x11\x01\x03\x11\x01"
)
return b"\xff\xd8" + app0 + sof0 + b"\xff\xd9"
def gif_bytes(width: int = 320, height: int = 200) -> bytes:
"""A GIF header with the given logical screen size."""
import struct
return b"GIF89a" + struct.pack("<HH", width, height) + b"\xf7\x00\x00" + b"\x00" * 20
def webp_bytes(width: int = 800, height: int = 400) -> bytes:
"""A lossless VP8X WebP header with the given canvas size (stored minus one)."""
import struct
return (
b"RIFF"
+ struct.pack("<I", 22)
+ b"WEBP"
+ b"VP8X"
+ struct.pack("<I", 10)
+ b"\x00\x00\x00\x00"
+ (width - 1).to_bytes(3, "little")
+ (height - 1).to_bytes(3, "little")
)

487
tests/test_config.py Normal file
View File

@@ -0,0 +1,487 @@
"""Tests for configuration loading, interpolation and validation."""
from __future__ import annotations
import json
import os
import tempfile
import unittest
from pathlib import Path
from app.config import (
DEFAULT_BIND_HOST,
ConfigError,
describe_config,
load_config,
normalise_api_base,
password_digest,
resolve_config_path,
)
# Fixture credentials. These are throwaway values that exercise the schema; no
# real account is involved and neither value is a secret. They are applied
# through the helpers below so every test states its intent in one line.
FIXTURE_UPSTREAM_VALUE = "upstream-fixture-value"
FIXTURE_READER_VALUE = "reader-fixture-value"
REPO_ROOT = Path(__file__).resolve().parent.parent
EXAMPLE_CONFIG = REPO_ROOT / "config.example.json"
def with_upstream_credential(document: dict, value: str) -> dict:
"""Set the upstream FreshRSS API credential under test."""
document["freshrss"]["api_password"] = value
return document
def with_reader_credential(document: dict, value: str, username: str = "reader") -> dict:
"""Replace auth.users with one user carrying a clear-text credential."""
document["auth"]["users"] = [{"username": username, "password": value}]
return document
def with_reader_digest(document: dict, digest: str, username: str = "reader") -> dict:
"""Replace auth.users with one user carrying a pre-computed digest."""
document["auth"]["users"] = [{"username": username, "password_sha256": digest}]
return document
def with_conflicting_credentials(document: dict, value: str) -> dict:
"""A deliberately over-specified user, to exercise the conflict rule."""
document["auth"]["users"] = [
{
"username": "reader",
"password": value,
"password_sha256": password_digest(value),
}
]
return document
def base_config() -> dict:
"""A minimal valid configuration, ready to be mutated by a test."""
document = {
"server": {"port": 8080},
"freshrss": {
"url": "https://rss.example.com",
"username": "tyler",
"api_password": FIXTURE_UPSTREAM_VALUE,
},
"auth": {"users": [{"username": "tyler", "password": FIXTURE_READER_VALUE}]},
}
return document
class ConfigTestCase(unittest.TestCase):
"""Writes a config document into a temporary directory and loads it."""
def setUp(self):
self._dir = tempfile.TemporaryDirectory()
self.addCleanup(self._dir.cleanup)
self.dir = Path(self._dir.name)
def write(self, document, name: str = "config.json") -> str:
path = self.dir / name
if isinstance(document, str):
path.write_text(document, encoding="utf-8")
else:
path.write_text(json.dumps(document), encoding="utf-8")
return str(path)
def load(self, document, name: str = "config.json"):
return load_config(self.write(document, name))
def expect_problem(self, document, fragment: str) -> ConfigError:
"""Assert the config is rejected, and return the error for more checks."""
with self.assertRaises(ConfigError) as caught:
self.load(document)
joined = "\n".join(caught.exception.problems)
self.assertIn(fragment, joined)
return caught.exception
class LoadingTests(ConfigTestCase):
def test_minimal_config_uses_documented_defaults(self):
config = self.load(base_config())
self.assertEqual(config.server.host, DEFAULT_BIND_HOST)
self.assertEqual(config.server.port, 8080)
self.assertEqual(config.freshrss.items_per_page, 30)
self.assertTrue(config.freshrss.mark_read)
self.assertTrue(config.epub.include_images)
self.assertEqual(config.auth.users[0].username, "tyler")
self.assertEqual(len(config.auth.users[0].password_sha256), 64)
def test_default_bind_address_is_loopback(self):
# The wildcard belongs in the deployment's config file, not in the code:
# a stray `docker run -p 8080:8080` should not silently be reachable.
self.assertEqual(DEFAULT_BIND_HOST, "127.0.0.1")
def test_missing_file_is_a_clear_error(self):
with self.assertRaises(ConfigError) as caught:
load_config(str(self.dir / "nope.json"))
joined = "\n".join(caught.exception.problems)
self.assertIn("not found", joined)
self.assertIn("CONFIG_PATH", joined)
def test_invalid_json_reports_position(self):
error = self.expect_problem('{"server": {,}', "not valid JSON")
self.assertIn("line 1", "\n".join(error.problems))
def test_non_object_root_is_rejected(self):
self.expect_problem("[1, 2, 3]", "top level")
def test_source_path_is_recorded(self):
path = self.write(base_config())
self.assertEqual(load_config(path).source_path, str(Path(path).resolve()))
def test_resolve_config_path_prefers_explicit_then_env(self):
explicit = self.write(base_config(), "explicit.json")
self.assertEqual(resolve_config_path(explicit), explicit)
from_env = str(self.dir / "from-env.json")
os.environ["CONFIG_PATH"] = from_env
self.addCleanup(os.environ.pop, "CONFIG_PATH", None)
self.assertEqual(resolve_config_path(), from_env)
class ExampleConfigTests(unittest.TestCase):
"""The shipped example must load: a broken example is a broken first run.
This is also where the container's wildcard bind address is asserted. That
literal belongs in the deployment's configuration file, not in Python, and the
test proves the file really does override the loopback default.
"""
def setUp(self):
# The example deliberately leaves secrets to the environment.
os.environ["FRESHRSS_API_PASSWORD"] = FIXTURE_UPSTREAM_VALUE
os.environ["OPDS_PASSWORD_SHA256"] = password_digest(FIXTURE_READER_VALUE)
self.addCleanup(os.environ.pop, "FRESHRSS_API_PASSWORD", None)
self.addCleanup(os.environ.pop, "OPDS_PASSWORD_SHA256", None)
self.document = json.loads(EXAMPLE_CONFIG.read_text(encoding="utf-8"))
def test_example_config_exists_and_is_valid(self):
config = load_config(str(EXAMPLE_CONFIG))
self.assertEqual(config.warnings, [])
self.assertEqual(len(config.auth.users), 1)
def test_example_config_overrides_the_loopback_default(self):
config = load_config(str(EXAMPLE_CONFIG))
self.assertNotEqual(config.server.host, DEFAULT_BIND_HOST)
self.assertEqual(config.server.host, self.document["server"]["host"])
def test_example_keeps_secrets_out_of_the_file(self):
self.assertTrue(self.document["freshrss"]["api_password"].startswith("${"))
self.assertTrue(self.document["auth"]["users"][0]["password_sha256"].startswith("${"))
def test_example_is_self_documenting(self):
comments = [key for key in self.document if key.startswith("_")]
self.assertTrue(comments)
self.assertTrue(self.document["server"].get("_host"))
def test_example_interpolates_its_credentials(self):
self.assertEqual(
load_config(str(EXAMPLE_CONFIG)).freshrss.api_password, FIXTURE_UPSTREAM_VALUE
)
class InterpolationTests(ConfigTestCase):
def test_environment_variable_is_expanded(self):
os.environ["TEST_FRESHRSS_CREDENTIAL"] = "from-env"
self.addCleanup(os.environ.pop, "TEST_FRESHRSS_CREDENTIAL", None)
document = with_upstream_credential(base_config(), "${TEST_FRESHRSS_CREDENTIAL}")
self.assertEqual(self.load(document).freshrss.api_password, "from-env")
def test_environment_variable_default_is_used_when_unset(self):
os.environ.pop("TEST_UNSET_VAR", None)
document = with_upstream_credential(base_config(), "${TEST_UNSET_VAR:-fallback}")
self.assertEqual(self.load(document).freshrss.api_password, "fallback")
def test_missing_environment_variable_names_itself(self):
os.environ.pop("TEST_DEFINITELY_UNSET", None)
document = with_upstream_credential(base_config(), "${TEST_DEFINITELY_UNSET}")
error = self.expect_problem(document, "TEST_DEFINITELY_UNSET")
self.assertIn("freshrss.api_password", "\n".join(error.problems))
def test_secret_file_is_read_and_trimmed(self):
(self.dir / "upstream-secret").write_text("file-secret\n", encoding="utf-8")
# A relative path resolves against the config file's own directory.
document = with_upstream_credential(base_config(), "${file:upstream-secret}")
self.assertEqual(self.load(document).freshrss.api_password, "file-secret")
def test_secret_file_may_be_absolute(self):
absolute = self.dir / "absolute-secret"
absolute.write_text("abs", encoding="utf-8")
document = with_upstream_credential(base_config(), f"${{file:{absolute}}}")
self.assertEqual(self.load(document).freshrss.api_password, "abs")
def test_missing_secret_file_is_an_error(self):
document = with_upstream_credential(base_config(), "${file:not-there}")
self.expect_problem(document, "cannot read secret file")
def test_escaped_dollar_is_literal(self):
document = with_upstream_credential(base_config(), "$${NOT_A_REFERENCE}")
self.assertEqual(self.load(document).freshrss.api_password, "${NOT_A_REFERENCE}")
def test_interpolation_reaches_nested_arrays(self):
os.environ.pop("TEST_PW", None)
document = with_reader_credential(base_config(), "${TEST_PW:-pw}")
self.assertEqual(self.load(document).auth.users[0].password_sha256, password_digest("pw"))
def test_empty_reference_is_an_error(self):
document = with_upstream_credential(base_config(), "${}")
self.expect_problem(document, "empty ${}")
class FreshRSSTests(ConfigTestCase):
def test_api_base_appends_the_greader_suffix(self):
self.assertEqual(
normalise_api_base("https://rss.example.com"),
"https://rss.example.com/api/greader.php",
)
def test_api_base_keeps_an_existing_suffix(self):
self.assertEqual(
normalise_api_base("https://rss.example.com/api/greader.php"),
"https://rss.example.com/api/greader.php",
)
def test_api_base_preserves_a_path_prefix(self):
self.assertEqual(
normalise_api_base("https://example.com/freshrss/"),
"https://example.com/freshrss/api/greader.php",
)
def test_api_base_is_case_insensitive_about_the_suffix(self):
self.assertEqual(
normalise_api_base("https://e.com/API/GReader.php"),
"https://e.com/API/GReader.php",
)
def test_url_without_scheme_is_rejected(self):
document = base_config()
document["freshrss"]["url"] = "rss.example.com"
self.expect_problem(document, "must start with http:// or https://")
def test_required_fields_are_reported_together(self):
document = {"server": {}, "auth": {"users": [{"username": "u", "password": "p"}]}}
with self.assertRaises(ConfigError) as caught:
self.load(document)
joined = "\n".join(caught.exception.problems)
for fragment in ("freshrss.url", "freshrss.username", "freshrss.api_password"):
self.assertIn(fragment, joined)
def test_items_per_page_must_fit_the_request_ceiling(self):
document = base_config()
document["freshrss"]["items_per_page"] = 100
document["freshrss"]["max_items_per_request"] = 50
self.expect_problem(document, "must be at least")
def test_retries_and_timeouts_are_validated(self):
document = base_config()
document["freshrss"]["timeout_seconds"] = 0
document["freshrss"]["retries"] = -1
with self.assertRaises(ConfigError) as caught:
self.load(document)
joined = "\n".join(caught.exception.problems)
self.assertIn("timeout_seconds", joined)
self.assertIn("retries", joined)
def test_mark_read_can_be_disabled(self):
document = base_config()
document["freshrss"]["mark_read"] = False
self.assertFalse(self.load(document).freshrss.mark_read)
class TypeValidationTests(ConfigTestCase):
def test_port_must_be_a_number_in_range(self):
for value, fragment in (
("8080", "whole number"),
(-1, "at least"),
(70000, "at most"),
):
document = base_config()
document["server"]["port"] = value
self.expect_problem(document, fragment)
def test_port_zero_asks_the_operating_system_for_a_free_port(self):
# Useful for a local run or a test; a normal deployment sets a real port.
document = base_config()
document["server"]["port"] = 0
self.assertEqual(self.load(document).server.port, 0)
def test_boolean_field_rejects_a_number(self):
document = base_config()
document["freshrss"]["mark_read"] = 1
self.expect_problem(document, "must be true or false")
def test_integer_field_rejects_a_boolean(self):
# bool is an int subclass in Python; `true` must not become 1.
document = base_config()
document["server"]["port"] = True
self.expect_problem(document, "whole number")
def test_string_field_rejects_a_list(self):
document = base_config()
document["catalog"] = {"title": ["not", "a", "string"]}
self.expect_problem(document, "catalog.title: must be a string")
def test_string_list_rejects_scalars(self):
document = base_config()
document["catalog"] = {"exclude_feeds": "one-feed"}
self.expect_problem(document, "must be an array of strings")
def test_section_must_be_an_object(self):
document = base_config()
document["cache"] = "nope"
self.expect_problem(document, "cache: must be a JSON object")
def test_log_level_is_restricted(self):
document = base_config()
document["server"]["log_level"] = "chatty"
self.expect_problem(document, "must be one of")
def test_ttl_must_be_positive(self):
document = base_config()
document["cache"] = {"stream_ttl_seconds": 0}
self.expect_problem(document, "greater than 0")
class AuthTests(ConfigTestCase):
def test_users_are_required(self):
document = base_config()
del document["auth"]
error = self.expect_problem(document, "auth.users: required")
self.assertIn("credentials", "\n".join(error.problems))
def test_empty_user_list_is_rejected(self):
document = base_config()
document["auth"]["users"] = []
self.expect_problem(document, "at least one user")
def test_clear_text_password_is_hashed(self):
document = with_reader_credential(base_config(), FIXTURE_READER_VALUE)
config = self.load(document)
self.assertEqual(
config.auth.users[0].password_sha256, password_digest(FIXTURE_READER_VALUE)
)
def test_clear_text_password_is_never_kept_in_the_config_object(self):
document = with_reader_credential(base_config(), FIXTURE_READER_VALUE)
config = self.load(document)
self.assertNotIn(FIXTURE_READER_VALUE, json.dumps(config.auth.users[0].__dict__))
def test_sha256_password_is_accepted_and_lowercased(self):
document = with_reader_digest(base_config(), password_digest("s3cret").upper())
self.assertEqual(
self.load(document).auth.users[0].password_sha256, password_digest("s3cret")
)
def test_both_password_forms_is_an_error(self):
document = with_conflicting_credentials(base_config(), FIXTURE_READER_VALUE)
self.expect_problem(document, "not both")
def test_neither_password_form_is_an_error(self):
document = base_config()
document["auth"]["users"] = [{"username": "a"}]
self.expect_problem(document, "needs password or password_sha256")
def test_malformed_digest_is_rejected(self):
document = with_reader_digest(base_config(), "deadbeef")
self.expect_problem(document, "64 hex characters")
def test_duplicate_usernames_are_rejected(self):
document = base_config()
document["auth"]["users"] = [
{"username": "a", "password": FIXTURE_READER_VALUE},
{"username": "a", "password": FIXTURE_UPSTREAM_VALUE},
]
self.expect_problem(document, "duplicate username")
def test_user_must_be_an_object(self):
document = base_config()
document["auth"]["users"] = ["tyler"]
self.expect_problem(document, "must be a JSON object")
def test_realm_is_configurable(self):
document = base_config()
document["auth"]["realm"] = "Reading"
self.assertEqual(self.load(document).auth.realm, "Reading")
class TlsTests(ConfigTestCase):
def test_cert_without_key_is_rejected(self):
document = base_config()
document["server"]["tls"] = {"cert_file": str(self.dir / "cert.pem")}
self.expect_problem(document, "must be set together")
def test_missing_certificate_file_is_rejected(self):
document = base_config()
document["server"]["tls"] = {
"cert_file": str(self.dir / "absent-cert.pem"),
"key_file": str(self.dir / "absent-key.pem"),
}
self.expect_problem(document, "no such file")
def test_present_certificate_pair_is_accepted(self):
cert = self.dir / "cert.pem"
key = self.dir / "key.pem"
cert.write_text("dummy", encoding="utf-8")
key.write_text("dummy", encoding="utf-8")
document = base_config()
document["server"]["tls"] = {"cert_file": str(cert), "key_file": str(key)}
config = self.load(document)
self.assertEqual(config.server.tls_cert_file, str(cert))
class WarningTests(ConfigTestCase):
def test_unknown_key_is_reported_with_a_suggestion(self):
document = base_config()
document["freshrss"]["mark-read"] = True
config = self.load(document)
self.assertTrue(config.warnings)
self.assertIn("freshrss.mark-read", config.warnings[0])
self.assertIn("mark_read", config.warnings[0])
def test_unknown_user_key_is_reported(self):
document = base_config()
document["auth"]["users"] = [
{"username": "a", "password": FIXTURE_READER_VALUE, "passwrod": "typo"}
]
config = self.load(document)
self.assertIn("auth.users[0].passwrod", config.warnings[0])
def test_unknown_tls_key_is_reported(self):
document = base_config()
document["server"]["tls"] = {"certfile": "/x", "keyfile": "/y"}
config = self.load(document)
self.assertEqual(len(config.warnings), 2)
def test_unknown_section_is_reported(self):
document = base_config()
document["cach"] = {"max_entries": 10}
config = self.load(document)
self.assertIn("cach:", config.warnings[0])
self.assertIn("cache", config.warnings[0])
def test_warnings_do_not_block_a_valid_config(self):
document = base_config()
document["nonsense"] = 1
self.assertEqual(len(self.load(document).warnings), 1)
class DescribeConfigTests(ConfigTestCase):
def test_password_is_redacted_in_the_summary(self):
summary = describe_config(self.load(base_config()))
self.assertNotIn(FIXTURE_UPSTREAM_VALUE, summary)
self.assertIn("rss.example.com/api/greader.php", summary)
def test_summary_lists_users(self):
self.assertIn("tyler", describe_config(self.load(base_config())))
def test_summary_notes_tls_state(self):
summary = describe_config(self.load(base_config()))
self.assertIn("127.0.0.1:8080", summary)
if __name__ == "__main__":
unittest.main()

533
tests/test_epub.py Normal file
View File

@@ -0,0 +1,533 @@
"""Tests for image metadata and EPUB generation.
The EPUB checks are structural on purpose. A book is rejected by a reader for
shape reasons -- a deflated mimetype, a manifest href with no entry, a document
that will not parse -- far more often than for content reasons, so those are the
properties asserted here.
"""
from __future__ import annotations
import unittest
import zipfile
from datetime import UTC, datetime
from io import BytesIO
from app import epub, images
from app.fetch import FetchedImage
from app.xhtml import parse_article
from tests.support import (
gif_bytes,
jpeg_bytes,
parse_xml,
png_bytes,
webp_bytes,
)
MODIFIED = datetime(2026, 11, 15, 12, 30, 45, tzinfo=UTC)
PUBLISHED = datetime(2026, 11, 14, 9, 0, 0, tzinfo=UTC)
class ImageMetadataTests(unittest.TestCase):
"""Sniffing and sizing, against structurally real image bytes."""
def test_sniffing_each_supported_format(self):
cases = {
"image/png": png_bytes(),
"image/jpeg": jpeg_bytes(),
"image/gif": gif_bytes(),
"image/webp": webp_bytes(),
}
for expected, data in cases.items():
self.assertEqual(images.sniff_media_type(data), expected)
def test_dimensions_are_read_for_each_format(self):
cases = [
(png_bytes(800, 400), (800, 400)),
(jpeg_bytes(640, 480), (640, 480)),
(gif_bytes(320, 200), (320, 200)),
(webp_bytes(1024, 512), (1024, 512)),
]
for data, expected in cases:
self.assertEqual(images.dimensions(data), expected)
def test_unknown_bytes_have_no_media_type(self):
self.assertIsNone(images.sniff_media_type(b"not an image at all"))
self.assertIsNone(images.sniff_media_type(b""))
self.assertIsNone(images.sniff_media_type(b"\x89PNG"))
def test_truncated_images_do_not_crash_the_sizer(self):
for data in (png_bytes()[:12], jpeg_bytes()[:6], gif_bytes()[:8], webp_bytes()[:20]):
self.assertIsNone(images.dimensions(data))
def test_svg_is_not_an_embeddable_type(self):
# SVG can carry script and external references; it is not on the list.
self.assertIsNone(images.normalise_media_type("image/svg+xml"))
def test_content_type_parameters_are_ignored(self):
self.assertEqual(images.normalise_media_type("image/jpeg; charset=binary"), "image/jpeg")
def test_aliases_are_normalised(self):
self.assertEqual(images.normalise_media_type("IMAGE/JPG"), "image/jpeg")
self.assertEqual(images.normalise_media_type("image/x-png"), "image/png")
def test_bytes_win_over_a_wrong_declared_type(self):
# A mislabelled payload must not produce an invalid manifest entry.
self.assertEqual(images.identify(png_bytes(), "text/html"), "image/png")
self.assertEqual(images.identify(b"unknown", "image/png"), "image/png")
self.assertIsNone(images.identify(b"unknown", "text/html"))
def test_cover_heuristic(self):
self.assertTrue(images.looks_like_a_cover(800, 400))
self.assertTrue(images.looks_like_a_cover(400, 200))
self.assertFalse(images.looks_like_a_cover(399, 200))
self.assertFalse(images.looks_like_a_cover(1, 1))
self.assertFalse(images.looks_like_a_cover(800, 100))
def make_book(**overrides) -> epub.Book:
values = {
"identifier": "urn:uuid:12345678-1234-1234-1234-123456789abc",
"title": "A Test Article & Its Friends",
"language": "en",
"modified": MODIFIED,
"published": PUBLISHED,
"content_xhtml": '<div class="article-body">\n<p>Body text.</p>\n</div>',
"author": "A. Writer",
"publisher": "Example Press",
"source_url": "https://example.com/articles/1",
"feed_title": "Example Tech",
}
values.update(overrides)
return epub.Book(**values)
class EpubContainerTests(unittest.TestCase):
"""The ZIP-level rules that make a file an EPUB at all."""
def setUp(self):
self.data = epub.build_epub(make_book())
self.archive = zipfile.ZipFile(BytesIO(self.data))
self.addCleanup(self.archive.close)
def test_mimetype_is_the_first_entry(self):
self.assertEqual(self.archive.namelist()[0], "mimetype")
def test_mimetype_is_stored_not_deflated(self):
info = self.archive.getinfo("mimetype")
self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
self.assertEqual(self.archive.read("mimetype"), b"application/epub+zip")
def test_mimetype_has_no_extra_field(self):
# An extra field in the local header is a known cause of readers refusing
# a book, so the raw bytes are inspected rather than the parsed record.
offset = self.archive.getinfo("mimetype").header_offset
local_header = self.data[offset : offset + 30]
extra_length = int.from_bytes(local_header[28:30], "little")
self.assertEqual(extra_length, 0)
def test_container_names_the_package_document(self):
xml = self.archive.read("META-INF/container.xml").decode("utf-8")
root = parse_xml(xml)
rootfile = root.must_find("rootfiles").must_find("rootfile")
self.assertEqual(rootfile.must_attr("full-path"), epub.OPF_PATH)
self.assertEqual(rootfile.must_attr("media-type"), "application/oebps-package+xml")
# Whatever it points at must actually exist.
self.assertIn(rootfile.must_attr("full-path"), self.archive.namelist())
def test_expected_parts_are_present(self):
names = set(self.archive.namelist())
for path in (
"META-INF/container.xml",
epub.OPF_PATH,
epub.NAV_PATH,
epub.NCX_PATH,
epub.CSS_PATH,
epub.COVER_PATH,
epub.ARTICLE_PATH,
):
self.assertIn(path, names)
class EpubDocumentTests(unittest.TestCase):
"""Every text part has to parse with a strict XML parser."""
def setUp(self):
self.data = epub.build_epub(make_book())
self.archive = zipfile.ZipFile(BytesIO(self.data))
self.addCleanup(self.archive.close)
def test_every_xml_part_is_well_formed(self):
for path in (
epub.OPF_PATH,
epub.NAV_PATH,
epub.NCX_PATH,
epub.COVER_PATH,
epub.ARTICLE_PATH,
"META-INF/container.xml",
):
with self.subTest(path=path):
parse_xml(self.archive.read(path))
def test_all_xml_starts_with_a_declaration(self):
for path in (
epub.OPF_PATH,
epub.NAV_PATH,
epub.NCX_PATH,
epub.COVER_PATH,
epub.ARTICLE_PATH,
):
with self.subTest(path=path):
text = self.archive.read(path).decode("utf-8")
self.assertTrue(text.startswith('<?xml version="1.0" encoding="utf-8"?>'))
def test_xhtml_documents_declare_the_namespace(self):
for path in (epub.NAV_PATH, epub.COVER_PATH, epub.ARTICLE_PATH):
with self.subTest(path=path):
root = parse_xml(self.archive.read(path))
self.assertEqual(root.must_attr("xmlns"), epub.XHTML_NAMESPACE)
def test_no_doctype_is_emitted(self):
# A DOCTYPE would have to be resolved by the reader's XML parser; EPUB 3
# makes it optional, so it is left out entirely.
for path in (epub.OPF_PATH, epub.NAV_PATH, epub.ARTICLE_PATH):
with self.subTest(path=path):
self.assertNotIn("<!DOCTYPE", self.archive.read(path).decode("utf-8"))
class EpubPackageTests(unittest.TestCase):
"""OPF metadata, manifest and spine."""
def setUp(self):
self.data = epub.build_epub(make_book())
self.archive = zipfile.ZipFile(BytesIO(self.data))
self.addCleanup(self.archive.close)
self.opf = parse_xml(self.archive.read(epub.OPF_PATH))
def manifest(self) -> dict:
items = self.opf.must_find("manifest").must_find_all("item")
return {item.must_attr("id"): item for item in items}
def test_package_declares_version_and_unique_identifier(self):
self.assertEqual(self.opf.must_attr("version"), "3.0")
self.assertEqual(self.opf.must_attr("unique-identifier"), "pub-id")
def test_required_dublin_core_metadata_is_present(self):
metadata = self.opf.must_find("metadata")
identifier = metadata.must_find("identifier")
self.assertEqual(identifier.must_attr("id"), "pub-id")
self.assertEqual(identifier.all_text(), make_book().identifier)
self.assertEqual(metadata.must_find("title").all_text(), make_book().title)
self.assertEqual(metadata.must_find("language").all_text(), "en")
def test_optional_metadata_is_present_when_known(self):
metadata = self.opf.must_find("metadata")
self.assertEqual(metadata.must_find("creator").all_text(), "A. Writer")
self.assertEqual(metadata.must_find("publisher").all_text(), "Example Press")
self.assertEqual(metadata.must_find("source").all_text(), "https://example.com/articles/1")
self.assertEqual(metadata.must_find("date").all_text(), "2026-11-14T09:00:00Z")
def test_dcterms_modified_has_the_required_format(self):
import re
meta = [
item
for item in self.opf.must_find("metadata").must_find_all("meta")
if item.attr("property") == "dcterms:modified"
]
self.assertEqual(len(meta), 1)
value = meta[0].all_text()
self.assertRegex(value, r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
self.assertEqual(value, "2026-11-15T12:30:45Z")
def test_absent_metadata_is_omitted_rather_than_emptied(self):
data = epub.build_epub(make_book(author=None, publisher=None, source_url=None))
opf = parse_xml(zipfile.ZipFile(BytesIO(data)).read(epub.OPF_PATH))
metadata = opf.must_find("metadata")
self.assertIsNone(metadata.find("creator"))
self.assertIsNone(metadata.find("publisher"))
self.assertIsNone(metadata.find("source"))
def test_navigation_document_is_marked(self):
nav = self.manifest()["nav"]
self.assertEqual(nav.must_attr("href"), "nav.xhtml")
self.assertEqual(nav.must_attr("properties"), "nav")
def test_spine_toc_points_at_the_ncx(self):
self.assertEqual(self.opf.must_find("spine").must_attr("toc"), "ncx")
def test_every_manifest_href_exists_in_the_zip(self):
for identifier, item in self.manifest().items():
with self.subTest(item=identifier):
self.assertIn("OEBPS/" + item.must_attr("href"), self.archive.namelist())
def test_every_spine_idref_resolves(self):
manifest = self.manifest()
for itemref in self.opf.must_find("spine").must_find_all("itemref"):
self.assertIn(itemref.must_attr("idref"), manifest)
def test_every_manifest_item_has_a_media_type(self):
for identifier, item in self.manifest().items():
with self.subTest(item=identifier):
self.assertTrue(item.must_attr("media-type"))
def test_guide_points_at_the_cover_page(self):
reference = self.opf.must_find("guide").must_find("reference")
self.assertEqual(reference.must_attr("type"), "cover")
self.assertEqual(reference.must_attr("href"), "text/cover.xhtml")
def test_ncx_has_matching_uid_and_navpoints(self):
ncx = parse_xml(self.archive.read(epub.NCX_PATH))
self.assertEqual(ncx.must_attr("version"), "2005-1")
uid = [
m for m in ncx.must_find("head").must_find_all("meta") if m.attr("name") == "dtb:uid"
]
self.assertEqual(uid[0].must_attr("content"), make_book().identifier)
self.assertEqual(len(ncx.must_find("navMap").must_find_all("navPoint")), 2)
class EpubContentTests(unittest.TestCase):
def build(self, **overrides) -> tuple[zipfile.ZipFile, bytes]:
data = epub.build_epub(make_book(**overrides))
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
return archive, data
def test_article_document_contains_the_body_and_title(self):
archive, _ = self.build()
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("Body text.", text)
self.assertIn("A Test Article &amp; Its Friends", text)
def test_article_document_carries_a_byline_and_date(self):
archive, _ = self.build()
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("A. Writer", text)
self.assertIn("2026-11-14", text)
def test_source_link_can_be_suppressed(self):
archive, _ = self.build(include_source_link=False)
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertNotIn("Originally published", text)
def test_empty_body_still_produces_a_readable_document(self):
archive, _ = self.build(content_xhtml="")
parse_xml(archive.read(epub.ARTICLE_PATH))
self.assertIn("no content", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
def test_title_with_markup_is_escaped(self):
archive, _ = self.build(title='<script>alert("x")</script>')
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertNotIn("<script>", text)
parse_xml(archive.read(epub.ARTICLE_PATH))
def test_stylesheet_is_embedded(self):
archive, _ = self.build(stylesheet="body { color: red; }")
self.assertEqual(archive.read(epub.CSS_PATH).decode("utf-8"), "body { color: red; }")
def test_default_stylesheet_is_not_empty(self):
archive, _ = self.build()
self.assertIn("line-height", archive.read(epub.CSS_PATH).decode("utf-8"))
class EpubImageTests(unittest.TestCase):
def test_images_are_embedded_and_declared(self):
cover = epub.BookImage(
href="images/img-1.png", media_type="image/png", data=png_bytes(800, 400)
)
data = epub.build_epub(make_book(images=[cover]))
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
self.assertIn("OEBPS/images/img-1.png", archive.namelist())
opf = parse_xml(archive.read(epub.OPF_PATH))
item = opf.must_find("manifest").must_find("item")
self.assertIsNotNone(item)
def test_cover_image_is_marked_and_referenced(self):
cover = epub.BookImage(
href="images/img-1.png",
media_type="image/png",
data=png_bytes(800, 400),
is_cover=True,
)
data = epub.build_epub(make_book(images=[cover]))
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
opf = parse_xml(archive.read(epub.OPF_PATH))
covers = [
item
for item in opf.must_find("manifest").must_find_all("item")
if item.attr("properties") == "cover-image"
]
self.assertEqual(len(covers), 1)
self.assertEqual(covers[0].must_attr("id"), "cover-image")
# The EPUB 2 spelling for older readers.
metas = [
m for m in opf.must_find("metadata").must_find_all("meta") if m.attr("name") == "cover"
]
self.assertEqual(metas[0].must_attr("content"), "cover-image")
# And the cover page shows it, with a path relative to text/.
self.assertIn("../images/img-1.png", archive.read(epub.COVER_PATH).decode("utf-8"))
def test_cover_image_is_not_promoted_without_a_real_cover(self):
small = epub.BookImage(
href="images/img-1.png", media_type="image/png", data=png_bytes(16, 16)
)
book = make_book(images=[small])
_choose = getattr(epub, "_choose_cover")
_choose(book)
self.assertIsNone(book.cover_image)
def test_deterministic_output_for_identical_input(self):
left = epub.build_epub(make_book())
right = epub.build_epub(make_book())
self.assertEqual(left, right)
class BuildArticleEpubTests(unittest.TestCase):
"""The end-to-end assembly step, with and without an image fetcher."""
class Item:
def __init__(self, content: str, title: str = "An Article") -> None:
self.id = "tag:google.com,2005:reader/item/0000000000000001"
self.title = title
self.content_html = content
self.author = "A. Writer"
self.url = "https://example.com/a/1"
self.feed_title = "Example Tech"
self.published = 1_700_000_000
self.updated = 1_700_000_000
self.published_at = datetime.fromtimestamp(1_700_000_000, tz=UTC)
self.updated_at = datetime.fromtimestamp(1_700_000_000, tz=UTC)
class Config:
include_images = True
max_images_per_article = 20
cover_image = True
include_source_link = True
stylesheet = None
max_epub_bytes = 33_554_432
class Catalog:
language = "en"
author = "FreshRSS"
publisher = "Example Press"
class FakeFetcher:
"""Returns one PNG for every URL, recording what it was asked for."""
def __init__(self, fail: bool = False) -> None:
self.asked: list[str] = []
self.fail = fail
def fetch_all(self, urls):
self.asked.extend(urls)
if self.fail:
return {url: None for url in urls}
return {
url: FetchedImage(url=url, media_type="image/png", data=png_bytes(800, 400))
for url in urls
}
def build(self, **overrides) -> tuple[zipfile.ZipFile, bytes]:
item = overrides.pop("item", self.Item("<p>Hello</p>"))
config = overrides.pop("config", self.Config())
fetcher = overrides.pop("fetcher", None)
data = epub.build_article_epub(
item, epub_config=config, catalog_config=self.Catalog(), fetcher=fetcher
)
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
return archive, data
def test_book_is_valid_without_a_fetcher(self):
archive, _ = self.build()
parse_xml(archive.read(epub.OPF_PATH))
self.assertIn("Hello", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
def test_article_identifier_is_stable(self):
_, first = self.build()
_, second = self.build()
self.assertEqual(first, second)
def test_remote_images_are_embedded_when_a_fetcher_is_supplied(self):
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
fetcher = self.FakeFetcher()
archive, _ = self.build(item=item, fetcher=fetcher)
self.assertEqual(fetcher.asked, ["https://cdn.example.com/a.png"])
self.assertIn("OEBPS/images/img-1.png", archive.namelist())
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("../images/img-1.png", article)
self.assertNotIn("https://cdn.example.com", article)
def test_unfetchable_images_are_dropped_from_the_body(self):
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
archive, _ = self.build(item=item, fetcher=self.FakeFetcher(fail=True))
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertNotIn("<img", article)
self.assertNotIn("cdn.example.com", article)
def test_lead_image_becomes_the_cover(self):
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
archive, _ = self.build(item=item, fetcher=self.FakeFetcher())
opf = parse_xml(archive.read(epub.OPF_PATH))
covers = [
item
for item in opf.must_find("manifest").must_find_all("item")
if item.attr("properties") == "cover-image"
]
self.assertEqual(len(covers), 1)
def test_images_are_left_remote_when_inlining_is_off(self):
class NoImages(self.Config):
include_images = False
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
archive, _ = self.build(item=item, config=NoImages(), fetcher=self.FakeFetcher())
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("https://cdn.example.com/a.png", article)
def test_image_count_is_capped(self):
class OneImage(self.Config):
max_images_per_article = 1
item = self.Item(
'<img src="https://cdn.example.com/a.png"><img src="https://cdn.example.com/b.png">'
)
fetcher = self.FakeFetcher()
archive, _ = self.build(item=item, config=OneImage(), fetcher=fetcher)
self.assertEqual(len(fetcher.asked), 1)
self.assertEqual(len(archive.namelist()), 9)
def test_oversized_book_is_refused(self):
class TinyLimit(self.Config):
max_epub_bytes = 1024
with self.assertRaises(epub.EpubTooLargeError):
self.build(config=TinyLimit(), fetcher=self.FakeFetcher())
def test_stylesheet_path_falls_back_when_unreadable(self):
class BadStylesheet(self.Config):
stylesheet = "/nonexistent/stylesheet.css"
archive, _ = self.build(config=BadStylesheet())
self.assertIn("line-height", archive.read(epub.CSS_PATH).decode("utf-8"))
def test_sanitised_body_has_no_scripts(self):
item = self.Item("<p>ok</p><script>alert(1)</script>")
archive, _ = self.build(item=item)
self.assertNotIn("alert", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
def test_parsed_content_respects_the_byte_cap(self):
item = self.Item("<p>" + "word " * 20_000 + "</p>")
data = epub.build_article_epub(
item,
epub_config=self.Config(),
catalog_config=self.Catalog(),
max_content_bytes=4096,
)
self.assertLess(len(data), 100_000)
if __name__ == "__main__":
unittest.main()

414
tests/test_freshrss.py Normal file
View File

@@ -0,0 +1,414 @@
"""Tests for the FreshRSS client, against an in-process FreshRSS stand-in.
These assert on the wire format as much as on the parsed result: the query
parameters the client sends (``r=d``, ``n=``, ``xt=``, ``ck=``) and the form body
it posts for ``edit-tag`` are the parts that silently break against a real server,
so they are checked explicitly.
"""
from __future__ import annotations
import unittest
from datetime import UTC, datetime
from app.config import FreshRSSConfig
from app.freshrss import (
READ_TAG,
READING_LIST,
STARRED,
AuthenticationError,
FreshRSSClient,
FreshRSSError,
NotFoundError,
TransientError,
label_stream,
)
from tests.stub_freshrss import build_demo_catalog
TECH_FEED = "feed/https://example.com/tech/atom.xml"
NEWS_FEED = "feed/https://news.example.org/rss?source=daily"
SOLO_FEED = "feed/https://solo.example.net/feed.xml"
def make_config(stub, **overrides) -> FreshRSSConfig:
"""A client config aimed at the stub, with fast, non-retrying defaults."""
values = {
"api_base": stub.api_url,
"username": stub.username,
"api_password": stub.password,
"timeout_seconds": 5.0,
"retries": 0,
}
values.update(overrides)
return FreshRSSConfig(**values)
class ClientTestCase(unittest.TestCase):
"""Starts a fresh stub per test."""
def setUp(self):
self.stub = build_demo_catalog().start()
self.addCleanup(self.stub.stop)
def client(self, **overrides) -> FreshRSSClient:
client = FreshRSSClient(
make_config(self.stub, **overrides),
# Retry timing is not what these tests are about, and sleeping would
# only make the suite slow.
sleep=lambda _seconds: None,
)
self.addCleanup(client.close)
return client
class AuthenticationTests(ClientTestCase):
def test_login_posts_credentials_and_reads_the_auth_line(self):
self.client().list_subscriptions()
logins = self.stub.find_requests("/accounts/ClientLogin")
self.assertEqual(len(logins), 1)
self.assertEqual(logins[0]["method"], "POST")
self.assertIn("Email=tyler", logins[0]["body"])
self.assertIn("Passwd=api-password", logins[0]["body"])
def test_subsequent_calls_use_the_googlelogin_header(self):
client = self.client()
client.list_subscriptions()
client.list_subscriptions()
requests = self.stub.find_requests("/subscription/list")
self.assertEqual(len(requests), 2)
for entry in requests:
self.assertEqual(entry["authorization"], "GoogleLogin auth=stub-auth-token")
# The token is cached, so only one login happened.
self.assertEqual(len(self.stub.find_requests("/accounts/ClientLogin")), 1)
def test_bad_password_raises_authentication_error(self):
client = self.client(api_password="wrong")
with self.assertRaises(AuthenticationError):
client.list_subscriptions()
def test_credentials_are_form_encoded(self):
# A password with & or = must not corrupt the form body.
self.stub.password = "p&ss=word+plus"
client = self.client(api_password="p&ss=word+plus")
client.list_subscriptions()
body = self.stub.find_requests("/accounts/ClientLogin")[0]["body"]
self.assertIn("Passwd=p%26ss%3Dword%2Bplus", body)
class SubscriptionTests(ClientTestCase):
def test_subscriptions_are_parsed_with_categories(self):
feeds = {feed.id: feed for feed in self.client().list_subscriptions()}
self.assertEqual(len(feeds), 3)
self.assertEqual(feeds[TECH_FEED].title, "Example Tech")
self.assertEqual(feeds[TECH_FEED].categories, ["Tech"])
self.assertEqual(feeds[NEWS_FEED].categories, ["News"])
def test_feed_without_a_category_is_recognised(self):
feeds = {feed.id: feed for feed in self.client().list_subscriptions()}
self.assertEqual(feeds[SOLO_FEED].categories, [])
self.assertEqual(feeds[SOLO_FEED].primary_category, "Uncategorised")
def test_subscriptions_are_sorted_by_title(self):
titles = [feed.title for feed in self.client().list_subscriptions()]
self.assertEqual(titles, sorted(titles, key=str.casefold))
def test_query_asks_for_json(self):
self.client().list_subscriptions()
query = self.stub.find_requests("/subscription/list")[0]["query"]
self.assertEqual(query["output"], ["json"])
def test_non_json_response_is_reported_clearly(self):
self.stub.corrupt_json = True
with self.assertRaises(FreshRSSError) as caught:
self.client().list_subscriptions()
self.assertIn("not JSON", str(caught.exception))
def test_probe_reports_success_and_feed_count(self):
ok, detail = self.client().probe()
self.assertTrue(ok)
self.assertEqual(detail, "3 feeds")
def test_probe_reports_failure_without_raising(self):
ok, detail = self.client(api_password="wrong").probe()
self.assertFalse(ok)
self.assertTrue(detail)
class StreamTests(ClientTestCase):
def test_stream_query_parameters(self):
self.client().stream_items(TECH_FEED, count=7, exclude_read=True)
request = self.stub.find_requests("/stream/contents/")[0]
self.assertEqual(request["query"]["output"], ["json"])
self.assertEqual(request["query"]["r"], ["d"])
self.assertEqual(request["query"]["n"], ["7"])
self.assertEqual(request["query"]["xt"], [READ_TAG])
self.assertTrue(request["query"]["ck"][0].isdigit())
def test_stream_id_is_path_encoded(self):
self.client().stream_items(NEWS_FEED, count=1)
raw_path = self.stub.find_requests("/stream/contents/")[0]["raw_path"]
# The feed URL contains "?" and "/" which must not leak into the path.
self.assertNotIn("?", raw_path.split("?")[0].split("/api/greader.php")[1])
self.assertIn("%3F", raw_path)
def test_unread_filter_is_applied_upstream(self):
client = self.client()
everything = client.stream_items(TECH_FEED, count=50)
unread = client.stream_items(TECH_FEED, count=50, exclude_read=True)
self.assertEqual(len(everything), 3)
self.assertEqual(len(unread), 2)
self.assertNotIn("Already Read Post", [item.title for item in unread])
def test_count_bounds_the_result(self):
items = self.client().stream_items(TECH_FEED, count=2)
self.assertEqual(len(items), 2)
def test_newest_first(self):
items = self.client().stream_items(TECH_FEED, count=10)
published = [item.published for item in items]
self.assertEqual(published, sorted(published, reverse=True))
def test_older_than_is_passed_through(self):
self.client().stream_items(TECH_FEED, count=5, older_than=1_699_500_000)
self.assertEqual(
self.stub.find_requests("/stream/contents/")[0]["query"]["ot"],
["1699500000"],
)
def test_label_stream_selects_label_articles(self):
items = self.client().stream_items(label_stream("News"), count=10)
self.assertEqual({item.feed_id for item in items}, {NEWS_FEED})
def test_reading_list_covers_every_feed(self):
items = self.client().stream_items(READING_LIST, count=50)
self.assertEqual({item.feed_id for item in items}, {TECH_FEED, NEWS_FEED, SOLO_FEED})
def test_starred_stream_returns_only_starred(self):
items = self.client().stream_items(STARRED, count=10)
self.assertEqual([item.title for item in items], ["Starred Investigation"])
self.assertTrue(items[0].is_starred)
def test_missing_stream_is_treated_as_empty(self):
# A feed unsubscribed upstream must not break the catalog.
self.stub.fail_next(404, path="/stream/contents/")
self.assertEqual(self.client().stream_items(TECH_FEED, count=5), [])
self.stub.fail_next(400, path="/stream/contents/")
self.assertEqual(self.client().stream_items(TECH_FEED, count=5), [])
class ItemParsingTests(ClientTestCase):
def items_by_title(self, stream=TECH_FEED) -> dict:
return {item.title: item for item in self.client().stream_items(stream, count=50)}
def test_full_content_is_preferred_over_the_summary(self):
self.stub.add_item(
TECH_FEED,
99,
"Full versus summary",
content="<p>FULL BODY</p>",
summary="<p>teaser</p>",
)
item = self.items_by_title()["Full versus summary"]
self.assertIn("FULL BODY", item.content_html)
def test_summary_is_used_when_there_is_no_content(self):
self.stub.add_item(
TECH_FEED,
98,
"Summary only",
summary="<p>only teaser</p>",
)
self.stub.items[TECH_FEED][-1].pop("content")
item = self.items_by_title()["Summary only"]
self.assertIn("only teaser", item.content_html)
def test_item_fields_are_parsed(self):
item = self.items_by_title()["A Tech Post & Its Friends"]
self.assertEqual(item.feed_id, TECH_FEED)
self.assertEqual(item.feed_title, "Example Tech")
self.assertEqual(item.author, "A. Writer")
self.assertEqual(item.url, "https://example.com/articles/1")
self.assertEqual(item.published, 1_700_000_100)
self.assertTrue(item.title.startswith("A Tech Post"))
def test_missing_author_is_allowed(self):
item = self.items_by_title(NEWS_FEED)["Markets Dip Slightly"]
self.assertIsNone(item.author)
def test_read_state_is_parsed(self):
items = self.items_by_title()
self.assertTrue(items["Already Read Post"].is_read)
self.assertFalse(items["A Tech Post & Its Friends"].is_read)
def test_kept_unread_overrides_the_read_tag(self):
# FreshRSS keeps both tags when a read item is explicitly marked unread.
item = self.items_by_title()["Already Read Post"]
item.categories.append("user/-/state/com.google/kept-unread")
self.assertFalse(item.is_read)
def test_malformed_entries_are_skipped(self):
self.stub.inject_raw_item(TECH_FEED, {"title": "no id"})
self.stub.inject_raw_item(TECH_FEED, "not even an object")
titles = [item.title for item in self.client().stream_items(TECH_FEED, count=50)]
self.assertNotIn("no id", titles)
self.assertEqual(len(titles), 3)
def test_untitled_item_gets_a_placeholder(self):
self.stub.add_item(TECH_FEED, 97, "", content="<p>x</p>")
titles = [item.title for item in self.client().stream_items(TECH_FEED, count=50)]
self.assertIn("(untitled)", titles)
def test_timestamps_are_utc_datetimes(self):
item = self.items_by_title()["A Tech Post & Its Friends"]
self.assertEqual(item.published_at, datetime(2023, 11, 14, 22, 15, 0, tzinfo=UTC))
self.assertEqual(item.published_at.tzinfo, UTC)
def test_broken_timestamp_falls_back_to_the_epoch(self):
# Every date field has to be unusable for the epoch fallback to apply;
# while `updated` survives, using it is the correct behaviour.
self.stub.add_item(TECH_FEED, 96, "Broken date", content="<p>x</p>")
entry = self.stub.items[TECH_FEED][-1]
for field in ("published", "updated", "timestampUsec"):
entry.pop(field, None)
item = self.items_by_title()["Broken date"]
self.assertEqual(item.published, 0)
self.assertEqual(item.published_at, datetime(1970, 1, 1, tzinfo=UTC))
def test_unusable_publication_date_falls_back_to_updated(self):
self.stub.add_item(TECH_FEED, 94, "Only updated", content="<p>x</p>")
entry = self.stub.items[TECH_FEED][-1]
entry["published"] = "not-a-number"
entry.pop("timestampUsec", None)
entry["updated"] = 1_600_000_000
item = self.items_by_title()["Only updated"]
self.assertEqual(item.published_at, datetime(2020, 9, 13, 12, 26, 40, tzinfo=UTC))
def test_timestamp_usec_is_used_when_published_is_absent(self):
self.stub.add_item(TECH_FEED, 95, "Usec only", content="<p>x</p>")
entry = self.stub.items[TECH_FEED][-1]
entry.pop("published")
entry["timestampUsec"] = "1699999999000000"
item = self.items_by_title()["Usec only"]
self.assertEqual(item.published, 1_699_999_999)
class UnreadCountTests(ClientTestCase):
def test_unread_counts_are_parsed(self):
counts = self.client().unread_counts()
self.assertEqual(counts[TECH_FEED], 2)
self.assertEqual(counts[NEWS_FEED], 2)
def test_unsupported_counts_raise_not_found(self):
# The caller latches this and carries on without counts.
self.stub.supports_unread_counts = False
with self.assertRaises(NotFoundError):
self.client().unread_counts()
class MarkReadTests(ClientTestCase):
def test_mark_read_fetches_the_write_token_and_posts_ids(self):
client = self.client()
identifier = self.stub.items[TECH_FEED][0]["id"]
self.assertEqual(client.mark_read([identifier]), 1)
token_requests = self.stub.find_requests("/reader/api/0/token")
self.assertEqual(len(token_requests), 1)
edits = self.stub.find_requests("/reader/api/0/edit-tag")
self.assertEqual(len(edits), 1)
self.assertEqual(edits[0]["method"], "POST")
body = edits[0]["body"]
self.assertIn("T=stub-write-token", body)
self.assertIn("a=user%2F-%2Fstate%2Fcom.google%2Fread", body)
self.assertIn("i=tag%3Agoogle.com%2C2005%3Areader%2Fitem", body)
self.assertIn(identifier, self.stub.read)
def test_mark_read_sends_one_i_parameter_per_id(self):
ids = [entry["id"] for entry in self.stub.items[TECH_FEED]]
self.assertEqual(self.client().mark_read(ids), 3)
body = self.stub.find_requests("/reader/api/0/edit-tag")[0]["body"]
self.assertEqual(body.count("&i="), 3)
def test_item_id_survives_form_encoding(self):
identifier = "tag:google.com,2005:reader/item/00000000000000ff"
self.client().mark_read([identifier])
from urllib.parse import parse_qs
body = self.stub.find_requests("/reader/api/0/edit-tag")[0]["body"]
self.assertEqual(parse_qs(body)["i"], [identifier])
def test_mark_read_is_idempotent(self):
client = self.client()
identifier = self.stub.items[TECH_FEED][0]["id"]
client.mark_read([identifier])
client.mark_read([identifier])
self.assertIn(identifier, self.stub.read)
self.assertEqual(len(self.stub.find_requests("/reader/api/0/edit-tag")), 2)
def test_mark_read_with_no_ids_makes_no_requests(self):
self.assertEqual(self.client().mark_read([]), 0)
self.assertEqual(self.stub.requests, [])
def test_stale_write_token_is_refreshed_and_retried(self):
client = self.client()
identifier = self.stub.items[TECH_FEED][0]["id"]
# Warm the token cache, then rotate the token upstream so the cached one
# is rejected. The client must re-authenticate and retry rather than lose
# the mark-read.
client.mark_read([identifier])
self.stub.write_token = "rotated-token"
self.stub.clear_read()
client.mark_read([identifier])
self.assertIn(identifier, self.stub.read)
self.assertEqual(len(self.stub.find_requests("/reader/api/0/edit-tag")), 3)
def test_marking_read_removes_the_item_from_the_unread_stream(self):
client = self.client()
identifier = self.stub.items[TECH_FEED][0]["id"]
client.mark_read([identifier])
unread = client.stream_items(TECH_FEED, count=50, exclude_read=True)
self.assertNotIn(identifier, [item.id for item in unread])
class FailureHandlingTests(ClientTestCase):
def test_transient_status_is_retried(self):
client = self.client(retries=2)
self.stub.fail_next(503, path="/subscription/list")
feeds = client.list_subscriptions()
self.assertEqual(len(feeds), 3)
self.assertEqual(len(self.stub.find_requests("/subscription/list")), 2)
def test_retries_are_bounded(self):
client = self.client(retries=1)
self.stub.fail_next(503, count=5, path="/subscription/list")
with self.assertRaises(TransientError):
client.list_subscriptions()
self.assertEqual(len(self.stub.find_requests("/subscription/list")), 2)
def test_server_error_is_not_swallowed(self):
self.stub.fail_next(500, path="/subscription/list")
with self.assertRaises(FreshRSSError) as caught:
self.client(retries=0).list_subscriptions()
self.assertEqual(caught.exception.status, 500)
def test_unreachable_upstream_raises_transient_error(self):
# Bind a port, then close it, so nothing is listening.
import socket
probe = socket.socket()
probe.bind(("127.0.0.1", 0))
port = probe.getsockname()[1]
probe.close()
config = make_config(self.stub, api_base=f"http://127.0.0.1:{port}/api/greader.php")
with self.assertRaises(TransientError):
FreshRSSClient(config).list_subscriptions()
def test_error_body_is_included_in_the_message(self):
self.stub.fail_next(500, path="/subscription/list")
with self.assertRaises(FreshRSSError) as caught:
self.client(retries=0).list_subscriptions()
self.assertIn("injected failure", str(caught.exception))
if __name__ == "__main__":
unittest.main()

543
tests/test_server.py Normal file
View File

@@ -0,0 +1,543 @@
"""End-to-end tests: a real server, talking to a real (stand-in) FreshRSS.
These run the whole stack rather than units of it: the configuration is loaded
through its validation path, the HTTP server serves on a socket, the FreshRSS
client speaks HTTP to the stub, and the EPUB that comes back is opened and
checked. The claim that matters most -- that downloading a book marks the
matching article read upstream, and only a completed download does -- can only be
verified here, by looking at what the stub received.
Image embedding is turned off in this configuration so the suite stays offline;
image handling has its own tests with a stand-in fetcher.
"""
from __future__ import annotations
import base64
import json
import tempfile
import threading
import time
import unittest
import urllib.error
import urllib.parse
import urllib.request
import zipfile
from io import BytesIO
from pathlib import Path
from app import server as app_server
from app.config import load_config, password_digest
from app.epub import ARTICLE_PATH, OPF_PATH
from tests.stub_freshrss import READING_LIST, build_demo_catalog, item_id
from tests.support import parse_xml
class _NoRedirects(urllib.request.HTTPRedirectHandler):
"""Surface a redirect as its own status instead of following it.
urlopen follows redirects transparently, so a test that asserts on a 302 would
otherwise be looking at the status of whatever the redirect pointed at.
"""
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
USERNAME = "reader"
PASSWORD = "reader-password"
TECH_FEED = "feed/https://example.com/tech/atom.xml"
class ServerTestCase(unittest.TestCase):
"""Starts a stub FreshRSS and the real server for each test."""
include_images = False
def setUp(self):
self.stub = build_demo_catalog().start()
self.addCleanup(self.stub.stop)
# The demo account ships one already-read article, so assertions about
# marking read are written against the change, not the absolute state.
self.baseline_read = set(self.stub.read)
self._dir = tempfile.TemporaryDirectory()
self.addCleanup(self._dir.cleanup)
self._no_redirects = urllib.request.build_opener(_NoRedirects)
self.config = self._write_config()
self.httpd, self.application = app_server.build_server(self.config)
self.port = self.httpd.server_address[1]
self.thread = threading.Thread(
target=self.httpd.serve_forever, kwargs={"poll_interval": 0.02}, daemon=True
)
self.thread.start()
self.addCleanup(self._stop_server)
def _stop_server(self):
self.httpd.shutdown()
self.httpd.server_close()
self.thread.join(timeout=5)
@property
def base(self) -> str:
return f"http://127.0.0.1:{self.port}"
def _write_config(self):
document = {
"server": {"host": "127.0.0.1", "port": 0, "log_level": "critical"},
"freshrss": {
"url": self.stub.api_url,
"username": self.stub.username,
"api_password": self.stub.password,
"items_per_page": 2,
"timeout_seconds": 5,
"retries": 0,
},
"catalog": {"title": "Test Catalog", "author": "Tester"},
"epub": {"include_images": self.include_images},
"auth": {
"users": [{"username": USERNAME, "password_sha256": password_digest(PASSWORD)}]
},
"cache": {"subscriptions_ttl_seconds": 1, "stream_ttl_seconds": 1},
}
path = Path(self._dir.name) / "config.json"
path.write_text(json.dumps(document), encoding="utf-8")
return load_config(str(path))
# -- HTTP helpers ----------------------------------------------------
def request(self, path, *, auth=True, method="GET", headers=None, follow_redirects=True):
request = urllib.request.Request(self.base + path, method=method)
if auth:
token = base64.b64encode(f"{USERNAME}:{PASSWORD}".encode()).decode()
request.add_header("Authorization", f"Basic {token}")
for key, value in (headers or {}).items():
request.add_header(key, value)
opener = urllib.request.urlopen if follow_redirects else self._no_redirects.open
try:
with opener(request, timeout=10) as response:
return response.status, dict(response.headers), response.read()
except urllib.error.HTTPError as error:
# With redirects suppressed, a 3xx arrives here, which is the point.
# The error object owns the response socket, so it is closed rather
# than left for the garbage collector.
try:
body = error.read()
finally:
error.close()
return error.code, dict(error.headers), body
def wait_for_mark_read(self, expected: set[str], timeout: float = 5.0) -> set[str]:
"""Wait for a mark-read to land upstream (returns what landed).
The server marks an article read only after the body has been delivered,
so from the client's side that upstream effect is asynchronous: a test
asserting immediately would be racing the handler thread. It is
deliberately not the other way round -- marking before delivery would
consume articles whose transfer failed.
"""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
landed = self.stub.read - self.baseline_read
if landed >= expected:
return landed
time.sleep(0.02)
return self.stub.read - self.baseline_read
def get_xml(self, path, *, auth=True):
status, headers, body = self.request(path, auth=auth)
self.assertEqual(status, 200, body[:400])
node = parse_xml(body)
return node, headers
def entries(self, node):
return {entry.must_find("title").all_text(): entry for entry in node.must_find_all("entry")}
def follow(self, entry, rel: str) -> str:
for link in entry.must_find_all("link"):
if link.attr("rel") == rel:
return link.must_attr("href")
raise AssertionError(f"no {rel!r} link in entry")
def acquisition_path(self, section="unread", label="Tech", feed_title="Example Tech") -> str:
"""Walk the catalog down to one feed's acquisition feed.
Shared by the suite rather than owned by one test class: the download,
pagination, search and serialisation tests all start from a feed.
"""
node, _ = self.get_xml(f"/opds/{section}")
category = self.follow(self.entries(node)[label], "subsection")
node, _ = self.get_xml(urllib.parse.urlsplit(category).path)
return urllib.parse.urlsplit(self.follow(self.entries(node)[feed_title], "subsection")).path
def download_path(self, title="A Tech Post & Its Friends", section="unread") -> str:
"""The EPUB acquisition URL for one article."""
node, _ = self.get_xml(self.acquisition_path(section=section))
entry = self.entries(node)[title]
return urllib.parse.urlsplit(
self.follow(entry, "http://opds-spec.org/acquisition/open-access")
).path
class AuthenticationTests(ServerTestCase):
def test_catalog_requires_credentials(self):
status, headers, _body = self.request("/opds", auth=False)
self.assertEqual(status, 401)
self.assertIn("Basic", headers.get("WWW-Authenticate", ""))
self.assertIn("Test" if False else "OPDS", headers.get("WWW-Authenticate", ""))
def test_wrong_password_is_rejected(self):
token = base64.b64encode(f"{USERNAME}:wrong".encode()).decode()
status, _headers, _body = self.request(
"/opds", auth=False, headers={"Authorization": f"Basic {token}"}
)
self.assertEqual(status, 401)
def test_health_needs_no_credentials_and_touches_nothing(self):
status, _headers, body = self.request("/healthz", auth=False)
self.assertEqual(status, 200)
self.assertEqual(json.loads(body), {"status": "ok"})
self.assertEqual(self.stub.requests, [])
def test_malformed_authorization_header_is_rejected(self):
for value in ("Basic", "Bearer xyz", "Basic !!!not-base64", "Basic " + "aGk="):
with self.subTest(value=value):
status, _headers, _body = self.request(
"/opds", auth=False, headers={"Authorization": value}
)
self.assertEqual(status, 401)
def test_diagnostics_requires_credentials(self):
self.assertEqual(self.request("/opds/healthz", auth=False)[0], 401)
def test_diagnostics_reports_upstream_state(self):
status, _headers, body = self.request("/opds/healthz")
self.assertEqual(status, 200)
payload = json.loads(body)
self.assertTrue(payload["upstream"]["reachable"])
self.assertEqual(payload["catalog"]["feeds"], 3)
self.assertTrue(payload["mark_read"])
class NavigationTests(ServerTestCase):
def test_bare_root_redirects_to_the_catalog(self):
# Redirects must be suppressed, or this would see the 401 that /opds
# answers an unauthenticated follower with.
status, headers, _body = self.request("/", auth=False, follow_redirects=False)
self.assertEqual(status, 302)
self.assertTrue(headers["Location"].endswith("/opds"))
def test_root_lists_the_sections(self):
node, headers = self.get_xml("/opds")
self.assertEqual(node.must_find("title").all_text(), "Test Catalog")
titles = set(self.entries(node))
self.assertIn("Unread", titles)
self.assertIn("Read", titles)
self.assertIn("Starred", titles)
def test_navigation_feed_declares_its_profile(self):
_node, headers = self.get_xml("/opds")
self.assertEqual(
headers["Content-Type"],
"application/atom+xml;profile=opds-catalog;kind=navigation",
)
def test_root_offers_a_search_link(self):
node, _headers = self.get_xml("/opds")
rels = {link.attr("rel") for link in node.must_find_all("link")}
self.assertIn("search", rels)
self.assertIn("self", rels)
def test_section_lists_categories_with_unread_counts(self):
node, _headers = self.get_xml("/opds/unread")
entries = self.entries(node)
self.assertIn("Tech", entries)
self.assertIn("News", entries)
self.assertIn("Uncategorised", entries)
self.assertIn("unread", entries["Tech"].must_find("content").all_text())
def test_category_lists_its_feeds(self):
category = self.follow(self.entries(self.get_xml("/opds/unread")[0])["Tech"], "subsection")
node, _headers = self.get_xml(urllib.parse.urlsplit(category).path)
titles = set(self.entries(node))
self.assertIn("Example Tech", titles)
self.assertIn("All in this category", titles)
def test_unknown_path_is_404(self):
self.assertEqual(self.request("/opds/nope")[0], 404)
def test_undecodable_identifier_is_404(self):
self.assertEqual(self.request("/opds/unread/c/!!!not-base64")[0], 404)
def test_post_is_not_allowed(self):
self.assertEqual(self.request("/opds", method="POST")[0], 405)
def test_opensearch_descriptor_is_served(self):
node, headers = self.get_xml("/opds/opensearch.xml")
self.assertEqual(headers["Content-Type"], "application/opensearchdescription+xml")
url = node.must_find("Url")
self.assertIn("searchTerms", url.must_attr("template"))
class AcquisitionTests(ServerTestCase):
def test_acquisition_feed_lists_articles(self):
node, headers = self.get_xml(self.acquisition_path())
self.assertEqual(
headers["Content-Type"],
"application/atom+xml;profile=opds-catalog;kind=acquisition",
)
titles = set(self.entries(node))
self.assertIn("A Tech Post & Its Friends", titles)
def test_read_articles_are_absent_from_the_unread_feed(self):
titles = set(self.entries(self.get_xml(self.acquisition_path())[0]))
self.assertNotIn("Already Read Post", titles)
def test_read_section_contains_read_articles(self):
path = self.acquisition_path(section="read")
titles = set(self.entries(self.get_xml(path)[0]))
self.assertIn("Already Read Post", titles)
def test_entry_offers_an_epub_acquisition_link(self):
node, _headers = self.get_xml(self.acquisition_path())
entry = self.entries(node)["A Tech Post & Its Friends"]
href = self.follow(entry, "http://opds-spec.org/acquisition/open-access")
self.assertTrue(href.endswith(".epub"))
types = {link.attr("type") for link in entry.must_find_all("link")}
self.assertIn("application/epub+zip", types)
# The generic relation is offered too, because readers disagree.
self.assertIn(
"http://opds-spec.org/acquisition",
{link.attr("rel") for link in entry.must_find_all("link")},
)
def test_entry_links_back_to_the_original_article(self):
node, _headers = self.get_xml(self.acquisition_path())
entry = self.entries(node)["A Tech Post & Its Friends"]
self.assertTrue(self.follow(entry, "alternate").startswith("https://"))
def test_entry_carries_its_feed_as_a_category(self):
node, _headers = self.get_xml(self.acquisition_path())
entry = self.entries(node)["A Tech Post & Its Friends"]
labels = {category.attr("label") for category in entry.must_find_all("category")}
self.assertIn("Example Tech", labels)
def test_articles_are_newest_first(self):
node, _headers = self.get_xml(self.acquisition_path())
stamps = [entry.must_find("updated").all_text() for entry in node.must_find_all("entry")]
self.assertEqual(stamps, sorted(stamps, reverse=True))
def test_pagination_link_appears_when_there_is_more(self):
# The test configuration serves two items per page, and the reading list
# holds five unread articles.
node, _headers = self.get_xml("/opds/unread/all")
rels = {link.attr("rel") for link in node.must_find_all("link")}
self.assertIn("next", rels)
self.assertIn("first", rels)
self.assertNotIn("previous", rels)
def test_second_page_is_reachable_and_different(self):
path = "/opds/unread/all"
first, _ = self.get_xml(path)
second, _ = self.get_xml(path + "?page=2")
first_ids = {entry.must_find("id").all_text() for entry in first.must_find_all("entry")}
second_ids = {entry.must_find("id").all_text() for entry in second.must_find_all("entry")}
self.assertTrue(second_ids)
self.assertFalse(first_ids & second_ids)
def test_starred_section_lists_starred_articles(self):
titles = set(self.entries(self.get_xml("/opds/starred")[0]))
self.assertIn("Starred Investigation", titles)
def test_search_finds_articles_in_the_window(self):
node, _headers = self.get_xml("/opds/search?q=Tech+Post")
titles = set(self.entries(node))
self.assertIn("A Tech Post & Its Friends", titles)
self.assertNotIn("Markets Dip Slightly", titles)
def test_empty_search_returns_an_empty_feed(self):
# A search with no term is a valid, empty acquisition feed rather than an
# error, so a reader that sends one sees an empty list.
node, _headers = self.get_xml("/opds/search?q=")
self.assertEqual(node.find_all("entry"), [])
def test_bad_page_number_falls_back_to_the_first_page(self):
node, _headers = self.get_xml(self.acquisition_path() + "?page=nonsense")
self.assertTrue(node.must_find_all("entry"))
class DownloadTests(ServerTestCase):
"""The claim under test: a completed download is what marks an article read."""
def test_download_returns_an_epub(self):
status, headers, body = self.request(self.download_path())
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "application/epub+zip")
disposition = headers["Content-Disposition"]
self.assertTrue(disposition.startswith("attachment;"))
# Both spellings are sent: a plain name every reader understands, and an
# RFC 5987 one so a non-ASCII title survives.
self.assertIn('filename="a-tech-post-its-friends.epub"', disposition)
self.assertIn("filename*=UTF-8''", disposition)
def test_downloaded_file_is_a_valid_epub(self):
_status, _headers, body = self.request(self.download_path())
archive = zipfile.ZipFile(BytesIO(body))
self.addCleanup(archive.close)
self.assertEqual(archive.namelist()[0], "mimetype")
self.assertEqual(archive.read("mimetype"), b"application/epub+zip")
self.assertEqual(archive.getinfo("mimetype").compress_type, zipfile.ZIP_STORED)
opf = parse_xml(archive.read(OPF_PATH))
self.assertEqual(opf.must_find("title").all_text(), "A Tech Post & Its Friends")
# The body went through the sanitiser on the way in.
article = archive.read(ARTICLE_PATH).decode("utf-8")
self.assertNotIn("alert(", article)
self.assertIn("Entities: &amp;", article)
def test_download_marks_the_article_read_upstream(self):
self.request(self.download_path())
self.assertEqual(self.wait_for_mark_read({item_id(1)}), {item_id(1)})
edits = self.stub.find_requests("/reader/api/0/edit-tag")
self.assertEqual(len(edits), 1)
self.assertIn("i=tag%3Agoogle.com", edits[0]["body"])
def test_downloads_mark_only_their_own_article(self):
self.request(self.download_path())
self.assertEqual(self.wait_for_mark_read({item_id(1)}), {item_id(1)})
def test_head_does_not_mark_read(self):
# A reader probing the URL, or a CDN, must not consume the article.
status, headers, body = self.request(self.download_path(), method="HEAD")
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "application/epub+zip")
self.assertEqual(body, b"")
# Deterministic rather than a race: a HEAD never reaches the mark-read.
self.assertEqual(self.stub.read - self.baseline_read, set())
self.assertEqual(self.stub.find_requests("/reader/api/0/edit-tag"), [])
def test_missing_article_is_404(self):
stream = base64.urlsafe_b64encode(TECH_FEED.encode()).decode().rstrip("=")
absent = (
base64.urlsafe_b64encode(b"tag:google.com,2005:reader/item/nope").decode().rstrip("=")
)
status, _headers, _body = self.request(f"/opds/d/{stream}/{absent}/nope.epub")
self.assertEqual(status, 404)
def test_article_id_is_encoded_in_the_url_and_survives(self):
# FreshRSS ids contain ':' and '/', so the URL must carry them encoded.
path = self.download_path()
self.assertNotIn("tag:google.com", path)
self.assertEqual(path.count("/"), 5)
def test_marked_read_article_leaves_the_unread_queue(self):
path = self.download_path()
self.request(path)
self.wait_for_mark_read({item_id(1)})
# The stream cache is invalidated by the mark-read, so the next browse
# reflects it without waiting for a TTL.
titles = set(self.entries(self.get_xml(self.acquisition_path())[0]))
self.assertNotIn("A Tech Post & Its Friends", titles)
def test_marked_read_article_appears_in_the_read_section(self):
self.request(self.download_path())
self.wait_for_mark_read({item_id(1)})
path = self.acquisition_path(section="read")
titles = set(self.entries(self.get_xml(path)[0]))
self.assertIn("A Tech Post & Its Friends", titles)
def test_mark_read_can_be_disabled(self):
self.application.config.freshrss.mark_read = False
self.request(self.download_path())
self.assertEqual(self.stub.read - self.baseline_read, set())
self.assertEqual(self.stub.find_requests("/reader/api/0/edit-tag"), [])
def test_advertised_unread_count_refreshes_after_a_download(self):
"""The count must not lag the queue it describes.
Unread counts come from the cached account snapshot, so a mark-read has to
drop that cache too -- otherwise a reader downloads an article and is still
told the old total, with one fewer entry underneath it.
"""
def advertised_unread() -> int:
node, _ = self.get_xml("/opds/unread")
for entry in node.must_find_all("entry"):
if entry.must_find("title").all_text() == "All feeds":
return int(entry.must_find("content").all_text().split()[0])
raise AssertionError("no 'All feeds' entry in the unread catalog")
before = advertised_unread()
self.request(self.download_path())
self.wait_for_mark_read({item_id(1)})
self.assertEqual(advertised_unread(), before - 1)
def test_second_download_is_still_served_after_the_queue_moved_on(self):
path = self.download_path()
self.request(path)
status, _headers, body = self.request(path)
self.assertEqual(status, 200)
self.assertTrue(body.startswith(b"PK"))
class UpstreamFailureTests(ServerTestCase):
def test_upstream_outage_is_a_bad_gateway(self):
self.stub.fail_next(503, count=10, path="/subscription/list")
status, _headers, _body = self.request("/opds")
self.assertEqual(status, 502)
def test_catalog_recovers_once_upstream_returns(self):
# Exactly one failure, so the next attempt succeeds.
self.stub.fail_next(503, path="/subscription/list")
self.assertEqual(self.request("/opds")[0], 502)
self.application.invalidate()
self.assertEqual(self.request("/opds")[0], 200)
def test_unread_counts_missing_is_not_fatal(self):
# The feature is optional; the catalog must still browse without it.
self.stub.supports_unread_counts = False
self.application.invalidate()
status, _headers, body = self.request("/opds/unread")
self.assertEqual(status, 200)
node = parse_xml(body)
titles = {entry.must_find("title").all_text() for entry in node.must_find_all("entry")}
self.assertIn("Tech", titles)
def test_unknown_host_is_refused_when_an_allowlist_is_set(self):
self.application.config.server.allowed_hosts = ["catalog.example.com"]
status, _headers, _body = self.request("/opds", auth=False)
self.assertEqual(status, 500)
def test_public_base_url_controls_the_links(self):
self.application.config.server.public_base_url = "https://catalog.example.com/books"
node, _headers = self.get_xml("/opds")
hrefs = {link.must_attr("href") for link in node.must_find_all("link")}
self.assertTrue(
any(href.startswith("https://catalog.example.com/books/opds") for href in hrefs)
)
class SerializationTests(ServerTestCase):
"""Responses must be correct XML even when titles contain markup."""
def test_titles_with_markup_are_escaped(self):
self.stub.subscriptions[0]["title"] = 'Feed & <Friends> "quoted"'
self.application.invalidate()
category = urllib.parse.urlsplit(
self.follow(self.entries(self.get_xml("/opds/unread")[0])["Tech"], "subsection")
).path
node, _headers = self.get_xml(category)
titles = {entry.must_find("title").all_text() for entry in node.must_find_all("entry")}
self.assertIn('Feed & <Friends> "quoted"', titles)
def test_feed_ids_are_stable_across_requests(self):
first, _ = self.get_xml("/opds/unread")
second, _ = self.get_xml("/opds/unread")
self.assertEqual(first.must_find("id").all_text(), second.must_find("id").all_text())
def test_entry_ids_are_unique_within_a_feed(self):
node, _headers = self.get_xml(self.acquisition_path())
ids = [entry.must_find("id").all_text() for entry in node.must_find_all("entry")]
self.assertEqual(len(ids), len(set(ids)))
if __name__ == "__main__":
unittest.main()

399
tests/test_xhtml.py Normal file
View File

@@ -0,0 +1,399 @@
"""Tests for the HTML to XHTML normaliser.
The bar every case is held to: the output must parse with a strict XML parser,
because that is what an EPUB reader does with it.
"""
from __future__ import annotations
import unittest
from app.xhtml import parse_article
from tests.support import assert_well_formed, body, collapse, parse_xml
BASE = "https://example.com/posts/one?utm=x"
def render(raw: str, image_map=None, base: str = BASE, max_input_bytes: int = 0) -> str:
"""Parse and serialise, asserting well-formedness on the way out."""
article = parse_article(raw, base, max_input_bytes=max_input_bytes)
fragment = article.to_xhtml(image_map)
assert_well_formed(unittest.TestCase(), fragment)
return fragment
class MalformedInputTests(unittest.TestCase):
"""Feed HTML is broken in predictable ways; none of it may reach the output."""
def test_unclosed_paragraphs_are_balanced(self):
out = render("<p>one<p>two<p>three")
self.assertEqual(body(out).count("<p>"), 3)
self.assertEqual(body(out).count("</p>"), 3)
self.assertIn("three", out)
def test_block_element_closes_open_paragraph(self):
# A naive stack would nest the div inside the paragraph.
out = render("<p>lead<div>block</div>trailing")
self.assertIn("<p>lead</p>", out)
self.assertIn("<div>block</div>", out)
self.assertNotIn("<p>lead<div>", out)
def test_unclosed_list_items(self):
out = render("<ul><li>a<li>b<li>c</ul>")
self.assertEqual(body(out).count("<li>"), 3)
self.assertEqual(body(out).count("</li>"), 3)
def test_stray_end_tag_is_ignored(self):
out = render("<p>text</span></div></p>")
self.assertIn("<p>text</p>", out)
def test_unclosed_table_row_and_cells(self):
out = render("<table><tr><td>a<td>b<tr><td>c</table>")
self.assertEqual(body(out).count("<tr>"), 2)
self.assertEqual(body(out).count("<td>"), 3)
self.assertIn("<td>c</td>", out)
def test_self_closing_non_void_element_swallows_nothing(self):
# `<div/>` must not open a subtree that eats the rest of the document, and
# the empty element it leaves behind is noise worth dropping.
out = render("<div/>after")
self.assertIn("after", out)
self.assertEqual(body(out), "after")
def test_void_elements_are_self_closed(self):
out = render("a<br>b<hr>c")
self.assertIn("<br/>", out)
self.assertIn("<hr/>", out)
self.assertNotIn("<br>", out)
def test_control_characters_are_removed(self):
# NUL and vertical tab cannot be represented in XML even as entities.
out = render("<p>be\x00fore\x0baft\x0cft</p>")
self.assertNotIn("\x00", out)
self.assertNotIn("\x0b", out)
self.assertNotIn("\x0c", out)
self.assertIn("before", out)
def test_unpaired_surrogate_is_removed(self):
out = render("<p>ok\ud800tail</p>")
self.assertNotIn("\ud800", out)
self.assertIn("oktail", out)
def test_zero_width_characters_are_removed(self):
out = render("<p>a\u200bb\ufeffc</p>")
self.assertEqual(collapse(parse_xml(out).all_text()), "abc")
def test_comments_are_dropped(self):
out = render("<p>keep<!-- drop me --> this</p>")
self.assertNotIn("drop me", out)
def test_processing_instruction_and_doctype_are_dropped(self):
out = render("<?xml version='1.0'?><!DOCTYPE html><p>body</p>")
self.assertNotIn("DOCTYPE", out)
self.assertIn("body", out)
def test_deeply_nested_wrappers_keep_their_text(self):
out = render("<section><article><aside>deep</aside></article></section>")
self.assertIn("deep", out)
self.assertNotIn("<section>", out)
self.assertNotIn("<aside>", out)
class EscapingTests(unittest.TestCase):
"""Nothing in the output may be interpretable as markup or as an entity."""
def test_bare_ampersand_is_escaped(self):
out = render("<p>Tom & Jerry</p>")
self.assertIn("Tom &amp; Jerry", out)
def test_named_and_numeric_entities_are_decoded_then_reescaped(self):
out = render("<p>&nbsp;&#233;&amp;</p>")
# &nbsp; must not survive as an entity, and must not break the parse.
self.assertNotIn("&nbsp;", out)
self.assertIn("\u00a0", out)
self.assertIn("\u00e9", out)
def test_text_that_looks_like_xml_is_escaped(self):
out = render("<p>&lt;script&gt;alert(1)&lt;/script&gt;</p>")
self.assertIn("&lt;script&gt;", out)
self.assertEqual(body(out).count("<script>"), 0)
def test_quotes_in_attributes_are_escaped(self):
out = render('<p><a href="https://e.com/?q=&quot;x&quot;" title="a&quot;b">t</a></p>')
self.assertIn("&quot;", out)
def test_cdata_section_is_unwrapped_to_text(self):
out = render("<p><![CDATA[raw text]]></p>")
self.assertIn("raw text", out)
def test_gets_escaped_inside_attributes_only_once(self):
out = render('<p><a href="https://e.com/a?b=1&amp;c=2">t</a></p>')
self.assertIn('href="https://e.com/a?b=1&amp;c=2"', out)
class DangerousContentTests(unittest.TestCase):
"""Scriptable and resource-fetching constructs must not survive."""
def test_script_element_and_body_are_removed(self):
out = render("<p>before</p><script>alert('xss')</script><p>after</p>")
self.assertNotIn("script", out.lower())
self.assertIn("before", out)
self.assertIn("after", out)
def test_unclosed_script_does_not_swallow_forever(self):
out = render("<script>alert(1)</script><p>visible</p>")
self.assertIn("visible", out)
def test_style_element_is_removed(self):
out = render("<style>p{color:red}</style><p>text</p>")
self.assertNotIn("color:red", out)
def test_iframe_and_object_are_removed_with_contents(self):
out = render('<iframe src="https://e.com/x"></iframe><p>t</p>')
self.assertNotIn("iframe", out)
def test_style_attribute_is_dropped(self):
out = render('<p style="background:url(javascript:1)">t</p>')
self.assertNotIn("style", out)
def test_event_handler_attributes_are_dropped(self):
out = render('<p onclick="evil()" onmouseover="evil()">t</p>')
self.assertNotIn("onclick", out)
self.assertNotIn("onmouseover", out)
def test_class_attribute_is_dropped(self):
# class fights the book's own stylesheet and means nothing offline.
out = render('<p class="x">t</p>')
self.assertNotIn("class", body(out))
def test_id_is_kept_for_anchor_targets(self):
out = render('<p id="section-2">t</p>')
self.assertIn('id="section-2"', out)
def test_invalid_id_is_dropped(self):
# An id has to be a usable XML name; anything else is not an anchor. The
# quote case needs single-quoted markup, or the parser would read a
# shorter, valid value and the case would prove nothing.
cases = {
"leading digit": '<p id="9lives">t</p>',
"space": '<p id="has space">t</p>',
"slash": '<p id="sla/sh">t</p>',
"quote": "<p id='quote\"d'>t</p>",
"empty": '<p id="">t</p>',
"too long": f'<p id="{"a" * 100}">t</p>',
}
for label, markup in cases.items():
self.assertNotIn("id=", body(render(markup)), label)
def test_javascript_url_is_dropped(self):
out = render('<a href="javascript:alert(1)">click</a>')
self.assertNotIn("javascript", out)
self.assertIn("click", out)
def test_data_url_link_is_dropped(self):
out = render('<a href="data:text/html,<script>x</script>">t</a>')
self.assertNotIn("data:", out)
def test_data_url_image_is_dropped(self):
out = render('<img src="data:image/png;base64,AAAA"><p>t</p>')
self.assertNotIn("data:", out)
self.assertNotIn("<img", out)
def test_img_without_source_is_dropped(self):
out = render('<p><img alt="no source"></p>')
self.assertNotIn("<img", out)
def test_only_href_survives_on_links(self):
out = render('<a href="https://e.com/a" target="_blank" rel="noopener" download>x</a>')
self.assertIn('href="https://e.com/a"', out)
self.assertNotIn("target", out)
self.assertNotIn("rel=", out)
class UrlResolutionTests(unittest.TestCase):
"""Bodies are re-based into the book, so relative URLs must be resolved."""
def test_root_relative_link_uses_article_origin(self):
out = render('<a href="/about">about</a>')
self.assertIn('href="https://example.com/about"', out)
def test_document_relative_link_uses_article_directory(self):
out = render('<a href="sibling.html">s</a>')
self.assertIn('href="https://example.com/posts/sibling.html"', out)
def test_protocol_relative_link_is_resolved(self):
out = render('<a href="//cdn.example.com/x">x</a>')
self.assertIn('href="https://cdn.example.com/x"', out)
def test_mailto_link_is_kept(self):
out = render('<a href="mailto:someone@example.com">mail</a>')
self.assertIn('href="mailto:someone@example.com"', out)
def test_anchor_only_link_stays_in_document(self):
# A same-document link must not be rewritten to the source web page: the
# matching id is kept, so it jumps within the book instead.
out = render('<p id="section">target</p><a href="#section">jump</a>')
self.assertIn('href="#section"', out)
self.assertIn('id="section"', out)
def test_relative_image_is_absolutised(self):
article = parse_article('<img src="images/a.png">', BASE)
self.assertEqual(article.image_urls, ["https://example.com/posts/images/a.png"])
def test_blockquote_cite_is_validated(self):
out = render('<blockquote cite="javascript:x">q</blockquote>')
self.assertNotIn("cite", out)
out = render('<blockquote cite="https://e.com/src">q</blockquote>')
self.assertIn('cite="https://e.com/src"', out)
class AnchorIdTests(unittest.TestCase):
"""Ids survive so an article's own table of contents works offline."""
def test_duplicate_ids_are_renamed_not_dropped(self):
out = render('<p id="a">1</p><p id="a">2</p><p id="a">3</p>')
self.assertIn('id="a"', out)
self.assertIn('id="a-2"', out)
self.assertIn('id="a-3"', out)
def test_id_on_void_element_is_dropped(self):
out = render('<img id="x" src="a.png">')
self.assertNotIn('id="x"', out)
def test_renamed_ids_do_not_collide_with_existing_ones(self):
out = render('<p id="a">1</p><p id="a-2">2</p><p id="a">3</p>')
self.assertEqual(body(out).count('id="a-2"'), 1)
self.assertIn('id="a-3"', out)
def test_unique_ids_are_left_alone(self):
out = render('<p id="one">1</p><p id="two">2</p>')
self.assertIn('id="one"', out)
self.assertIn('id="two"', out)
def test_an_element_keeping_only_an_id_is_not_pruned(self):
out = render('<div id="anchor"></div><p>text</p>')
self.assertIn('id="anchor"', out)
class ImageHandlingTests(unittest.TestCase):
"""Images are collected for the fetcher and rewritten by the caller."""
def test_image_urls_are_collected_in_order_and_deduplicated(self):
article = parse_article('<p><img src="a.png"><img src="a.png"><img src="b.png"></p>', BASE)
self.assertEqual(
article.image_urls,
["https://example.com/posts/a.png", "https://example.com/posts/b.png"],
)
def test_remote_urls_are_kept_when_no_map_is_given(self):
out = render('<img src="a.png" alt="alt text">')
self.assertIn('src="https://example.com/posts/a.png"', out)
self.assertIn('alt="alt text"', out)
def test_map_rewrites_src_to_local_path(self):
url = "https://example.com/posts/a.png"
out = render('<img src="a.png">', image_map={url: "images/img-1.png"})
self.assertIn('src="images/img-1.png"', out)
def test_none_in_map_drops_image_and_empty_wrapper(self):
url = "https://example.com/posts/a.png"
out = render('<div><img src="a.png"></div><p>text</p>', image_map={url: None})
self.assertNotIn("<img", out)
self.assertNotIn("<div>", out)
self.assertIn("<p>text</p>", out)
def test_alt_and_title_are_preserved_for_localised_images(self):
url = "https://example.com/posts/a.png"
out = render('<img src="a.png" alt="A" title="T">', image_map={url: "images/i.png"})
self.assertIn('alt="A"', out)
self.assertIn('title="T"', out)
class WhitespaceAndStructureTests(unittest.TestCase):
def test_whitespace_is_collapsed_outside_pre(self):
out = render("<p>one\n\n two\t\tthree</p>")
self.assertIn("<p>one two three</p>", out)
def test_whitespace_is_preserved_inside_pre(self):
out = render("<pre>def f():\n return 1</pre>")
self.assertIn("def f():\n return 1", out)
def test_empty_paragraphs_are_pruned(self):
out = render("<p> </p><p>&nbsp;</p><p>real</p>")
self.assertEqual(body(out).count("<p>"), 1)
self.assertIn("real", out)
def test_empty_table_cells_are_kept(self):
# An empty cell is layout, not noise, so pruning must leave it alone.
out = render("<table><tr><td></td><td>b</td></tr></table>")
self.assertEqual(body(out).count("<td></td>"), 1)
def test_plain_text_body_is_wrapped_in_a_paragraph(self):
out = render("just words with no markup")
self.assertEqual(body(out), "<p>just words with no markup</p>")
def test_whitespace_only_body_renders_empty(self):
article = parse_article(" \n ", BASE)
self.assertEqual(article.to_xhtml(), "")
def test_empty_input_renders_empty(self):
self.assertEqual(parse_article("", BASE).to_xhtml(), "")
def test_heading_levels_survive(self):
out = render("<h1>a</h1><h2>b</h2><h3>c</h3>")
for tag in ("h1", "h2", "h3"):
self.assertIn(f"<{tag}>", out)
def test_colspan_is_clamped(self):
out = render('<td colspan="99999">x</td>')
self.assertNotIn("colspan", out)
out = render('<td colspan="2">x</td>')
self.assertIn('colspan="2"', out)
def test_element_structure_is_preserved(self):
out = render("<ul><li>a<ul><li>b</li></ul></li></ul>")
tree = parse_xml(out)
# The fragment's own root is the article wrapper.
self.assertEqual(tree.must_attr("class"), "article-body")
outer = tree.must_find("ul")
items = outer.must_find_all("li")
self.assertEqual(len(items), 2)
# The nested list belongs to the first item, not to the outer list.
self.assertTrue(items[0].find_all("ul"))
class ArticleApiTests(unittest.TestCase):
def test_text_extraction_flattens_and_collapses(self):
article = parse_article("<p>One</p> <p>Two &amp; three</p>", BASE)
self.assertEqual(article.text(), "One Two & three")
def test_text_extraction_is_limited_at_a_word_boundary(self):
article = parse_article("<p>alpha beta gamma delta</p>", BASE)
text = article.text(limit=12)
self.assertTrue(text.endswith("\u2026"))
self.assertLessEqual(len(text), 13)
self.assertIn("alpha beta", text)
def test_truncation_flag_and_output_validity(self):
article = parse_article("<p>" + "word " * 5000 + "</p>", BASE, max_input_bytes=2000)
self.assertTrue(article.truncated)
assert_well_formed(self, article.to_xhtml())
def test_no_truncation_when_under_limit(self):
article = parse_article("<p>short</p>", BASE, max_input_bytes=10_000)
self.assertFalse(article.truncated)
def test_output_is_deterministic(self):
raw = '<p>a &amp; b</p><div><img src="i.png"><ul><li>x</ul></div>'
self.assertEqual(render(raw), render(raw))
def test_image_urls_are_cached_between_calls(self):
article = parse_article('<img src="a.png">', BASE)
self.assertIs(article.image_urls, article.image_urls)
def test_output_wrapper_has_the_expected_class(self):
self.assertTrue(render("<p>x</p>").startswith('<div class="article-body">'))
if __name__ == "__main__":
unittest.main()