Initial commit

This commit is contained in:
2026-11-15 18:15:03 -05:00
commit b76aea2899
36 changed files with 9424 additions and 0 deletions

42
app/__init__.py Normal file
View File

@@ -0,0 +1,42 @@
"""OPDS server backed by FreshRSS.
An OPDS 1.2 catalog whose shelves mirror FreshRSS categories and feeds, serving
one EPUB per article, generated in memory at download time. Nothing is written to
disk: the catalog is rebuilt from the FreshRSS API as it is browsed, and a
completed EPUB download marks the corresponding article read upstream.
Layout, in dependency order:
``config``
The JSON document, its interpolation (``${ENV}``, ``${file:...}``) and its
validation. Everything else is configured from here.
``urls``
Identifier and filename helpers: base64url for path identifiers, stable
``urn:uuid`` values for Atom ids, and safe download filenames.
``cache``
A bounded in-memory TTL cache with per-key single-flight, so a reader
browsing quickly does not turn into a burst of upstream requests.
``xhtml``
Feed HTML into well-formed XHTML. Everything the reader's XML parser sees
has been rebuilt by this module.
``images``
Media-type sniffing and dimension parsing, so a book's manifest is honest
about its bytes and a cover can be chosen without an imaging library.
``fetch``
The one place that fetches a host the operator did not choose. Article
images are fetched through an SSRF guard that dials a validated address.
``epub``
The EPUB 3 writer, assembled entirely in memory.
``freshrss``
The FreshRSS client, over its Google Reader compatible API.
``auth``
HTTP Basic authentication, with a throttle on failed attempts.
``opds``
OPDS 1.2 Atom document builders.
``server``
Routing, the catalog logic, and EPUB delivery.
Run it with ``python -m app --config config.json``.
"""
__version__ = "1.0.0"

127
app/__main__.py Normal file
View File

@@ -0,0 +1,127 @@
"""Entry point: load the configuration, then serve.
Startup is deliberately noisy about anything that will be hard to diagnose later:
warnings from configuration validation, the effective settings (with secrets
redacted), and any degraded optional feature. An OPDS server that is reachable
but shows an empty catalog should say why in its own log, not require a
debugging session.
"""
from __future__ import annotations
import argparse
import logging
import signal
import ssl
import sys
import threading
import app.server as app_server
from app import __version__
from app import config as app_config
LOGGER = logging.getLogger("app")
_LOG_FORMAT = "%(asctime)s %(levelname)-7s %(name)s: %(message)s"
_LOG_DATEFORMAT = "%Y-%m-%dT%H:%M:%S%z"
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="opds-from-rss",
description="Serve a FreshRSS account as an OPDS 1.2 catalog of EPUBs.",
)
parser.add_argument(
"--config",
metavar="PATH",
help="configuration file (default: $CONFIG_PATH, /config/config.json, config.json)",
)
parser.add_argument(
"--check-config",
action="store_true",
help="validate the configuration and print it, then exit",
)
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
return parser
def configure_logging(level: str) -> None:
logging.basicConfig(level=level.upper(), format=_LOG_FORMAT, datefmt=_LOG_DATEFORMAT)
def main(argv: list[str] | None = None) -> int:
arguments = build_parser().parse_args(argv)
# Logging is configured twice: once coarsely so a configuration failure is
# reported properly, then again at the level the configuration asks for.
configure_logging("info")
try:
settings = app_config.load_config(arguments.config)
except app_config.ConfigError as exc:
# The exception already lists every problem with its path.
print(str(exc), file=sys.stderr)
return 2
configure_logging(settings.server.log_level)
for warning in settings.warnings:
LOGGER.warning("configuration: %s", warning)
for line in app_config.describe_config(settings).splitlines():
LOGGER.info("%s", line)
if arguments.check_config:
LOGGER.info("configuration is valid; not starting the server")
return 0
httpd, application = app_server.build_server(settings)
if settings.server.tls_cert_file and settings.server.tls_key_file:
_enable_tls(httpd, settings.server.tls_cert_file, settings.server.tls_key_file)
LOGGER.info("serving HTTPS directly (no reverse proxy required)")
else:
LOGGER.warning(
"serving plain HTTP: put a TLS-terminating proxy in front of this, "
"because HTTP Basic credentials are only base64-encoded"
)
_install_signal_handlers(httpd)
LOGGER.info(
"listening on %s:%s; catalog at %s/opds",
settings.server.host,
settings.server.port,
settings.server.public_base_url or f"http://{settings.server.host}:{settings.server.port}",
)
try:
httpd.serve_forever(poll_interval=0.25)
finally:
LOGGER.info("shutting down")
httpd.server_close()
application.client.close()
return 0
def _enable_tls(httpd, cert_file: str, key_file: str) -> None:
"""Wrap the listening socket, so the service can be secure on its own.
The paths are passed in rather than read off the configuration so that the
"both or neither" guarantee made during validation is visible here too.
"""
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(cert_file, key_file)
httpd.socket = context.wrap_socket(httpd.socket, server_side=True)
def _install_signal_handlers(httpd) -> None:
"""Stop cleanly on SIGINT/SIGTERM, which is how a container is stopped.
``shutdown`` blocks until the serve loop exits and must not be called from the
thread running it, so the handler hands the work to a short-lived thread.
"""
def stop(signum: int, _frame: object) -> None:
LOGGER.info("received %s", signal.Signals(signum).name)
threading.Thread(target=httpd.shutdown, daemon=True).start()
for handled in (signal.SIGINT, signal.SIGTERM):
signal.signal(handled, stop)
if __name__ == "__main__":
sys.exit(main())

186
app/auth.py Normal file
View File

@@ -0,0 +1,186 @@
"""HTTP Basic authentication, with a throttle on failed attempts.
Basic is the only scheme every OPDS reader UI can drive: the reader prompts for a
user name and password, and the server answers an unauthenticated request with
``401`` and a ``WWW-Authenticate`` header. That header is also the whole reason
this service must sit behind TLS -- the credentials travel base64-encoded, which
is encoding, not encryption.
Two things this module is careful about:
*The comparison is constant time.* Passwords are stored as sha256 digests
(``make hash-password``) and compared with :func:`hmac.compare_digest`, and an
unknown user name is compared against a dummy digest so that a missing account and
a wrong password take the same time. Otherwise the response time tells an
attacker which half of the guess was right.
*Failures are throttled.* Basic auth can be guessed at line speed, so an address
that fails repeatedly is refused for a cooling-off period before its credentials
are even examined.
"""
from __future__ import annotations
import base64
import binascii
import hashlib
import hmac
import logging
import threading
import time
from dataclasses import dataclass, field
from app import config as app_config
LOGGER = logging.getLogger(__name__)
# Compared against when the user name is unknown, so that the work done -- and
# therefore the time taken -- does not depend on whether the account exists.
_DUMMY_DIGEST = hashlib.sha256(b"no such user").hexdigest()
class UnauthorizedError(Exception):
"""Credentials were missing, malformed, or wrong. Answered with 401."""
def __init__(self, message: str = "authentication required") -> None:
super().__init__(message)
class RateLimitedError(Exception):
"""Too many failed attempts from this address. Answered with 429."""
def __init__(self, retry_after: int) -> None:
super().__init__("too many failed authentication attempts")
self.retry_after = max(1, int(retry_after))
@dataclass
class _Attempts:
"""Failed-attempt bookkeeping for one client address."""
timestamps: list[float] = field(default_factory=list)
def within_window(self, now: float, window: float) -> int:
return sum(1 for moment in self.timestamps if now - moment < window)
class Authenticator:
"""Checks Basic credentials and throttles repeated failures."""
def __init__(
self,
config: app_config.AuthConfig,
*,
clock=time.monotonic,
) -> None:
self._config = config
self._clock = clock
# Digest lookups avoid a linear scan that would leak user names by timing.
self._users = {user.username: user.password_sha256 for user in config.users}
self._lock = threading.Lock()
self._attempts: dict[str, _Attempts] = {}
@property
def challenge(self) -> str:
"""The ``WWW-Authenticate`` value that makes a reader prompt."""
realm = self._config.realm.replace('"', "")
return f'Basic realm="{realm}", charset="UTF-8"'
def authenticate(self, authorization: str | None, client: str) -> str:
"""Return the authenticated user name, or raise.
``client`` identifies the address for throttling; only failed attempts are
counted, and a success clears the record.
"""
self._check_throttle(client)
username, password = _parse_basic(authorization)
if username is None or not self._matches(username, password):
self._record_failure(client)
# The source address is what matters for spotting a brute-force run, so
# that is the warning. The attempted name is attacker-supplied and
# close to an identifier, so it is kept for debugging only, and the
# password is never logged at any level.
LOGGER.warning("rejected authentication attempt from %s", client)
LOGGER.debug("rejected credentials for user %r from %s", username, client)
raise UnauthorizedError
self._clear_failures(client)
return username
def _matches(self, username: str, password: str | None) -> bool:
if password is None:
return False
expected = self._users.get(username, _DUMMY_DIGEST)
offered = hashlib.sha256(password.encode("utf-8")).hexdigest()
return hmac.compare_digest(expected, offered)
# -- throttling ------------------------------------------------------
def _check_throttle(self, client: str) -> None:
window = self._config.throttle_seconds
with self._lock:
record = self._attempts.get(client)
if record is None:
return
now = self._clock()
recent = record.within_window(now, window)
if recent < self._config.max_failures_per_minute:
return
oldest = min(record.timestamps)
retry_after = max(1, int(window - (now - oldest)) + 1)
raise RateLimitedError(retry_after)
def _record_failure(self, client: str) -> None:
now = self._clock()
window = self._config.throttle_seconds
with self._lock:
record = self._attempts.setdefault(client, _Attempts())
record.timestamps.append(now)
# Trim anything that has aged out, and drop the record entirely when
# it is empty, so a long uptime cannot accumulate addresses.
record.timestamps = [moment for moment in record.timestamps if now - moment < window]
if not record.timestamps:
del self._attempts[client]
elif len(self._attempts) > 4096:
self._evict_oldest(now, window)
def _clear_failures(self, client: str) -> None:
with self._lock:
self._attempts.pop(client, None)
def _evict_oldest(self, now: float, window: float) -> None:
"""Bound the table: a flood of addresses must not become a memory leak."""
stale = [
address
for address, record in self._attempts.items()
if not record.within_window(now, window)
]
for address in stale:
del self._attempts[address]
while len(self._attempts) > 4096:
oldest = min(
self._attempts,
key=lambda address: min(self._attempts[address].timestamps),
)
del self._attempts[oldest]
def _parse_basic(authorization: str | None) -> tuple[str | None, str | None]:
"""Split a Basic header into its credentials.
Returns ``(None, None)`` for anything that is not a well-formed Basic header,
which the caller treats exactly like a wrong password.
"""
if not authorization:
return None, None
scheme, _, payload = authorization.partition(" ")
if scheme.lower() != "basic" or not payload:
return None, None
try:
decoded = base64.b64decode(payload.strip(), validate=True).decode("utf-8")
except (binascii.Error, UnicodeDecodeError):
return None, None
# Only the first colon separates: a password may contain colons.
username, separator, password = decoded.partition(":")
if not separator or not username:
return None, None
return username, password

157
app/cache.py Normal file
View File

@@ -0,0 +1,157 @@
"""Bounded in-memory TTL cache with per-key single-flight.
The server keeps nothing on disk, which means every navigation click and every
download is backed by a live FreshRSS API call. An OPDS reader browses a tree
and re-requests feeds freely, so without a cache a single reader produces a small
burst of API traffic per tap. This cache absorbs that burst and nothing more:
entries expire on a short TTL and are evicted when the cache is full.
Single-flight matters as much as caching here. Readers often open several
connections at once (browsing navigation feeds in parallel, or prefetching cover
requests), and without it a cold cache would issue the same FreshRSS request
several times over. ``get_or_compute`` guarantees one in-flight computation per
key while still letting unrelated keys run concurrently.
"""
from __future__ import annotations
import logging
import threading
import time
from collections.abc import Callable
from typing import Any
LOGGER = logging.getLogger(__name__)
class TTLCache:
"""A thread-safe, size-bounded cache of values with a per-entry TTL."""
def __init__(self, max_entries: int = 512) -> None:
# Config validation guarantees an int here, so no defensive coercion.
self._max_entries = max(1, max_entries)
self._lock = threading.Lock()
self._values: dict[str, tuple[float, Any]] = {}
self._key_locks: dict[str, threading.Lock] = {}
self.hits = 0
self.misses = 0
self.evictions = 0
def get(self, key: str, *, now: float | None = None) -> Any | None:
"""Return a live value, or None when missing or expired."""
moment = time.monotonic() if now is None else now
with self._lock:
entry = self._values.get(key)
if entry is None:
self.misses += 1
return None
expires_at, value = entry
if expires_at <= moment:
del self._values[key]
self.misses += 1
return None
# Re-insert so the newest entries sit at the end of the ordering and
# the eviction below drops genuinely cold keys first.
del self._values[key]
self._values[key] = (expires_at, value)
self.hits += 1
return value
def set(self, key: str, value: Any, ttl_seconds: float) -> None:
"""Store a value, evicting the coldest entry if the cache is full."""
if ttl_seconds <= 0:
return
expires_at = time.monotonic() + ttl_seconds
with self._lock:
self._values.pop(key, None)
self._values[key] = (expires_at, value)
while len(self._values) > self._max_entries:
oldest, _ = next(iter(self._values.items()))
del self._values[oldest]
self.evictions += 1
def get_or_compute(self, key: str, ttl_seconds: float, compute: Callable[[], Any]) -> Any:
"""Return a cached value, computing it at most once per key at a time.
Exceptions from ``compute`` are never cached, so a transient FreshRSS
failure is retried on the next request instead of being remembered.
"""
cached = self.get(key)
if cached is not None:
return cached
with self._lock:
key_lock = self._key_locks.get(key)
if key_lock is None:
key_lock = threading.Lock()
self._key_locks[key] = key_lock
with key_lock:
# Another thread may have filled the key while this one waited.
cached = self.get(key)
if cached is not None:
return cached
value = compute()
self.set(key, value, ttl_seconds)
return value
def invalidate(self, prefix: str = "") -> int:
"""Drop every key starting with ``prefix`` (all keys when empty)."""
with self._lock:
if not prefix:
removed = len(self._values)
self._values.clear()
else:
doomed = [key for key in self._values if key.startswith(prefix)]
for key in doomed:
del self._values[key]
removed = len(doomed)
# Collapse the lock table too: keys are unbounded over a long uptime
# and a stale lock object is pure dead weight.
if not prefix or removed:
self._key_locks.clear()
return removed
def stats(self) -> dict[str, int]:
"""Return a snapshot for the diagnostics endpoint."""
with self._lock:
return {
"entries": len(self._values),
"max_entries": self._max_entries,
"hits": self.hits,
"misses": self.misses,
"evictions": self.evictions,
}
class FailureLatch:
"""Suppress a repeated failing optional call for a bounded period.
Unread counts and upstream search are enhancements: when one of them is
unsupported by the FreshRSS build in front of us, retrying on every request
would add latency to every page load for no benefit. The latch remembers the
failure for a TTL, reports it once on the way in, and then stays quiet.
"""
def __init__(self, ttl_seconds: float = 300.0) -> None:
self._ttl = ttl_seconds
self._locked_until: dict[str, float] = {}
self._lock = threading.Lock()
def is_open(self, key: str) -> bool:
"""True when calls for ``key`` should be attempted again."""
with self._lock:
until = self._locked_until.get(key, 0.0)
return time.monotonic() >= until
def trip(self, key: str) -> bool:
"""Mark ``key`` failed; return True the first time within the window."""
with self._lock:
now = time.monotonic()
first = now >= self._locked_until.get(key, 0.0)
self._locked_until[key] = now + self._ttl
return first
def clear(self, key: str) -> None:
with self._lock:
self._locked_until.pop(key, None)

841
app/config.py Normal file
View File

@@ -0,0 +1,841 @@
"""JSON configuration: loading, interpolation, validation.
Configuration is a single JSON document, which keeps deployment to "mount a file
and run the container". Two things make that practical for a service holding
credentials:
*Interpolation.* Any string value may contain ``${VAR}`` (an environment
variable, optionally with a ``:-default``), ``${file:/run/secrets/x}`` (a file,
which is how Docker secrets arrive), or ``$${`` for a literal ``${``. Secrets
therefore never have to be written into the JSON document that gets committed.
*Aggregated validation.* Every problem found is reported in one pass with the
JSON path that caused it, because a container that exits on the first of five
typos is a slow way to configure a service. Unknown keys are reported too, as
warnings: a misspelled ``mark_read`` that silently defaulted to something else is
exactly the kind of thing that costs an hour.
*Comments.* JSON has no comment syntax, so any key beginning with an underscore
is treated as one: it is ignored, never interpolated, and never reported as
unknown. That lets ``config.example.json`` explain itself.
"""
from __future__ import annotations
import hashlib
import json
import logging
import os
import re
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
LOGGER = logging.getLogger(__name__)
CONFIG_PATH_ENV = "CONFIG_PATH"
# Default listen address. Loopback is the safe default for a service that holds
# a whole reading queue: running it directly on a host binds only to that host.
# A container is the one case that needs the wildcard, because the published port
# only reaches the process through the container's own interface -- so the
# shipped config.example.json and docker-compose.yaml set it explicitly, and there
# is exactly one place (the mounted configuration) that decides exposure.
DEFAULT_BIND_HOST = "127.0.0.1"
_DEFAULT_PATHS = ("/config/config.json", "config.json")
_INTERPOLATION_RE = re.compile(r"\$\{([^{}]*)\}")
_SHA256_RE = re.compile(r"\A[0-9a-fA-F]{64}\Z")
_ESCAPED_DOLLAR = "\x00escaped-dollar\x00"
GREADER_SUFFIX = "/api/greader.php"
_LOG_LEVELS = frozenset({"debug", "info", "warning", "error", "critical"})
class ConfigError(Exception):
"""Raised when the configuration cannot be used.
Carries every problem found, so a human sees the whole list at once.
"""
def __init__(self, problems: list[str], source: str = "") -> None:
self.problems = problems
self.source = source
where = f" in {source}" if source else ""
detail = "\n".join(f" - {problem}" for problem in problems)
super().__init__(f"invalid configuration{where}:\n{detail}")
@dataclass
class ServerConfig:
"""HTTP listener settings."""
host: str = DEFAULT_BIND_HOST
port: int = 8080
log_level: str = "info"
# Absolute base URL for links. When unset, links are built from the request
# Host header, which is fine behind a proxy but lets a spoofed Host shape the
# catalog URLs -- so a public deployment should set this.
public_base_url: str | None = None
# Honour X-Forwarded-Proto/Host/Prefix. Only enable behind a proxy you trust,
# because the headers are client-controlled otherwise.
trust_proxy_headers: bool = False
# Optional Host allowlist; a request with any other Host is refused.
allowed_hosts: list[str] = field(default_factory=list)
tls_cert_file: str | None = None
tls_key_file: str | None = None
@dataclass
class FreshRSSConfig:
"""Upstream FreshRSS connection."""
api_base: str
username: str
api_password: str
verify_tls: bool = True
ca_file: str | None = None
timeout_seconds: float = 15.0
retries: int = 2
# Mark the article read upstream once its EPUB has been delivered.
mark_read: bool = True
items_per_page: int = 30
max_items_per_request: int = 200
# How far back the Read section looks. Read items are filtered client-side
# (only the unread filter is a documented upstream parameter), so this bounds
# the history that section can show.
read_window_items: int = 200
# Article body cap before parsing; prevents one enormous feed item from
# dominating memory.
max_content_bytes: int = 1_048_576
@dataclass
class CatalogConfig:
"""OPDS catalog presentation."""
title: str = "FreshRSS OPDS"
author: str = "FreshRSS"
language: str = "en"
publisher: str | None = None
show_unread_counts: bool = True
show_read_section: bool = True
show_starred: bool = True
hide_empty_categories: bool = False
exclude_categories: list[str] = field(default_factory=list)
exclude_feeds: list[str] = field(default_factory=list)
@dataclass
class EpubConfig:
"""EPUB generation."""
include_images: bool = True
allowed_image_hosts: list[str] = field(default_factory=list)
blocked_image_hosts: list[str] = field(default_factory=list)
max_images_per_article: int = 20
max_image_bytes: int = 2_097_152
max_total_image_bytes: int = 16_777_216
image_timeout_seconds: float = 10.0
cover_image: bool = True
include_source_link: bool = True
stylesheet: str | None = None
max_epub_bytes: int = 33_554_432
@dataclass
class AuthUser:
"""One Basic-auth principal. Passwords are always compared as digests."""
username: str
password_sha256: str
@dataclass
class AuthConfig:
"""HTTP Basic credentials.
Authentication is mandatory by design: this catalog exposes the whole of a
reading queue, and an unauthenticated OPDS endpoint on a routable address is
an open book. Basic auth must therefore sit behind TLS (a reverse proxy, or
the TLS options under ``server``), because the header is not encrypted.
"""
realm: str = "OPDS"
users: list[AuthUser] = field(default_factory=list)
max_failures_per_minute: int = 10
throttle_seconds: float = 60.0
@dataclass
class CacheConfig:
"""In-memory cache lifetimes. Nothing is ever written to disk."""
subscriptions_ttl_seconds: float = 300.0
stream_ttl_seconds: float = 60.0
item_ttl_seconds: float = 900.0
max_entries: int = 512
@dataclass
class Config:
"""The fully resolved configuration."""
server: ServerConfig
freshrss: FreshRSSConfig
catalog: CatalogConfig
epub: EpubConfig
auth: AuthConfig
cache: CacheConfig
source_path: str = ""
warnings: list[str] = field(default_factory=list)
def resolve_config_path(explicit: str | None = None) -> str:
"""Find the configuration file: flag, then env, then the usual locations."""
if explicit:
return explicit
from_env = os.environ.get(CONFIG_PATH_ENV)
if from_env:
return from_env
for candidate in _DEFAULT_PATHS:
if Path(candidate).is_file():
return candidate
return _DEFAULT_PATHS[0]
def load_config(path: str | None = None) -> Config:
"""Load, interpolate and validate the configuration file."""
source = resolve_config_path(path)
raw_text = _read_config_text(source)
try:
document = json.loads(raw_text)
except json.JSONDecodeError as exc:
detail = f"not valid JSON: {exc.msg} (line {exc.lineno}, column {exc.colno})"
raise ConfigError([detail], source) from exc
if not isinstance(document, dict):
raise ConfigError(["the top level of the configuration must be a JSON object"], source)
base_dir = Path(source).resolve().parent
problems: list[str] = []
interpolated = _interpolate(document, "", base_dir, problems)
if problems:
raise ConfigError(problems, source)
if not isinstance(interpolated, dict): # pragma: no cover - shape is preserved
raise ConfigError(["internal error: configuration root is not an object"], source)
warnings: list[str] = []
validator = _Validator()
config = Config(
server=_build_server(validator, interpolated),
freshrss=_build_freshrss(validator, interpolated),
catalog=_build_catalog(validator, interpolated),
epub=_build_epub(validator, interpolated),
auth=_build_auth(validator, interpolated),
cache=_build_cache(validator, interpolated),
source_path=str(Path(source).resolve()),
warnings=warnings,
)
warnings.extend(_unknown_keys(interpolated, _KNOWN_KEYS))
if validator.problems:
raise ConfigError(validator.problems, source)
return config
def password_digest(password: str) -> str:
"""Digest a password for storage in ``auth.users[].password_sha256``."""
return hashlib.sha256(password.encode("utf-8")).hexdigest()
def normalise_api_base(url: str) -> str:
"""Normalise a FreshRSS URL to its Google Reader API base.
Mirrors the firmware client's rule so the same address works in both places:
the ``/api/greader.php`` suffix is appended when absent and left alone when
present, and any path prefix in between (``/freshrss``) is preserved.
"""
base = url.strip().rstrip("/")
if base.lower().endswith(GREADER_SUFFIX):
return base
return base + GREADER_SUFFIX
def describe_config(config: Config) -> str:
"""Render the effective configuration for logs, with secrets redacted."""
users = ", ".join(user.username for user in config.auth.users) or "(none)"
lines = [
f"config file: {config.source_path}",
f"listener: {config.server.host}:{config.server.port} "
f"(log level {config.server.log_level}"
+ (", TLS on" if config.server.tls_cert_file else "")
+ ")",
f"freshrss: {config.freshrss.api_base} as {config.freshrss.username} "
f"(password: {_mask(config.freshrss.api_password)}, mark read: "
f"{'yes' if config.freshrss.mark_read else 'no'})",
f"catalog: {config.catalog.title!r}, {config.freshrss.items_per_page} items/page, "
f"read section: {'yes' if config.catalog.show_read_section else 'no'}",
f"epub images: {'yes' if config.epub.include_images else 'no'}",
f"auth users: {users}",
]
return "\n".join(lines)
def _mask(secret: str) -> str:
if not secret:
return "(empty)"
return "***" if len(secret) <= 8 else f"***({len(secret)} chars)"
def _read_config_text(source: str) -> str:
try:
return Path(source).read_text(encoding="utf-8")
except FileNotFoundError as exc:
raise ConfigError(
[
f"configuration file not found: {source}",
f"set {CONFIG_PATH_ENV} or pass --config; "
"see config.example.json for a starting point",
],
source,
) from exc
except OSError as exc:
raise ConfigError([f"cannot read configuration file: {exc.strerror}"], source) from exc
# -- interpolation -------------------------------------------------------
def _interpolate(value: Any, path: str, base_dir: Path, problems: list[str]) -> Any:
"""Recursively expand ``${VAR}`` and ``${file:...}`` in every string."""
if isinstance(value, str):
return _interpolate_string(value, path, base_dir, problems)
if isinstance(value, dict):
return {
key: (
# A `_`-prefixed key is a comment: left exactly as written, so a
# comment that mentions ${SOMETHING} stays inert.
item
if key.startswith("_")
else _interpolate(item, f"{path}.{key}" if path else key, base_dir, problems)
)
for key, item in value.items()
}
if isinstance(value, list):
return [
_interpolate(item, f"{path}[{index}]", base_dir, problems)
for index, item in enumerate(value)
]
return value
def _interpolate_string(value: str, path: str, base_dir: Path, problems: list[str]) -> str:
if "${" not in value:
return value
# Protect an escaped literal `${` before scanning for real references.
text = value.replace("$${", _ESCAPED_DOLLAR + "{")
def replace(match: re.Match[str]) -> str:
expression = match.group(1).strip()
if expression.startswith("file:"):
return _read_secret_file(expression[len("file:") :].strip(), path, base_dir, problems)
name, separator, default = expression.partition(":-")
name = name.strip()
if not name:
problems.append(f"{path}: empty ${{}} reference")
return ""
found = os.environ.get(name)
if found is not None:
return found
if separator:
return default
problems.append(
f"{path}: environment variable {name} is not set (use ${{{name}:-default}} "
"to make it optional, or ${file:path} to read a secret from a file)"
)
return ""
result = _INTERPOLATION_RE.sub(replace, text)
return result.replace(_ESCAPED_DOLLAR + "{", "${")
def _read_secret_file(filename: str, path: str, base_dir: Path, problems: list[str]) -> str:
if not filename:
problems.append(f"{path}: ${{file:}} needs a path")
return ""
candidate = Path(filename)
if not candidate.is_absolute():
candidate = base_dir / candidate
try:
# Strip the trailing newline every secret-management tool adds.
return candidate.read_text(encoding="utf-8").strip()
except OSError as exc:
problems.append(f"{path}: cannot read secret file {candidate}: {exc.strerror}")
return ""
# -- validation ----------------------------------------------------------
class _Validator:
"""Collects type and range problems while building the config dataclasses.
Problems are qualified by the section being validated, so a message reads
``freshrss.url: required, but missing`` rather than the ambiguous
``url: required, but missing``. That matters as soon as two sections share a
key name, which they do (``timeout_seconds`` lives in three of them).
"""
def __init__(self) -> None:
self.problems: list[str] = []
self._prefix = ""
@contextmanager
def within(self, path: str) -> Iterator[None]:
"""Qualify every problem reported inside this block with ``path``."""
previous = self._prefix
self._prefix = f"{previous}.{path}" if previous else path
try:
yield
finally:
self._prefix = previous
def qualify(self, key: str | None = None) -> str:
"""Full path for a key, in the section currently being validated."""
if not key:
return self._prefix
return f"{self._prefix}.{key}" if self._prefix else key
def problem(self, message: str, key: str | None = None) -> None:
"""Report a problem against the current section, or against one key."""
location = self.qualify(key)
self.problems.append(f"{location}: {message}" if location else message)
def section(self, raw: dict[str, Any], name: str) -> dict[str, Any]:
value = raw.get(name)
if value is None:
return {}
if not isinstance(value, dict):
self.problem("must be a JSON object", key=name)
return {}
return value
def string(
self,
raw: dict[str, Any],
key: str,
default: str | None = None,
*,
required: bool = False,
) -> Any:
if key not in raw or raw[key] is None:
if required:
self.problem("required, but missing", key=key)
return default
value = raw[key]
if not isinstance(value, str):
self.problem("must be a string", key=key)
return default
text = value.strip()
if not text:
if required:
self.problem("required, but empty", key=key)
return default
return text
def one_of(self, raw: dict[str, Any], key: str, allowed: frozenset[str], default: str) -> str:
value = self.string(raw, key, default)
if value is None or value not in allowed:
self.problem(f"must be one of {', '.join(sorted(allowed))} (got {value!r})", key=key)
return default
return value
def boolean(self, raw: dict[str, Any], key: str, default: bool) -> bool:
if key not in raw or raw[key] is None:
return default
value = raw[key]
if not isinstance(value, bool):
self.problem("must be true or false", key=key)
return default
return value
def integer(
self,
raw: dict[str, Any],
key: str,
default: int,
*,
minimum: int | None = None,
maximum: int | None = None,
) -> int:
if key not in raw or raw[key] is None:
return default
value = raw[key]
# A bool is an int in Python; accepting it here would turn `true` into 1.
if isinstance(value, bool) or not isinstance(value, int):
self.problem("must be a whole number", key=key)
return default
if minimum is not None and value < minimum:
self.problem(f"must be at least {minimum} (got {value})", key=key)
return default
if maximum is not None and value > maximum:
self.problem(f"must be at most {maximum} (got {value})", key=key)
return default
return value
def number(self, raw: dict[str, Any], key: str, default: float, *, above: float = 0) -> float:
if key not in raw or raw[key] is None:
return default
value = raw[key]
if isinstance(value, bool) or not isinstance(value, (int, float)):
self.problem("must be a number", key=key)
return default
if value <= above:
self.problem(f"must be greater than {above:g} (got {value:g})", key=key)
return default
return float(value)
def string_list(self, raw: dict[str, Any], key: str) -> list[str]:
if key not in raw or raw[key] is None:
return []
value = raw[key]
if not isinstance(value, list):
self.problem("must be an array of strings", key=key)
return []
items: list[str] = []
for index, item in enumerate(value):
if not isinstance(item, str):
self.problem("must be a string", key=f"{key}[{index}]")
continue
if item.strip():
items.append(item.strip())
return items
def _build_server(validator: _Validator, raw: dict[str, Any]) -> ServerConfig:
section = validator.section(raw, "server")
with validator.within("server"):
with validator.within("tls"):
tls = validator.section(section, "tls")
cert = validator.string(tls, "cert_file")
key = validator.string(tls, "key_file")
if bool(cert) != bool(key):
validator.problem("cert_file and key_file must be set together")
elif cert:
_require_readable_file(validator, cert, "cert_file")
_require_readable_file(validator, key, "key_file")
return ServerConfig(
host=validator.string(section, "host", DEFAULT_BIND_HOST) or DEFAULT_BIND_HOST,
# Port 0 asks the operating system for a free port, which is what the test
# suite uses and what a socket-activated or ephemeral deployment wants.
port=validator.integer(section, "port", 8080, minimum=0, maximum=65535),
log_level=validator.one_of(section, "log_level", _LOG_LEVELS, "info"),
public_base_url=validator.string(section, "public_base_url"),
trust_proxy_headers=validator.boolean(section, "trust_proxy_headers", False),
allowed_hosts=[
host.lower() for host in validator.string_list(section, "allowed_hosts")
],
tls_cert_file=cert,
tls_key_file=key,
)
def _build_freshrss(validator: _Validator, raw: dict[str, Any]) -> FreshRSSConfig:
section = validator.section(raw, "freshrss")
with validator.within("freshrss"):
url = validator.string(section, "url", required=True)
if url and not url.lower().startswith(("http://", "https://")):
validator.problem(f"must start with http:// or https:// (got {url!r})", key="url")
username = validator.string(section, "username", required=True)
password = validator.string(section, "api_password", required=True)
ca_file = validator.string(section, "ca_file")
if ca_file:
_require_readable_file(validator, ca_file, "ca_file")
items_per_page = validator.integer(section, "items_per_page", 30, minimum=1, maximum=500)
max_items = validator.integer(
section, "max_items_per_request", 200, minimum=1, maximum=5000
)
if max_items < items_per_page:
validator.problem(
f"max_items_per_request ({max_items}) must be at least "
f"items_per_page ({items_per_page})"
)
return FreshRSSConfig(
api_base=normalise_api_base(url) if url else "",
username=username or "",
api_password=password or "",
verify_tls=validator.boolean(section, "verify_tls", True),
ca_file=ca_file,
timeout_seconds=validator.number(section, "timeout_seconds", 15.0),
retries=validator.integer(section, "retries", 2, minimum=0, maximum=10),
mark_read=validator.boolean(section, "mark_read", True),
items_per_page=items_per_page,
max_items_per_request=max_items,
read_window_items=validator.integer(
section, "read_window_items", 200, minimum=1, maximum=5000
),
max_content_bytes=validator.integer(
section, "max_content_bytes", 1_048_576, minimum=4096, maximum=67_108_864
),
)
def _build_catalog(validator: _Validator, raw: dict[str, Any]) -> CatalogConfig:
section = validator.section(raw, "catalog")
with validator.within("catalog"):
return CatalogConfig(
title=validator.string(section, "title", "FreshRSS OPDS") or "FreshRSS OPDS",
author=validator.string(section, "author", "FreshRSS") or "FreshRSS",
language=validator.string(section, "language", "en") or "en",
publisher=validator.string(section, "publisher"),
show_unread_counts=validator.boolean(section, "show_unread_counts", True),
show_read_section=validator.boolean(section, "show_read_section", True),
show_starred=validator.boolean(section, "show_starred", True),
hide_empty_categories=validator.boolean(section, "hide_empty_categories", False),
exclude_categories=validator.string_list(section, "exclude_categories"),
exclude_feeds=validator.string_list(section, "exclude_feeds"),
)
def _build_epub(validator: _Validator, raw: dict[str, Any]) -> EpubConfig:
section = validator.section(raw, "epub")
with validator.within("epub"):
stylesheet = validator.string(section, "stylesheet")
if stylesheet:
_require_readable_file(validator, stylesheet, "stylesheet")
return EpubConfig(
include_images=validator.boolean(section, "include_images", True),
allowed_image_hosts=[
host.lower() for host in validator.string_list(section, "allowed_image_hosts")
],
blocked_image_hosts=[
host.lower() for host in validator.string_list(section, "blocked_image_hosts")
],
max_images_per_article=validator.integer(
section, "max_images_per_article", 20, minimum=0, maximum=200
),
max_image_bytes=validator.integer(
section, "max_image_bytes", 2_097_152, minimum=1024, maximum=52_428_800
),
max_total_image_bytes=validator.integer(
section,
"max_total_image_bytes",
16_777_216,
minimum=1024,
maximum=268_435_456,
),
image_timeout_seconds=validator.number(section, "image_timeout_seconds", 10.0),
cover_image=validator.boolean(section, "cover_image", True),
include_source_link=validator.boolean(section, "include_source_link", True),
stylesheet=stylesheet,
max_epub_bytes=validator.integer(
section, "max_epub_bytes", 33_554_432, minimum=4096, maximum=268_435_456
),
)
def _build_auth(validator: _Validator, raw: dict[str, Any]) -> AuthConfig:
section = validator.section(raw, "auth")
users: list[AuthUser] = []
with validator.within("auth"):
raw_users = section.get("users")
if raw_users is None:
validator.problem(
"required -- this catalog serves your entire reading queue, so it "
"will not start without credentials "
"(generate a digest with: make hash-password)",
key="users",
)
elif not isinstance(raw_users, list):
validator.problem("must be an array of user objects", key="users")
elif not raw_users:
validator.problem("must contain at least one user", key="users")
else:
seen: set[str] = set()
for index, entry in enumerate(raw_users):
with validator.within(f"users[{index}]"):
users.append(_build_auth_user(validator, entry, seen))
return AuthConfig(
realm=validator.string(section, "realm", "OPDS") or "OPDS",
users=users,
max_failures_per_minute=validator.integer(
section, "max_failures_per_minute", 10, minimum=1, maximum=10_000
),
throttle_seconds=validator.number(section, "throttle_seconds", 60.0),
)
def _build_auth_user(validator: _Validator, entry: Any, seen: set[str]) -> AuthUser:
"""Build one user; the caller has already entered its ``users[i]`` context."""
if not isinstance(entry, dict):
validator.problem("must be a JSON object")
return AuthUser("", "")
username = validator.string(entry, "username", required=True) or ""
if username and username in seen:
validator.problem(f"duplicate username {username!r}", key="username")
seen.add(username)
plain = _validated_plain_password(validator, entry)
digest = _validated_digest(validator, entry)
if plain is not None and digest is not None:
validator.problem("set either password or password_sha256, not both")
return AuthUser(username, "")
if digest is not None:
return AuthUser(username, digest)
if plain is not None:
LOGGER.warning(
"%s.password is in clear text in the configuration file; "
"prefer password_sha256 (make hash-password)",
validator.qualify(),
)
return AuthUser(username, password_digest(plain))
validator.problem("needs password or password_sha256")
return AuthUser(username, "")
def _validated_plain_password(validator: _Validator, entry: dict[str, Any]) -> str | None:
"""Return a clear-text password from config, or None when absent/invalid."""
value = entry.get("password")
if value is None:
return None
if not isinstance(value, str) or not value:
validator.problem("must be a non-empty string", key="password")
return None
return value
def _validated_digest(validator: _Validator, entry: dict[str, Any]) -> str | None:
"""Return a validated sha256 digest from config, or None when absent/invalid."""
value = entry.get("password_sha256")
if value is None:
return None
if not isinstance(value, str) or not _SHA256_RE.match(value):
validator.problem("must be 64 hex characters (make hash-password)", key="password_sha256")
return None
return value.lower()
def _build_cache(validator: _Validator, raw: dict[str, Any]) -> CacheConfig:
section = validator.section(raw, "cache")
with validator.within("cache"):
return CacheConfig(
subscriptions_ttl_seconds=validator.number(section, "subscriptions_ttl_seconds", 300.0),
stream_ttl_seconds=validator.number(section, "stream_ttl_seconds", 60.0),
item_ttl_seconds=validator.number(section, "item_ttl_seconds", 900.0),
max_entries=validator.integer(section, "max_entries", 512, minimum=16, maximum=100_000),
)
def _require_readable_file(validator: _Validator, filename: str, key: str) -> None:
if not Path(filename).is_file():
validator.problem(f"no such file: {filename}", key=key)
_KNOWN_KEYS: dict[str, set[str]] = {
"server": {
"host",
"port",
"log_level",
"public_base_url",
"trust_proxy_headers",
"allowed_hosts",
"tls",
},
"server.tls": {"cert_file", "key_file"},
"freshrss": {
"url",
"username",
"api_password",
"verify_tls",
"ca_file",
"timeout_seconds",
"retries",
"mark_read",
"items_per_page",
"max_items_per_request",
"read_window_items",
"max_content_bytes",
},
"catalog": {
"title",
"author",
"language",
"publisher",
"show_unread_counts",
"show_read_section",
"show_starred",
"hide_empty_categories",
"exclude_categories",
"exclude_feeds",
},
"epub": {
"include_images",
"allowed_image_hosts",
"blocked_image_hosts",
"max_images_per_article",
"max_image_bytes",
"max_total_image_bytes",
"image_timeout_seconds",
"cover_image",
"include_source_link",
"stylesheet",
"max_epub_bytes",
},
"auth": {"realm", "users", "max_failures_per_minute", "throttle_seconds"},
"auth.users[]": {"username", "password", "password_sha256"},
"cache": {
"subscriptions_ttl_seconds",
"stream_ttl_seconds",
"item_ttl_seconds",
"max_entries",
},
}
def _unknown_keys(document: dict[str, Any], known: dict[str, set[str]]) -> list[str]:
"""Report keys the server would otherwise silently ignore."""
warnings: list[str] = []
sections = {name for name in known if "." not in name}
for key in document:
if key in sections or key.startswith("_"):
continue
suggestions = _closest(key, sections)
hint = f" (did you mean {suggestions}?)" if suggestions else ""
warnings.append(f"{key}: unknown top-level setting, ignored{hint}")
for section, allowed in known.items():
if "." in section:
continue
value = document.get(section)
if not isinstance(value, dict):
continue
for key in value:
if key not in allowed and not key.startswith("_"):
suggestions = _closest(key, allowed)
hint = f" (did you mean {suggestions}?)" if suggestions else ""
warnings.append(f"{section}.{key}: unknown setting, ignored{hint}")
if section == "server" and isinstance(value.get("tls"), dict):
for key in value["tls"]:
if key not in known["server.tls"] and not key.startswith("_"):
warnings.append(f"server.tls.{key}: unknown setting, ignored")
if section == "auth" and isinstance(value.get("users"), list):
for index, entry in enumerate(value["users"]):
if not isinstance(entry, dict):
continue
for key in entry:
if key not in known["auth.users[]"] and not key.startswith("_"):
warnings.append(f"auth.users[{index}].{key}: unknown setting, ignored")
return warnings
def _closest(key: str, candidates: set[str]) -> str:
"""Suggest a near-miss key name, cheaply."""
lowered = key.lower().replace("-", "_")
for candidate in sorted(candidates):
if candidate.lower().replace("-", "_") == lowered:
return repr(candidate)
for candidate in sorted(candidates):
if candidate.lower().startswith(lowered[:4]) and len(lowered) >= 4:
return repr(candidate)
return ""

492
app/epub.py Normal file
View File

@@ -0,0 +1,492 @@
"""EPUB 3 generation, entirely in memory.
An EPUB is a ZIP with a strict shape, and a reader that rejects a book usually
does so because of a shape violation rather than because of its content. The
rules that matter here, and that the tests pin down:
* ``mimetype`` is the first entry, stored uncompressed, with no extra field. A
ZIP where it is deflated or not first is not an EPUB.
* ``META-INF/container.xml`` names the OPF, which is the package document.
* XML everywhere: the OPF, the navigation document, the NCX fallback and every
content document have to parse with a strict XML parser. Article bodies are
normalised by :mod:`app.xhtml` before they come anywhere near this module.
* A publication needs ``dc:identifier``, ``dc:title``, ``dc:language`` and a
``dcterms:modified`` timestamp; a navigation document marked ``properties="nav"``
is mandatory, and the NCX is kept as the EPUB 2 fallback because older readers
still look for it.
* Every manifest href resolves to a real entry, and every spine idref resolves to
a manifest item. Both are checked in the tests.
Nothing touches the filesystem: the ZIP is assembled in a ``BytesIO`` and handed
back as ``bytes``.
"""
from __future__ import annotations
import io
import logging
import zipfile
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from app import images, urls, xhtml
LOGGER = logging.getLogger(__name__)
MIMETYPE = "application/epub+zip"
# Where each kind of part lives inside the package.
OPF_PATH = "OEBPS/content.opf"
NAV_PATH = "OEBPS/nav.xhtml"
NCX_PATH = "OEBPS/toc.ncx"
CSS_PATH = "OEBPS/styles/main.css"
COVER_PATH = "OEBPS/text/cover.xhtml"
ARTICLE_PATH = "OEBPS/text/article.xhtml"
IMAGE_DIR = "OEBPS/images"
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
DC_NAMESPACE = "http://purl.org/dc/elements/1.1/"
NCX_NAMESPACE = "http://www.daisy.org/z3986/2005/ncx/"
OPS_NAMESPACE = "http://www.idpf.org/2007/ops"
# A deliberately plain stylesheet. It sets a readable measure and gives code,
# quotes and images somewhere sane to sit, and it never fights the reader's own
# typography: no colours, no font families, no absolute sizes.
DEFAULT_STYLESHEET = """\
body { margin: 5%; line-height: 1.5; }
h1, h2, h3 { line-height: 1.25; page-break-after: avoid; }
.article-title { margin-bottom: 0.2em; }
.byline { margin-top: 0; font-size: 0.9em; opacity: 0.8; }
img { max-width: 100%; height: auto; }
figure { margin: 1em 0; }
figcaption { font-size: 0.9em; font-style: italic; }
blockquote { margin: 1em 1.5em; }
pre { white-space: pre-wrap; word-wrap: break-word; }
code, pre { font-family: monospace; }
table { border-collapse: collapse; max-width: 100%; }
td, th { border: 1px solid currentColor; padding: 0.3em 0.5em; }
hr { border: none; border-top: 1px solid currentColor; }
.source { margin-top: 2em; font-size: 0.85em; opacity: 0.8; }
.cover { text-align: center; }
.cover img { max-height: 90%; }
"""
# Content documents sit one directory below the images.
_TEXT_PREFIX = "../"
# A literal middle dot rather than the HTML entity &middot;: XML predefines only
# amp, lt, gt, quot and apos, and this package ships no DOCTYPE that could define
# any others, so an HTML entity here would make the document unparseable.
BYLINE_SEPARATOR = " \u00b7 "
class EpubTooLargeError(Exception):
"""The assembled book exceeded the configured size cap."""
@dataclass(frozen=True)
class BookImage:
"""An image embedded in the book."""
href: str # relative to the OPF, e.g. "images/img-1.png"
media_type: str
data: bytes
is_cover: bool = False
@property
def document_href(self) -> str:
"""The same image, as referenced from a document in ``text/``."""
return _TEXT_PREFIX + self.href
@dataclass
class Book:
"""Everything needed to write one EPUB."""
identifier: str
title: str
language: str
modified: datetime
content_xhtml: str
published: datetime | None = None
author: str | None = None
publisher: str | None = None
source_url: str | None = None
feed_title: str | None = None
images: list[BookImage] = field(default_factory=list)
stylesheet: str = DEFAULT_STYLESHEET
include_source_link: bool = True
@property
def cover_image(self) -> BookImage | None:
for image in self.images:
if image.is_cover:
return image
return None
def build_epub(book: Book) -> bytes:
"""Assemble a book into EPUB bytes."""
buffer = io.BytesIO()
# ZIP entry timestamps come from the book so that identical input produces
# identical bytes; a book that changes only because it was rebuilt would
# otherwise defeat every cache between here and the reader.
stamp = _zip_timestamp(book.modified)
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
# The mimetype entry must come first and must not be compressed, which is
# why it is written through an explicit ZipInfo rather than writestr.
info = zipfile.ZipInfo("mimetype", date_time=stamp)
info.compress_type = zipfile.ZIP_STORED
info.external_attr = 0o644 << 16
archive.writestr(info, MIMETYPE)
_write(archive, "META-INF/container.xml", _container_xml(), stamp)
_write(archive, OPF_PATH, _opf_document(book), stamp)
_write(archive, NAV_PATH, _nav_document(book), stamp)
_write(archive, NCX_PATH, _ncx_document(book), stamp)
_write(archive, CSS_PATH, book.stylesheet, stamp)
_write(archive, COVER_PATH, _cover_document(book), stamp)
_write(archive, ARTICLE_PATH, _article_document(book), stamp)
for image in book.images:
package_dir = OPF_PATH.rsplit("/", 1)[0]
_write(archive, f"{package_dir}/{image.href}", image.data, stamp)
return buffer.getvalue()
def build_article_epub(
item,
*,
epub_config,
catalog_config,
fetcher=None,
max_content_bytes: int = 1_048_576,
) -> bytes:
"""Turn one FreshRSS item into an EPUB.
The article body comes from FreshRSS, normalised by :mod:`app.xhtml`. Images
are embedded when a fetcher is supplied, and any that cannot be fetched are
dropped from the body rather than left pointing at the network -- a book that
claims to work offline should.
"""
article = xhtml.parse_article(
item.content_html, base_url=item.url or "", max_input_bytes=max_content_bytes
)
book_images: list[BookImage] = []
image_map: dict[str, str | None] | None = None
if epub_config.include_images and fetcher is not None:
wanted = article.image_urls[: epub_config.max_images_per_article]
fetched = fetcher.fetch_all(wanted)
image_map = {}
for url in wanted:
image = fetched.get(url)
if image is None:
image_map[url] = None
continue
index = len(book_images) + 1
href = f"images/img-{index}{images.MEDIA_TYPES[image.media_type]}"
book_images.append(BookImage(href=href, media_type=image.media_type, data=image.data))
image_map[url] = _TEXT_PREFIX + href
elif article.image_urls:
# Keeping remote URLs means the OPF has to declare remote resources, which
# is handled by the caller not setting include_images.
image_map = None
book = Book(
identifier=urls.stable_urn("book", item.id),
title=item.title,
language=catalog_config.language,
modified=item.updated_at,
published=item.published_at,
content_xhtml=article.to_xhtml(image_map),
author=item.author or catalog_config.author,
publisher=catalog_config.publisher,
source_url=item.url,
feed_title=item.feed_title,
images=book_images,
include_source_link=epub_config.include_source_link,
stylesheet=_load_stylesheet(epub_config.stylesheet),
)
if epub_config.cover_image:
_choose_cover(book)
data = build_epub(book)
if len(data) > epub_config.max_epub_bytes:
raise EpubTooLargeError(
f"book is {len(data)} bytes, over the {epub_config.max_epub_bytes} byte cap"
)
return data
def _load_stylesheet(path: str | None) -> str:
"""Use the configured stylesheet, falling back to the built-in one."""
if not path:
return DEFAULT_STYLESHEET
try:
return Path(path).read_text(encoding="utf-8")
except OSError as exc:
LOGGER.warning("cannot read stylesheet %s (%s); using the built-in one", path, exc)
return DEFAULT_STYLESHEET
def _choose_cover(book: Book) -> None:
"""Promote the first suitably large image to be the book's cover.
A reader's library view shows a cover or a blank tile, and a feed's lead image
is almost always the right one. Images whose size cannot be read are skipped:
promoting an unknown image risks a cover that is a tracking pixel.
"""
for index, image in enumerate(book.images):
size = images.dimensions(image.data)
if size is None:
continue
width, height = size
if images.looks_like_a_cover(width, height):
book.images[index] = BookImage(
href=image.href,
media_type=image.media_type,
data=image.data,
is_cover=True,
)
return
def _write(
archive: zipfile.ZipFile,
path: str,
content: str | bytes,
stamp: tuple[int, int, int, int, int, int],
) -> None:
"""Write one entry, encoding text parts and passing binary parts through."""
info = zipfile.ZipInfo(path, date_time=stamp)
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o644 << 16
data = content if isinstance(content, bytes) else content.encode("utf-8")
archive.writestr(info, data)
def _zip_timestamp(moment: datetime) -> tuple[int, int, int, int, int, int]:
"""A ZIP timestamp, clamped to the range the format can express (from 1980)."""
year = max(1980, min(2107, moment.year))
return (year, moment.month, moment.day, moment.hour, moment.minute, moment.second)
# -- documents -----------------------------------------------------------
def _container_xml() -> str:
return (
'<?xml version="1.0" encoding="utf-8"?>\n'
'<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">\n'
" <rootfiles>\n"
f' <rootfile full-path="{OPF_PATH}" media-type="application/oebps-package+xml"/>\n'
" </rootfiles>\n"
"</container>\n"
)
def _opf_document(book: Book) -> str:
metadata = [
f' <dc:identifier id="pub-id">{_escape(book.identifier)}</dc:identifier>',
f" <dc:title>{_escape(book.title)}</dc:title>",
f" <dc:language>{_escape(book.language)}</dc:language>",
]
if book.author:
metadata.append(f" <dc:creator>{_escape(book.author)}</dc:creator>")
if book.publisher:
metadata.append(f" <dc:publisher>{_escape(book.publisher)}</dc:publisher>")
if book.published:
metadata.append(f" <dc:date>{_timestamp(book.published)}</dc:date>")
if book.source_url:
metadata.append(f" <dc:source>{_escape(book.source_url)}</dc:source>")
metadata.append(f' <meta property="dcterms:modified">{_timestamp(book.modified)}</meta>')
if book.cover_image is not None:
# The EPUB 2 spelling of the same statement, for readers that predate the
# cover-image property below.
metadata.append(' <meta name="cover" content="cover-image"/>')
manifest = [
' <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>',
' <item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>',
' <item id="css" href="styles/main.css" media-type="text/css"/>',
' <item id="cover" href="text/cover.xhtml" media-type="application/xhtml+xml"/>',
' <item id="article" href="text/article.xhtml" media-type="application/xhtml+xml"/>',
]
cover = book.cover_image
for index, image in enumerate(book.images, start=1):
properties = ' properties="cover-image"' if image.is_cover else ""
identifier = "cover-image" if image.is_cover else f"image-{index}"
manifest.append(
f' <item id="{identifier}" href="{_escape(image.href)}" '
f'media-type="{image.media_type}"{properties}/>'
)
if cover is None and book.images:
# Images are embedded rather than linked, so nothing to declare here; the
# branch exists to keep the intent obvious if remote images are ever kept.
pass
spine = [
' <itemref idref="cover"/>',
' <itemref idref="article"/>',
]
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<package xmlns="{OPF_NAMESPACE}" version="3.0" '
f'unique-identifier="pub-id" xml:lang="{_escape(book.language)}">',
f' <metadata xmlns:dc="{DC_NAMESPACE}">',
*metadata,
" </metadata>",
" <manifest>",
*manifest,
" </manifest>",
' <spine toc="ncx">',
*spine,
" </spine>",
" <guide>",
' <reference type="cover" title="Cover" href="text/cover.xhtml"/>',
" </guide>",
"</package>",
"",
]
)
def _nav_document(book: Book) -> str:
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<html xmlns="{XHTML_NAMESPACE}" xmlns:epub="{OPS_NAMESPACE}" '
f'xml:lang="{_escape(book.language)}" lang="{_escape(book.language)}">',
" <head>",
" <title>Contents</title>",
' <meta charset="utf-8"/>',
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
" </head>",
" <body>",
' <nav epub:type="toc" id="toc">',
" <h1>Contents</h1>",
" <ol>",
f' <li><a href="text/cover.xhtml">{_escape(book.title)}</a></li>',
' <li><a href="text/article.xhtml">Article</a></li>',
" </ol>",
" </nav>",
" </body>",
"</html>",
"",
]
)
def _ncx_document(book: Book) -> str:
points = [
("navpoint-1", 1, "Cover", "text/cover.xhtml"),
("navpoint-2", 2, "Article", "text/article.xhtml"),
]
body = []
for identifier, order, label, href in points:
body.extend(
[
f' <navPoint id="{identifier}" playOrder="{order}">',
f" <navLabel><text>{_escape(label)}</text></navLabel>",
f' <content src="{href}"/>',
" </navPoint>",
]
)
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<ncx xmlns="{NCX_NAMESPACE}" version="2005-1">',
" <head>",
f' <meta name="dtb:uid" content="{_escape(book.identifier)}"/>',
' <meta name="dtb:depth" content="1"/>',
' <meta name="dtb:totalPageCount" content="0"/>',
' <meta name="dtb:maxPageNumber" content="0"/>',
" </head>",
f" <docTitle><text>{_escape(book.title)}</text></docTitle>",
" <navMap>",
*body,
" </navMap>",
"</ncx>",
"",
]
)
def _cover_document(book: Book) -> str:
lines = [
'<?xml version="1.0" encoding="utf-8"?>',
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
f'lang="{_escape(book.language)}">',
" <head>",
f" <title>{_escape(book.title)}</title>",
' <meta charset="utf-8"/>',
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
" </head>",
' <body class="cover">',
]
cover = book.cover_image
if cover is not None:
lines.append(f' <img src="{_escape(cover.document_href)}" alt="{_escape(book.title)}"/>')
lines.append(f' <h1 class="article-title">{_escape(book.title)}</h1>')
if book.author:
lines.append(f' <p class="byline">{_escape(book.author)}</p>')
if book.feed_title:
lines.append(f' <p class="byline">{_escape(book.feed_title)}</p>')
if book.published:
lines.append(f' <p class="byline">{_date_only(book.published)}</p>')
lines.extend([" </body>", "</html>", ""])
return "\n".join(lines)
def _article_document(book: Book) -> str:
byline_bits = []
if book.author:
byline_bits.append(_escape(book.author))
if book.published:
byline_bits.append(_date_only(book.published))
lines = [
'<?xml version="1.0" encoding="utf-8"?>',
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
f'lang="{_escape(book.language)}">',
" <head>",
f" <title>{_escape(book.title)}</title>",
' <meta charset="utf-8"/>',
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
" </head>",
" <body>",
f' <h1 class="article-title">{_escape(book.title)}</h1>',
]
if byline_bits:
lines.append(f' <p class="byline">{BYLINE_SEPARATOR.join(byline_bits)}</p>')
if book.content_xhtml:
lines.append(book.content_xhtml)
else:
# Never ship an empty content document: a reader that opens a blank page
# looks broken, and the original is the most useful thing to show instead.
lines.append(" <p>This article had no content to download.</p>")
if book.include_source_link and book.source_url:
lines.append(
' <p class="source">Originally published at '
f'<a href="{_escape(book.source_url)}">{_escape(book.source_url)}</a>.</p>'
)
lines.extend([" </body>", "</html>", ""])
return "\n".join(lines)
# -- formatting ----------------------------------------------------------
def _escape(text: str) -> str:
"""XML-escape text for a document body or an attribute value."""
return xhtml.escape_text(text).replace('"', "&quot;")
def _timestamp(moment: datetime) -> str:
"""An EPUB date: UTC, seconds precision, ``Z`` suffix."""
return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def _date_only(moment: datetime) -> str:
return moment.astimezone(UTC).strftime("%Y-%m-%d")

339
app/fetch.py Normal file
View File

@@ -0,0 +1,339 @@
"""Fetching article images for embedding, with an SSRF guard.
Image URLs come from feed content, which means they are attacker-controlled by
anyone who can publish to a feed this account follows. That makes this the one
place in the server that makes an outbound request to a host the operator did not
choose, so the guard is the point of the module rather than an extra.
What is enforced, and why:
*Only http and https*, so ``file:``, ``gopher:`` and friends are unreachable.
*Every address a name resolves to is checked* against the private, loopback,
link-local, multicast and reserved ranges. If a name resolves to a mixture, the
whole fetch is refused rather than picking the acceptable address -- a mixed
answer is itself the attack.
*The connection is dialled to the validated address, not to the name.* This is
what closes the DNS-rebinding window: a name that answered with a public address
during validation cannot answer with ``169.254.169.254`` a millisecond later,
because the name is never resolved a second time. TLS still verifies the name
the URL used, so a valid certificate for the real host is required.
*Redirects are followed manually*, so each hop is validated again; ``urllib``'s
automatic redirect handling would happily follow the first hop to
``http://127.0.0.1:8080/``.
*Bodies are capped*, per image and across the whole article, so one hostile feed
cannot exhaust memory.
A refusal is a normal outcome: the image is dropped from the book and the article
is still built.
"""
from __future__ import annotations
import http.client
import ipaddress
import logging
import socket
import ssl
import threading
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from app import images
LOGGER = logging.getLogger(__name__)
USER_AGENT = "odps-from-rss/1.0 (+image fetch for EPUB embedding)"
_ALLOWED_SCHEMES = ("http", "https")
_MAX_REDIRECTS = 3
_READ_CHUNK = 64 * 1024
# Refuse to even start reading a body larger than the cap, when the server tells
# us the size up front. A little slack covers a mislabelled Content-Length.
_CONTENT_LENGTH_SLACK = 4096
class ImageFetchError(Exception):
"""The URL was refused by the guard, or the fetch failed."""
@dataclass(frozen=True)
class FetchedImage:
"""An image that was fetched, validated and identified."""
url: str
media_type: str
data: bytes
@property
def filename(self) -> str:
"""A safe file name for the EPUB, with the extension from the media type."""
return "image" + images.MEDIA_TYPES.get(self.media_type, ".bin")
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
"""HTTPS to a validated address, verifying the hostname from the URL.
``HTTPSConnection`` would use its own host for SNI and certificate
verification, which here is a bare IP address and would always fail. Dialling
the address but verifying the name keeps both properties: no second DNS
lookup, and a real certificate check.
"""
def __init__(
self,
address: str,
port: int,
*,
server_hostname: str,
context: ssl.SSLContext,
timeout: float,
) -> None:
super().__init__(address, port, timeout=timeout, context=context)
self._verify_hostname = server_hostname
def connect(self) -> None:
http.client.HTTPConnection.connect(self)
self.sock = self._context.wrap_socket( # type: ignore[union-attr]
self.sock, server_hostname=self._verify_hostname
)
class ImageFetcher:
"""Fetches article images concurrently, refusing anything unsafe."""
def __init__(
self,
*,
allowed_hosts: list[str] | None = None,
blocked_hosts: list[str] | None = None,
max_image_bytes: int = 2_097_152,
max_total_bytes: int = 16_777_216,
timeout: float = 10.0,
max_workers: int = 4,
allow_private_networks: bool = False,
) -> None:
self._allowed_hosts = [host.lower() for host in (allowed_hosts or [])]
self._blocked_hosts = [host.lower() for host in (blocked_hosts or [])]
self._max_image_bytes = max_image_bytes
self._max_total_bytes = max_total_bytes
self._timeout = timeout
self._max_workers = max(1, max_workers)
# The upstream FreshRSS may be on a LAN, but article images never have a
# legitimate reason to point into one. Kept as a parameter so a test can
# exercise the guard, not as a configuration option.
self._allow_private_networks = allow_private_networks
self._tls_context = ssl.create_default_context()
self._budget_lock = threading.Lock()
self._budget_used = 0
def fetch_all(self, urls: list[str]) -> dict[str, FetchedImage | None]:
"""Fetch each URL, mapping it to an image or to None when refused.
Order is preserved by the caller, which walks the document's images; this
method only has to answer "did it arrive?". Concurrent, because a article
with a dozen images should not cost a dozen sequential round trips.
"""
unique: list[str] = []
for url in urls:
if url not in unique:
unique.append(url)
if not unique:
return {}
with self._budget_lock:
self._budget_used = 0
workers = min(self._max_workers, len(unique))
results: dict[str, FetchedImage | None] = {}
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="image") as pool:
for url, result in zip(unique, pool.map(self._fetch_safely, unique), strict=True):
results[url] = result
return results
# -- internals -------------------------------------------------------
def _fetch_safely(self, url: str) -> FetchedImage | None:
"""Fetch one image, turning every failure into a dropped image."""
try:
return self._fetch(url)
except ImageFetchError as exc:
LOGGER.info("not embedding image %s: %s", url, exc)
except (OSError, http.client.HTTPException, ssl.SSLError) as exc:
LOGGER.info("could not fetch image %s: %s", url, exc)
except Exception:
# One bad image must never break the book it was going into.
LOGGER.exception("unexpected failure fetching image %s", url)
return None
def _fetch(self, url: str) -> FetchedImage:
target = url
for _ in range(_MAX_REDIRECTS + 1):
parsed = urllib.parse.urlsplit(target)
scheme = parsed.scheme.lower()
host = parsed.hostname
if scheme not in _ALLOWED_SCHEMES:
raise ImageFetchError(f"scheme {scheme or '(none)'!r} is not allowed")
if not host:
raise ImageFetchError("no host in URL")
if parsed.username or parsed.password:
raise ImageFetchError("credentials in an image URL are not followed")
self._check_host_allowed(host)
address = self._resolve(host)
port = parsed.port or (443 if scheme == "https" else 80)
status, headers, body = self._request_once(scheme, address, host, port, parsed)
if status in (301, 302, 303, 307, 308):
location = headers.get("location")
if not location:
raise ImageFetchError("redirect without a location")
# Resolve relative redirects against the URL that produced them.
target = urllib.parse.urljoin(target, location)
LOGGER.debug("following image redirect to %s", target)
continue
if status != 200:
raise ImageFetchError(f"server answered {status}")
declared = headers.get("content-type")
media_type = images.identify(body, declared)
if media_type is None:
raise ImageFetchError(f"not an embeddable image ({declared or 'no type'})")
return FetchedImage(url=url, media_type=media_type, data=body)
raise ImageFetchError("too many redirects")
def _check_host_allowed(self, host: str) -> None:
lowered = host.lower()
if any(_host_matches(lowered, blocked) for blocked in self._blocked_hosts):
raise ImageFetchError("host is on the blocked list")
if self._allowed_hosts and not any(
_host_matches(lowered, allowed) for allowed in self._allowed_hosts
):
raise ImageFetchError("host is not on the allowed list")
def _resolve(self, host: str) -> str:
"""Validate every address for a host and return the one to dial."""
try:
infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
except socket.gaierror as exc:
raise ImageFetchError(f"cannot resolve host ({exc.strerror or exc})") from exc
addresses: list[str] = []
for info in infos:
address = str(info[4][0])
if address not in addresses:
addresses.append(address)
if not addresses:
raise ImageFetchError("host resolved to no addresses")
for address in addresses:
reason = self._address_refusal(address)
if reason is not None:
# Refuse the whole host rather than picking a good address: a
# mixed answer is the signature of a rebinding attempt.
raise ImageFetchError(f"{address} is refused ({reason})")
return addresses[0]
def _address_refusal(self, address: str) -> str | None:
try:
parsed = ipaddress.ip_address(address)
except ValueError:
return "unparseable address"
if self._allow_private_networks:
return None
# A v4-mapped v6 address is judged as the v4 address it is, and this has to
# come first: ::ffff:127.0.0.1 is neither loopback nor private when it is
# tested as a v6 address, so the checks below would let it through.
if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None:
return self._address_refusal(str(parsed.ipv4_mapped))
if parsed.is_loopback:
return "loopback"
if parsed.is_link_local:
return "link-local"
if parsed.is_private:
return "private network"
if parsed.is_multicast:
return "multicast"
if parsed.is_reserved or parsed.is_unspecified:
return "reserved"
return None
def _request_once(
self,
scheme: str,
address: str,
host: str,
port: int,
parsed: urllib.parse.SplitResult,
) -> tuple[int, dict[str, str], bytes]:
"""One request, to a pinned address, reading at most the byte cap."""
if scheme == "https":
connection: http.client.HTTPConnection = _PinnedHTTPSConnection(
address,
port,
server_hostname=host,
context=self._tls_context,
timeout=self._timeout,
)
else:
connection = http.client.HTTPConnection(address, port, timeout=self._timeout)
path = urllib.parse.urlunsplit(("", "", parsed.path or "/", parsed.query, ""))
headers = {
"Host": _host_header(host, port, scheme),
"User-Agent": USER_AGENT,
"Accept": "image/*",
"Connection": "close",
}
try:
connection.request("GET", path, headers=headers)
response = connection.getresponse()
status = response.status
response_headers = {key.lower(): value for key, value in response.getheaders()}
if status != 200:
response.read(_READ_CHUNK)
return status, response_headers, b""
return status, response_headers, self._read_capped(response, response_headers)
finally:
connection.close()
def _read_capped(self, response: http.client.HTTPResponse, headers: dict[str, str]) -> bytes:
"""Read a response body, refusing one that exceeds the budget."""
limit = self._image_limit(headers)
chunks: list[bytes] = []
total = 0
while True:
chunk = response.read(_READ_CHUNK)
if not chunk:
break
total += len(chunk)
if total > limit:
raise ImageFetchError(f"image is larger than {limit} bytes")
chunks.append(chunk)
self._spend(total)
return b"".join(chunks)
def _image_limit(self, headers: dict[str, str]) -> int:
"""The smaller of the per-image cap and what is left of the total budget."""
declared = headers.get("content-length")
limit = self._max_image_bytes
if declared and declared.isdigit():
size = int(declared)
if size > self._max_image_bytes:
raise ImageFetchError(f"image declares {size} bytes")
with self._budget_lock:
remaining = max(0, self._max_total_bytes - self._budget_used)
if remaining == 0:
# The article's total allowance is spent; the rest of its images are
# dropped rather than growing the book further.
raise ImageFetchError("the article's image budget is exhausted")
return min(limit, remaining)
def _spend(self, size: int) -> None:
with self._budget_lock:
self._budget_used += size
def _host_matches(host: str, pattern: str) -> bool:
"""Exact host or subdomain match, so ``example.com`` covers ``cdn.example.com``."""
return host == pattern or host.endswith("." + pattern)
def _host_header(host: str, port: int, scheme: str) -> str:
"""A Host header, including the port only when it is not the default."""
default = 443 if scheme == "https" else 80
if port == default:
return host
return f"{host}:{port}"

598
app/freshrss.py Normal file
View File

@@ -0,0 +1,598 @@
"""FreshRSS client over its Google Reader compatible API.
Only the endpoints the firmware client already proved out against a real server
are used (see ``lib/FreshRss/src/FreshRssSync.cpp`` in the reader tree):
* ``POST accounts/ClientLogin`` returns the ``Auth`` token that goes into an
``Authorization: GoogleLogin auth=...`` header.
* ``GET reader/api/0/subscription/list?output=json`` lists feeds, each with its
category labels.
* ``GET reader/api/0/stream/contents/<stream>?output=json&r=d&n=&xt=`` fetches
items. ``r=d`` is newest first, ``n`` bounds the count, and ``xt`` excludes
items carrying a tag -- which is how the unread queue is filtered server-side.
* ``GET reader/api/0/token`` returns the write token, and
``POST reader/api/0/edit-tag`` with ``T=`` and repeated ``i=`` marks items read.
Deliberately absent: anything that would need an undocumented parameter. Where a
filter is not expressible with these calls (read items), the caller filters
locally rather than assuming an endpoint behaves a certain way.
Note the two endpoints this module never touches: ``unread-count`` is optional
and its failure is tolerated, and nothing here fetches article bodies from the
open internet -- the body FreshRSS already stores is the source of truth.
"""
from __future__ import annotations
import json
import logging
import ssl
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
from app import config as app_config
LOGGER = logging.getLogger(__name__)
# Google Reader stream and tag identifiers.
READING_LIST = "user/-/state/com.google/reading-list"
STARRED = "user/-/state/com.google/starred"
READ_TAG = "user/-/state/com.google/read"
STARRED_TAG = "user/-/state/com.google/starred"
KEPT_UNREAD_TAG = "user/-/state/com.google/kept-unread"
USER_AGENT = "odps-from-rss/1.0 (+https://github.com/; OPDS 1.2 catalog)"
# Statuses worth retrying: the upstream is briefly unhappy, not wrong.
_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504, 507, 509})
_AUTH_STATUSES = frozenset({401, 403})
_MISSING_STATUSES = frozenset({400, 404})
class FreshRSSError(Exception):
"""An upstream call failed."""
def __init__(self, message: str, *, status: int | None = None) -> None:
super().__init__(message)
self.status = status
class AuthenticationError(FreshRSSError):
"""The API password or username was rejected."""
class TransientError(FreshRSSError):
"""The upstream could not be reached, or answered with a retryable status."""
class NotFoundError(FreshRSSError):
"""The requested stream or item does not exist upstream."""
@dataclass
class Subscription:
"""One feed in the FreshRSS account."""
id: str
title: str
categories: list[str] = field(default_factory=list)
html_url: str | None = None
unread: int | None = None
@property
def primary_category(self) -> str:
"""The label this feed is shelved under, or a stable placeholder."""
return self.categories[0] if self.categories else UNCATEGORISED
@dataclass
class Item:
"""One article."""
id: str
title: str
published: int
updated: int
feed_id: str
feed_title: str
content_html: str = ""
author: str | None = None
url: str | None = None
feed_url: str | None = None
categories: list[str] = field(default_factory=list)
@property
def is_read(self) -> bool:
"""True when the item is read upstream.
FreshRSS keeps both the read and the kept-unread tag on an item that was
read and then explicitly marked unread again, and there the explicit
signal is the one that reflects what the reader wants.
"""
if KEPT_UNREAD_TAG in self.categories:
return False
return READ_TAG in self.categories
@property
def is_starred(self) -> bool:
return STARRED_TAG in self.categories
@property
def published_at(self) -> datetime:
"""Publication time as an aware UTC datetime.
Falls back to the epoch rather than raising: a feed with a broken date is
not a reason to fail the whole catalog.
"""
stamp = self.published or self.updated or 0
try:
return datetime.fromtimestamp(stamp, tz=UTC)
except (OverflowError, OSError, ValueError):
return datetime(1970, 1, 1, tzinfo=UTC)
@property
def updated_at(self) -> datetime:
stamp = self.updated or self.published or 0
try:
return datetime.fromtimestamp(stamp, tz=UTC)
except (OverflowError, OSError, ValueError):
return self.published_at
UNCATEGORISED = "Uncategorised"
def label_stream(label: str) -> str:
"""Stream id for a category label."""
return f"user/-/label/{label}"
class FreshRSSClient:
"""A small client for one FreshRSS account.
Instances are safe to share between threads: the only mutable state is the
cached session token, which is guarded by a lock.
"""
def __init__(
self,
config: app_config.FreshRSSConfig,
*,
clock: Any = time.time,
sleep: Any = time.sleep,
) -> None:
self._config = config
self._clock = clock
# Injectable so tests can exercise retry logic without waiting it out.
self._sleep = sleep
self._auth_token: str | None = None
self._write_token: str | None = None
self._write_token_fetched_at = 0.0
self._token_lock = threading.Lock()
self._tls_context = _build_tls_context(config)
self._opener = urllib.request.build_opener(
urllib.request.HTTPSHandler(context=self._tls_context)
)
# -- public API ------------------------------------------------------
def list_subscriptions(self) -> list[Subscription]:
"""Every feed in the account, with its category labels."""
payload = self._get_json("/reader/api/0/subscription/list")
raw_subscriptions = payload.get("subscriptions")
if not isinstance(raw_subscriptions, list):
raise FreshRSSError("subscription list had no subscriptions array")
subscriptions: list[Subscription] = []
for entry in raw_subscriptions:
parsed = _parse_subscription(entry)
if parsed is not None:
subscriptions.append(parsed)
subscriptions.sort(key=lambda feed: feed.title.casefold())
return subscriptions
def unread_counts(self) -> dict[str, int]:
"""Unread counts keyed by stream id. Optional upstream feature.
Raises whatever the client raises, so the caller can latch the failure and
carry on without counts -- the catalog is perfectly usable without them.
"""
payload = self._get_json("/reader/api/0/unread-count", {"all": "true"})
counts: dict[str, int] = {}
for entry in payload.get("unreadcounts") or []:
if not isinstance(entry, dict):
continue
stream_id = entry.get("id")
count = entry.get("count")
if isinstance(stream_id, str) and isinstance(count, int):
counts[stream_id] = count
return counts
def stream_items(
self,
stream_id: str,
*,
count: int,
exclude_read: bool = False,
older_than: int | None = None,
) -> list[Item]:
"""Fetch items from one stream, newest first.
A stream that has disappeared upstream (``400``/``404``) is reported as an
empty result rather than an error: a feed you unsubscribed from should not
break the catalog that still lists it.
"""
params = {
"output": "json",
"r": "d",
"n": str(max(1, count)),
# Cache killer: asks FreshRSS not to serve a cached response.
"ck": str(int(self._clock())),
}
if exclude_read:
params["xt"] = READ_TAG
if older_than:
params["ot"] = str(int(older_than))
path = "/reader/api/0/stream/contents/" + urllib.parse.quote(stream_id, safe="")
try:
payload = self._get_json(path, params)
except NotFoundError:
LOGGER.info("stream %s is gone upstream; treating it as empty", stream_id)
return []
raw_items = payload.get("items")
if not isinstance(raw_items, list):
return []
items = [item for item in (_parse_item(entry) for entry in raw_items) if item]
return items
def mark_read(self, item_ids: list[str]) -> int:
"""Mark items read upstream. Returns the number of ids sent.
Idempotent: adding the read tag to an already-read item is a no-op
upstream, so a repeated download of the same book is harmless.
"""
wanted = [item_id for item_id in item_ids if item_id]
if not wanted:
return 0
token = self._write_token_value()
form: list[tuple[str, str]] = [
("T", token),
("a", READ_TAG),
]
form.extend(("i", item_id) for item_id in wanted)
# edit-tag needs both the session header and the write token: the header
# authenticates the caller, the token authorises the write.
try:
self._request(
"POST",
"/reader/api/0/edit-tag",
body=urllib.parse.urlencode(form),
authenticated=True,
)
except AuthenticationError:
# Either the session or the write token outlived its validity; drop
# both, fetch fresh ones, and retry the write once.
self._forget_tokens()
form[0] = ("T", self._write_token_value())
self._request(
"POST",
"/reader/api/0/edit-tag",
body=urllib.parse.urlencode(form),
authenticated=True,
)
return len(wanted)
def probe(self) -> tuple[bool, str]:
"""Check credentials and reachability, for the diagnostics endpoint."""
try:
subscriptions = self.list_subscriptions()
except FreshRSSError as exc:
return False, str(exc)
return True, f"{len(subscriptions)} feeds"
def close(self) -> None:
"""Release upstream resources (connections are per-request)."""
self._forget_tokens()
# -- transport -------------------------------------------------------
def _forget_tokens(self) -> None:
with self._token_lock:
self._auth_token = None
self._write_token = None
self._write_token_fetched_at = 0.0
def _auth_header(self) -> dict[str, str]:
with self._token_lock:
token = self._auth_token
if token is None:
token = self._login()
with self._token_lock:
self._auth_token = token
return {"Authorization": f"GoogleLogin auth={token}"}
def _write_token_value(self) -> str:
with self._token_lock:
token = self._write_token
fetched_at = self._write_token_fetched_at
# Refresh hourly: the token is cheap to fetch and this bounds how long a
# stale one can survive a password change upstream.
if token and self._clock() - fetched_at < 3600:
return token
response = self._request("GET", "/reader/api/0/token", authenticated=True)
token = response.decode("utf-8", "replace").strip()
if not token:
raise FreshRSSError("upstream returned an empty write token")
with self._token_lock:
self._write_token = token
self._write_token_fetched_at = self._clock()
return token
def _login(self) -> str:
"""Exchange the API password for an Auth token."""
form = urllib.parse.urlencode(
{"Email": self._config.username, "Passwd": self._config.api_password}
)
response = self._request("POST", "/accounts/ClientLogin", body=form)
text = response.decode("utf-8", "replace")
for line in text.splitlines():
if line.startswith("Auth="):
value = line[len("Auth=") :].strip()
if value:
LOGGER.debug("authenticated with FreshRSS as %s", self._config.username)
return value
raise AuthenticationError("FreshRSS rejected the API password")
def _get_json(self, path: str, params: dict[str, str] | None = None) -> dict[str, Any]:
query = dict(params or {})
query.setdefault("output", "json")
response = self._request("GET", path, query=query, authenticated=True)
try:
payload = json.loads(response.decode("utf-8", "replace"))
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
raise FreshRSSError(f"upstream sent a response that is not JSON ({exc})") from exc
if not isinstance(payload, dict):
raise FreshRSSError("upstream sent a JSON value that is not an object")
return payload
def _request(
self,
method: str,
path: str,
*,
query: dict[str, str] | None = None,
body: str | None = None,
authenticated: bool = False,
) -> bytes:
"""Perform one upstream request, retrying transient failures."""
url = self._config.api_base + path
if query:
url = f"{url}?{urllib.parse.urlencode(query)}"
data = body.encode("utf-8") if body is not None else None
attempts = max(1, self._config.retries + 1)
last_error: Exception | None = None
for attempt in range(attempts):
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
if body is not None:
headers["Content-Type"] = "application/x-www-form-urlencoded"
if authenticated:
headers.update(self._auth_header())
request = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
with self._opener.open(request, timeout=self._config.timeout_seconds) as reply:
return reply.read()
except urllib.error.HTTPError as exc:
status = exc.code
try:
detail = _read_error_body(exc)
finally:
# The error object owns the response socket. Leaving it open
# leaks one connection for every failed upstream request.
exc.close()
if status in _AUTH_STATUSES:
raise AuthenticationError(
f"FreshRSS refused the request ({status}); check the API password",
status=status,
) from exc
if status in _MISSING_STATUSES:
raise NotFoundError(
f"upstream has no such resource ({status})", status=status
) from exc
if status in _RETRY_STATUSES:
# A retryable status stays transient even once the attempts run
# out, so the caller can tell "upstream is unwell" from
# "upstream says this request is wrong".
error = TransientError(f"upstream returned {status}{detail}", status=status)
if attempt + 1 < attempts:
last_error = error
self._backoff(attempt, status)
continue
raise error from exc
raise FreshRSSError(f"upstream returned {status}{detail}", status=status) from exc
except urllib.error.URLError as exc:
last_error = TransientError(f"cannot reach FreshRSS: {exc.reason}")
if attempt + 1 < attempts:
self._backoff(attempt, None)
continue
raise last_error from exc
except TimeoutError as exc:
last_error = TransientError("the upstream request timed out")
if attempt + 1 < attempts:
self._backoff(attempt, None)
continue
raise last_error from exc
except ssl.SSLError as exc:
raise FreshRSSError(f"TLS failure talking to FreshRSS: {exc}") from exc
# Only reachable if the loop exhausted its attempts without raising.
raise last_error or TransientError("the upstream request failed")
def _backoff(self, attempt: int, status: int | None) -> None:
delay = min(2.0, 0.2 * (2**attempt))
LOGGER.warning(
"upstream %s, retrying in %.1fs%s",
f"returned {status}" if status else "unreachable",
delay,
f" (attempt {attempt + 1})" if attempt else "",
)
self._sleep(delay)
def _build_tls_context(config: app_config.FreshRSSConfig) -> ssl.SSLContext:
"""Build the TLS context for upstream calls.
An operator pointing at a LAN instance with a private CA sets ``ca_file``;
disabling verification is possible but logged loudly, because it silently
removes the protection against a machine-in-the-middle reading the API
password.
"""
if not config.verify_tls:
LOGGER.warning(
"freshrss.verify_tls is false: TLS certificates are not verified. "
"The API password and article contents are exposed to anyone able to "
"intercept the connection."
)
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
return context
context = ssl.create_default_context()
if config.ca_file:
context.load_verify_locations(cafile=config.ca_file)
return context
def _read_error_body(error: urllib.error.HTTPError) -> str:
"""Include a short, sanitised excerpt of an error body in the message."""
try:
raw = error.read(512)
except Exception:
# A diagnostic excerpt must never mask the error it is describing.
return ""
text = raw.decode("utf-8", "replace").strip().replace("\n", " ")
if not text:
return ""
return f": {text[:200]}"
def _text(value: Any) -> str:
"""A trimmed string from upstream JSON, or an empty string.
Upstream is an arbitrary FreshRSS build, so every field is treated as
untrusted input of unknown type rather than as the shape the docs suggest.
"""
return value.strip() if isinstance(value, str) else ""
def _as_int(value: Any) -> int:
"""Best-effort integer from upstream JSON, tolerating strings."""
if isinstance(value, bool):
return 0
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
if isinstance(value, str):
try:
return int(value)
except ValueError:
return 0
return 0
def _parse_subscription(entry: Any) -> Subscription | None:
if not isinstance(entry, dict):
return None
stream_id = entry.get("id")
if not isinstance(stream_id, str) or not stream_id:
return None
categories: list[str] = []
for category in entry.get("categories") or []:
if not isinstance(category, dict):
continue
label = _text(category.get("label"))
if label and label not in categories:
categories.append(label)
return Subscription(
id=stream_id,
title=_text(entry.get("title")) or stream_id,
categories=categories,
html_url=_text(entry.get("htmlUrl")) or None,
)
def _parse_item(entry: Any) -> Item | None:
if not isinstance(entry, dict):
return None
item_id = entry.get("id")
if not isinstance(item_id, str) or not item_id:
return None
raw_origin = entry.get("origin")
origin: dict[str, Any] = raw_origin if isinstance(raw_origin, dict) else {}
feed_id = _text(origin.get("streamId"))
return Item(
id=item_id,
title=_text(entry.get("title")) or "(untitled)",
published=_as_int(entry.get("published")) or _timestamp_usec(entry),
updated=_as_int(entry.get("updated")) or _timestamp_usec(entry),
feed_id=feed_id,
feed_title=_text(origin.get("title")) or feed_id,
content_html=_extract_content(entry),
author=_text(entry.get("author")) or None,
url=_extract_url(entry, origin),
feed_url=_text(origin.get("htmlUrl")) or None,
categories=[tag for tag in (entry.get("categories") or []) if isinstance(tag, str)],
)
def _timestamp_usec(entry: dict[str, Any]) -> int:
"""Derive seconds from Google Reader's microsecond timestamp, when present."""
raw = entry.get("timestampUsec")
if isinstance(raw, str) and raw.isdigit():
return int(raw) // 1_000_000
if isinstance(raw, int):
return raw // 1_000_000
return 0
def _extract_content(entry: dict[str, Any]) -> str:
"""Prefer the full body over the summary.
FreshRSS stores whichever it has; ``content`` is the article body when the
feed provides one, and ``summary`` is the fallback.
"""
for key in ("content", "summary"):
block = entry.get(key)
if isinstance(block, str) and block.strip():
return block
if isinstance(block, dict):
inner = block.get("content")
if isinstance(inner, str) and inner.strip():
return inner
return ""
def _extract_url(entry: dict[str, Any], origin: dict[str, Any]) -> str | None:
alternates = entry.get("alternate")
if isinstance(alternates, list):
for candidate in alternates:
if not isinstance(candidate, dict):
continue
href = candidate.get("href")
if not isinstance(href, str) or not href:
continue
kind = candidate.get("type")
if kind in (None, "text/html"):
return href
for candidate in alternates:
if isinstance(candidate, dict) and isinstance(candidate.get("href"), str):
return candidate["href"]
html_url = origin.get("htmlUrl")
return html_url if isinstance(html_url, str) and html_url else None

193
app/images.py Normal file
View File

@@ -0,0 +1,193 @@
"""Image metadata: media type sniffing and dimension parsing.
Two small jobs, both in service of the EPUB writer.
*Sniffing.* A feed image is as likely to arrive as ``application/octet-stream``
or ``text/plain`` as with a correct ``Content-Type``. An EPUB manifest entry
whose media type does not match the bytes is invalid, so the type is decided from
the actual magic bytes and the response header is only a fallback.
*Dimensions.* Picking a cover means knowing which image is big enough to be one.
Reading a PNG header or a JPEG start-of-frame costs nothing and avoids pulling in
an imaging library (and its C dependencies) just to answer "is this wide?".
"""
from __future__ import annotations
import struct
# Types the EPUB writer will embed. SVG is deliberately absent: it can carry
# script and external references, and an image in an article body has no reason
# to need either.
MEDIA_TYPES = {
"image/jpeg": ".jpg",
"image/png": ".png",
"image/gif": ".gif",
"image/webp": ".webp",
}
_MEDIA_TYPE_ALIASES = {
"image/jpg": "image/jpeg",
"image/pjpeg": "image/jpeg",
"image/x-png": "image/png",
"image/x-gif": "image/gif",
}
def normalise_media_type(raw: str | None) -> str | None:
"""Map a Content-Type header onto a media type this server will embed."""
if not raw:
return None
# Strip parameters: "image/jpeg; charset=binary" is still a JPEG.
base = raw.split(";", 1)[0].strip().lower()
base = _MEDIA_TYPE_ALIASES.get(base, base)
return base if base in MEDIA_TYPES else None
def sniff_media_type(data: bytes) -> str | None:
"""Identify an image from its leading bytes, or None if unrecognised."""
if len(data) < 12:
return None
if data.startswith(b"\xff\xd8\xff"):
return "image/jpeg"
if data.startswith(b"\x89PNG\r\n\x1a\n"):
return "image/png"
if data.startswith((b"GIF87a", b"GIF89a")):
return "image/gif"
if data.startswith(b"RIFF") and data[8:12] == b"WEBP":
return "image/webp"
return None
def identify(data: bytes, declared: str | None) -> str | None:
"""Decide the media type for a fetched image.
The bytes win. A declared type is used only when the bytes are recognisable
as nothing else, and never to *override* a successful sniff -- otherwise a
mislabelled payload would produce an invalid EPUB manifest.
"""
sniffed = sniff_media_type(data)
if sniffed is not None:
return sniffed
return normalise_media_type(declared)
def dimensions(data: bytes) -> tuple[int, int] | None:
"""Return ``(width, height)`` in pixels, or None when it cannot be read.
Only the container headers are parsed: the value is used to decide whether an
image is a plausible cover, so "unknown" is a perfectly good answer.
"""
sniffed = sniff_media_type(data)
try:
if sniffed == "image/png":
return _png_dimensions(data)
if sniffed == "image/gif":
return _gif_dimensions(data)
if sniffed == "image/jpeg":
return _jpeg_dimensions(data)
if sniffed == "image/webp":
return _webp_dimensions(data)
except (struct.error, IndexError, ValueError):
return None
return None
def _png_dimensions(data: bytes) -> tuple[int, int] | None:
# IHDR is required to be the first chunk: 8 signature bytes, then length and
# type, so width and height sit at a fixed offset.
if len(data) < 24:
return None
width, height = struct.unpack(">II", data[16:24])
return (width, height) if width and height else None
def _gif_dimensions(data: bytes) -> tuple[int, int] | None:
if len(data) < 10:
return None
width, height = struct.unpack("<HH", data[6:10])
return (width, height) if width and height else None
def _webp_dimensions(data: bytes) -> tuple[int, int] | None:
if len(data) < 30:
return None
chunk = data[12:16]
if chunk == b"VP8X":
width = int.from_bytes(data[24:27], "little") + 1
height = int.from_bytes(data[27:30], "little") + 1
return (width, height)
if chunk == b"VP8 ":
# Lossy: dimensions are 14 bytes into the frame header, after the
# start code, with 14 bits of width and 14 of height.
if len(data) < 30:
return None
start = 26
width = int.from_bytes(data[start : start + 2], "little") & 0x3FFF
height = int.from_bytes(data[start + 2 : start + 4], "little") & 0x3FFF
return (width, height) if width and height else None
if chunk == b"VP8L":
bits = int.from_bytes(data[21:25], "little")
width = (bits & 0x3FFF) + 1
height = ((bits >> 14) & 0x3FFF) + 1
return (width, height)
return None
# JPEG start-of-frame markers, which carry the dimensions.
_SOF_MARKERS = frozenset(
{
0xC0,
0xC1,
0xC2,
0xC3,
0xC5,
0xC6,
0xC7,
0xC9,
0xCA,
0xCB,
0xCD,
0xCE,
0xCF,
}
)
def _jpeg_dimensions(data: bytes) -> tuple[int, int] | None:
"""Walk the JPEG segment chain to the start-of-frame marker.
A JPEG has no fixed header offset, so the segments have to be stepped over
until the frame header appears. This is bounded work: it stops at the first
frame marker, which always precedes the image data.
"""
index = 2 # skip the SOI marker (FF D8)
limit = len(data)
while index + 9 < limit:
if data[index] != 0xFF:
index += 1
continue
marker = data[index + 1]
if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7:
index += 2
continue
if marker == 0xDA: # start of scan: no frame header found
return None
length = struct.unpack(">H", data[index + 2 : index + 4])[0]
if length < 2:
return None
if marker in _SOF_MARKERS:
height, width = struct.unpack(">HH", data[index + 5 : index + 9])
return (width, height) if width and height else None
index += 2 + length
return None
def looks_like_a_cover(width: int, height: int) -> bool:
"""Whether an image is a plausible book cover.
A lead image is usually landscape and wide; a thumbnail, icon or tracking
pixel is not. The bar is deliberately low -- it only has to beat a blank
tile in a reader's library view.
"""
return width >= 400 and height >= 200

308
app/opds.py Normal file
View File

@@ -0,0 +1,308 @@
"""OPDS 1.2 catalog documents.
The dialect is Atom with the OPDS profile, because that is what ebook readers
actually implement: KOReader, Calibre, Moon+ Reader, Librera, FBReader and Aldiko
all browse OPDS 1.2, and its navigation/acquisition split maps exactly onto the
category/feed/article tree this server mirrors out of FreshRSS.
The two document kinds, and the rules that matter:
*Navigation feeds* (``kind=navigation``) are directories. Each entry is a
``rel="subsection"`` link to another feed, carrying a text description such as an
unread count.
*Acquisition feeds* (``kind=acquisition``) list articles. Each entry carries an
``rel="http://opds-spec.org/acquisition/open-access"`` link with an EPUB media
type, which is what a reader downloads. A plain ``http://opds-spec.org/acquisition``
link to the same URL is also emitted: readers disagree about which relation to
look for, and both are valid ways to say the same thing. An ``alternate`` link
points at the original article, which readers show as "open in browser".
Every feed carries ``rel="self"`` with the profile in its type, plus ``rel="start"``
back to the root, because readers navigate by these rather than by URL. Pagination
uses ``rel="next"``/``rel="previous"``/``rel="first"``.
Documents are built as strings rather than through a DOM library: the output is
small, fully determined, and easy to read in a test failure. Every interpolated
value goes through an escaper, and the tests parse the result with a strict XML
parser.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import UTC, datetime
from app import xhtml
ATOM_NAMESPACE = "http://www.w3.org/2005/Atom"
OPDS_NAMESPACE = "http://opds-spec.org/2010/catalog"
OPENSEARCH_NAMESPACE = "http://a9.com/-/spec/opensearch/1.1/"
DC_NAMESPACE = "http://purl.org/dc/elements/1.1/"
NAVIGATION_TYPE = "application/atom+xml;profile=opds-catalog;kind=navigation"
ACQUISITION_TYPE = "application/atom+xml;profile=opds-catalog;kind=acquisition"
OPENSEARCH_TYPE = "application/opensearchdescription+xml"
EPUB_TYPE = "application/epub+zip"
HTML_TYPE = "text/html"
ACQUISITION_REL = "http://opds-spec.org/acquisition"
OPEN_ACCESS_REL = "http://opds-spec.org/acquisition/open-access"
SUBSECTION_REL = "subsection"
ALTERNATE_REL = "alternate"
@dataclass(frozen=True)
class NavEntry:
"""One row of a navigation feed: a link deeper into the catalog."""
title: str
entry_id: str
href: str
updated: datetime
description: str | None = None
@dataclass(frozen=True)
class AcquisitionEntry:
"""One row of an acquisition feed: an article, as a downloadable book."""
title: str
entry_id: str
updated: datetime
download_href: str
author: str | None = None
summary: str | None = None
language: str | None = None
alternate_href: str | None = None
categories: tuple[tuple[str, str], ...] = ()
@dataclass(frozen=True)
class PageLinks:
"""Pagination targets, already absolute. Omitting one omits its link."""
first: str | None = None
previous: str | None = None
next: str | None = None
@dataclass(frozen=True)
class FeedLinks:
"""Everything a reader needs to place a feed in the catalog.
Pagination lives only in :attr:`pages`: an earlier version also carried
``next_url``/``previous_url``, and the renderer read those while callers set
``pages``, so the links silently never appeared.
"""
self_url: str
start_url: str | None = None
search_url: str | None = None
pages: PageLinks = field(default_factory=PageLinks)
def navigation_feed(
*,
feed_id: str,
title: str,
updated: datetime,
links: FeedLinks,
entries: list[NavEntry],
author: str = "OPDS",
subtitle: str | None = None,
) -> str:
"""Render a navigation feed."""
return _feed(
feed_id=feed_id,
title=title,
updated=updated,
links=links,
entries=[_nav_entry_xml(entry) for entry in entries],
kind_type=NAVIGATION_TYPE,
author=author,
subtitle=subtitle,
)
def acquisition_feed(
*,
feed_id: str,
title: str,
updated: datetime,
links: FeedLinks,
entries: list[AcquisitionEntry],
author: str = "OPDS",
subtitle: str | None = None,
) -> str:
"""Render an acquisition feed."""
return _feed(
feed_id=feed_id,
title=title,
updated=updated,
links=links,
entries=[_acquisition_entry_xml(entry) for entry in entries],
kind_type=ACQUISITION_TYPE,
author=author,
subtitle=subtitle,
)
def opensearch_description(
*,
search_url: str,
short_name: str,
description: str,
output_type: str = ACQUISITION_TYPE,
) -> str:
"""Render an OpenSearch description document.
Readers that support search advertise it by fetching this document, which is
why the root navigation feed links to it with ``rel="search"``.
"""
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<OpenSearchDescription xmlns="{OPENSEARCH_NAMESPACE}">',
f" <ShortName>{_escape(short_name)}</ShortName>",
f" <Description>{_escape(description)}</Description>",
" <InputEncoding>UTF-8</InputEncoding>",
" <OutputEncoding>UTF-8</OutputEncoding>",
f' <Url type="{_attribute(output_type)}" '
f'template="{_attribute(search_url)}?q={{searchTerms}}&amp;page={{startPage?}}"/>',
"</OpenSearchDescription>",
"",
]
)
# -- feed assembly -------------------------------------------------------
def _feed(
*,
feed_id: str,
title: str,
updated: datetime,
links: FeedLinks,
entries: list[str],
kind_type: str,
author: str,
subtitle: str | None,
) -> str:
lines = [
'<?xml version="1.0" encoding="utf-8"?>',
f'<feed xmlns="{ATOM_NAMESPACE}" xmlns:opds="{OPDS_NAMESPACE}" xmlns:dc="{DC_NAMESPACE}">',
f" <id>{_escape(feed_id)}</id>",
f" <title>{_escape(title)}</title>",
f" <updated>{_timestamp(updated)}</updated>",
]
if subtitle:
lines.append(f" <subtitle>{_escape(subtitle)}</subtitle>")
lines.append(f" <author><name>{_escape(author)}</name></author>")
lines.extend(_feed_links_xml(links, kind_type))
lines.extend(entries)
lines.extend(["</feed>", ""])
return "\n".join(lines)
def _feed_links_xml(links: FeedLinks, kind_type: str) -> list[str]:
"""Self, start, search and pagination links, in that order."""
rendered = [
f' <link rel="self" href="{_attribute(links.self_url)}" type="{_attribute(kind_type)}"/>'
]
if links.start_url:
rendered.append(
f' <link rel="start" href="{_attribute(links.start_url)}" '
f'type="{_attribute(NAVIGATION_TYPE)}"/>'
)
if links.search_url:
rendered.append(
f' <link rel="search" href="{_attribute(links.search_url)}" '
f'type="{_attribute(OPENSEARCH_TYPE)}"/>'
)
first = links.pages.first
if first:
rendered.append(
f' <link rel="first" href="{_attribute(first)}" type="{_attribute(kind_type)}"/>'
)
if links.pages.previous:
rendered.append(
f' <link rel="previous" href="{_attribute(links.pages.previous)}" '
f'type="{_attribute(kind_type)}"/>'
)
if links.pages.next:
rendered.append(
f' <link rel="next" href="{_attribute(links.pages.next)}" '
f'type="{_attribute(kind_type)}"/>'
)
return rendered
def _nav_entry_xml(entry: NavEntry) -> str:
lines = [
" <entry>",
f" <title>{_escape(entry.title)}</title>",
f" <id>{_escape(entry.entry_id)}</id>",
f" <updated>{_timestamp(entry.updated)}</updated>",
]
if entry.description:
lines.append(f' <content type="text">{_escape(entry.description)}</content>')
lines.extend(
[
f' <link rel="{SUBSECTION_REL}" href="{_attribute(entry.href)}" '
f'type="{_attribute(NAVIGATION_TYPE)}"/>',
" </entry>",
]
)
return "\n".join(lines)
def _acquisition_entry_xml(entry: AcquisitionEntry) -> str:
lines = [
" <entry>",
f" <title>{_escape(entry.title)}</title>",
f" <id>{_escape(entry.entry_id)}</id>",
f" <updated>{_timestamp(entry.updated)}</updated>",
]
if entry.author:
lines.append(f" <author><name>{_escape(entry.author)}</name></author>")
if entry.summary:
lines.append(f' <summary type="text">{_escape(entry.summary)}</summary>')
if entry.language:
lines.append(f" <dc:language>{_escape(entry.language)}</dc:language>")
for term, label in entry.categories:
lines.append(f' <category term="{_attribute(term)}" label="{_attribute(label)}"/>')
# Two acquisition relations to the same resource, because readers disagree
# about which one to look for; both are valid.
for rel in (OPEN_ACCESS_REL, ACQUISITION_REL):
lines.append(
f' <link rel="{rel}" href="{_attribute(entry.download_href)}" '
f'type="{_attribute(EPUB_TYPE)}"/>'
)
if entry.alternate_href:
lines.append(
f' <link rel="{ALTERNATE_REL}" href="{_attribute(entry.alternate_href)}" '
f'type="{_attribute(HTML_TYPE)}"/>'
)
lines.append(" </entry>")
return "\n".join(lines)
# -- formatting ----------------------------------------------------------
def _escape(text: str) -> str:
"""Escape text for a document body."""
return xhtml.escape_text(text)
def _attribute(value: str) -> str:
"""Escape a value for a double-quoted attribute."""
return xhtml.escape_attribute(value)
def _timestamp(moment: datetime) -> str:
"""Atom dates are RFC 3339 in UTC."""
return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")

1075
app/server.py Normal file

File diff suppressed because it is too large Load Diff

120
app/urls.py Normal file
View File

@@ -0,0 +1,120 @@
"""Identifier, slug and URL helpers.
FreshRSS ids do not survive a URL path or a Content-Disposition filename
unharmed. Item ids look like ``tag:google.com,2005:reader/item/0000000000000abc``
and stream ids look like ``feed/https://example.com/atom.xml?x=1``, so both
carry characters that are structural in a path (``/``, ``?``, ``#``) and are
long enough to blow past filename limits.
The rule this module enforces: every opaque id that appears in a path is
base64url-encoded, and the human-readable part of a download URL is a slug that
the server ignores when it looks the item up. That keeps routes unambiguous
without any escaping tricks, and lets a reader name the saved file sensibly.
"""
from __future__ import annotations
import base64
import binascii
import re
import unicodedata
import uuid
# Fixed namespace for every derived urn:uuid in this server. Hardcoded so that
# feed and entry ids stay stable across restarts and across hostnames -- a client
# that cached an entry id must still recognise it tomorrow.
_UUID_NAMESPACE = uuid.UUID("b7437da1-d469-46b1-a582-0af386fadb9d")
_B64URL_RE = re.compile(r"\A[A-Za-z0-9_-]+\Z")
# Everything that is not an ASCII lowercase letter or digit becomes a dash, so
# the result is safe in a path segment, a filename, and a shell.
_SLUG_JUNK_RE = re.compile(r"[^a-z0-9]+")
def b64url_encode(value: str) -> str:
"""Encode a UTF-8 string as unpadded base64url (path-segment safe)."""
raw = value.encode("utf-8")
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
def b64url_decode(value: str) -> str:
"""Decode an unpadded base64url string, raising ValueError on junk.
Strict on purpose: this parses untrusted path input, so anything that is not
a well-formed unpadded base64url token -- or that decodes to something other
than UTF-8 -- is rejected instead of being coerced.
"""
if not value or not _B64URL_RE.match(value):
raise ValueError("not a base64url token")
padding = "=" * (-len(value) % 4)
try:
raw = base64.urlsafe_b64decode(value + padding)
except (binascii.Error, ValueError) as exc:
raise ValueError("not a base64url token") from exc
try:
return raw.decode("utf-8")
except UnicodeDecodeError as exc:
raise ValueError("base64url token is not UTF-8") from exc
def slugify(text: str, max_length: int = 60) -> str:
"""Reduce a title to a conservative filename slug.
Accented characters are folded to ASCII rather than dropped, non-ASCII
scripts fall back to ``article``, and the result is length-capped because a
filesystem name is the eventual destination.
"""
folded = unicodedata.normalize("NFKD", text or "")
ascii_text = folded.encode("ascii", "ignore").decode("ascii")
slug = _SLUG_JUNK_RE.sub("-", ascii_text.lower()).strip("-")
if len(slug) > max_length:
slug = slug[:max_length].rstrip("-")
return slug or "article"
def stable_urn(kind: str, value: str) -> str:
"""Return a deterministic ``urn:uuid`` for a feed, entry or item.
Atom requires every feed and entry to carry a permanent id. Deriving it from
the FreshRSS id rather than from the request URL means the same article keeps
one identity no matter which category path the reader reached it through.
"""
derived = uuid.uuid5(_UUID_NAMESPACE, f"{kind}:{value}")
return f"urn:uuid:{derived}"
def content_disposition(ascii_name: str, unicode_name: str | None = None) -> str:
"""Build an attachment Content-Disposition with an RFC 5987 fallback.
Every OPDS reader understands ``filename``; readers that save a title with
non-ASCII characters need ``filename*``. Sending both is the only way to get
a correct name everywhere, so both are always emitted.
"""
ascii_name = ascii_name.replace('"', "").replace("\\", "") or "article.epub"
header = f'attachment; filename="{ascii_name}"'
if unicode_name and unicode_name != ascii_name:
encoded = _quote_rfc5987(unicode_name.replace('"', "").replace("\\", ""))
if encoded:
header += f"; filename*=UTF-8''{encoded}"
return header
def _quote_rfc5987(value: str) -> str:
out = []
for byte in value.encode("utf-8"):
char = chr(byte)
if char.isalnum() and byte < 0x80 or char in "-._~":
out.append(char)
else:
out.append(f"%{byte:02X}")
return "".join(out)
def join_url(base: str, *parts: str) -> str:
"""Join URL path parts onto a base, collapsing duplicate slashes."""
segments = [base.rstrip("/")]
for part in parts:
cleaned = part.strip("/")
if cleaned:
segments.append(cleaned)
return "/".join(segments)

747
app/xhtml.py Normal file
View File

@@ -0,0 +1,747 @@
"""Turn arbitrary feed HTML into well-formed XHTML for the EPUB pipeline.
EPUB is XML end to end: the OPF, the NCX, the navigation document and every
content document are parsed by a strict XML parser (the reader firmware this
server was written alongside uses expat). Feed article bodies are the opposite
of strict -- unclosed ``<li>``, stray ``&``, ``<br>`` instead of ``<br/>``,
control characters, embedded scripts and tracking pixels. Handing that to the
EPUB writer would produce a book that some readers refuse to open, so everything
goes through this normaliser first.
How it stays safe, in order of importance:
* The input is parsed into an in-memory tree and re-serialised from scratch.
Any malformedness in the source therefore cannot reach the output: whatever the
parser tolerated is rebuilt with balanced tags, quoted attributes and escaped
text. There is no pass-through of raw markup.
* Characters that XML 1.0 forbids outright (most C0 controls, unpaired
surrogates) are removed, because they are not expressible even when escaped.
* Entities are decoded to real Unicode by the parser and re-escaped on output, so
a stray ``&`` in the source cannot produce a document that will not parse.
* Scriptable and metadata elements are dropped with their contents; every other
unknown element is unwrapped, keeping its text.
* Only URLs that survive scheme validation reach an attribute, which keeps
``javascript:`` and ``data:`` payloads out of the book.
The tree is kept around after parsing so a caller can collect image URLs, fetch
them, and then serialise with the fetched images rewritten to local paths.
"""
from __future__ import annotations
import html
import logging
import re
from collections.abc import Iterator
from html.parser import HTMLParser
from urllib.parse import urljoin, urlsplit
LOGGER = logging.getLogger(__name__)
# Elements kept in the output. Mirrors the firmware reader's sanitiser
# (lib/FreshRss/src/FreshRssArticle.cpp) with the additions that make sense on a
# device with a real layout engine: images, and the small semantic set EPUB
# readers style (q, cite, time, abbr, del, ins, small, mark).
KEPT_TAGS = frozenset(
{
"p",
"div",
"span",
"br",
"hr",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"ul",
"ol",
"li",
"dl",
"dt",
"dd",
"blockquote",
"pre",
"code",
"em",
"i",
"strong",
"b",
"u",
"s",
"sub",
"sup",
"a",
"img",
"figure",
"figcaption",
"table",
"thead",
"tbody",
"tfoot",
"tr",
"td",
"th",
"caption",
"small",
"mark",
"cite",
"time",
"abbr",
"q",
"ins",
"del",
}
)
# Dropped along with everything inside them. These are either scriptable,
# fetch extra resources, or are document-level metadata that has no place inside
# an EPUB content document.
DROP_WITH_CONTENT = frozenset(
{
"script",
"style",
"iframe",
"frame",
"frameset",
"object",
"embed",
"applet",
"svg",
"math",
"form",
"input",
"button",
"select",
"option",
"optgroup",
"textarea",
"label",
"fieldset",
"legend",
"noscript",
"template",
"canvas",
"video",
"audio",
"source",
"track",
"map",
"area",
"head",
"title",
"meta",
"link",
"base",
"dialog",
"slot",
}
)
# HTML void elements: no end tag exists, so the parser must not expect one.
VOID_TAGS = frozenset({"br", "hr", "img"})
_HTML_VOID = frozenset(
{
"area",
"base",
"br",
"col",
"embed",
"hr",
"img",
"input",
"link",
"meta",
"param",
"source",
"track",
"wbr",
}
)
# Elements that silently end an open element in HTML. Feeds are full of
# unclosed <p> and <li>, and the browser-correct fix is to close the open
# element rather than to nest one block inside another (which is what a naive
# stack would do, producing <p>text<div>...</div></p>).
_IMPLICIT_CLOSE: dict[str, frozenset[str]] = {
"p": frozenset(
{
"address",
"article",
"aside",
"blockquote",
"details",
"div",
"dl",
"fieldset",
"figcaption",
"figure",
"footer",
"form",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"header",
"hr",
"main",
"nav",
"ol",
"p",
"pre",
"section",
"table",
"ul",
}
),
"li": frozenset({"li"}),
"dt": frozenset({"dt", "dd"}),
"dd": frozenset({"dt", "dd"}),
"td": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
"th": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
"tr": frozenset({"tr", "tbody", "tfoot", "thead"}),
"option": frozenset({"option", "optgroup"}),
"thead": frozenset({"tbody", "tfoot"}),
"tbody": frozenset({"tbody", "tfoot"}),
}
# Attributes kept, per element. Everything else is discarded: inline styles and
# layout attributes fight the EPUB stylesheet, and event handlers, tracking
# attributes and class names have no meaning in a book.
_KEPT_ATTRS: dict[str, frozenset[str]] = {
"a": frozenset({"href", "title"}),
"img": frozenset({"src", "alt", "title"}),
"blockquote": frozenset({"cite"}),
"q": frozenset({"cite"}),
"time": frozenset({"datetime"}),
"ins": frozenset({"datetime"}),
"del": frozenset({"datetime"}),
"abbr": frozenset({"title"}),
"td": frozenset({"colspan", "rowspan"}),
"th": frozenset({"colspan", "rowspan"}),
"ol": frozenset({"start"}),
}
# Empty structural elements worth removing once their contents are gone (for
# example a wrapper whose only child was a rejected image). Table cells are
# deliberately absent: an empty cell is layout, not noise.
_DROPPABLE_WHEN_EMPTY = frozenset(
{
"p",
"div",
"span",
"li",
"ul",
"ol",
"dl",
"blockquote",
"figure",
"figcaption",
"section",
"article",
}
)
_LINK_SCHEMES = ("http", "https", "mailto")
_IMAGE_SCHEMES = ("http", "https")
# XML 1.0 permits #x9, #xA, #xD, #x20-#xD7FF, #xE000-#xFFFD, #x10000-#x10FFFF.
# Everything else -- NUL bytes, vertical tabs from mangled feeds, form feeds,
# unpaired surrogates -- cannot be represented even as a character reference and
# must be deleted before serialisation.
_ILLEGAL_XML_RE = re.compile(
"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]"
)
# Collapses runs of layout whitespace. A non-breaking space is deliberately not
# matched: feeds use it for intentional spacing, and reflowable text keeps it.
_WHITESPACE_RE = re.compile(r"[^\S\xa0]+")
# Trims layout whitespace from the edges of a block. Python's str.strip() treats
# a non-breaking space as whitespace and would eat one the author typed on
# purpose, so the pattern excludes it here too.
_BLOCK_TRIM_RE = re.compile(r"\A[^\S\xa0]+|[^\S\xa0]+\Z")
_ZERO_WIDTH_RE = re.compile("[\u200b\u200c\u200d\u2060\ufeff]")
# XML name rules for an id: starts with a letter or underscore, then a bounded run
# of name characters. Anything else is not a usable anchor target.
_ID_RE = re.compile(r"\A[A-Za-z_][A-Za-z0-9_.-]{0,63}\Z")
_ID_ATTRS = frozenset({"id"})
# Wrappers that need the attribute filter relaxed for integer attributes.
_INT_ATTR_LIMITS = {"colspan": (1, 100), "rowspan": (1, 100), "start": (-10000, 10000)}
class _Element:
"""One element node in the parsed tree."""
__slots__ = ("attrs", "children", "tag")
def __init__(self, tag: str) -> None:
self.tag = tag
self.attrs: dict[str, str] = {}
self.children: list[_Element | _Text] = []
class _Text:
"""One text node in the parsed tree."""
__slots__ = ("text",)
def __init__(self, text: str) -> None:
self.text = text
class _TreeBuilder(HTMLParser):
"""Build a normalised element tree from feed HTML.
The parser is tolerant by construction -- it never raises on malformed input,
it only decides what to keep -- so this class is the single place where all
of the "this feed is broken" handling lives.
"""
def __init__(self) -> None:
# convert_charrefs decodes &amp;/&#233;/&nbsp; to Unicode for us, so the
# serialiser only ever has to escape what it writes itself.
super().__init__(convert_charrefs=True)
self.root = _Element("#root")
self._stack: list[_Element] = [self.root]
# Names of open drop-with-content elements; while non-empty, all input is
# discarded except the end tag that closes the innermost one.
self._suppressed: list[str] = []
# -- parser callbacks -------------------------------------------------
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if self._suppressed:
if tag in DROP_WITH_CONTENT and tag not in _HTML_VOID:
self._suppressed.append(tag)
return
if tag in DROP_WITH_CONTENT:
if tag not in _HTML_VOID:
self._suppressed.append(tag)
return
self._close_implied(tag)
if tag in _HTML_VOID and tag not in VOID_TAGS:
# Void but not kept (<col>, <wbr>, <param>): it cannot have content,
# so it must never open a subtree that would swallow the rest.
# br/hr/img are void *and* kept, so they fall through to be emitted.
return
element = _Element(tag)
element.attrs = self._filter_attrs(tag, attrs)
self._stack[-1].children.append(element)
if tag not in VOID_TAGS:
self._stack.append(element)
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
# <br/> lands here. A self-closing element is complete: it is appended
# and never pushed, so there is nothing left open. A self-closing
# drop-with-content element has no content to discard and must not open a
# suppression region, or the rest of the document would be swallowed.
if self._suppressed or tag in DROP_WITH_CONTENT or tag in _HTML_VOID:
return
self._close_implied(tag)
element = _Element(tag)
element.attrs = self._filter_attrs(tag, attrs)
self._stack[-1].children.append(element)
def handle_endtag(self, tag: str) -> None:
if self._suppressed:
if tag == self._suppressed[-1]:
self._suppressed.pop()
return
if tag in _HTML_VOID:
return
# Close down to the matching open element, tolerating unclosed tags in
# between (this is exactly what browsers do).
for index in range(len(self._stack) - 1, 0, -1):
if self._stack[index].tag == tag:
del self._stack[index:]
return
# Stray end tag with no matching open element: ignore it.
def handle_data(self, data: str) -> None:
if self._suppressed or not data:
return
self._stack[-1].children.append(_Text(data))
def handle_comment(self, data: str) -> None:
return
def handle_decl(self, decl: str) -> None:
return
def handle_pi(self, data: str) -> None:
return
def unknown_decl(self, data: str) -> None:
if self._suppressed:
return
# An HTML parser treats <![CDATA[...]]> as a bogus comment and discards
# it, but a feed body that passed through an XML parser upstream can
# legitimately contain one, and its payload is real content. The parser
# hands over the section body with the closing ]] already stripped.
marker = "CDATA["
if data.startswith(marker):
inner = data[len(marker) :]
if inner:
self._stack[-1].children.append(_Text(inner))
# -- helpers ---------------------------------------------------------
def _close_implied(self, tag: str) -> None:
"""Close open elements that HTML would close implicitly."""
while len(self._stack) > 1:
conditions = _IMPLICIT_CLOSE.get(self._stack[-1].tag)
if conditions is None or tag not in conditions:
return
self._stack.pop()
def _filter_attrs(self, tag: str, attrs: list[tuple[str, str | None]]) -> dict[str, str]:
if tag not in KEPT_TAGS:
# The element is unwrapped later, so its attributes are never used.
return {}
allowed = _KEPT_ATTRS.get(tag, frozenset())
if tag not in VOID_TAGS:
# Anchor targets survive so an in-book table of contents still jumps
# to its section when the article is read offline.
allowed = allowed | _ID_ATTRS
kept: dict[str, str] = {}
for raw_name, raw_value in attrs:
name = raw_name.lower()
if name not in allowed:
continue
value = html.unescape(raw_value or "") if raw_value is not None else ""
value = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", value)).strip()
if name == "id":
if not _ID_RE.match(value):
continue
kept[name] = value
continue
if name in _INT_ATTR_LIMITS:
bounds = _INT_ATTR_LIMITS[name]
try:
number = int(value)
except ValueError:
continue
if not bounds[0] <= number <= bounds[1]:
continue
value = str(number)
if not value:
continue
kept[name] = value
return kept
class Article:
"""A parsed, normalised article body ready for EPUB serialisation."""
def __init__(self, tree: _Element, base_url: str, truncated: bool) -> None:
self._tree = tree
self._base_url = base_url
self.truncated = truncated
self._image_urls: list[str] | None = None
@property
def image_urls(self) -> list[str]:
"""Absolute http(s) image URLs, in document order, de-duplicated."""
if self._image_urls is None:
seen: list[str] = []
known: set[str] = set()
for element in _iter_elements(self._tree):
if element.tag != "img":
continue
src = element.attrs.get("src", "")
if src and src not in known:
known.add(src)
seen.append(src)
self._image_urls = seen
return self._image_urls
def text(self, limit: int = 600) -> str:
"""Flattened plain text, for summaries and the search index."""
parts: list[str] = []
for element in _iter_text(self._tree):
parts.append(element.text)
collapsed = _WHITESPACE_RE.sub(" ", _ILLEGAL_XML_RE.sub("", "".join(parts))).strip()
if len(collapsed) <= limit:
return collapsed
# Cut at a word boundary so a summary never ends mid-word.
cut = collapsed[:limit].rsplit(" ", 1)[0]
return (cut or collapsed[:limit]).rstrip() + "\u2026"
def to_xhtml(self, image_map: dict[str, str | None] | None = None) -> str:
"""Serialise the body as an XHTML fragment.
``image_map`` maps an original absolute image URL to the local href to
use in its place; a value of None drops that image (and its wrapper, if it
becomes empty). Pass None to leave remote image URLs untouched -- the
caller must then declare remote resources in the OPF manifest.
Applying a map rewrites the tree in place, so an Article is rendered
once. The server builds a book and serialises it exactly once.
"""
if image_map is not None:
_apply_image_map(self._tree, image_map)
# A dropped image can leave an empty wrapper behind.
_prune_empty(self._tree)
body = _serialise_children(self._tree, inside_pre=False)
if not body.strip():
return ""
return f'<div class="article-body">\n{body}\n</div>'
class _Resolve:
"""Decides which URLs survive normalisation, and where they point."""
def __init__(self, base_url: str) -> None:
self.base_url = base_url
def link(self, raw: str) -> str | None:
# A same-document reference is left alone so it resolves against the book
# rather than against the web page the article came from. The matching id
# was kept by the attribute filter.
if raw.startswith("#"):
return raw
resolved = _absolutise(raw, self.base_url)
if resolved is None:
return None
if urlsplit(resolved).scheme.lower() not in _LINK_SCHEMES:
return None
return resolved
def image(self, raw: str) -> str | None:
resolved = _absolutise(raw, self.base_url)
if resolved is None:
return None
if urlsplit(resolved).scheme.lower() not in _IMAGE_SCHEMES:
return None
return resolved
def parse_article(raw_html: str, base_url: str = "", max_input_bytes: int = 0) -> Article:
"""Parse a feed article body into a normalised tree.
``base_url`` is the article's own URL; relative links and images inside the
body are resolved against it, which is the only way they keep working once
the body is inside a book with a different base.
"""
text = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", raw_html or ""))
truncated = False
if max_input_bytes > 0 and len(text.encode("utf-8", "ignore")) > max_input_bytes:
# Cut on a character boundary, not a byte boundary, and do not try to be
# clever about tags: the parser tolerates a truncated tail.
while len(text.encode("utf-8", "ignore")) > max_input_bytes:
text = text[: int(len(text) * 0.98) or 1]
truncated = True
LOGGER.warning("article body truncated to %d bytes before parsing", max_input_bytes)
builder = _TreeBuilder()
builder.feed(text)
builder.close()
tree = builder.root
if not _has_elements(tree):
# A body with no markup at all (plain text feed) still needs block
# structure, or the EPUB would contain a bare text node.
paragraph = _Element("p")
paragraph.children = list(tree.children)
tree.children = [paragraph]
_resolve_urls(tree, _Resolve(base_url), set())
_prune_empty(tree)
return Article(tree, base_url, truncated)
def _absolutise(raw: str, base_url: str) -> str | None:
candidate = (raw or "").strip()
if not candidate:
return None
# Control characters and raw whitespace inside a URL are always corruption.
candidate = _ILLEGAL_XML_RE.sub("", candidate)
if any(char.isspace() for char in candidate):
candidate = candidate.split()[0]
try:
if base_url:
return urljoin(base_url, candidate)
return candidate
except ValueError:
return None
def _resolve_urls(element: _Element, resolve: _Resolve, seen_ids: set[str]) -> None:
"""Validate URLs and de-duplicate anchor ids, in document order.
Runs once, at parse time, so everything downstream (the image collector, the
fetcher, the OPF manifest) sees final absolute URLs and there is no second
chance for a bad scheme to slip through.
"""
for child in element.children:
if isinstance(child, _Text):
continue
_dedupe_id(child, seen_ids)
if child.tag == "a":
href = child.attrs.get("href")
if href is not None:
target = resolve.link(href)
if target is None:
child.attrs.pop("href", None)
else:
child.attrs["href"] = target
elif child.tag == "img":
src = child.attrs.get("src")
target = resolve.image(src) if src else None
if target is None:
# Mark it for pruning: an image with no usable source is noise.
child.tag = "#dropped-image"
else:
child.attrs["src"] = target
elif child.tag in ("blockquote", "q"):
cite = child.attrs.get("cite")
if cite is not None:
target = resolve.link(cite)
if target is None:
child.attrs.pop("cite", None)
else:
child.attrs["cite"] = target
_resolve_urls(child, resolve, seen_ids)
def _dedupe_id(element: _Element, seen_ids: set[str]) -> None:
"""Make an element's id unique within the document.
Duplicate ids are invalid and would make the first anchor win, so a repeat is
renamed rather than dropped -- dropping it would break the link pointing at
it. One book holds one article, so a document-wide set is enough.
"""
anchor = element.attrs.get("id")
if anchor is None:
return
if anchor not in seen_ids:
seen_ids.add(anchor)
return
index = 2
while f"{anchor}-{index}" in seen_ids:
index += 1
renamed = f"{anchor}-{index}"
seen_ids.add(renamed)
element.attrs["id"] = renamed
def _apply_image_map(element: _Element, image_map: dict[str, str | None]) -> None:
"""Point images at their local copies inside the book.
A URL mapped to None means the fetch failed or the image was refused, so the
element is marked for pruning instead of being left pointing at the network.
"""
for child in element.children:
if isinstance(child, _Text):
continue
if child.tag == "img":
src = child.attrs.get("src")
if src is not None and src in image_map:
target = image_map[src]
if target is None:
child.tag = "#dropped-image"
else:
child.attrs["src"] = target
_apply_image_map(child, image_map)
def _prune_empty(node: _Element) -> None:
"""Remove rejected images and the empty wrappers they leave behind."""
kept: list[_Element | _Text] = []
for child in node.children:
if isinstance(child, _Element):
if child.tag == "#dropped-image":
continue
_prune_empty(child)
if _is_noise(child):
continue
kept.append(child)
node.children = kept
def _is_noise(element: _Element) -> bool:
"""True for a droppable wrapper with no content left inside it.
Whitespace-only text does not count as content (``<p>&nbsp;</p>`` is noise),
but a retained ``id`` does: it is the target of a link.
"""
if element.tag not in _DROPPABLE_WHEN_EMPTY or element.attrs:
return False
for child in element.children:
if isinstance(child, _Element):
return False
if child.text.strip():
return False
return True
def _has_elements(element: _Element) -> bool:
"""True when the tree contains at least one element node."""
return any(True for _ in _iter_elements(element))
def _serialise_children(node: _Element, inside_pre: bool) -> str:
return "".join(_serialise(child, inside_pre) for child in node.children)
def _serialise(node: _Element | _Text, inside_pre: bool) -> str:
if isinstance(node, _Text):
if inside_pre:
return escape_text(node.text)
# Collapse whitespace outside <pre>: feed HTML arrives indented, and that
# indentation is pure bytes in the book.
return escape_text(_WHITESPACE_RE.sub(" ", node.text))
if node.tag not in KEPT_TAGS:
# Unwrapped elements keep their children but lose their own tag, so no
# unknown element can appear in the output.
return _serialise_children(node, inside_pre)
child_pre = inside_pre or node.tag == "pre"
inner = _serialise_children(node, child_pre)
if node.tag in ("p", "div", "li", "td", "th", "blockquote", "figcaption", "caption"):
inner = _BLOCK_TRIM_RE.sub("", inner)
attributes = "".join(
f' {name}="{escape_attribute(value)}"' for name, value in _ordered_attrs(node)
)
if node.tag in VOID_TAGS:
return f"<{node.tag}{attributes}/>"
return f"<{node.tag}{attributes}>{inner}</{node.tag}>"
def _ordered_attrs(element: _Element) -> list[tuple[str, str]]:
"""Stable attribute order, with the most meaningful attribute first."""
priority = {"id": 0, "href": 1, "src": 2, "alt": 3, "title": 4}
return sorted(element.attrs.items(), key=lambda item: (priority.get(item[0], 9), item[0]))
def escape_text(text: str) -> str:
return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
def escape_attribute(value: str) -> str:
# Attribute values are normalised by XML parsers anyway (tab/newline become
# spaces), so do it explicitly rather than leaving it implicit.
text = value.replace("\t", " ").replace("\n", " ").replace("\r", " ")
return (
text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
)
def _iter_elements(node: _Element) -> Iterator[_Element]:
"""Depth-first traversal over element nodes only."""
for child in node.children:
if isinstance(child, _Element):
yield child
yield from _iter_elements(child)
def _iter_text(node: _Element) -> Iterator[_Text]:
"""Depth-first traversal over text nodes only."""
for child in node.children:
if isinstance(child, _Text):
yield child
else:
yield from _iter_text(child)