Initial commit
This commit is contained in:
42
app/__init__.py
Normal file
42
app/__init__.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""OPDS server backed by FreshRSS.
|
||||
|
||||
An OPDS 1.2 catalog whose shelves mirror FreshRSS categories and feeds, serving
|
||||
one EPUB per article, generated in memory at download time. Nothing is written to
|
||||
disk: the catalog is rebuilt from the FreshRSS API as it is browsed, and a
|
||||
completed EPUB download marks the corresponding article read upstream.
|
||||
|
||||
Layout, in dependency order:
|
||||
|
||||
``config``
|
||||
The JSON document, its interpolation (``${ENV}``, ``${file:...}``) and its
|
||||
validation. Everything else is configured from here.
|
||||
``urls``
|
||||
Identifier and filename helpers: base64url for path identifiers, stable
|
||||
``urn:uuid`` values for Atom ids, and safe download filenames.
|
||||
``cache``
|
||||
A bounded in-memory TTL cache with per-key single-flight, so a reader
|
||||
browsing quickly does not turn into a burst of upstream requests.
|
||||
``xhtml``
|
||||
Feed HTML into well-formed XHTML. Everything the reader's XML parser sees
|
||||
has been rebuilt by this module.
|
||||
``images``
|
||||
Media-type sniffing and dimension parsing, so a book's manifest is honest
|
||||
about its bytes and a cover can be chosen without an imaging library.
|
||||
``fetch``
|
||||
The one place that fetches a host the operator did not choose. Article
|
||||
images are fetched through an SSRF guard that dials a validated address.
|
||||
``epub``
|
||||
The EPUB 3 writer, assembled entirely in memory.
|
||||
``freshrss``
|
||||
The FreshRSS client, over its Google Reader compatible API.
|
||||
``auth``
|
||||
HTTP Basic authentication, with a throttle on failed attempts.
|
||||
``opds``
|
||||
OPDS 1.2 Atom document builders.
|
||||
``server``
|
||||
Routing, the catalog logic, and EPUB delivery.
|
||||
|
||||
Run it with ``python -m app --config config.json``.
|
||||
"""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
127
app/__main__.py
Normal file
127
app/__main__.py
Normal file
@@ -0,0 +1,127 @@
|
||||
"""Entry point: load the configuration, then serve.
|
||||
|
||||
Startup is deliberately noisy about anything that will be hard to diagnose later:
|
||||
warnings from configuration validation, the effective settings (with secrets
|
||||
redacted), and any degraded optional feature. An OPDS server that is reachable
|
||||
but shows an empty catalog should say why in its own log, not require a
|
||||
debugging session.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import signal
|
||||
import ssl
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import app.server as app_server
|
||||
from app import __version__
|
||||
from app import config as app_config
|
||||
|
||||
LOGGER = logging.getLogger("app")
|
||||
|
||||
_LOG_FORMAT = "%(asctime)s %(levelname)-7s %(name)s: %(message)s"
|
||||
_LOG_DATEFORMAT = "%Y-%m-%dT%H:%M:%S%z"
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="opds-from-rss",
|
||||
description="Serve a FreshRSS account as an OPDS 1.2 catalog of EPUBs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
metavar="PATH",
|
||||
help="configuration file (default: $CONFIG_PATH, /config/config.json, config.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check-config",
|
||||
action="store_true",
|
||||
help="validate the configuration and print it, then exit",
|
||||
)
|
||||
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
||||
return parser
|
||||
|
||||
|
||||
def configure_logging(level: str) -> None:
|
||||
logging.basicConfig(level=level.upper(), format=_LOG_FORMAT, datefmt=_LOG_DATEFORMAT)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
arguments = build_parser().parse_args(argv)
|
||||
# Logging is configured twice: once coarsely so a configuration failure is
|
||||
# reported properly, then again at the level the configuration asks for.
|
||||
configure_logging("info")
|
||||
try:
|
||||
settings = app_config.load_config(arguments.config)
|
||||
except app_config.ConfigError as exc:
|
||||
# The exception already lists every problem with its path.
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
configure_logging(settings.server.log_level)
|
||||
for warning in settings.warnings:
|
||||
LOGGER.warning("configuration: %s", warning)
|
||||
for line in app_config.describe_config(settings).splitlines():
|
||||
LOGGER.info("%s", line)
|
||||
|
||||
if arguments.check_config:
|
||||
LOGGER.info("configuration is valid; not starting the server")
|
||||
return 0
|
||||
|
||||
httpd, application = app_server.build_server(settings)
|
||||
if settings.server.tls_cert_file and settings.server.tls_key_file:
|
||||
_enable_tls(httpd, settings.server.tls_cert_file, settings.server.tls_key_file)
|
||||
LOGGER.info("serving HTTPS directly (no reverse proxy required)")
|
||||
else:
|
||||
LOGGER.warning(
|
||||
"serving plain HTTP: put a TLS-terminating proxy in front of this, "
|
||||
"because HTTP Basic credentials are only base64-encoded"
|
||||
)
|
||||
_install_signal_handlers(httpd)
|
||||
|
||||
LOGGER.info(
|
||||
"listening on %s:%s; catalog at %s/opds",
|
||||
settings.server.host,
|
||||
settings.server.port,
|
||||
settings.server.public_base_url or f"http://{settings.server.host}:{settings.server.port}",
|
||||
)
|
||||
try:
|
||||
httpd.serve_forever(poll_interval=0.25)
|
||||
finally:
|
||||
LOGGER.info("shutting down")
|
||||
httpd.server_close()
|
||||
application.client.close()
|
||||
return 0
|
||||
|
||||
|
||||
def _enable_tls(httpd, cert_file: str, key_file: str) -> None:
|
||||
"""Wrap the listening socket, so the service can be secure on its own.
|
||||
|
||||
The paths are passed in rather than read off the configuration so that the
|
||||
"both or neither" guarantee made during validation is visible here too.
|
||||
"""
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(cert_file, key_file)
|
||||
httpd.socket = context.wrap_socket(httpd.socket, server_side=True)
|
||||
|
||||
|
||||
def _install_signal_handlers(httpd) -> None:
|
||||
"""Stop cleanly on SIGINT/SIGTERM, which is how a container is stopped.
|
||||
|
||||
``shutdown`` blocks until the serve loop exits and must not be called from the
|
||||
thread running it, so the handler hands the work to a short-lived thread.
|
||||
"""
|
||||
|
||||
def stop(signum: int, _frame: object) -> None:
|
||||
LOGGER.info("received %s", signal.Signals(signum).name)
|
||||
threading.Thread(target=httpd.shutdown, daemon=True).start()
|
||||
|
||||
for handled in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(handled, stop)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
186
app/auth.py
Normal file
186
app/auth.py
Normal file
@@ -0,0 +1,186 @@
|
||||
"""HTTP Basic authentication, with a throttle on failed attempts.
|
||||
|
||||
Basic is the only scheme every OPDS reader UI can drive: the reader prompts for a
|
||||
user name and password, and the server answers an unauthenticated request with
|
||||
``401`` and a ``WWW-Authenticate`` header. That header is also the whole reason
|
||||
this service must sit behind TLS -- the credentials travel base64-encoded, which
|
||||
is encoding, not encryption.
|
||||
|
||||
Two things this module is careful about:
|
||||
|
||||
*The comparison is constant time.* Passwords are stored as sha256 digests
|
||||
(``make hash-password``) and compared with :func:`hmac.compare_digest`, and an
|
||||
unknown user name is compared against a dummy digest so that a missing account and
|
||||
a wrong password take the same time. Otherwise the response time tells an
|
||||
attacker which half of the guess was right.
|
||||
|
||||
*Failures are throttled.* Basic auth can be guessed at line speed, so an address
|
||||
that fails repeatedly is refused for a cooling-off period before its credentials
|
||||
are even examined.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import hmac
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from app import config as app_config
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# Compared against when the user name is unknown, so that the work done -- and
|
||||
# therefore the time taken -- does not depend on whether the account exists.
|
||||
_DUMMY_DIGEST = hashlib.sha256(b"no such user").hexdigest()
|
||||
|
||||
|
||||
class UnauthorizedError(Exception):
|
||||
"""Credentials were missing, malformed, or wrong. Answered with 401."""
|
||||
|
||||
def __init__(self, message: str = "authentication required") -> None:
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class RateLimitedError(Exception):
|
||||
"""Too many failed attempts from this address. Answered with 429."""
|
||||
|
||||
def __init__(self, retry_after: int) -> None:
|
||||
super().__init__("too many failed authentication attempts")
|
||||
self.retry_after = max(1, int(retry_after))
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Attempts:
|
||||
"""Failed-attempt bookkeeping for one client address."""
|
||||
|
||||
timestamps: list[float] = field(default_factory=list)
|
||||
|
||||
def within_window(self, now: float, window: float) -> int:
|
||||
return sum(1 for moment in self.timestamps if now - moment < window)
|
||||
|
||||
|
||||
class Authenticator:
|
||||
"""Checks Basic credentials and throttles repeated failures."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: app_config.AuthConfig,
|
||||
*,
|
||||
clock=time.monotonic,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._clock = clock
|
||||
# Digest lookups avoid a linear scan that would leak user names by timing.
|
||||
self._users = {user.username: user.password_sha256 for user in config.users}
|
||||
self._lock = threading.Lock()
|
||||
self._attempts: dict[str, _Attempts] = {}
|
||||
|
||||
@property
|
||||
def challenge(self) -> str:
|
||||
"""The ``WWW-Authenticate`` value that makes a reader prompt."""
|
||||
realm = self._config.realm.replace('"', "")
|
||||
return f'Basic realm="{realm}", charset="UTF-8"'
|
||||
|
||||
def authenticate(self, authorization: str | None, client: str) -> str:
|
||||
"""Return the authenticated user name, or raise.
|
||||
|
||||
``client`` identifies the address for throttling; only failed attempts are
|
||||
counted, and a success clears the record.
|
||||
"""
|
||||
self._check_throttle(client)
|
||||
username, password = _parse_basic(authorization)
|
||||
if username is None or not self._matches(username, password):
|
||||
self._record_failure(client)
|
||||
# The source address is what matters for spotting a brute-force run, so
|
||||
# that is the warning. The attempted name is attacker-supplied and
|
||||
# close to an identifier, so it is kept for debugging only, and the
|
||||
# password is never logged at any level.
|
||||
LOGGER.warning("rejected authentication attempt from %s", client)
|
||||
LOGGER.debug("rejected credentials for user %r from %s", username, client)
|
||||
raise UnauthorizedError
|
||||
self._clear_failures(client)
|
||||
return username
|
||||
|
||||
def _matches(self, username: str, password: str | None) -> bool:
|
||||
if password is None:
|
||||
return False
|
||||
expected = self._users.get(username, _DUMMY_DIGEST)
|
||||
offered = hashlib.sha256(password.encode("utf-8")).hexdigest()
|
||||
return hmac.compare_digest(expected, offered)
|
||||
|
||||
# -- throttling ------------------------------------------------------
|
||||
|
||||
def _check_throttle(self, client: str) -> None:
|
||||
window = self._config.throttle_seconds
|
||||
with self._lock:
|
||||
record = self._attempts.get(client)
|
||||
if record is None:
|
||||
return
|
||||
now = self._clock()
|
||||
recent = record.within_window(now, window)
|
||||
if recent < self._config.max_failures_per_minute:
|
||||
return
|
||||
oldest = min(record.timestamps)
|
||||
retry_after = max(1, int(window - (now - oldest)) + 1)
|
||||
raise RateLimitedError(retry_after)
|
||||
|
||||
def _record_failure(self, client: str) -> None:
|
||||
now = self._clock()
|
||||
window = self._config.throttle_seconds
|
||||
with self._lock:
|
||||
record = self._attempts.setdefault(client, _Attempts())
|
||||
record.timestamps.append(now)
|
||||
# Trim anything that has aged out, and drop the record entirely when
|
||||
# it is empty, so a long uptime cannot accumulate addresses.
|
||||
record.timestamps = [moment for moment in record.timestamps if now - moment < window]
|
||||
if not record.timestamps:
|
||||
del self._attempts[client]
|
||||
elif len(self._attempts) > 4096:
|
||||
self._evict_oldest(now, window)
|
||||
|
||||
def _clear_failures(self, client: str) -> None:
|
||||
with self._lock:
|
||||
self._attempts.pop(client, None)
|
||||
|
||||
def _evict_oldest(self, now: float, window: float) -> None:
|
||||
"""Bound the table: a flood of addresses must not become a memory leak."""
|
||||
stale = [
|
||||
address
|
||||
for address, record in self._attempts.items()
|
||||
if not record.within_window(now, window)
|
||||
]
|
||||
for address in stale:
|
||||
del self._attempts[address]
|
||||
while len(self._attempts) > 4096:
|
||||
oldest = min(
|
||||
self._attempts,
|
||||
key=lambda address: min(self._attempts[address].timestamps),
|
||||
)
|
||||
del self._attempts[oldest]
|
||||
|
||||
|
||||
def _parse_basic(authorization: str | None) -> tuple[str | None, str | None]:
|
||||
"""Split a Basic header into its credentials.
|
||||
|
||||
Returns ``(None, None)`` for anything that is not a well-formed Basic header,
|
||||
which the caller treats exactly like a wrong password.
|
||||
"""
|
||||
if not authorization:
|
||||
return None, None
|
||||
scheme, _, payload = authorization.partition(" ")
|
||||
if scheme.lower() != "basic" or not payload:
|
||||
return None, None
|
||||
try:
|
||||
decoded = base64.b64decode(payload.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError):
|
||||
return None, None
|
||||
# Only the first colon separates: a password may contain colons.
|
||||
username, separator, password = decoded.partition(":")
|
||||
if not separator or not username:
|
||||
return None, None
|
||||
return username, password
|
||||
157
app/cache.py
Normal file
157
app/cache.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""Bounded in-memory TTL cache with per-key single-flight.
|
||||
|
||||
The server keeps nothing on disk, which means every navigation click and every
|
||||
download is backed by a live FreshRSS API call. An OPDS reader browses a tree
|
||||
and re-requests feeds freely, so without a cache a single reader produces a small
|
||||
burst of API traffic per tap. This cache absorbs that burst and nothing more:
|
||||
entries expire on a short TTL and are evicted when the cache is full.
|
||||
|
||||
Single-flight matters as much as caching here. Readers often open several
|
||||
connections at once (browsing navigation feeds in parallel, or prefetching cover
|
||||
requests), and without it a cold cache would issue the same FreshRSS request
|
||||
several times over. ``get_or_compute`` guarantees one in-flight computation per
|
||||
key while still letting unrelated keys run concurrently.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TTLCache:
|
||||
"""A thread-safe, size-bounded cache of values with a per-entry TTL."""
|
||||
|
||||
def __init__(self, max_entries: int = 512) -> None:
|
||||
# Config validation guarantees an int here, so no defensive coercion.
|
||||
self._max_entries = max(1, max_entries)
|
||||
self._lock = threading.Lock()
|
||||
self._values: dict[str, tuple[float, Any]] = {}
|
||||
self._key_locks: dict[str, threading.Lock] = {}
|
||||
self.hits = 0
|
||||
self.misses = 0
|
||||
self.evictions = 0
|
||||
|
||||
def get(self, key: str, *, now: float | None = None) -> Any | None:
|
||||
"""Return a live value, or None when missing or expired."""
|
||||
moment = time.monotonic() if now is None else now
|
||||
with self._lock:
|
||||
entry = self._values.get(key)
|
||||
if entry is None:
|
||||
self.misses += 1
|
||||
return None
|
||||
expires_at, value = entry
|
||||
if expires_at <= moment:
|
||||
del self._values[key]
|
||||
self.misses += 1
|
||||
return None
|
||||
# Re-insert so the newest entries sit at the end of the ordering and
|
||||
# the eviction below drops genuinely cold keys first.
|
||||
del self._values[key]
|
||||
self._values[key] = (expires_at, value)
|
||||
self.hits += 1
|
||||
return value
|
||||
|
||||
def set(self, key: str, value: Any, ttl_seconds: float) -> None:
|
||||
"""Store a value, evicting the coldest entry if the cache is full."""
|
||||
if ttl_seconds <= 0:
|
||||
return
|
||||
expires_at = time.monotonic() + ttl_seconds
|
||||
with self._lock:
|
||||
self._values.pop(key, None)
|
||||
self._values[key] = (expires_at, value)
|
||||
while len(self._values) > self._max_entries:
|
||||
oldest, _ = next(iter(self._values.items()))
|
||||
del self._values[oldest]
|
||||
self.evictions += 1
|
||||
|
||||
def get_or_compute(self, key: str, ttl_seconds: float, compute: Callable[[], Any]) -> Any:
|
||||
"""Return a cached value, computing it at most once per key at a time.
|
||||
|
||||
Exceptions from ``compute`` are never cached, so a transient FreshRSS
|
||||
failure is retried on the next request instead of being remembered.
|
||||
"""
|
||||
cached = self.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
with self._lock:
|
||||
key_lock = self._key_locks.get(key)
|
||||
if key_lock is None:
|
||||
key_lock = threading.Lock()
|
||||
self._key_locks[key] = key_lock
|
||||
|
||||
with key_lock:
|
||||
# Another thread may have filled the key while this one waited.
|
||||
cached = self.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
value = compute()
|
||||
self.set(key, value, ttl_seconds)
|
||||
return value
|
||||
|
||||
def invalidate(self, prefix: str = "") -> int:
|
||||
"""Drop every key starting with ``prefix`` (all keys when empty)."""
|
||||
with self._lock:
|
||||
if not prefix:
|
||||
removed = len(self._values)
|
||||
self._values.clear()
|
||||
else:
|
||||
doomed = [key for key in self._values if key.startswith(prefix)]
|
||||
for key in doomed:
|
||||
del self._values[key]
|
||||
removed = len(doomed)
|
||||
# Collapse the lock table too: keys are unbounded over a long uptime
|
||||
# and a stale lock object is pure dead weight.
|
||||
if not prefix or removed:
|
||||
self._key_locks.clear()
|
||||
return removed
|
||||
|
||||
def stats(self) -> dict[str, int]:
|
||||
"""Return a snapshot for the diagnostics endpoint."""
|
||||
with self._lock:
|
||||
return {
|
||||
"entries": len(self._values),
|
||||
"max_entries": self._max_entries,
|
||||
"hits": self.hits,
|
||||
"misses": self.misses,
|
||||
"evictions": self.evictions,
|
||||
}
|
||||
|
||||
|
||||
class FailureLatch:
|
||||
"""Suppress a repeated failing optional call for a bounded period.
|
||||
|
||||
Unread counts and upstream search are enhancements: when one of them is
|
||||
unsupported by the FreshRSS build in front of us, retrying on every request
|
||||
would add latency to every page load for no benefit. The latch remembers the
|
||||
failure for a TTL, reports it once on the way in, and then stays quiet.
|
||||
"""
|
||||
|
||||
def __init__(self, ttl_seconds: float = 300.0) -> None:
|
||||
self._ttl = ttl_seconds
|
||||
self._locked_until: dict[str, float] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def is_open(self, key: str) -> bool:
|
||||
"""True when calls for ``key`` should be attempted again."""
|
||||
with self._lock:
|
||||
until = self._locked_until.get(key, 0.0)
|
||||
return time.monotonic() >= until
|
||||
|
||||
def trip(self, key: str) -> bool:
|
||||
"""Mark ``key`` failed; return True the first time within the window."""
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
first = now >= self._locked_until.get(key, 0.0)
|
||||
self._locked_until[key] = now + self._ttl
|
||||
return first
|
||||
|
||||
def clear(self, key: str) -> None:
|
||||
with self._lock:
|
||||
self._locked_until.pop(key, None)
|
||||
841
app/config.py
Normal file
841
app/config.py
Normal file
@@ -0,0 +1,841 @@
|
||||
"""JSON configuration: loading, interpolation, validation.
|
||||
|
||||
Configuration is a single JSON document, which keeps deployment to "mount a file
|
||||
and run the container". Two things make that practical for a service holding
|
||||
credentials:
|
||||
|
||||
*Interpolation.* Any string value may contain ``${VAR}`` (an environment
|
||||
variable, optionally with a ``:-default``), ``${file:/run/secrets/x}`` (a file,
|
||||
which is how Docker secrets arrive), or ``$${`` for a literal ``${``. Secrets
|
||||
therefore never have to be written into the JSON document that gets committed.
|
||||
|
||||
*Aggregated validation.* Every problem found is reported in one pass with the
|
||||
JSON path that caused it, because a container that exits on the first of five
|
||||
typos is a slow way to configure a service. Unknown keys are reported too, as
|
||||
warnings: a misspelled ``mark_read`` that silently defaulted to something else is
|
||||
exactly the kind of thing that costs an hour.
|
||||
|
||||
*Comments.* JSON has no comment syntax, so any key beginning with an underscore
|
||||
is treated as one: it is ignored, never interpolated, and never reported as
|
||||
unknown. That lets ``config.example.json`` explain itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
CONFIG_PATH_ENV = "CONFIG_PATH"
|
||||
|
||||
# Default listen address. Loopback is the safe default for a service that holds
|
||||
# a whole reading queue: running it directly on a host binds only to that host.
|
||||
# A container is the one case that needs the wildcard, because the published port
|
||||
# only reaches the process through the container's own interface -- so the
|
||||
# shipped config.example.json and docker-compose.yaml set it explicitly, and there
|
||||
# is exactly one place (the mounted configuration) that decides exposure.
|
||||
DEFAULT_BIND_HOST = "127.0.0.1"
|
||||
_DEFAULT_PATHS = ("/config/config.json", "config.json")
|
||||
|
||||
_INTERPOLATION_RE = re.compile(r"\$\{([^{}]*)\}")
|
||||
_SHA256_RE = re.compile(r"\A[0-9a-fA-F]{64}\Z")
|
||||
_ESCAPED_DOLLAR = "\x00escaped-dollar\x00"
|
||||
|
||||
GREADER_SUFFIX = "/api/greader.php"
|
||||
|
||||
_LOG_LEVELS = frozenset({"debug", "info", "warning", "error", "critical"})
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
"""Raised when the configuration cannot be used.
|
||||
|
||||
Carries every problem found, so a human sees the whole list at once.
|
||||
"""
|
||||
|
||||
def __init__(self, problems: list[str], source: str = "") -> None:
|
||||
self.problems = problems
|
||||
self.source = source
|
||||
where = f" in {source}" if source else ""
|
||||
detail = "\n".join(f" - {problem}" for problem in problems)
|
||||
super().__init__(f"invalid configuration{where}:\n{detail}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ServerConfig:
|
||||
"""HTTP listener settings."""
|
||||
|
||||
host: str = DEFAULT_BIND_HOST
|
||||
port: int = 8080
|
||||
log_level: str = "info"
|
||||
# Absolute base URL for links. When unset, links are built from the request
|
||||
# Host header, which is fine behind a proxy but lets a spoofed Host shape the
|
||||
# catalog URLs -- so a public deployment should set this.
|
||||
public_base_url: str | None = None
|
||||
# Honour X-Forwarded-Proto/Host/Prefix. Only enable behind a proxy you trust,
|
||||
# because the headers are client-controlled otherwise.
|
||||
trust_proxy_headers: bool = False
|
||||
# Optional Host allowlist; a request with any other Host is refused.
|
||||
allowed_hosts: list[str] = field(default_factory=list)
|
||||
tls_cert_file: str | None = None
|
||||
tls_key_file: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class FreshRSSConfig:
|
||||
"""Upstream FreshRSS connection."""
|
||||
|
||||
api_base: str
|
||||
username: str
|
||||
api_password: str
|
||||
verify_tls: bool = True
|
||||
ca_file: str | None = None
|
||||
timeout_seconds: float = 15.0
|
||||
retries: int = 2
|
||||
# Mark the article read upstream once its EPUB has been delivered.
|
||||
mark_read: bool = True
|
||||
items_per_page: int = 30
|
||||
max_items_per_request: int = 200
|
||||
# How far back the Read section looks. Read items are filtered client-side
|
||||
# (only the unread filter is a documented upstream parameter), so this bounds
|
||||
# the history that section can show.
|
||||
read_window_items: int = 200
|
||||
# Article body cap before parsing; prevents one enormous feed item from
|
||||
# dominating memory.
|
||||
max_content_bytes: int = 1_048_576
|
||||
|
||||
|
||||
@dataclass
|
||||
class CatalogConfig:
|
||||
"""OPDS catalog presentation."""
|
||||
|
||||
title: str = "FreshRSS OPDS"
|
||||
author: str = "FreshRSS"
|
||||
language: str = "en"
|
||||
publisher: str | None = None
|
||||
show_unread_counts: bool = True
|
||||
show_read_section: bool = True
|
||||
show_starred: bool = True
|
||||
hide_empty_categories: bool = False
|
||||
exclude_categories: list[str] = field(default_factory=list)
|
||||
exclude_feeds: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EpubConfig:
|
||||
"""EPUB generation."""
|
||||
|
||||
include_images: bool = True
|
||||
allowed_image_hosts: list[str] = field(default_factory=list)
|
||||
blocked_image_hosts: list[str] = field(default_factory=list)
|
||||
max_images_per_article: int = 20
|
||||
max_image_bytes: int = 2_097_152
|
||||
max_total_image_bytes: int = 16_777_216
|
||||
image_timeout_seconds: float = 10.0
|
||||
cover_image: bool = True
|
||||
include_source_link: bool = True
|
||||
stylesheet: str | None = None
|
||||
max_epub_bytes: int = 33_554_432
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuthUser:
|
||||
"""One Basic-auth principal. Passwords are always compared as digests."""
|
||||
|
||||
username: str
|
||||
password_sha256: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuthConfig:
|
||||
"""HTTP Basic credentials.
|
||||
|
||||
Authentication is mandatory by design: this catalog exposes the whole of a
|
||||
reading queue, and an unauthenticated OPDS endpoint on a routable address is
|
||||
an open book. Basic auth must therefore sit behind TLS (a reverse proxy, or
|
||||
the TLS options under ``server``), because the header is not encrypted.
|
||||
"""
|
||||
|
||||
realm: str = "OPDS"
|
||||
users: list[AuthUser] = field(default_factory=list)
|
||||
max_failures_per_minute: int = 10
|
||||
throttle_seconds: float = 60.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CacheConfig:
|
||||
"""In-memory cache lifetimes. Nothing is ever written to disk."""
|
||||
|
||||
subscriptions_ttl_seconds: float = 300.0
|
||||
stream_ttl_seconds: float = 60.0
|
||||
item_ttl_seconds: float = 900.0
|
||||
max_entries: int = 512
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
"""The fully resolved configuration."""
|
||||
|
||||
server: ServerConfig
|
||||
freshrss: FreshRSSConfig
|
||||
catalog: CatalogConfig
|
||||
epub: EpubConfig
|
||||
auth: AuthConfig
|
||||
cache: CacheConfig
|
||||
source_path: str = ""
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def resolve_config_path(explicit: str | None = None) -> str:
|
||||
"""Find the configuration file: flag, then env, then the usual locations."""
|
||||
if explicit:
|
||||
return explicit
|
||||
from_env = os.environ.get(CONFIG_PATH_ENV)
|
||||
if from_env:
|
||||
return from_env
|
||||
for candidate in _DEFAULT_PATHS:
|
||||
if Path(candidate).is_file():
|
||||
return candidate
|
||||
return _DEFAULT_PATHS[0]
|
||||
|
||||
|
||||
def load_config(path: str | None = None) -> Config:
|
||||
"""Load, interpolate and validate the configuration file."""
|
||||
source = resolve_config_path(path)
|
||||
raw_text = _read_config_text(source)
|
||||
try:
|
||||
document = json.loads(raw_text)
|
||||
except json.JSONDecodeError as exc:
|
||||
detail = f"not valid JSON: {exc.msg} (line {exc.lineno}, column {exc.colno})"
|
||||
raise ConfigError([detail], source) from exc
|
||||
if not isinstance(document, dict):
|
||||
raise ConfigError(["the top level of the configuration must be a JSON object"], source)
|
||||
|
||||
base_dir = Path(source).resolve().parent
|
||||
problems: list[str] = []
|
||||
interpolated = _interpolate(document, "", base_dir, problems)
|
||||
if problems:
|
||||
raise ConfigError(problems, source)
|
||||
if not isinstance(interpolated, dict): # pragma: no cover - shape is preserved
|
||||
raise ConfigError(["internal error: configuration root is not an object"], source)
|
||||
|
||||
warnings: list[str] = []
|
||||
validator = _Validator()
|
||||
config = Config(
|
||||
server=_build_server(validator, interpolated),
|
||||
freshrss=_build_freshrss(validator, interpolated),
|
||||
catalog=_build_catalog(validator, interpolated),
|
||||
epub=_build_epub(validator, interpolated),
|
||||
auth=_build_auth(validator, interpolated),
|
||||
cache=_build_cache(validator, interpolated),
|
||||
source_path=str(Path(source).resolve()),
|
||||
warnings=warnings,
|
||||
)
|
||||
warnings.extend(_unknown_keys(interpolated, _KNOWN_KEYS))
|
||||
if validator.problems:
|
||||
raise ConfigError(validator.problems, source)
|
||||
return config
|
||||
|
||||
|
||||
def password_digest(password: str) -> str:
|
||||
"""Digest a password for storage in ``auth.users[].password_sha256``."""
|
||||
return hashlib.sha256(password.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def normalise_api_base(url: str) -> str:
|
||||
"""Normalise a FreshRSS URL to its Google Reader API base.
|
||||
|
||||
Mirrors the firmware client's rule so the same address works in both places:
|
||||
the ``/api/greader.php`` suffix is appended when absent and left alone when
|
||||
present, and any path prefix in between (``/freshrss``) is preserved.
|
||||
"""
|
||||
base = url.strip().rstrip("/")
|
||||
if base.lower().endswith(GREADER_SUFFIX):
|
||||
return base
|
||||
return base + GREADER_SUFFIX
|
||||
|
||||
|
||||
def describe_config(config: Config) -> str:
|
||||
"""Render the effective configuration for logs, with secrets redacted."""
|
||||
users = ", ".join(user.username for user in config.auth.users) or "(none)"
|
||||
lines = [
|
||||
f"config file: {config.source_path}",
|
||||
f"listener: {config.server.host}:{config.server.port} "
|
||||
f"(log level {config.server.log_level}"
|
||||
+ (", TLS on" if config.server.tls_cert_file else "")
|
||||
+ ")",
|
||||
f"freshrss: {config.freshrss.api_base} as {config.freshrss.username} "
|
||||
f"(password: {_mask(config.freshrss.api_password)}, mark read: "
|
||||
f"{'yes' if config.freshrss.mark_read else 'no'})",
|
||||
f"catalog: {config.catalog.title!r}, {config.freshrss.items_per_page} items/page, "
|
||||
f"read section: {'yes' if config.catalog.show_read_section else 'no'}",
|
||||
f"epub images: {'yes' if config.epub.include_images else 'no'}",
|
||||
f"auth users: {users}",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _mask(secret: str) -> str:
|
||||
if not secret:
|
||||
return "(empty)"
|
||||
return "***" if len(secret) <= 8 else f"***({len(secret)} chars)"
|
||||
|
||||
|
||||
def _read_config_text(source: str) -> str:
|
||||
try:
|
||||
return Path(source).read_text(encoding="utf-8")
|
||||
except FileNotFoundError as exc:
|
||||
raise ConfigError(
|
||||
[
|
||||
f"configuration file not found: {source}",
|
||||
f"set {CONFIG_PATH_ENV} or pass --config; "
|
||||
"see config.example.json for a starting point",
|
||||
],
|
||||
source,
|
||||
) from exc
|
||||
except OSError as exc:
|
||||
raise ConfigError([f"cannot read configuration file: {exc.strerror}"], source) from exc
|
||||
|
||||
|
||||
# -- interpolation -------------------------------------------------------
|
||||
|
||||
|
||||
def _interpolate(value: Any, path: str, base_dir: Path, problems: list[str]) -> Any:
|
||||
"""Recursively expand ``${VAR}`` and ``${file:...}`` in every string."""
|
||||
if isinstance(value, str):
|
||||
return _interpolate_string(value, path, base_dir, problems)
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: (
|
||||
# A `_`-prefixed key is a comment: left exactly as written, so a
|
||||
# comment that mentions ${SOMETHING} stays inert.
|
||||
item
|
||||
if key.startswith("_")
|
||||
else _interpolate(item, f"{path}.{key}" if path else key, base_dir, problems)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [
|
||||
_interpolate(item, f"{path}[{index}]", base_dir, problems)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
return value
|
||||
|
||||
|
||||
def _interpolate_string(value: str, path: str, base_dir: Path, problems: list[str]) -> str:
|
||||
if "${" not in value:
|
||||
return value
|
||||
# Protect an escaped literal `${` before scanning for real references.
|
||||
text = value.replace("$${", _ESCAPED_DOLLAR + "{")
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
expression = match.group(1).strip()
|
||||
if expression.startswith("file:"):
|
||||
return _read_secret_file(expression[len("file:") :].strip(), path, base_dir, problems)
|
||||
name, separator, default = expression.partition(":-")
|
||||
name = name.strip()
|
||||
if not name:
|
||||
problems.append(f"{path}: empty ${{}} reference")
|
||||
return ""
|
||||
found = os.environ.get(name)
|
||||
if found is not None:
|
||||
return found
|
||||
if separator:
|
||||
return default
|
||||
problems.append(
|
||||
f"{path}: environment variable {name} is not set (use ${{{name}:-default}} "
|
||||
"to make it optional, or ${file:path} to read a secret from a file)"
|
||||
)
|
||||
return ""
|
||||
|
||||
result = _INTERPOLATION_RE.sub(replace, text)
|
||||
return result.replace(_ESCAPED_DOLLAR + "{", "${")
|
||||
|
||||
|
||||
def _read_secret_file(filename: str, path: str, base_dir: Path, problems: list[str]) -> str:
|
||||
if not filename:
|
||||
problems.append(f"{path}: ${{file:}} needs a path")
|
||||
return ""
|
||||
candidate = Path(filename)
|
||||
if not candidate.is_absolute():
|
||||
candidate = base_dir / candidate
|
||||
try:
|
||||
# Strip the trailing newline every secret-management tool adds.
|
||||
return candidate.read_text(encoding="utf-8").strip()
|
||||
except OSError as exc:
|
||||
problems.append(f"{path}: cannot read secret file {candidate}: {exc.strerror}")
|
||||
return ""
|
||||
|
||||
|
||||
# -- validation ----------------------------------------------------------
|
||||
|
||||
|
||||
class _Validator:
|
||||
"""Collects type and range problems while building the config dataclasses.
|
||||
|
||||
Problems are qualified by the section being validated, so a message reads
|
||||
``freshrss.url: required, but missing`` rather than the ambiguous
|
||||
``url: required, but missing``. That matters as soon as two sections share a
|
||||
key name, which they do (``timeout_seconds`` lives in three of them).
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.problems: list[str] = []
|
||||
self._prefix = ""
|
||||
|
||||
@contextmanager
|
||||
def within(self, path: str) -> Iterator[None]:
|
||||
"""Qualify every problem reported inside this block with ``path``."""
|
||||
previous = self._prefix
|
||||
self._prefix = f"{previous}.{path}" if previous else path
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._prefix = previous
|
||||
|
||||
def qualify(self, key: str | None = None) -> str:
|
||||
"""Full path for a key, in the section currently being validated."""
|
||||
if not key:
|
||||
return self._prefix
|
||||
return f"{self._prefix}.{key}" if self._prefix else key
|
||||
|
||||
def problem(self, message: str, key: str | None = None) -> None:
|
||||
"""Report a problem against the current section, or against one key."""
|
||||
location = self.qualify(key)
|
||||
self.problems.append(f"{location}: {message}" if location else message)
|
||||
|
||||
def section(self, raw: dict[str, Any], name: str) -> dict[str, Any]:
|
||||
value = raw.get(name)
|
||||
if value is None:
|
||||
return {}
|
||||
if not isinstance(value, dict):
|
||||
self.problem("must be a JSON object", key=name)
|
||||
return {}
|
||||
return value
|
||||
|
||||
def string(
|
||||
self,
|
||||
raw: dict[str, Any],
|
||||
key: str,
|
||||
default: str | None = None,
|
||||
*,
|
||||
required: bool = False,
|
||||
) -> Any:
|
||||
if key not in raw or raw[key] is None:
|
||||
if required:
|
||||
self.problem("required, but missing", key=key)
|
||||
return default
|
||||
value = raw[key]
|
||||
if not isinstance(value, str):
|
||||
self.problem("must be a string", key=key)
|
||||
return default
|
||||
text = value.strip()
|
||||
if not text:
|
||||
if required:
|
||||
self.problem("required, but empty", key=key)
|
||||
return default
|
||||
return text
|
||||
|
||||
def one_of(self, raw: dict[str, Any], key: str, allowed: frozenset[str], default: str) -> str:
|
||||
value = self.string(raw, key, default)
|
||||
if value is None or value not in allowed:
|
||||
self.problem(f"must be one of {', '.join(sorted(allowed))} (got {value!r})", key=key)
|
||||
return default
|
||||
return value
|
||||
|
||||
def boolean(self, raw: dict[str, Any], key: str, default: bool) -> bool:
|
||||
if key not in raw or raw[key] is None:
|
||||
return default
|
||||
value = raw[key]
|
||||
if not isinstance(value, bool):
|
||||
self.problem("must be true or false", key=key)
|
||||
return default
|
||||
return value
|
||||
|
||||
def integer(
|
||||
self,
|
||||
raw: dict[str, Any],
|
||||
key: str,
|
||||
default: int,
|
||||
*,
|
||||
minimum: int | None = None,
|
||||
maximum: int | None = None,
|
||||
) -> int:
|
||||
if key not in raw or raw[key] is None:
|
||||
return default
|
||||
value = raw[key]
|
||||
# A bool is an int in Python; accepting it here would turn `true` into 1.
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
self.problem("must be a whole number", key=key)
|
||||
return default
|
||||
if minimum is not None and value < minimum:
|
||||
self.problem(f"must be at least {minimum} (got {value})", key=key)
|
||||
return default
|
||||
if maximum is not None and value > maximum:
|
||||
self.problem(f"must be at most {maximum} (got {value})", key=key)
|
||||
return default
|
||||
return value
|
||||
|
||||
def number(self, raw: dict[str, Any], key: str, default: float, *, above: float = 0) -> float:
|
||||
if key not in raw or raw[key] is None:
|
||||
return default
|
||||
value = raw[key]
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
self.problem("must be a number", key=key)
|
||||
return default
|
||||
if value <= above:
|
||||
self.problem(f"must be greater than {above:g} (got {value:g})", key=key)
|
||||
return default
|
||||
return float(value)
|
||||
|
||||
def string_list(self, raw: dict[str, Any], key: str) -> list[str]:
|
||||
if key not in raw or raw[key] is None:
|
||||
return []
|
||||
value = raw[key]
|
||||
if not isinstance(value, list):
|
||||
self.problem("must be an array of strings", key=key)
|
||||
return []
|
||||
items: list[str] = []
|
||||
for index, item in enumerate(value):
|
||||
if not isinstance(item, str):
|
||||
self.problem("must be a string", key=f"{key}[{index}]")
|
||||
continue
|
||||
if item.strip():
|
||||
items.append(item.strip())
|
||||
return items
|
||||
|
||||
|
||||
def _build_server(validator: _Validator, raw: dict[str, Any]) -> ServerConfig:
|
||||
section = validator.section(raw, "server")
|
||||
with validator.within("server"):
|
||||
with validator.within("tls"):
|
||||
tls = validator.section(section, "tls")
|
||||
cert = validator.string(tls, "cert_file")
|
||||
key = validator.string(tls, "key_file")
|
||||
if bool(cert) != bool(key):
|
||||
validator.problem("cert_file and key_file must be set together")
|
||||
elif cert:
|
||||
_require_readable_file(validator, cert, "cert_file")
|
||||
_require_readable_file(validator, key, "key_file")
|
||||
return ServerConfig(
|
||||
host=validator.string(section, "host", DEFAULT_BIND_HOST) or DEFAULT_BIND_HOST,
|
||||
# Port 0 asks the operating system for a free port, which is what the test
|
||||
# suite uses and what a socket-activated or ephemeral deployment wants.
|
||||
port=validator.integer(section, "port", 8080, minimum=0, maximum=65535),
|
||||
log_level=validator.one_of(section, "log_level", _LOG_LEVELS, "info"),
|
||||
public_base_url=validator.string(section, "public_base_url"),
|
||||
trust_proxy_headers=validator.boolean(section, "trust_proxy_headers", False),
|
||||
allowed_hosts=[
|
||||
host.lower() for host in validator.string_list(section, "allowed_hosts")
|
||||
],
|
||||
tls_cert_file=cert,
|
||||
tls_key_file=key,
|
||||
)
|
||||
|
||||
|
||||
def _build_freshrss(validator: _Validator, raw: dict[str, Any]) -> FreshRSSConfig:
|
||||
section = validator.section(raw, "freshrss")
|
||||
with validator.within("freshrss"):
|
||||
url = validator.string(section, "url", required=True)
|
||||
if url and not url.lower().startswith(("http://", "https://")):
|
||||
validator.problem(f"must start with http:// or https:// (got {url!r})", key="url")
|
||||
username = validator.string(section, "username", required=True)
|
||||
password = validator.string(section, "api_password", required=True)
|
||||
ca_file = validator.string(section, "ca_file")
|
||||
if ca_file:
|
||||
_require_readable_file(validator, ca_file, "ca_file")
|
||||
items_per_page = validator.integer(section, "items_per_page", 30, minimum=1, maximum=500)
|
||||
max_items = validator.integer(
|
||||
section, "max_items_per_request", 200, minimum=1, maximum=5000
|
||||
)
|
||||
if max_items < items_per_page:
|
||||
validator.problem(
|
||||
f"max_items_per_request ({max_items}) must be at least "
|
||||
f"items_per_page ({items_per_page})"
|
||||
)
|
||||
return FreshRSSConfig(
|
||||
api_base=normalise_api_base(url) if url else "",
|
||||
username=username or "",
|
||||
api_password=password or "",
|
||||
verify_tls=validator.boolean(section, "verify_tls", True),
|
||||
ca_file=ca_file,
|
||||
timeout_seconds=validator.number(section, "timeout_seconds", 15.0),
|
||||
retries=validator.integer(section, "retries", 2, minimum=0, maximum=10),
|
||||
mark_read=validator.boolean(section, "mark_read", True),
|
||||
items_per_page=items_per_page,
|
||||
max_items_per_request=max_items,
|
||||
read_window_items=validator.integer(
|
||||
section, "read_window_items", 200, minimum=1, maximum=5000
|
||||
),
|
||||
max_content_bytes=validator.integer(
|
||||
section, "max_content_bytes", 1_048_576, minimum=4096, maximum=67_108_864
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _build_catalog(validator: _Validator, raw: dict[str, Any]) -> CatalogConfig:
|
||||
section = validator.section(raw, "catalog")
|
||||
with validator.within("catalog"):
|
||||
return CatalogConfig(
|
||||
title=validator.string(section, "title", "FreshRSS OPDS") or "FreshRSS OPDS",
|
||||
author=validator.string(section, "author", "FreshRSS") or "FreshRSS",
|
||||
language=validator.string(section, "language", "en") or "en",
|
||||
publisher=validator.string(section, "publisher"),
|
||||
show_unread_counts=validator.boolean(section, "show_unread_counts", True),
|
||||
show_read_section=validator.boolean(section, "show_read_section", True),
|
||||
show_starred=validator.boolean(section, "show_starred", True),
|
||||
hide_empty_categories=validator.boolean(section, "hide_empty_categories", False),
|
||||
exclude_categories=validator.string_list(section, "exclude_categories"),
|
||||
exclude_feeds=validator.string_list(section, "exclude_feeds"),
|
||||
)
|
||||
|
||||
|
||||
def _build_epub(validator: _Validator, raw: dict[str, Any]) -> EpubConfig:
|
||||
section = validator.section(raw, "epub")
|
||||
with validator.within("epub"):
|
||||
stylesheet = validator.string(section, "stylesheet")
|
||||
if stylesheet:
|
||||
_require_readable_file(validator, stylesheet, "stylesheet")
|
||||
return EpubConfig(
|
||||
include_images=validator.boolean(section, "include_images", True),
|
||||
allowed_image_hosts=[
|
||||
host.lower() for host in validator.string_list(section, "allowed_image_hosts")
|
||||
],
|
||||
blocked_image_hosts=[
|
||||
host.lower() for host in validator.string_list(section, "blocked_image_hosts")
|
||||
],
|
||||
max_images_per_article=validator.integer(
|
||||
section, "max_images_per_article", 20, minimum=0, maximum=200
|
||||
),
|
||||
max_image_bytes=validator.integer(
|
||||
section, "max_image_bytes", 2_097_152, minimum=1024, maximum=52_428_800
|
||||
),
|
||||
max_total_image_bytes=validator.integer(
|
||||
section,
|
||||
"max_total_image_bytes",
|
||||
16_777_216,
|
||||
minimum=1024,
|
||||
maximum=268_435_456,
|
||||
),
|
||||
image_timeout_seconds=validator.number(section, "image_timeout_seconds", 10.0),
|
||||
cover_image=validator.boolean(section, "cover_image", True),
|
||||
include_source_link=validator.boolean(section, "include_source_link", True),
|
||||
stylesheet=stylesheet,
|
||||
max_epub_bytes=validator.integer(
|
||||
section, "max_epub_bytes", 33_554_432, minimum=4096, maximum=268_435_456
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _build_auth(validator: _Validator, raw: dict[str, Any]) -> AuthConfig:
|
||||
section = validator.section(raw, "auth")
|
||||
users: list[AuthUser] = []
|
||||
with validator.within("auth"):
|
||||
raw_users = section.get("users")
|
||||
if raw_users is None:
|
||||
validator.problem(
|
||||
"required -- this catalog serves your entire reading queue, so it "
|
||||
"will not start without credentials "
|
||||
"(generate a digest with: make hash-password)",
|
||||
key="users",
|
||||
)
|
||||
elif not isinstance(raw_users, list):
|
||||
validator.problem("must be an array of user objects", key="users")
|
||||
elif not raw_users:
|
||||
validator.problem("must contain at least one user", key="users")
|
||||
else:
|
||||
seen: set[str] = set()
|
||||
for index, entry in enumerate(raw_users):
|
||||
with validator.within(f"users[{index}]"):
|
||||
users.append(_build_auth_user(validator, entry, seen))
|
||||
return AuthConfig(
|
||||
realm=validator.string(section, "realm", "OPDS") or "OPDS",
|
||||
users=users,
|
||||
max_failures_per_minute=validator.integer(
|
||||
section, "max_failures_per_minute", 10, minimum=1, maximum=10_000
|
||||
),
|
||||
throttle_seconds=validator.number(section, "throttle_seconds", 60.0),
|
||||
)
|
||||
|
||||
|
||||
def _build_auth_user(validator: _Validator, entry: Any, seen: set[str]) -> AuthUser:
|
||||
"""Build one user; the caller has already entered its ``users[i]`` context."""
|
||||
if not isinstance(entry, dict):
|
||||
validator.problem("must be a JSON object")
|
||||
return AuthUser("", "")
|
||||
username = validator.string(entry, "username", required=True) or ""
|
||||
if username and username in seen:
|
||||
validator.problem(f"duplicate username {username!r}", key="username")
|
||||
seen.add(username)
|
||||
|
||||
plain = _validated_plain_password(validator, entry)
|
||||
digest = _validated_digest(validator, entry)
|
||||
if plain is not None and digest is not None:
|
||||
validator.problem("set either password or password_sha256, not both")
|
||||
return AuthUser(username, "")
|
||||
if digest is not None:
|
||||
return AuthUser(username, digest)
|
||||
if plain is not None:
|
||||
LOGGER.warning(
|
||||
"%s.password is in clear text in the configuration file; "
|
||||
"prefer password_sha256 (make hash-password)",
|
||||
validator.qualify(),
|
||||
)
|
||||
return AuthUser(username, password_digest(plain))
|
||||
validator.problem("needs password or password_sha256")
|
||||
return AuthUser(username, "")
|
||||
|
||||
|
||||
def _validated_plain_password(validator: _Validator, entry: dict[str, Any]) -> str | None:
|
||||
"""Return a clear-text password from config, or None when absent/invalid."""
|
||||
value = entry.get("password")
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str) or not value:
|
||||
validator.problem("must be a non-empty string", key="password")
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _validated_digest(validator: _Validator, entry: dict[str, Any]) -> str | None:
|
||||
"""Return a validated sha256 digest from config, or None when absent/invalid."""
|
||||
value = entry.get("password_sha256")
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str) or not _SHA256_RE.match(value):
|
||||
validator.problem("must be 64 hex characters (make hash-password)", key="password_sha256")
|
||||
return None
|
||||
return value.lower()
|
||||
|
||||
|
||||
def _build_cache(validator: _Validator, raw: dict[str, Any]) -> CacheConfig:
|
||||
section = validator.section(raw, "cache")
|
||||
with validator.within("cache"):
|
||||
return CacheConfig(
|
||||
subscriptions_ttl_seconds=validator.number(section, "subscriptions_ttl_seconds", 300.0),
|
||||
stream_ttl_seconds=validator.number(section, "stream_ttl_seconds", 60.0),
|
||||
item_ttl_seconds=validator.number(section, "item_ttl_seconds", 900.0),
|
||||
max_entries=validator.integer(section, "max_entries", 512, minimum=16, maximum=100_000),
|
||||
)
|
||||
|
||||
|
||||
def _require_readable_file(validator: _Validator, filename: str, key: str) -> None:
|
||||
if not Path(filename).is_file():
|
||||
validator.problem(f"no such file: {filename}", key=key)
|
||||
|
||||
|
||||
_KNOWN_KEYS: dict[str, set[str]] = {
|
||||
"server": {
|
||||
"host",
|
||||
"port",
|
||||
"log_level",
|
||||
"public_base_url",
|
||||
"trust_proxy_headers",
|
||||
"allowed_hosts",
|
||||
"tls",
|
||||
},
|
||||
"server.tls": {"cert_file", "key_file"},
|
||||
"freshrss": {
|
||||
"url",
|
||||
"username",
|
||||
"api_password",
|
||||
"verify_tls",
|
||||
"ca_file",
|
||||
"timeout_seconds",
|
||||
"retries",
|
||||
"mark_read",
|
||||
"items_per_page",
|
||||
"max_items_per_request",
|
||||
"read_window_items",
|
||||
"max_content_bytes",
|
||||
},
|
||||
"catalog": {
|
||||
"title",
|
||||
"author",
|
||||
"language",
|
||||
"publisher",
|
||||
"show_unread_counts",
|
||||
"show_read_section",
|
||||
"show_starred",
|
||||
"hide_empty_categories",
|
||||
"exclude_categories",
|
||||
"exclude_feeds",
|
||||
},
|
||||
"epub": {
|
||||
"include_images",
|
||||
"allowed_image_hosts",
|
||||
"blocked_image_hosts",
|
||||
"max_images_per_article",
|
||||
"max_image_bytes",
|
||||
"max_total_image_bytes",
|
||||
"image_timeout_seconds",
|
||||
"cover_image",
|
||||
"include_source_link",
|
||||
"stylesheet",
|
||||
"max_epub_bytes",
|
||||
},
|
||||
"auth": {"realm", "users", "max_failures_per_minute", "throttle_seconds"},
|
||||
"auth.users[]": {"username", "password", "password_sha256"},
|
||||
"cache": {
|
||||
"subscriptions_ttl_seconds",
|
||||
"stream_ttl_seconds",
|
||||
"item_ttl_seconds",
|
||||
"max_entries",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _unknown_keys(document: dict[str, Any], known: dict[str, set[str]]) -> list[str]:
|
||||
"""Report keys the server would otherwise silently ignore."""
|
||||
warnings: list[str] = []
|
||||
sections = {name for name in known if "." not in name}
|
||||
for key in document:
|
||||
if key in sections or key.startswith("_"):
|
||||
continue
|
||||
suggestions = _closest(key, sections)
|
||||
hint = f" (did you mean {suggestions}?)" if suggestions else ""
|
||||
warnings.append(f"{key}: unknown top-level setting, ignored{hint}")
|
||||
for section, allowed in known.items():
|
||||
if "." in section:
|
||||
continue
|
||||
value = document.get(section)
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
for key in value:
|
||||
if key not in allowed and not key.startswith("_"):
|
||||
suggestions = _closest(key, allowed)
|
||||
hint = f" (did you mean {suggestions}?)" if suggestions else ""
|
||||
warnings.append(f"{section}.{key}: unknown setting, ignored{hint}")
|
||||
if section == "server" and isinstance(value.get("tls"), dict):
|
||||
for key in value["tls"]:
|
||||
if key not in known["server.tls"] and not key.startswith("_"):
|
||||
warnings.append(f"server.tls.{key}: unknown setting, ignored")
|
||||
if section == "auth" and isinstance(value.get("users"), list):
|
||||
for index, entry in enumerate(value["users"]):
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
for key in entry:
|
||||
if key not in known["auth.users[]"] and not key.startswith("_"):
|
||||
warnings.append(f"auth.users[{index}].{key}: unknown setting, ignored")
|
||||
return warnings
|
||||
|
||||
|
||||
def _closest(key: str, candidates: set[str]) -> str:
|
||||
"""Suggest a near-miss key name, cheaply."""
|
||||
lowered = key.lower().replace("-", "_")
|
||||
for candidate in sorted(candidates):
|
||||
if candidate.lower().replace("-", "_") == lowered:
|
||||
return repr(candidate)
|
||||
for candidate in sorted(candidates):
|
||||
if candidate.lower().startswith(lowered[:4]) and len(lowered) >= 4:
|
||||
return repr(candidate)
|
||||
return ""
|
||||
492
app/epub.py
Normal file
492
app/epub.py
Normal file
@@ -0,0 +1,492 @@
|
||||
"""EPUB 3 generation, entirely in memory.
|
||||
|
||||
An EPUB is a ZIP with a strict shape, and a reader that rejects a book usually
|
||||
does so because of a shape violation rather than because of its content. The
|
||||
rules that matter here, and that the tests pin down:
|
||||
|
||||
* ``mimetype`` is the first entry, stored uncompressed, with no extra field. A
|
||||
ZIP where it is deflated or not first is not an EPUB.
|
||||
* ``META-INF/container.xml`` names the OPF, which is the package document.
|
||||
* XML everywhere: the OPF, the navigation document, the NCX fallback and every
|
||||
content document have to parse with a strict XML parser. Article bodies are
|
||||
normalised by :mod:`app.xhtml` before they come anywhere near this module.
|
||||
* A publication needs ``dc:identifier``, ``dc:title``, ``dc:language`` and a
|
||||
``dcterms:modified`` timestamp; a navigation document marked ``properties="nav"``
|
||||
is mandatory, and the NCX is kept as the EPUB 2 fallback because older readers
|
||||
still look for it.
|
||||
* Every manifest href resolves to a real entry, and every spine idref resolves to
|
||||
a manifest item. Both are checked in the tests.
|
||||
|
||||
Nothing touches the filesystem: the ZIP is assembled in a ``BytesIO`` and handed
|
||||
back as ``bytes``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app import images, urls, xhtml
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
MIMETYPE = "application/epub+zip"
|
||||
|
||||
# Where each kind of part lives inside the package.
|
||||
OPF_PATH = "OEBPS/content.opf"
|
||||
NAV_PATH = "OEBPS/nav.xhtml"
|
||||
NCX_PATH = "OEBPS/toc.ncx"
|
||||
CSS_PATH = "OEBPS/styles/main.css"
|
||||
COVER_PATH = "OEBPS/text/cover.xhtml"
|
||||
ARTICLE_PATH = "OEBPS/text/article.xhtml"
|
||||
IMAGE_DIR = "OEBPS/images"
|
||||
|
||||
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
|
||||
OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
|
||||
DC_NAMESPACE = "http://purl.org/dc/elements/1.1/"
|
||||
NCX_NAMESPACE = "http://www.daisy.org/z3986/2005/ncx/"
|
||||
OPS_NAMESPACE = "http://www.idpf.org/2007/ops"
|
||||
|
||||
# A deliberately plain stylesheet. It sets a readable measure and gives code,
|
||||
# quotes and images somewhere sane to sit, and it never fights the reader's own
|
||||
# typography: no colours, no font families, no absolute sizes.
|
||||
DEFAULT_STYLESHEET = """\
|
||||
body { margin: 5%; line-height: 1.5; }
|
||||
h1, h2, h3 { line-height: 1.25; page-break-after: avoid; }
|
||||
.article-title { margin-bottom: 0.2em; }
|
||||
.byline { margin-top: 0; font-size: 0.9em; opacity: 0.8; }
|
||||
img { max-width: 100%; height: auto; }
|
||||
figure { margin: 1em 0; }
|
||||
figcaption { font-size: 0.9em; font-style: italic; }
|
||||
blockquote { margin: 1em 1.5em; }
|
||||
pre { white-space: pre-wrap; word-wrap: break-word; }
|
||||
code, pre { font-family: monospace; }
|
||||
table { border-collapse: collapse; max-width: 100%; }
|
||||
td, th { border: 1px solid currentColor; padding: 0.3em 0.5em; }
|
||||
hr { border: none; border-top: 1px solid currentColor; }
|
||||
.source { margin-top: 2em; font-size: 0.85em; opacity: 0.8; }
|
||||
.cover { text-align: center; }
|
||||
.cover img { max-height: 90%; }
|
||||
"""
|
||||
|
||||
# Content documents sit one directory below the images.
|
||||
_TEXT_PREFIX = "../"
|
||||
|
||||
# A literal middle dot rather than the HTML entity ·: XML predefines only
|
||||
# amp, lt, gt, quot and apos, and this package ships no DOCTYPE that could define
|
||||
# any others, so an HTML entity here would make the document unparseable.
|
||||
BYLINE_SEPARATOR = " \u00b7 "
|
||||
|
||||
|
||||
class EpubTooLargeError(Exception):
|
||||
"""The assembled book exceeded the configured size cap."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BookImage:
|
||||
"""An image embedded in the book."""
|
||||
|
||||
href: str # relative to the OPF, e.g. "images/img-1.png"
|
||||
media_type: str
|
||||
data: bytes
|
||||
is_cover: bool = False
|
||||
|
||||
@property
|
||||
def document_href(self) -> str:
|
||||
"""The same image, as referenced from a document in ``text/``."""
|
||||
return _TEXT_PREFIX + self.href
|
||||
|
||||
|
||||
@dataclass
|
||||
class Book:
|
||||
"""Everything needed to write one EPUB."""
|
||||
|
||||
identifier: str
|
||||
title: str
|
||||
language: str
|
||||
modified: datetime
|
||||
content_xhtml: str
|
||||
published: datetime | None = None
|
||||
author: str | None = None
|
||||
publisher: str | None = None
|
||||
source_url: str | None = None
|
||||
feed_title: str | None = None
|
||||
images: list[BookImage] = field(default_factory=list)
|
||||
stylesheet: str = DEFAULT_STYLESHEET
|
||||
include_source_link: bool = True
|
||||
|
||||
@property
|
||||
def cover_image(self) -> BookImage | None:
|
||||
for image in self.images:
|
||||
if image.is_cover:
|
||||
return image
|
||||
return None
|
||||
|
||||
|
||||
def build_epub(book: Book) -> bytes:
|
||||
"""Assemble a book into EPUB bytes."""
|
||||
buffer = io.BytesIO()
|
||||
# ZIP entry timestamps come from the book so that identical input produces
|
||||
# identical bytes; a book that changes only because it was rebuilt would
|
||||
# otherwise defeat every cache between here and the reader.
|
||||
stamp = _zip_timestamp(book.modified)
|
||||
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
# The mimetype entry must come first and must not be compressed, which is
|
||||
# why it is written through an explicit ZipInfo rather than writestr.
|
||||
info = zipfile.ZipInfo("mimetype", date_time=stamp)
|
||||
info.compress_type = zipfile.ZIP_STORED
|
||||
info.external_attr = 0o644 << 16
|
||||
archive.writestr(info, MIMETYPE)
|
||||
|
||||
_write(archive, "META-INF/container.xml", _container_xml(), stamp)
|
||||
_write(archive, OPF_PATH, _opf_document(book), stamp)
|
||||
_write(archive, NAV_PATH, _nav_document(book), stamp)
|
||||
_write(archive, NCX_PATH, _ncx_document(book), stamp)
|
||||
_write(archive, CSS_PATH, book.stylesheet, stamp)
|
||||
_write(archive, COVER_PATH, _cover_document(book), stamp)
|
||||
_write(archive, ARTICLE_PATH, _article_document(book), stamp)
|
||||
for image in book.images:
|
||||
package_dir = OPF_PATH.rsplit("/", 1)[0]
|
||||
_write(archive, f"{package_dir}/{image.href}", image.data, stamp)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def build_article_epub(
|
||||
item,
|
||||
*,
|
||||
epub_config,
|
||||
catalog_config,
|
||||
fetcher=None,
|
||||
max_content_bytes: int = 1_048_576,
|
||||
) -> bytes:
|
||||
"""Turn one FreshRSS item into an EPUB.
|
||||
|
||||
The article body comes from FreshRSS, normalised by :mod:`app.xhtml`. Images
|
||||
are embedded when a fetcher is supplied, and any that cannot be fetched are
|
||||
dropped from the body rather than left pointing at the network -- a book that
|
||||
claims to work offline should.
|
||||
"""
|
||||
article = xhtml.parse_article(
|
||||
item.content_html, base_url=item.url or "", max_input_bytes=max_content_bytes
|
||||
)
|
||||
book_images: list[BookImage] = []
|
||||
image_map: dict[str, str | None] | None = None
|
||||
|
||||
if epub_config.include_images and fetcher is not None:
|
||||
wanted = article.image_urls[: epub_config.max_images_per_article]
|
||||
fetched = fetcher.fetch_all(wanted)
|
||||
image_map = {}
|
||||
for url in wanted:
|
||||
image = fetched.get(url)
|
||||
if image is None:
|
||||
image_map[url] = None
|
||||
continue
|
||||
index = len(book_images) + 1
|
||||
href = f"images/img-{index}{images.MEDIA_TYPES[image.media_type]}"
|
||||
book_images.append(BookImage(href=href, media_type=image.media_type, data=image.data))
|
||||
image_map[url] = _TEXT_PREFIX + href
|
||||
elif article.image_urls:
|
||||
# Keeping remote URLs means the OPF has to declare remote resources, which
|
||||
# is handled by the caller not setting include_images.
|
||||
image_map = None
|
||||
|
||||
book = Book(
|
||||
identifier=urls.stable_urn("book", item.id),
|
||||
title=item.title,
|
||||
language=catalog_config.language,
|
||||
modified=item.updated_at,
|
||||
published=item.published_at,
|
||||
content_xhtml=article.to_xhtml(image_map),
|
||||
author=item.author or catalog_config.author,
|
||||
publisher=catalog_config.publisher,
|
||||
source_url=item.url,
|
||||
feed_title=item.feed_title,
|
||||
images=book_images,
|
||||
include_source_link=epub_config.include_source_link,
|
||||
stylesheet=_load_stylesheet(epub_config.stylesheet),
|
||||
)
|
||||
if epub_config.cover_image:
|
||||
_choose_cover(book)
|
||||
data = build_epub(book)
|
||||
if len(data) > epub_config.max_epub_bytes:
|
||||
raise EpubTooLargeError(
|
||||
f"book is {len(data)} bytes, over the {epub_config.max_epub_bytes} byte cap"
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def _load_stylesheet(path: str | None) -> str:
|
||||
"""Use the configured stylesheet, falling back to the built-in one."""
|
||||
if not path:
|
||||
return DEFAULT_STYLESHEET
|
||||
try:
|
||||
return Path(path).read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
LOGGER.warning("cannot read stylesheet %s (%s); using the built-in one", path, exc)
|
||||
return DEFAULT_STYLESHEET
|
||||
|
||||
|
||||
def _choose_cover(book: Book) -> None:
|
||||
"""Promote the first suitably large image to be the book's cover.
|
||||
|
||||
A reader's library view shows a cover or a blank tile, and a feed's lead image
|
||||
is almost always the right one. Images whose size cannot be read are skipped:
|
||||
promoting an unknown image risks a cover that is a tracking pixel.
|
||||
"""
|
||||
for index, image in enumerate(book.images):
|
||||
size = images.dimensions(image.data)
|
||||
if size is None:
|
||||
continue
|
||||
width, height = size
|
||||
if images.looks_like_a_cover(width, height):
|
||||
book.images[index] = BookImage(
|
||||
href=image.href,
|
||||
media_type=image.media_type,
|
||||
data=image.data,
|
||||
is_cover=True,
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
def _write(
|
||||
archive: zipfile.ZipFile,
|
||||
path: str,
|
||||
content: str | bytes,
|
||||
stamp: tuple[int, int, int, int, int, int],
|
||||
) -> None:
|
||||
"""Write one entry, encoding text parts and passing binary parts through."""
|
||||
info = zipfile.ZipInfo(path, date_time=stamp)
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.external_attr = 0o644 << 16
|
||||
data = content if isinstance(content, bytes) else content.encode("utf-8")
|
||||
archive.writestr(info, data)
|
||||
|
||||
|
||||
def _zip_timestamp(moment: datetime) -> tuple[int, int, int, int, int, int]:
|
||||
"""A ZIP timestamp, clamped to the range the format can express (from 1980)."""
|
||||
year = max(1980, min(2107, moment.year))
|
||||
return (year, moment.month, moment.day, moment.hour, moment.minute, moment.second)
|
||||
|
||||
|
||||
# -- documents -----------------------------------------------------------
|
||||
|
||||
|
||||
def _container_xml() -> str:
|
||||
return (
|
||||
'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
'<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">\n'
|
||||
" <rootfiles>\n"
|
||||
f' <rootfile full-path="{OPF_PATH}" media-type="application/oebps-package+xml"/>\n'
|
||||
" </rootfiles>\n"
|
||||
"</container>\n"
|
||||
)
|
||||
|
||||
|
||||
def _opf_document(book: Book) -> str:
|
||||
metadata = [
|
||||
f' <dc:identifier id="pub-id">{_escape(book.identifier)}</dc:identifier>',
|
||||
f" <dc:title>{_escape(book.title)}</dc:title>",
|
||||
f" <dc:language>{_escape(book.language)}</dc:language>",
|
||||
]
|
||||
if book.author:
|
||||
metadata.append(f" <dc:creator>{_escape(book.author)}</dc:creator>")
|
||||
if book.publisher:
|
||||
metadata.append(f" <dc:publisher>{_escape(book.publisher)}</dc:publisher>")
|
||||
if book.published:
|
||||
metadata.append(f" <dc:date>{_timestamp(book.published)}</dc:date>")
|
||||
if book.source_url:
|
||||
metadata.append(f" <dc:source>{_escape(book.source_url)}</dc:source>")
|
||||
metadata.append(f' <meta property="dcterms:modified">{_timestamp(book.modified)}</meta>')
|
||||
if book.cover_image is not None:
|
||||
# The EPUB 2 spelling of the same statement, for readers that predate the
|
||||
# cover-image property below.
|
||||
metadata.append(' <meta name="cover" content="cover-image"/>')
|
||||
|
||||
manifest = [
|
||||
' <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>',
|
||||
' <item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>',
|
||||
' <item id="css" href="styles/main.css" media-type="text/css"/>',
|
||||
' <item id="cover" href="text/cover.xhtml" media-type="application/xhtml+xml"/>',
|
||||
' <item id="article" href="text/article.xhtml" media-type="application/xhtml+xml"/>',
|
||||
]
|
||||
cover = book.cover_image
|
||||
for index, image in enumerate(book.images, start=1):
|
||||
properties = ' properties="cover-image"' if image.is_cover else ""
|
||||
identifier = "cover-image" if image.is_cover else f"image-{index}"
|
||||
manifest.append(
|
||||
f' <item id="{identifier}" href="{_escape(image.href)}" '
|
||||
f'media-type="{image.media_type}"{properties}/>'
|
||||
)
|
||||
if cover is None and book.images:
|
||||
# Images are embedded rather than linked, so nothing to declare here; the
|
||||
# branch exists to keep the intent obvious if remote images are ever kept.
|
||||
pass
|
||||
|
||||
spine = [
|
||||
' <itemref idref="cover"/>',
|
||||
' <itemref idref="article"/>',
|
||||
]
|
||||
return "\n".join(
|
||||
[
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<package xmlns="{OPF_NAMESPACE}" version="3.0" '
|
||||
f'unique-identifier="pub-id" xml:lang="{_escape(book.language)}">',
|
||||
f' <metadata xmlns:dc="{DC_NAMESPACE}">',
|
||||
*metadata,
|
||||
" </metadata>",
|
||||
" <manifest>",
|
||||
*manifest,
|
||||
" </manifest>",
|
||||
' <spine toc="ncx">',
|
||||
*spine,
|
||||
" </spine>",
|
||||
" <guide>",
|
||||
' <reference type="cover" title="Cover" href="text/cover.xhtml"/>',
|
||||
" </guide>",
|
||||
"</package>",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _nav_document(book: Book) -> str:
|
||||
return "\n".join(
|
||||
[
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<html xmlns="{XHTML_NAMESPACE}" xmlns:epub="{OPS_NAMESPACE}" '
|
||||
f'xml:lang="{_escape(book.language)}" lang="{_escape(book.language)}">',
|
||||
" <head>",
|
||||
" <title>Contents</title>",
|
||||
' <meta charset="utf-8"/>',
|
||||
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
|
||||
" </head>",
|
||||
" <body>",
|
||||
' <nav epub:type="toc" id="toc">',
|
||||
" <h1>Contents</h1>",
|
||||
" <ol>",
|
||||
f' <li><a href="text/cover.xhtml">{_escape(book.title)}</a></li>',
|
||||
' <li><a href="text/article.xhtml">Article</a></li>',
|
||||
" </ol>",
|
||||
" </nav>",
|
||||
" </body>",
|
||||
"</html>",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _ncx_document(book: Book) -> str:
|
||||
points = [
|
||||
("navpoint-1", 1, "Cover", "text/cover.xhtml"),
|
||||
("navpoint-2", 2, "Article", "text/article.xhtml"),
|
||||
]
|
||||
body = []
|
||||
for identifier, order, label, href in points:
|
||||
body.extend(
|
||||
[
|
||||
f' <navPoint id="{identifier}" playOrder="{order}">',
|
||||
f" <navLabel><text>{_escape(label)}</text></navLabel>",
|
||||
f' <content src="{href}"/>',
|
||||
" </navPoint>",
|
||||
]
|
||||
)
|
||||
return "\n".join(
|
||||
[
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<ncx xmlns="{NCX_NAMESPACE}" version="2005-1">',
|
||||
" <head>",
|
||||
f' <meta name="dtb:uid" content="{_escape(book.identifier)}"/>',
|
||||
' <meta name="dtb:depth" content="1"/>',
|
||||
' <meta name="dtb:totalPageCount" content="0"/>',
|
||||
' <meta name="dtb:maxPageNumber" content="0"/>',
|
||||
" </head>",
|
||||
f" <docTitle><text>{_escape(book.title)}</text></docTitle>",
|
||||
" <navMap>",
|
||||
*body,
|
||||
" </navMap>",
|
||||
"</ncx>",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _cover_document(book: Book) -> str:
|
||||
lines = [
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
|
||||
f'lang="{_escape(book.language)}">',
|
||||
" <head>",
|
||||
f" <title>{_escape(book.title)}</title>",
|
||||
' <meta charset="utf-8"/>',
|
||||
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
|
||||
" </head>",
|
||||
' <body class="cover">',
|
||||
]
|
||||
cover = book.cover_image
|
||||
if cover is not None:
|
||||
lines.append(f' <img src="{_escape(cover.document_href)}" alt="{_escape(book.title)}"/>')
|
||||
lines.append(f' <h1 class="article-title">{_escape(book.title)}</h1>')
|
||||
if book.author:
|
||||
lines.append(f' <p class="byline">{_escape(book.author)}</p>')
|
||||
if book.feed_title:
|
||||
lines.append(f' <p class="byline">{_escape(book.feed_title)}</p>')
|
||||
if book.published:
|
||||
lines.append(f' <p class="byline">{_date_only(book.published)}</p>')
|
||||
lines.extend([" </body>", "</html>", ""])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _article_document(book: Book) -> str:
|
||||
byline_bits = []
|
||||
if book.author:
|
||||
byline_bits.append(_escape(book.author))
|
||||
if book.published:
|
||||
byline_bits.append(_date_only(book.published))
|
||||
lines = [
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
|
||||
f'lang="{_escape(book.language)}">',
|
||||
" <head>",
|
||||
f" <title>{_escape(book.title)}</title>",
|
||||
' <meta charset="utf-8"/>',
|
||||
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
|
||||
" </head>",
|
||||
" <body>",
|
||||
f' <h1 class="article-title">{_escape(book.title)}</h1>',
|
||||
]
|
||||
if byline_bits:
|
||||
lines.append(f' <p class="byline">{BYLINE_SEPARATOR.join(byline_bits)}</p>')
|
||||
if book.content_xhtml:
|
||||
lines.append(book.content_xhtml)
|
||||
else:
|
||||
# Never ship an empty content document: a reader that opens a blank page
|
||||
# looks broken, and the original is the most useful thing to show instead.
|
||||
lines.append(" <p>This article had no content to download.</p>")
|
||||
if book.include_source_link and book.source_url:
|
||||
lines.append(
|
||||
' <p class="source">Originally published at '
|
||||
f'<a href="{_escape(book.source_url)}">{_escape(book.source_url)}</a>.</p>'
|
||||
)
|
||||
lines.extend([" </body>", "</html>", ""])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# -- formatting ----------------------------------------------------------
|
||||
|
||||
|
||||
def _escape(text: str) -> str:
|
||||
"""XML-escape text for a document body or an attribute value."""
|
||||
return xhtml.escape_text(text).replace('"', """)
|
||||
|
||||
|
||||
def _timestamp(moment: datetime) -> str:
|
||||
"""An EPUB date: UTC, seconds precision, ``Z`` suffix."""
|
||||
return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _date_only(moment: datetime) -> str:
|
||||
return moment.astimezone(UTC).strftime("%Y-%m-%d")
|
||||
339
app/fetch.py
Normal file
339
app/fetch.py
Normal file
@@ -0,0 +1,339 @@
|
||||
"""Fetching article images for embedding, with an SSRF guard.
|
||||
|
||||
Image URLs come from feed content, which means they are attacker-controlled by
|
||||
anyone who can publish to a feed this account follows. That makes this the one
|
||||
place in the server that makes an outbound request to a host the operator did not
|
||||
choose, so the guard is the point of the module rather than an extra.
|
||||
|
||||
What is enforced, and why:
|
||||
|
||||
*Only http and https*, so ``file:``, ``gopher:`` and friends are unreachable.
|
||||
*Every address a name resolves to is checked* against the private, loopback,
|
||||
link-local, multicast and reserved ranges. If a name resolves to a mixture, the
|
||||
whole fetch is refused rather than picking the acceptable address -- a mixed
|
||||
answer is itself the attack.
|
||||
*The connection is dialled to the validated address, not to the name.* This is
|
||||
what closes the DNS-rebinding window: a name that answered with a public address
|
||||
during validation cannot answer with ``169.254.169.254`` a millisecond later,
|
||||
because the name is never resolved a second time. TLS still verifies the name
|
||||
the URL used, so a valid certificate for the real host is required.
|
||||
*Redirects are followed manually*, so each hop is validated again; ``urllib``'s
|
||||
automatic redirect handling would happily follow the first hop to
|
||||
``http://127.0.0.1:8080/``.
|
||||
*Bodies are capped*, per image and across the whole article, so one hostile feed
|
||||
cannot exhaust memory.
|
||||
|
||||
A refusal is a normal outcome: the image is dropped from the book and the article
|
||||
is still built.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import ipaddress
|
||||
import logging
|
||||
import socket
|
||||
import ssl
|
||||
import threading
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app import images
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
USER_AGENT = "odps-from-rss/1.0 (+image fetch for EPUB embedding)"
|
||||
|
||||
_ALLOWED_SCHEMES = ("http", "https")
|
||||
_MAX_REDIRECTS = 3
|
||||
_READ_CHUNK = 64 * 1024
|
||||
# Refuse to even start reading a body larger than the cap, when the server tells
|
||||
# us the size up front. A little slack covers a mislabelled Content-Length.
|
||||
_CONTENT_LENGTH_SLACK = 4096
|
||||
|
||||
|
||||
class ImageFetchError(Exception):
|
||||
"""The URL was refused by the guard, or the fetch failed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FetchedImage:
|
||||
"""An image that was fetched, validated and identified."""
|
||||
|
||||
url: str
|
||||
media_type: str
|
||||
data: bytes
|
||||
|
||||
@property
|
||||
def filename(self) -> str:
|
||||
"""A safe file name for the EPUB, with the extension from the media type."""
|
||||
return "image" + images.MEDIA_TYPES.get(self.media_type, ".bin")
|
||||
|
||||
|
||||
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
|
||||
"""HTTPS to a validated address, verifying the hostname from the URL.
|
||||
|
||||
``HTTPSConnection`` would use its own host for SNI and certificate
|
||||
verification, which here is a bare IP address and would always fail. Dialling
|
||||
the address but verifying the name keeps both properties: no second DNS
|
||||
lookup, and a real certificate check.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
address: str,
|
||||
port: int,
|
||||
*,
|
||||
server_hostname: str,
|
||||
context: ssl.SSLContext,
|
||||
timeout: float,
|
||||
) -> None:
|
||||
super().__init__(address, port, timeout=timeout, context=context)
|
||||
self._verify_hostname = server_hostname
|
||||
|
||||
def connect(self) -> None:
|
||||
http.client.HTTPConnection.connect(self)
|
||||
self.sock = self._context.wrap_socket( # type: ignore[union-attr]
|
||||
self.sock, server_hostname=self._verify_hostname
|
||||
)
|
||||
|
||||
|
||||
class ImageFetcher:
|
||||
"""Fetches article images concurrently, refusing anything unsafe."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
allowed_hosts: list[str] | None = None,
|
||||
blocked_hosts: list[str] | None = None,
|
||||
max_image_bytes: int = 2_097_152,
|
||||
max_total_bytes: int = 16_777_216,
|
||||
timeout: float = 10.0,
|
||||
max_workers: int = 4,
|
||||
allow_private_networks: bool = False,
|
||||
) -> None:
|
||||
self._allowed_hosts = [host.lower() for host in (allowed_hosts or [])]
|
||||
self._blocked_hosts = [host.lower() for host in (blocked_hosts or [])]
|
||||
self._max_image_bytes = max_image_bytes
|
||||
self._max_total_bytes = max_total_bytes
|
||||
self._timeout = timeout
|
||||
self._max_workers = max(1, max_workers)
|
||||
# The upstream FreshRSS may be on a LAN, but article images never have a
|
||||
# legitimate reason to point into one. Kept as a parameter so a test can
|
||||
# exercise the guard, not as a configuration option.
|
||||
self._allow_private_networks = allow_private_networks
|
||||
self._tls_context = ssl.create_default_context()
|
||||
self._budget_lock = threading.Lock()
|
||||
self._budget_used = 0
|
||||
|
||||
def fetch_all(self, urls: list[str]) -> dict[str, FetchedImage | None]:
|
||||
"""Fetch each URL, mapping it to an image or to None when refused.
|
||||
|
||||
Order is preserved by the caller, which walks the document's images; this
|
||||
method only has to answer "did it arrive?". Concurrent, because a article
|
||||
with a dozen images should not cost a dozen sequential round trips.
|
||||
"""
|
||||
unique: list[str] = []
|
||||
for url in urls:
|
||||
if url not in unique:
|
||||
unique.append(url)
|
||||
if not unique:
|
||||
return {}
|
||||
with self._budget_lock:
|
||||
self._budget_used = 0
|
||||
workers = min(self._max_workers, len(unique))
|
||||
results: dict[str, FetchedImage | None] = {}
|
||||
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="image") as pool:
|
||||
for url, result in zip(unique, pool.map(self._fetch_safely, unique), strict=True):
|
||||
results[url] = result
|
||||
return results
|
||||
|
||||
# -- internals -------------------------------------------------------
|
||||
|
||||
def _fetch_safely(self, url: str) -> FetchedImage | None:
|
||||
"""Fetch one image, turning every failure into a dropped image."""
|
||||
try:
|
||||
return self._fetch(url)
|
||||
except ImageFetchError as exc:
|
||||
LOGGER.info("not embedding image %s: %s", url, exc)
|
||||
except (OSError, http.client.HTTPException, ssl.SSLError) as exc:
|
||||
LOGGER.info("could not fetch image %s: %s", url, exc)
|
||||
except Exception:
|
||||
# One bad image must never break the book it was going into.
|
||||
LOGGER.exception("unexpected failure fetching image %s", url)
|
||||
return None
|
||||
|
||||
def _fetch(self, url: str) -> FetchedImage:
|
||||
target = url
|
||||
for _ in range(_MAX_REDIRECTS + 1):
|
||||
parsed = urllib.parse.urlsplit(target)
|
||||
scheme = parsed.scheme.lower()
|
||||
host = parsed.hostname
|
||||
if scheme not in _ALLOWED_SCHEMES:
|
||||
raise ImageFetchError(f"scheme {scheme or '(none)'!r} is not allowed")
|
||||
if not host:
|
||||
raise ImageFetchError("no host in URL")
|
||||
if parsed.username or parsed.password:
|
||||
raise ImageFetchError("credentials in an image URL are not followed")
|
||||
self._check_host_allowed(host)
|
||||
address = self._resolve(host)
|
||||
port = parsed.port or (443 if scheme == "https" else 80)
|
||||
status, headers, body = self._request_once(scheme, address, host, port, parsed)
|
||||
if status in (301, 302, 303, 307, 308):
|
||||
location = headers.get("location")
|
||||
if not location:
|
||||
raise ImageFetchError("redirect without a location")
|
||||
# Resolve relative redirects against the URL that produced them.
|
||||
target = urllib.parse.urljoin(target, location)
|
||||
LOGGER.debug("following image redirect to %s", target)
|
||||
continue
|
||||
if status != 200:
|
||||
raise ImageFetchError(f"server answered {status}")
|
||||
declared = headers.get("content-type")
|
||||
media_type = images.identify(body, declared)
|
||||
if media_type is None:
|
||||
raise ImageFetchError(f"not an embeddable image ({declared or 'no type'})")
|
||||
return FetchedImage(url=url, media_type=media_type, data=body)
|
||||
raise ImageFetchError("too many redirects")
|
||||
|
||||
def _check_host_allowed(self, host: str) -> None:
|
||||
lowered = host.lower()
|
||||
if any(_host_matches(lowered, blocked) for blocked in self._blocked_hosts):
|
||||
raise ImageFetchError("host is on the blocked list")
|
||||
if self._allowed_hosts and not any(
|
||||
_host_matches(lowered, allowed) for allowed in self._allowed_hosts
|
||||
):
|
||||
raise ImageFetchError("host is not on the allowed list")
|
||||
|
||||
def _resolve(self, host: str) -> str:
|
||||
"""Validate every address for a host and return the one to dial."""
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
|
||||
except socket.gaierror as exc:
|
||||
raise ImageFetchError(f"cannot resolve host ({exc.strerror or exc})") from exc
|
||||
addresses: list[str] = []
|
||||
for info in infos:
|
||||
address = str(info[4][0])
|
||||
if address not in addresses:
|
||||
addresses.append(address)
|
||||
if not addresses:
|
||||
raise ImageFetchError("host resolved to no addresses")
|
||||
for address in addresses:
|
||||
reason = self._address_refusal(address)
|
||||
if reason is not None:
|
||||
# Refuse the whole host rather than picking a good address: a
|
||||
# mixed answer is the signature of a rebinding attempt.
|
||||
raise ImageFetchError(f"{address} is refused ({reason})")
|
||||
return addresses[0]
|
||||
|
||||
def _address_refusal(self, address: str) -> str | None:
|
||||
try:
|
||||
parsed = ipaddress.ip_address(address)
|
||||
except ValueError:
|
||||
return "unparseable address"
|
||||
if self._allow_private_networks:
|
||||
return None
|
||||
# A v4-mapped v6 address is judged as the v4 address it is, and this has to
|
||||
# come first: ::ffff:127.0.0.1 is neither loopback nor private when it is
|
||||
# tested as a v6 address, so the checks below would let it through.
|
||||
if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None:
|
||||
return self._address_refusal(str(parsed.ipv4_mapped))
|
||||
if parsed.is_loopback:
|
||||
return "loopback"
|
||||
if parsed.is_link_local:
|
||||
return "link-local"
|
||||
if parsed.is_private:
|
||||
return "private network"
|
||||
if parsed.is_multicast:
|
||||
return "multicast"
|
||||
if parsed.is_reserved or parsed.is_unspecified:
|
||||
return "reserved"
|
||||
return None
|
||||
|
||||
def _request_once(
|
||||
self,
|
||||
scheme: str,
|
||||
address: str,
|
||||
host: str,
|
||||
port: int,
|
||||
parsed: urllib.parse.SplitResult,
|
||||
) -> tuple[int, dict[str, str], bytes]:
|
||||
"""One request, to a pinned address, reading at most the byte cap."""
|
||||
if scheme == "https":
|
||||
connection: http.client.HTTPConnection = _PinnedHTTPSConnection(
|
||||
address,
|
||||
port,
|
||||
server_hostname=host,
|
||||
context=self._tls_context,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
else:
|
||||
connection = http.client.HTTPConnection(address, port, timeout=self._timeout)
|
||||
path = urllib.parse.urlunsplit(("", "", parsed.path or "/", parsed.query, ""))
|
||||
headers = {
|
||||
"Host": _host_header(host, port, scheme),
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "image/*",
|
||||
"Connection": "close",
|
||||
}
|
||||
try:
|
||||
connection.request("GET", path, headers=headers)
|
||||
response = connection.getresponse()
|
||||
status = response.status
|
||||
response_headers = {key.lower(): value for key, value in response.getheaders()}
|
||||
if status != 200:
|
||||
response.read(_READ_CHUNK)
|
||||
return status, response_headers, b""
|
||||
return status, response_headers, self._read_capped(response, response_headers)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def _read_capped(self, response: http.client.HTTPResponse, headers: dict[str, str]) -> bytes:
|
||||
"""Read a response body, refusing one that exceeds the budget."""
|
||||
limit = self._image_limit(headers)
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
while True:
|
||||
chunk = response.read(_READ_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
total += len(chunk)
|
||||
if total > limit:
|
||||
raise ImageFetchError(f"image is larger than {limit} bytes")
|
||||
chunks.append(chunk)
|
||||
self._spend(total)
|
||||
return b"".join(chunks)
|
||||
|
||||
def _image_limit(self, headers: dict[str, str]) -> int:
|
||||
"""The smaller of the per-image cap and what is left of the total budget."""
|
||||
declared = headers.get("content-length")
|
||||
limit = self._max_image_bytes
|
||||
if declared and declared.isdigit():
|
||||
size = int(declared)
|
||||
if size > self._max_image_bytes:
|
||||
raise ImageFetchError(f"image declares {size} bytes")
|
||||
with self._budget_lock:
|
||||
remaining = max(0, self._max_total_bytes - self._budget_used)
|
||||
if remaining == 0:
|
||||
# The article's total allowance is spent; the rest of its images are
|
||||
# dropped rather than growing the book further.
|
||||
raise ImageFetchError("the article's image budget is exhausted")
|
||||
return min(limit, remaining)
|
||||
|
||||
def _spend(self, size: int) -> None:
|
||||
with self._budget_lock:
|
||||
self._budget_used += size
|
||||
|
||||
|
||||
def _host_matches(host: str, pattern: str) -> bool:
|
||||
"""Exact host or subdomain match, so ``example.com`` covers ``cdn.example.com``."""
|
||||
return host == pattern or host.endswith("." + pattern)
|
||||
|
||||
|
||||
def _host_header(host: str, port: int, scheme: str) -> str:
|
||||
"""A Host header, including the port only when it is not the default."""
|
||||
default = 443 if scheme == "https" else 80
|
||||
if port == default:
|
||||
return host
|
||||
return f"{host}:{port}"
|
||||
598
app/freshrss.py
Normal file
598
app/freshrss.py
Normal file
@@ -0,0 +1,598 @@
|
||||
"""FreshRSS client over its Google Reader compatible API.
|
||||
|
||||
Only the endpoints the firmware client already proved out against a real server
|
||||
are used (see ``lib/FreshRss/src/FreshRssSync.cpp`` in the reader tree):
|
||||
|
||||
* ``POST accounts/ClientLogin`` returns the ``Auth`` token that goes into an
|
||||
``Authorization: GoogleLogin auth=...`` header.
|
||||
* ``GET reader/api/0/subscription/list?output=json`` lists feeds, each with its
|
||||
category labels.
|
||||
* ``GET reader/api/0/stream/contents/<stream>?output=json&r=d&n=&xt=`` fetches
|
||||
items. ``r=d`` is newest first, ``n`` bounds the count, and ``xt`` excludes
|
||||
items carrying a tag -- which is how the unread queue is filtered server-side.
|
||||
* ``GET reader/api/0/token`` returns the write token, and
|
||||
``POST reader/api/0/edit-tag`` with ``T=`` and repeated ``i=`` marks items read.
|
||||
|
||||
Deliberately absent: anything that would need an undocumented parameter. Where a
|
||||
filter is not expressible with these calls (read items), the caller filters
|
||||
locally rather than assuming an endpoint behaves a certain way.
|
||||
|
||||
Note the two endpoints this module never touches: ``unread-count`` is optional
|
||||
and its failure is tolerated, and nothing here fetches article bodies from the
|
||||
open internet -- the body FreshRSS already stores is the source of truth.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import ssl
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from app import config as app_config
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# Google Reader stream and tag identifiers.
|
||||
READING_LIST = "user/-/state/com.google/reading-list"
|
||||
STARRED = "user/-/state/com.google/starred"
|
||||
READ_TAG = "user/-/state/com.google/read"
|
||||
STARRED_TAG = "user/-/state/com.google/starred"
|
||||
KEPT_UNREAD_TAG = "user/-/state/com.google/kept-unread"
|
||||
|
||||
USER_AGENT = "odps-from-rss/1.0 (+https://github.com/; OPDS 1.2 catalog)"
|
||||
|
||||
# Statuses worth retrying: the upstream is briefly unhappy, not wrong.
|
||||
_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504, 507, 509})
|
||||
_AUTH_STATUSES = frozenset({401, 403})
|
||||
_MISSING_STATUSES = frozenset({400, 404})
|
||||
|
||||
|
||||
class FreshRSSError(Exception):
|
||||
"""An upstream call failed."""
|
||||
|
||||
def __init__(self, message: str, *, status: int | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
|
||||
|
||||
class AuthenticationError(FreshRSSError):
|
||||
"""The API password or username was rejected."""
|
||||
|
||||
|
||||
class TransientError(FreshRSSError):
|
||||
"""The upstream could not be reached, or answered with a retryable status."""
|
||||
|
||||
|
||||
class NotFoundError(FreshRSSError):
|
||||
"""The requested stream or item does not exist upstream."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Subscription:
|
||||
"""One feed in the FreshRSS account."""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
categories: list[str] = field(default_factory=list)
|
||||
html_url: str | None = None
|
||||
unread: int | None = None
|
||||
|
||||
@property
|
||||
def primary_category(self) -> str:
|
||||
"""The label this feed is shelved under, or a stable placeholder."""
|
||||
return self.categories[0] if self.categories else UNCATEGORISED
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
"""One article."""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
published: int
|
||||
updated: int
|
||||
feed_id: str
|
||||
feed_title: str
|
||||
content_html: str = ""
|
||||
author: str | None = None
|
||||
url: str | None = None
|
||||
feed_url: str | None = None
|
||||
categories: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def is_read(self) -> bool:
|
||||
"""True when the item is read upstream.
|
||||
|
||||
FreshRSS keeps both the read and the kept-unread tag on an item that was
|
||||
read and then explicitly marked unread again, and there the explicit
|
||||
signal is the one that reflects what the reader wants.
|
||||
"""
|
||||
if KEPT_UNREAD_TAG in self.categories:
|
||||
return False
|
||||
return READ_TAG in self.categories
|
||||
|
||||
@property
|
||||
def is_starred(self) -> bool:
|
||||
return STARRED_TAG in self.categories
|
||||
|
||||
@property
|
||||
def published_at(self) -> datetime:
|
||||
"""Publication time as an aware UTC datetime.
|
||||
|
||||
Falls back to the epoch rather than raising: a feed with a broken date is
|
||||
not a reason to fail the whole catalog.
|
||||
"""
|
||||
stamp = self.published or self.updated or 0
|
||||
try:
|
||||
return datetime.fromtimestamp(stamp, tz=UTC)
|
||||
except (OverflowError, OSError, ValueError):
|
||||
return datetime(1970, 1, 1, tzinfo=UTC)
|
||||
|
||||
@property
|
||||
def updated_at(self) -> datetime:
|
||||
stamp = self.updated or self.published or 0
|
||||
try:
|
||||
return datetime.fromtimestamp(stamp, tz=UTC)
|
||||
except (OverflowError, OSError, ValueError):
|
||||
return self.published_at
|
||||
|
||||
|
||||
UNCATEGORISED = "Uncategorised"
|
||||
|
||||
|
||||
def label_stream(label: str) -> str:
|
||||
"""Stream id for a category label."""
|
||||
return f"user/-/label/{label}"
|
||||
|
||||
|
||||
class FreshRSSClient:
|
||||
"""A small client for one FreshRSS account.
|
||||
|
||||
Instances are safe to share between threads: the only mutable state is the
|
||||
cached session token, which is guarded by a lock.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: app_config.FreshRSSConfig,
|
||||
*,
|
||||
clock: Any = time.time,
|
||||
sleep: Any = time.sleep,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._clock = clock
|
||||
# Injectable so tests can exercise retry logic without waiting it out.
|
||||
self._sleep = sleep
|
||||
self._auth_token: str | None = None
|
||||
self._write_token: str | None = None
|
||||
self._write_token_fetched_at = 0.0
|
||||
self._token_lock = threading.Lock()
|
||||
self._tls_context = _build_tls_context(config)
|
||||
self._opener = urllib.request.build_opener(
|
||||
urllib.request.HTTPSHandler(context=self._tls_context)
|
||||
)
|
||||
|
||||
# -- public API ------------------------------------------------------
|
||||
|
||||
def list_subscriptions(self) -> list[Subscription]:
|
||||
"""Every feed in the account, with its category labels."""
|
||||
payload = self._get_json("/reader/api/0/subscription/list")
|
||||
raw_subscriptions = payload.get("subscriptions")
|
||||
if not isinstance(raw_subscriptions, list):
|
||||
raise FreshRSSError("subscription list had no subscriptions array")
|
||||
subscriptions: list[Subscription] = []
|
||||
for entry in raw_subscriptions:
|
||||
parsed = _parse_subscription(entry)
|
||||
if parsed is not None:
|
||||
subscriptions.append(parsed)
|
||||
subscriptions.sort(key=lambda feed: feed.title.casefold())
|
||||
return subscriptions
|
||||
|
||||
def unread_counts(self) -> dict[str, int]:
|
||||
"""Unread counts keyed by stream id. Optional upstream feature.
|
||||
|
||||
Raises whatever the client raises, so the caller can latch the failure and
|
||||
carry on without counts -- the catalog is perfectly usable without them.
|
||||
"""
|
||||
payload = self._get_json("/reader/api/0/unread-count", {"all": "true"})
|
||||
counts: dict[str, int] = {}
|
||||
for entry in payload.get("unreadcounts") or []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
stream_id = entry.get("id")
|
||||
count = entry.get("count")
|
||||
if isinstance(stream_id, str) and isinstance(count, int):
|
||||
counts[stream_id] = count
|
||||
return counts
|
||||
|
||||
def stream_items(
|
||||
self,
|
||||
stream_id: str,
|
||||
*,
|
||||
count: int,
|
||||
exclude_read: bool = False,
|
||||
older_than: int | None = None,
|
||||
) -> list[Item]:
|
||||
"""Fetch items from one stream, newest first.
|
||||
|
||||
A stream that has disappeared upstream (``400``/``404``) is reported as an
|
||||
empty result rather than an error: a feed you unsubscribed from should not
|
||||
break the catalog that still lists it.
|
||||
"""
|
||||
params = {
|
||||
"output": "json",
|
||||
"r": "d",
|
||||
"n": str(max(1, count)),
|
||||
# Cache killer: asks FreshRSS not to serve a cached response.
|
||||
"ck": str(int(self._clock())),
|
||||
}
|
||||
if exclude_read:
|
||||
params["xt"] = READ_TAG
|
||||
if older_than:
|
||||
params["ot"] = str(int(older_than))
|
||||
path = "/reader/api/0/stream/contents/" + urllib.parse.quote(stream_id, safe="")
|
||||
try:
|
||||
payload = self._get_json(path, params)
|
||||
except NotFoundError:
|
||||
LOGGER.info("stream %s is gone upstream; treating it as empty", stream_id)
|
||||
return []
|
||||
raw_items = payload.get("items")
|
||||
if not isinstance(raw_items, list):
|
||||
return []
|
||||
items = [item for item in (_parse_item(entry) for entry in raw_items) if item]
|
||||
return items
|
||||
|
||||
def mark_read(self, item_ids: list[str]) -> int:
|
||||
"""Mark items read upstream. Returns the number of ids sent.
|
||||
|
||||
Idempotent: adding the read tag to an already-read item is a no-op
|
||||
upstream, so a repeated download of the same book is harmless.
|
||||
"""
|
||||
wanted = [item_id for item_id in item_ids if item_id]
|
||||
if not wanted:
|
||||
return 0
|
||||
token = self._write_token_value()
|
||||
form: list[tuple[str, str]] = [
|
||||
("T", token),
|
||||
("a", READ_TAG),
|
||||
]
|
||||
form.extend(("i", item_id) for item_id in wanted)
|
||||
# edit-tag needs both the session header and the write token: the header
|
||||
# authenticates the caller, the token authorises the write.
|
||||
try:
|
||||
self._request(
|
||||
"POST",
|
||||
"/reader/api/0/edit-tag",
|
||||
body=urllib.parse.urlencode(form),
|
||||
authenticated=True,
|
||||
)
|
||||
except AuthenticationError:
|
||||
# Either the session or the write token outlived its validity; drop
|
||||
# both, fetch fresh ones, and retry the write once.
|
||||
self._forget_tokens()
|
||||
form[0] = ("T", self._write_token_value())
|
||||
self._request(
|
||||
"POST",
|
||||
"/reader/api/0/edit-tag",
|
||||
body=urllib.parse.urlencode(form),
|
||||
authenticated=True,
|
||||
)
|
||||
return len(wanted)
|
||||
|
||||
def probe(self) -> tuple[bool, str]:
|
||||
"""Check credentials and reachability, for the diagnostics endpoint."""
|
||||
try:
|
||||
subscriptions = self.list_subscriptions()
|
||||
except FreshRSSError as exc:
|
||||
return False, str(exc)
|
||||
return True, f"{len(subscriptions)} feeds"
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release upstream resources (connections are per-request)."""
|
||||
self._forget_tokens()
|
||||
|
||||
# -- transport -------------------------------------------------------
|
||||
|
||||
def _forget_tokens(self) -> None:
|
||||
with self._token_lock:
|
||||
self._auth_token = None
|
||||
self._write_token = None
|
||||
self._write_token_fetched_at = 0.0
|
||||
|
||||
def _auth_header(self) -> dict[str, str]:
|
||||
with self._token_lock:
|
||||
token = self._auth_token
|
||||
if token is None:
|
||||
token = self._login()
|
||||
with self._token_lock:
|
||||
self._auth_token = token
|
||||
return {"Authorization": f"GoogleLogin auth={token}"}
|
||||
|
||||
def _write_token_value(self) -> str:
|
||||
with self._token_lock:
|
||||
token = self._write_token
|
||||
fetched_at = self._write_token_fetched_at
|
||||
# Refresh hourly: the token is cheap to fetch and this bounds how long a
|
||||
# stale one can survive a password change upstream.
|
||||
if token and self._clock() - fetched_at < 3600:
|
||||
return token
|
||||
response = self._request("GET", "/reader/api/0/token", authenticated=True)
|
||||
token = response.decode("utf-8", "replace").strip()
|
||||
if not token:
|
||||
raise FreshRSSError("upstream returned an empty write token")
|
||||
with self._token_lock:
|
||||
self._write_token = token
|
||||
self._write_token_fetched_at = self._clock()
|
||||
return token
|
||||
|
||||
def _login(self) -> str:
|
||||
"""Exchange the API password for an Auth token."""
|
||||
form = urllib.parse.urlencode(
|
||||
{"Email": self._config.username, "Passwd": self._config.api_password}
|
||||
)
|
||||
response = self._request("POST", "/accounts/ClientLogin", body=form)
|
||||
text = response.decode("utf-8", "replace")
|
||||
for line in text.splitlines():
|
||||
if line.startswith("Auth="):
|
||||
value = line[len("Auth=") :].strip()
|
||||
if value:
|
||||
LOGGER.debug("authenticated with FreshRSS as %s", self._config.username)
|
||||
return value
|
||||
raise AuthenticationError("FreshRSS rejected the API password")
|
||||
|
||||
def _get_json(self, path: str, params: dict[str, str] | None = None) -> dict[str, Any]:
|
||||
query = dict(params or {})
|
||||
query.setdefault("output", "json")
|
||||
response = self._request("GET", path, query=query, authenticated=True)
|
||||
try:
|
||||
payload = json.loads(response.decode("utf-8", "replace"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
raise FreshRSSError(f"upstream sent a response that is not JSON ({exc})") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise FreshRSSError("upstream sent a JSON value that is not an object")
|
||||
return payload
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
query: dict[str, str] | None = None,
|
||||
body: str | None = None,
|
||||
authenticated: bool = False,
|
||||
) -> bytes:
|
||||
"""Perform one upstream request, retrying transient failures."""
|
||||
url = self._config.api_base + path
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = body.encode("utf-8") if body is not None else None
|
||||
attempts = max(1, self._config.retries + 1)
|
||||
last_error: Exception | None = None
|
||||
|
||||
for attempt in range(attempts):
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
||||
if body is not None:
|
||||
headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||
if authenticated:
|
||||
headers.update(self._auth_header())
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with self._opener.open(request, timeout=self._config.timeout_seconds) as reply:
|
||||
return reply.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
status = exc.code
|
||||
try:
|
||||
detail = _read_error_body(exc)
|
||||
finally:
|
||||
# The error object owns the response socket. Leaving it open
|
||||
# leaks one connection for every failed upstream request.
|
||||
exc.close()
|
||||
if status in _AUTH_STATUSES:
|
||||
raise AuthenticationError(
|
||||
f"FreshRSS refused the request ({status}); check the API password",
|
||||
status=status,
|
||||
) from exc
|
||||
if status in _MISSING_STATUSES:
|
||||
raise NotFoundError(
|
||||
f"upstream has no such resource ({status})", status=status
|
||||
) from exc
|
||||
if status in _RETRY_STATUSES:
|
||||
# A retryable status stays transient even once the attempts run
|
||||
# out, so the caller can tell "upstream is unwell" from
|
||||
# "upstream says this request is wrong".
|
||||
error = TransientError(f"upstream returned {status}{detail}", status=status)
|
||||
if attempt + 1 < attempts:
|
||||
last_error = error
|
||||
self._backoff(attempt, status)
|
||||
continue
|
||||
raise error from exc
|
||||
raise FreshRSSError(f"upstream returned {status}{detail}", status=status) from exc
|
||||
except urllib.error.URLError as exc:
|
||||
last_error = TransientError(f"cannot reach FreshRSS: {exc.reason}")
|
||||
if attempt + 1 < attempts:
|
||||
self._backoff(attempt, None)
|
||||
continue
|
||||
raise last_error from exc
|
||||
except TimeoutError as exc:
|
||||
last_error = TransientError("the upstream request timed out")
|
||||
if attempt + 1 < attempts:
|
||||
self._backoff(attempt, None)
|
||||
continue
|
||||
raise last_error from exc
|
||||
except ssl.SSLError as exc:
|
||||
raise FreshRSSError(f"TLS failure talking to FreshRSS: {exc}") from exc
|
||||
|
||||
# Only reachable if the loop exhausted its attempts without raising.
|
||||
raise last_error or TransientError("the upstream request failed")
|
||||
|
||||
def _backoff(self, attempt: int, status: int | None) -> None:
|
||||
delay = min(2.0, 0.2 * (2**attempt))
|
||||
LOGGER.warning(
|
||||
"upstream %s, retrying in %.1fs%s",
|
||||
f"returned {status}" if status else "unreachable",
|
||||
delay,
|
||||
f" (attempt {attempt + 1})" if attempt else "",
|
||||
)
|
||||
self._sleep(delay)
|
||||
|
||||
|
||||
def _build_tls_context(config: app_config.FreshRSSConfig) -> ssl.SSLContext:
|
||||
"""Build the TLS context for upstream calls.
|
||||
|
||||
An operator pointing at a LAN instance with a private CA sets ``ca_file``;
|
||||
disabling verification is possible but logged loudly, because it silently
|
||||
removes the protection against a machine-in-the-middle reading the API
|
||||
password.
|
||||
"""
|
||||
if not config.verify_tls:
|
||||
LOGGER.warning(
|
||||
"freshrss.verify_tls is false: TLS certificates are not verified. "
|
||||
"The API password and article contents are exposed to anyone able to "
|
||||
"intercept the connection."
|
||||
)
|
||||
context = ssl.create_default_context()
|
||||
context.check_hostname = False
|
||||
context.verify_mode = ssl.CERT_NONE
|
||||
return context
|
||||
context = ssl.create_default_context()
|
||||
if config.ca_file:
|
||||
context.load_verify_locations(cafile=config.ca_file)
|
||||
return context
|
||||
|
||||
|
||||
def _read_error_body(error: urllib.error.HTTPError) -> str:
|
||||
"""Include a short, sanitised excerpt of an error body in the message."""
|
||||
try:
|
||||
raw = error.read(512)
|
||||
except Exception:
|
||||
# A diagnostic excerpt must never mask the error it is describing.
|
||||
return ""
|
||||
text = raw.decode("utf-8", "replace").strip().replace("\n", " ")
|
||||
if not text:
|
||||
return ""
|
||||
return f": {text[:200]}"
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
"""A trimmed string from upstream JSON, or an empty string.
|
||||
|
||||
Upstream is an arbitrary FreshRSS build, so every field is treated as
|
||||
untrusted input of unknown type rather than as the shape the docs suggest.
|
||||
"""
|
||||
return value.strip() if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def _as_int(value: Any) -> int:
|
||||
"""Best-effort integer from upstream JSON, tolerating strings."""
|
||||
if isinstance(value, bool):
|
||||
return 0
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_subscription(entry: Any) -> Subscription | None:
|
||||
if not isinstance(entry, dict):
|
||||
return None
|
||||
stream_id = entry.get("id")
|
||||
if not isinstance(stream_id, str) or not stream_id:
|
||||
return None
|
||||
categories: list[str] = []
|
||||
for category in entry.get("categories") or []:
|
||||
if not isinstance(category, dict):
|
||||
continue
|
||||
label = _text(category.get("label"))
|
||||
if label and label not in categories:
|
||||
categories.append(label)
|
||||
return Subscription(
|
||||
id=stream_id,
|
||||
title=_text(entry.get("title")) or stream_id,
|
||||
categories=categories,
|
||||
html_url=_text(entry.get("htmlUrl")) or None,
|
||||
)
|
||||
|
||||
|
||||
def _parse_item(entry: Any) -> Item | None:
|
||||
if not isinstance(entry, dict):
|
||||
return None
|
||||
item_id = entry.get("id")
|
||||
if not isinstance(item_id, str) or not item_id:
|
||||
return None
|
||||
|
||||
raw_origin = entry.get("origin")
|
||||
origin: dict[str, Any] = raw_origin if isinstance(raw_origin, dict) else {}
|
||||
feed_id = _text(origin.get("streamId"))
|
||||
return Item(
|
||||
id=item_id,
|
||||
title=_text(entry.get("title")) or "(untitled)",
|
||||
published=_as_int(entry.get("published")) or _timestamp_usec(entry),
|
||||
updated=_as_int(entry.get("updated")) or _timestamp_usec(entry),
|
||||
feed_id=feed_id,
|
||||
feed_title=_text(origin.get("title")) or feed_id,
|
||||
content_html=_extract_content(entry),
|
||||
author=_text(entry.get("author")) or None,
|
||||
url=_extract_url(entry, origin),
|
||||
feed_url=_text(origin.get("htmlUrl")) or None,
|
||||
categories=[tag for tag in (entry.get("categories") or []) if isinstance(tag, str)],
|
||||
)
|
||||
|
||||
|
||||
def _timestamp_usec(entry: dict[str, Any]) -> int:
|
||||
"""Derive seconds from Google Reader's microsecond timestamp, when present."""
|
||||
raw = entry.get("timestampUsec")
|
||||
if isinstance(raw, str) and raw.isdigit():
|
||||
return int(raw) // 1_000_000
|
||||
if isinstance(raw, int):
|
||||
return raw // 1_000_000
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_content(entry: dict[str, Any]) -> str:
|
||||
"""Prefer the full body over the summary.
|
||||
|
||||
FreshRSS stores whichever it has; ``content`` is the article body when the
|
||||
feed provides one, and ``summary`` is the fallback.
|
||||
"""
|
||||
for key in ("content", "summary"):
|
||||
block = entry.get(key)
|
||||
if isinstance(block, str) and block.strip():
|
||||
return block
|
||||
if isinstance(block, dict):
|
||||
inner = block.get("content")
|
||||
if isinstance(inner, str) and inner.strip():
|
||||
return inner
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_url(entry: dict[str, Any], origin: dict[str, Any]) -> str | None:
|
||||
alternates = entry.get("alternate")
|
||||
if isinstance(alternates, list):
|
||||
for candidate in alternates:
|
||||
if not isinstance(candidate, dict):
|
||||
continue
|
||||
href = candidate.get("href")
|
||||
if not isinstance(href, str) or not href:
|
||||
continue
|
||||
kind = candidate.get("type")
|
||||
if kind in (None, "text/html"):
|
||||
return href
|
||||
for candidate in alternates:
|
||||
if isinstance(candidate, dict) and isinstance(candidate.get("href"), str):
|
||||
return candidate["href"]
|
||||
html_url = origin.get("htmlUrl")
|
||||
return html_url if isinstance(html_url, str) and html_url else None
|
||||
193
app/images.py
Normal file
193
app/images.py
Normal file
@@ -0,0 +1,193 @@
|
||||
"""Image metadata: media type sniffing and dimension parsing.
|
||||
|
||||
Two small jobs, both in service of the EPUB writer.
|
||||
|
||||
*Sniffing.* A feed image is as likely to arrive as ``application/octet-stream``
|
||||
or ``text/plain`` as with a correct ``Content-Type``. An EPUB manifest entry
|
||||
whose media type does not match the bytes is invalid, so the type is decided from
|
||||
the actual magic bytes and the response header is only a fallback.
|
||||
|
||||
*Dimensions.* Picking a cover means knowing which image is big enough to be one.
|
||||
Reading a PNG header or a JPEG start-of-frame costs nothing and avoids pulling in
|
||||
an imaging library (and its C dependencies) just to answer "is this wide?".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
|
||||
# Types the EPUB writer will embed. SVG is deliberately absent: it can carry
|
||||
# script and external references, and an image in an article body has no reason
|
||||
# to need either.
|
||||
MEDIA_TYPES = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/gif": ".gif",
|
||||
"image/webp": ".webp",
|
||||
}
|
||||
|
||||
_MEDIA_TYPE_ALIASES = {
|
||||
"image/jpg": "image/jpeg",
|
||||
"image/pjpeg": "image/jpeg",
|
||||
"image/x-png": "image/png",
|
||||
"image/x-gif": "image/gif",
|
||||
}
|
||||
|
||||
|
||||
def normalise_media_type(raw: str | None) -> str | None:
|
||||
"""Map a Content-Type header onto a media type this server will embed."""
|
||||
if not raw:
|
||||
return None
|
||||
# Strip parameters: "image/jpeg; charset=binary" is still a JPEG.
|
||||
base = raw.split(";", 1)[0].strip().lower()
|
||||
base = _MEDIA_TYPE_ALIASES.get(base, base)
|
||||
return base if base in MEDIA_TYPES else None
|
||||
|
||||
|
||||
def sniff_media_type(data: bytes) -> str | None:
|
||||
"""Identify an image from its leading bytes, or None if unrecognised."""
|
||||
if len(data) < 12:
|
||||
return None
|
||||
if data.startswith(b"\xff\xd8\xff"):
|
||||
return "image/jpeg"
|
||||
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return "image/png"
|
||||
if data.startswith((b"GIF87a", b"GIF89a")):
|
||||
return "image/gif"
|
||||
if data.startswith(b"RIFF") and data[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
return None
|
||||
|
||||
|
||||
def identify(data: bytes, declared: str | None) -> str | None:
|
||||
"""Decide the media type for a fetched image.
|
||||
|
||||
The bytes win. A declared type is used only when the bytes are recognisable
|
||||
as nothing else, and never to *override* a successful sniff -- otherwise a
|
||||
mislabelled payload would produce an invalid EPUB manifest.
|
||||
"""
|
||||
sniffed = sniff_media_type(data)
|
||||
if sniffed is not None:
|
||||
return sniffed
|
||||
return normalise_media_type(declared)
|
||||
|
||||
|
||||
def dimensions(data: bytes) -> tuple[int, int] | None:
|
||||
"""Return ``(width, height)`` in pixels, or None when it cannot be read.
|
||||
|
||||
Only the container headers are parsed: the value is used to decide whether an
|
||||
image is a plausible cover, so "unknown" is a perfectly good answer.
|
||||
"""
|
||||
sniffed = sniff_media_type(data)
|
||||
try:
|
||||
if sniffed == "image/png":
|
||||
return _png_dimensions(data)
|
||||
if sniffed == "image/gif":
|
||||
return _gif_dimensions(data)
|
||||
if sniffed == "image/jpeg":
|
||||
return _jpeg_dimensions(data)
|
||||
if sniffed == "image/webp":
|
||||
return _webp_dimensions(data)
|
||||
except (struct.error, IndexError, ValueError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _png_dimensions(data: bytes) -> tuple[int, int] | None:
|
||||
# IHDR is required to be the first chunk: 8 signature bytes, then length and
|
||||
# type, so width and height sit at a fixed offset.
|
||||
if len(data) < 24:
|
||||
return None
|
||||
width, height = struct.unpack(">II", data[16:24])
|
||||
return (width, height) if width and height else None
|
||||
|
||||
|
||||
def _gif_dimensions(data: bytes) -> tuple[int, int] | None:
|
||||
if len(data) < 10:
|
||||
return None
|
||||
width, height = struct.unpack("<HH", data[6:10])
|
||||
return (width, height) if width and height else None
|
||||
|
||||
|
||||
def _webp_dimensions(data: bytes) -> tuple[int, int] | None:
|
||||
if len(data) < 30:
|
||||
return None
|
||||
chunk = data[12:16]
|
||||
if chunk == b"VP8X":
|
||||
width = int.from_bytes(data[24:27], "little") + 1
|
||||
height = int.from_bytes(data[27:30], "little") + 1
|
||||
return (width, height)
|
||||
if chunk == b"VP8 ":
|
||||
# Lossy: dimensions are 14 bytes into the frame header, after the
|
||||
# start code, with 14 bits of width and 14 of height.
|
||||
if len(data) < 30:
|
||||
return None
|
||||
start = 26
|
||||
width = int.from_bytes(data[start : start + 2], "little") & 0x3FFF
|
||||
height = int.from_bytes(data[start + 2 : start + 4], "little") & 0x3FFF
|
||||
return (width, height) if width and height else None
|
||||
if chunk == b"VP8L":
|
||||
bits = int.from_bytes(data[21:25], "little")
|
||||
width = (bits & 0x3FFF) + 1
|
||||
height = ((bits >> 14) & 0x3FFF) + 1
|
||||
return (width, height)
|
||||
return None
|
||||
|
||||
|
||||
# JPEG start-of-frame markers, which carry the dimensions.
|
||||
_SOF_MARKERS = frozenset(
|
||||
{
|
||||
0xC0,
|
||||
0xC1,
|
||||
0xC2,
|
||||
0xC3,
|
||||
0xC5,
|
||||
0xC6,
|
||||
0xC7,
|
||||
0xC9,
|
||||
0xCA,
|
||||
0xCB,
|
||||
0xCD,
|
||||
0xCE,
|
||||
0xCF,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _jpeg_dimensions(data: bytes) -> tuple[int, int] | None:
|
||||
"""Walk the JPEG segment chain to the start-of-frame marker.
|
||||
|
||||
A JPEG has no fixed header offset, so the segments have to be stepped over
|
||||
until the frame header appears. This is bounded work: it stops at the first
|
||||
frame marker, which always precedes the image data.
|
||||
"""
|
||||
index = 2 # skip the SOI marker (FF D8)
|
||||
limit = len(data)
|
||||
while index + 9 < limit:
|
||||
if data[index] != 0xFF:
|
||||
index += 1
|
||||
continue
|
||||
marker = data[index + 1]
|
||||
if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7:
|
||||
index += 2
|
||||
continue
|
||||
if marker == 0xDA: # start of scan: no frame header found
|
||||
return None
|
||||
length = struct.unpack(">H", data[index + 2 : index + 4])[0]
|
||||
if length < 2:
|
||||
return None
|
||||
if marker in _SOF_MARKERS:
|
||||
height, width = struct.unpack(">HH", data[index + 5 : index + 9])
|
||||
return (width, height) if width and height else None
|
||||
index += 2 + length
|
||||
return None
|
||||
|
||||
|
||||
def looks_like_a_cover(width: int, height: int) -> bool:
|
||||
"""Whether an image is a plausible book cover.
|
||||
|
||||
A lead image is usually landscape and wide; a thumbnail, icon or tracking
|
||||
pixel is not. The bar is deliberately low -- it only has to beat a blank
|
||||
tile in a reader's library view.
|
||||
"""
|
||||
return width >= 400 and height >= 200
|
||||
308
app/opds.py
Normal file
308
app/opds.py
Normal file
@@ -0,0 +1,308 @@
|
||||
"""OPDS 1.2 catalog documents.
|
||||
|
||||
The dialect is Atom with the OPDS profile, because that is what ebook readers
|
||||
actually implement: KOReader, Calibre, Moon+ Reader, Librera, FBReader and Aldiko
|
||||
all browse OPDS 1.2, and its navigation/acquisition split maps exactly onto the
|
||||
category/feed/article tree this server mirrors out of FreshRSS.
|
||||
|
||||
The two document kinds, and the rules that matter:
|
||||
|
||||
*Navigation feeds* (``kind=navigation``) are directories. Each entry is a
|
||||
``rel="subsection"`` link to another feed, carrying a text description such as an
|
||||
unread count.
|
||||
|
||||
*Acquisition feeds* (``kind=acquisition``) list articles. Each entry carries an
|
||||
``rel="http://opds-spec.org/acquisition/open-access"`` link with an EPUB media
|
||||
type, which is what a reader downloads. A plain ``http://opds-spec.org/acquisition``
|
||||
link to the same URL is also emitted: readers disagree about which relation to
|
||||
look for, and both are valid ways to say the same thing. An ``alternate`` link
|
||||
points at the original article, which readers show as "open in browser".
|
||||
|
||||
Every feed carries ``rel="self"`` with the profile in its type, plus ``rel="start"``
|
||||
back to the root, because readers navigate by these rather than by URL. Pagination
|
||||
uses ``rel="next"``/``rel="previous"``/``rel="first"``.
|
||||
|
||||
Documents are built as strings rather than through a DOM library: the output is
|
||||
small, fully determined, and easy to read in a test failure. Every interpolated
|
||||
value goes through an escaper, and the tests parse the result with a strict XML
|
||||
parser.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app import xhtml
|
||||
|
||||
ATOM_NAMESPACE = "http://www.w3.org/2005/Atom"
|
||||
OPDS_NAMESPACE = "http://opds-spec.org/2010/catalog"
|
||||
OPENSEARCH_NAMESPACE = "http://a9.com/-/spec/opensearch/1.1/"
|
||||
DC_NAMESPACE = "http://purl.org/dc/elements/1.1/"
|
||||
|
||||
NAVIGATION_TYPE = "application/atom+xml;profile=opds-catalog;kind=navigation"
|
||||
ACQUISITION_TYPE = "application/atom+xml;profile=opds-catalog;kind=acquisition"
|
||||
OPENSEARCH_TYPE = "application/opensearchdescription+xml"
|
||||
|
||||
EPUB_TYPE = "application/epub+zip"
|
||||
HTML_TYPE = "text/html"
|
||||
|
||||
ACQUISITION_REL = "http://opds-spec.org/acquisition"
|
||||
OPEN_ACCESS_REL = "http://opds-spec.org/acquisition/open-access"
|
||||
SUBSECTION_REL = "subsection"
|
||||
ALTERNATE_REL = "alternate"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NavEntry:
|
||||
"""One row of a navigation feed: a link deeper into the catalog."""
|
||||
|
||||
title: str
|
||||
entry_id: str
|
||||
href: str
|
||||
updated: datetime
|
||||
description: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AcquisitionEntry:
|
||||
"""One row of an acquisition feed: an article, as a downloadable book."""
|
||||
|
||||
title: str
|
||||
entry_id: str
|
||||
updated: datetime
|
||||
download_href: str
|
||||
author: str | None = None
|
||||
summary: str | None = None
|
||||
language: str | None = None
|
||||
alternate_href: str | None = None
|
||||
categories: tuple[tuple[str, str], ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PageLinks:
|
||||
"""Pagination targets, already absolute. Omitting one omits its link."""
|
||||
|
||||
first: str | None = None
|
||||
previous: str | None = None
|
||||
next: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FeedLinks:
|
||||
"""Everything a reader needs to place a feed in the catalog.
|
||||
|
||||
Pagination lives only in :attr:`pages`: an earlier version also carried
|
||||
``next_url``/``previous_url``, and the renderer read those while callers set
|
||||
``pages``, so the links silently never appeared.
|
||||
"""
|
||||
|
||||
self_url: str
|
||||
start_url: str | None = None
|
||||
search_url: str | None = None
|
||||
pages: PageLinks = field(default_factory=PageLinks)
|
||||
|
||||
|
||||
def navigation_feed(
|
||||
*,
|
||||
feed_id: str,
|
||||
title: str,
|
||||
updated: datetime,
|
||||
links: FeedLinks,
|
||||
entries: list[NavEntry],
|
||||
author: str = "OPDS",
|
||||
subtitle: str | None = None,
|
||||
) -> str:
|
||||
"""Render a navigation feed."""
|
||||
return _feed(
|
||||
feed_id=feed_id,
|
||||
title=title,
|
||||
updated=updated,
|
||||
links=links,
|
||||
entries=[_nav_entry_xml(entry) for entry in entries],
|
||||
kind_type=NAVIGATION_TYPE,
|
||||
author=author,
|
||||
subtitle=subtitle,
|
||||
)
|
||||
|
||||
|
||||
def acquisition_feed(
|
||||
*,
|
||||
feed_id: str,
|
||||
title: str,
|
||||
updated: datetime,
|
||||
links: FeedLinks,
|
||||
entries: list[AcquisitionEntry],
|
||||
author: str = "OPDS",
|
||||
subtitle: str | None = None,
|
||||
) -> str:
|
||||
"""Render an acquisition feed."""
|
||||
return _feed(
|
||||
feed_id=feed_id,
|
||||
title=title,
|
||||
updated=updated,
|
||||
links=links,
|
||||
entries=[_acquisition_entry_xml(entry) for entry in entries],
|
||||
kind_type=ACQUISITION_TYPE,
|
||||
author=author,
|
||||
subtitle=subtitle,
|
||||
)
|
||||
|
||||
|
||||
def opensearch_description(
|
||||
*,
|
||||
search_url: str,
|
||||
short_name: str,
|
||||
description: str,
|
||||
output_type: str = ACQUISITION_TYPE,
|
||||
) -> str:
|
||||
"""Render an OpenSearch description document.
|
||||
|
||||
Readers that support search advertise it by fetching this document, which is
|
||||
why the root navigation feed links to it with ``rel="search"``.
|
||||
"""
|
||||
return "\n".join(
|
||||
[
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<OpenSearchDescription xmlns="{OPENSEARCH_NAMESPACE}">',
|
||||
f" <ShortName>{_escape(short_name)}</ShortName>",
|
||||
f" <Description>{_escape(description)}</Description>",
|
||||
" <InputEncoding>UTF-8</InputEncoding>",
|
||||
" <OutputEncoding>UTF-8</OutputEncoding>",
|
||||
f' <Url type="{_attribute(output_type)}" '
|
||||
f'template="{_attribute(search_url)}?q={{searchTerms}}&page={{startPage?}}"/>',
|
||||
"</OpenSearchDescription>",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# -- feed assembly -------------------------------------------------------
|
||||
|
||||
|
||||
def _feed(
|
||||
*,
|
||||
feed_id: str,
|
||||
title: str,
|
||||
updated: datetime,
|
||||
links: FeedLinks,
|
||||
entries: list[str],
|
||||
kind_type: str,
|
||||
author: str,
|
||||
subtitle: str | None,
|
||||
) -> str:
|
||||
lines = [
|
||||
'<?xml version="1.0" encoding="utf-8"?>',
|
||||
f'<feed xmlns="{ATOM_NAMESPACE}" xmlns:opds="{OPDS_NAMESPACE}" xmlns:dc="{DC_NAMESPACE}">',
|
||||
f" <id>{_escape(feed_id)}</id>",
|
||||
f" <title>{_escape(title)}</title>",
|
||||
f" <updated>{_timestamp(updated)}</updated>",
|
||||
]
|
||||
if subtitle:
|
||||
lines.append(f" <subtitle>{_escape(subtitle)}</subtitle>")
|
||||
lines.append(f" <author><name>{_escape(author)}</name></author>")
|
||||
lines.extend(_feed_links_xml(links, kind_type))
|
||||
lines.extend(entries)
|
||||
lines.extend(["</feed>", ""])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _feed_links_xml(links: FeedLinks, kind_type: str) -> list[str]:
|
||||
"""Self, start, search and pagination links, in that order."""
|
||||
rendered = [
|
||||
f' <link rel="self" href="{_attribute(links.self_url)}" type="{_attribute(kind_type)}"/>'
|
||||
]
|
||||
if links.start_url:
|
||||
rendered.append(
|
||||
f' <link rel="start" href="{_attribute(links.start_url)}" '
|
||||
f'type="{_attribute(NAVIGATION_TYPE)}"/>'
|
||||
)
|
||||
if links.search_url:
|
||||
rendered.append(
|
||||
f' <link rel="search" href="{_attribute(links.search_url)}" '
|
||||
f'type="{_attribute(OPENSEARCH_TYPE)}"/>'
|
||||
)
|
||||
first = links.pages.first
|
||||
if first:
|
||||
rendered.append(
|
||||
f' <link rel="first" href="{_attribute(first)}" type="{_attribute(kind_type)}"/>'
|
||||
)
|
||||
if links.pages.previous:
|
||||
rendered.append(
|
||||
f' <link rel="previous" href="{_attribute(links.pages.previous)}" '
|
||||
f'type="{_attribute(kind_type)}"/>'
|
||||
)
|
||||
if links.pages.next:
|
||||
rendered.append(
|
||||
f' <link rel="next" href="{_attribute(links.pages.next)}" '
|
||||
f'type="{_attribute(kind_type)}"/>'
|
||||
)
|
||||
return rendered
|
||||
|
||||
|
||||
def _nav_entry_xml(entry: NavEntry) -> str:
|
||||
lines = [
|
||||
" <entry>",
|
||||
f" <title>{_escape(entry.title)}</title>",
|
||||
f" <id>{_escape(entry.entry_id)}</id>",
|
||||
f" <updated>{_timestamp(entry.updated)}</updated>",
|
||||
]
|
||||
if entry.description:
|
||||
lines.append(f' <content type="text">{_escape(entry.description)}</content>')
|
||||
lines.extend(
|
||||
[
|
||||
f' <link rel="{SUBSECTION_REL}" href="{_attribute(entry.href)}" '
|
||||
f'type="{_attribute(NAVIGATION_TYPE)}"/>',
|
||||
" </entry>",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _acquisition_entry_xml(entry: AcquisitionEntry) -> str:
|
||||
lines = [
|
||||
" <entry>",
|
||||
f" <title>{_escape(entry.title)}</title>",
|
||||
f" <id>{_escape(entry.entry_id)}</id>",
|
||||
f" <updated>{_timestamp(entry.updated)}</updated>",
|
||||
]
|
||||
if entry.author:
|
||||
lines.append(f" <author><name>{_escape(entry.author)}</name></author>")
|
||||
if entry.summary:
|
||||
lines.append(f' <summary type="text">{_escape(entry.summary)}</summary>')
|
||||
if entry.language:
|
||||
lines.append(f" <dc:language>{_escape(entry.language)}</dc:language>")
|
||||
for term, label in entry.categories:
|
||||
lines.append(f' <category term="{_attribute(term)}" label="{_attribute(label)}"/>')
|
||||
# Two acquisition relations to the same resource, because readers disagree
|
||||
# about which one to look for; both are valid.
|
||||
for rel in (OPEN_ACCESS_REL, ACQUISITION_REL):
|
||||
lines.append(
|
||||
f' <link rel="{rel}" href="{_attribute(entry.download_href)}" '
|
||||
f'type="{_attribute(EPUB_TYPE)}"/>'
|
||||
)
|
||||
if entry.alternate_href:
|
||||
lines.append(
|
||||
f' <link rel="{ALTERNATE_REL}" href="{_attribute(entry.alternate_href)}" '
|
||||
f'type="{_attribute(HTML_TYPE)}"/>'
|
||||
)
|
||||
lines.append(" </entry>")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# -- formatting ----------------------------------------------------------
|
||||
|
||||
|
||||
def _escape(text: str) -> str:
|
||||
"""Escape text for a document body."""
|
||||
return xhtml.escape_text(text)
|
||||
|
||||
|
||||
def _attribute(value: str) -> str:
|
||||
"""Escape a value for a double-quoted attribute."""
|
||||
return xhtml.escape_attribute(value)
|
||||
|
||||
|
||||
def _timestamp(moment: datetime) -> str:
|
||||
"""Atom dates are RFC 3339 in UTC."""
|
||||
return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
1075
app/server.py
Normal file
1075
app/server.py
Normal file
File diff suppressed because it is too large
Load Diff
120
app/urls.py
Normal file
120
app/urls.py
Normal file
@@ -0,0 +1,120 @@
|
||||
"""Identifier, slug and URL helpers.
|
||||
|
||||
FreshRSS ids do not survive a URL path or a Content-Disposition filename
|
||||
unharmed. Item ids look like ``tag:google.com,2005:reader/item/0000000000000abc``
|
||||
and stream ids look like ``feed/https://example.com/atom.xml?x=1``, so both
|
||||
carry characters that are structural in a path (``/``, ``?``, ``#``) and are
|
||||
long enough to blow past filename limits.
|
||||
|
||||
The rule this module enforces: every opaque id that appears in a path is
|
||||
base64url-encoded, and the human-readable part of a download URL is a slug that
|
||||
the server ignores when it looks the item up. That keeps routes unambiguous
|
||||
without any escaping tricks, and lets a reader name the saved file sensibly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import re
|
||||
import unicodedata
|
||||
import uuid
|
||||
|
||||
# Fixed namespace for every derived urn:uuid in this server. Hardcoded so that
|
||||
# feed and entry ids stay stable across restarts and across hostnames -- a client
|
||||
# that cached an entry id must still recognise it tomorrow.
|
||||
_UUID_NAMESPACE = uuid.UUID("b7437da1-d469-46b1-a582-0af386fadb9d")
|
||||
|
||||
_B64URL_RE = re.compile(r"\A[A-Za-z0-9_-]+\Z")
|
||||
# Everything that is not an ASCII lowercase letter or digit becomes a dash, so
|
||||
# the result is safe in a path segment, a filename, and a shell.
|
||||
_SLUG_JUNK_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
|
||||
def b64url_encode(value: str) -> str:
|
||||
"""Encode a UTF-8 string as unpadded base64url (path-segment safe)."""
|
||||
raw = value.encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
||||
|
||||
|
||||
def b64url_decode(value: str) -> str:
|
||||
"""Decode an unpadded base64url string, raising ValueError on junk.
|
||||
|
||||
Strict on purpose: this parses untrusted path input, so anything that is not
|
||||
a well-formed unpadded base64url token -- or that decodes to something other
|
||||
than UTF-8 -- is rejected instead of being coerced.
|
||||
"""
|
||||
if not value or not _B64URL_RE.match(value):
|
||||
raise ValueError("not a base64url token")
|
||||
padding = "=" * (-len(value) % 4)
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(value + padding)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise ValueError("not a base64url token") from exc
|
||||
try:
|
||||
return raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError("base64url token is not UTF-8") from exc
|
||||
|
||||
|
||||
def slugify(text: str, max_length: int = 60) -> str:
|
||||
"""Reduce a title to a conservative filename slug.
|
||||
|
||||
Accented characters are folded to ASCII rather than dropped, non-ASCII
|
||||
scripts fall back to ``article``, and the result is length-capped because a
|
||||
filesystem name is the eventual destination.
|
||||
"""
|
||||
folded = unicodedata.normalize("NFKD", text or "")
|
||||
ascii_text = folded.encode("ascii", "ignore").decode("ascii")
|
||||
slug = _SLUG_JUNK_RE.sub("-", ascii_text.lower()).strip("-")
|
||||
if len(slug) > max_length:
|
||||
slug = slug[:max_length].rstrip("-")
|
||||
return slug or "article"
|
||||
|
||||
|
||||
def stable_urn(kind: str, value: str) -> str:
|
||||
"""Return a deterministic ``urn:uuid`` for a feed, entry or item.
|
||||
|
||||
Atom requires every feed and entry to carry a permanent id. Deriving it from
|
||||
the FreshRSS id rather than from the request URL means the same article keeps
|
||||
one identity no matter which category path the reader reached it through.
|
||||
"""
|
||||
derived = uuid.uuid5(_UUID_NAMESPACE, f"{kind}:{value}")
|
||||
return f"urn:uuid:{derived}"
|
||||
|
||||
|
||||
def content_disposition(ascii_name: str, unicode_name: str | None = None) -> str:
|
||||
"""Build an attachment Content-Disposition with an RFC 5987 fallback.
|
||||
|
||||
Every OPDS reader understands ``filename``; readers that save a title with
|
||||
non-ASCII characters need ``filename*``. Sending both is the only way to get
|
||||
a correct name everywhere, so both are always emitted.
|
||||
"""
|
||||
ascii_name = ascii_name.replace('"', "").replace("\\", "") or "article.epub"
|
||||
header = f'attachment; filename="{ascii_name}"'
|
||||
if unicode_name and unicode_name != ascii_name:
|
||||
encoded = _quote_rfc5987(unicode_name.replace('"', "").replace("\\", ""))
|
||||
if encoded:
|
||||
header += f"; filename*=UTF-8''{encoded}"
|
||||
return header
|
||||
|
||||
|
||||
def _quote_rfc5987(value: str) -> str:
|
||||
out = []
|
||||
for byte in value.encode("utf-8"):
|
||||
char = chr(byte)
|
||||
if char.isalnum() and byte < 0x80 or char in "-._~":
|
||||
out.append(char)
|
||||
else:
|
||||
out.append(f"%{byte:02X}")
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def join_url(base: str, *parts: str) -> str:
|
||||
"""Join URL path parts onto a base, collapsing duplicate slashes."""
|
||||
segments = [base.rstrip("/")]
|
||||
for part in parts:
|
||||
cleaned = part.strip("/")
|
||||
if cleaned:
|
||||
segments.append(cleaned)
|
||||
return "/".join(segments)
|
||||
747
app/xhtml.py
Normal file
747
app/xhtml.py
Normal file
@@ -0,0 +1,747 @@
|
||||
"""Turn arbitrary feed HTML into well-formed XHTML for the EPUB pipeline.
|
||||
|
||||
EPUB is XML end to end: the OPF, the NCX, the navigation document and every
|
||||
content document are parsed by a strict XML parser (the reader firmware this
|
||||
server was written alongside uses expat). Feed article bodies are the opposite
|
||||
of strict -- unclosed ``<li>``, stray ``&``, ``<br>`` instead of ``<br/>``,
|
||||
control characters, embedded scripts and tracking pixels. Handing that to the
|
||||
EPUB writer would produce a book that some readers refuse to open, so everything
|
||||
goes through this normaliser first.
|
||||
|
||||
How it stays safe, in order of importance:
|
||||
|
||||
* The input is parsed into an in-memory tree and re-serialised from scratch.
|
||||
Any malformedness in the source therefore cannot reach the output: whatever the
|
||||
parser tolerated is rebuilt with balanced tags, quoted attributes and escaped
|
||||
text. There is no pass-through of raw markup.
|
||||
* Characters that XML 1.0 forbids outright (most C0 controls, unpaired
|
||||
surrogates) are removed, because they are not expressible even when escaped.
|
||||
* Entities are decoded to real Unicode by the parser and re-escaped on output, so
|
||||
a stray ``&`` in the source cannot produce a document that will not parse.
|
||||
* Scriptable and metadata elements are dropped with their contents; every other
|
||||
unknown element is unwrapped, keeping its text.
|
||||
* Only URLs that survive scheme validation reach an attribute, which keeps
|
||||
``javascript:`` and ``data:`` payloads out of the book.
|
||||
|
||||
The tree is kept around after parsing so a caller can collect image URLs, fetch
|
||||
them, and then serialise with the fetched images rewritten to local paths.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# Elements kept in the output. Mirrors the firmware reader's sanitiser
|
||||
# (lib/FreshRss/src/FreshRssArticle.cpp) with the additions that make sense on a
|
||||
# device with a real layout engine: images, and the small semantic set EPUB
|
||||
# readers style (q, cite, time, abbr, del, ins, small, mark).
|
||||
KEPT_TAGS = frozenset(
|
||||
{
|
||||
"p",
|
||||
"div",
|
||||
"span",
|
||||
"br",
|
||||
"hr",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"ul",
|
||||
"ol",
|
||||
"li",
|
||||
"dl",
|
||||
"dt",
|
||||
"dd",
|
||||
"blockquote",
|
||||
"pre",
|
||||
"code",
|
||||
"em",
|
||||
"i",
|
||||
"strong",
|
||||
"b",
|
||||
"u",
|
||||
"s",
|
||||
"sub",
|
||||
"sup",
|
||||
"a",
|
||||
"img",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"table",
|
||||
"thead",
|
||||
"tbody",
|
||||
"tfoot",
|
||||
"tr",
|
||||
"td",
|
||||
"th",
|
||||
"caption",
|
||||
"small",
|
||||
"mark",
|
||||
"cite",
|
||||
"time",
|
||||
"abbr",
|
||||
"q",
|
||||
"ins",
|
||||
"del",
|
||||
}
|
||||
)
|
||||
|
||||
# Dropped along with everything inside them. These are either scriptable,
|
||||
# fetch extra resources, or are document-level metadata that has no place inside
|
||||
# an EPUB content document.
|
||||
DROP_WITH_CONTENT = frozenset(
|
||||
{
|
||||
"script",
|
||||
"style",
|
||||
"iframe",
|
||||
"frame",
|
||||
"frameset",
|
||||
"object",
|
||||
"embed",
|
||||
"applet",
|
||||
"svg",
|
||||
"math",
|
||||
"form",
|
||||
"input",
|
||||
"button",
|
||||
"select",
|
||||
"option",
|
||||
"optgroup",
|
||||
"textarea",
|
||||
"label",
|
||||
"fieldset",
|
||||
"legend",
|
||||
"noscript",
|
||||
"template",
|
||||
"canvas",
|
||||
"video",
|
||||
"audio",
|
||||
"source",
|
||||
"track",
|
||||
"map",
|
||||
"area",
|
||||
"head",
|
||||
"title",
|
||||
"meta",
|
||||
"link",
|
||||
"base",
|
||||
"dialog",
|
||||
"slot",
|
||||
}
|
||||
)
|
||||
|
||||
# HTML void elements: no end tag exists, so the parser must not expect one.
|
||||
VOID_TAGS = frozenset({"br", "hr", "img"})
|
||||
_HTML_VOID = frozenset(
|
||||
{
|
||||
"area",
|
||||
"base",
|
||||
"br",
|
||||
"col",
|
||||
"embed",
|
||||
"hr",
|
||||
"img",
|
||||
"input",
|
||||
"link",
|
||||
"meta",
|
||||
"param",
|
||||
"source",
|
||||
"track",
|
||||
"wbr",
|
||||
}
|
||||
)
|
||||
|
||||
# Elements that silently end an open element in HTML. Feeds are full of
|
||||
# unclosed <p> and <li>, and the browser-correct fix is to close the open
|
||||
# element rather than to nest one block inside another (which is what a naive
|
||||
# stack would do, producing <p>text<div>...</div></p>).
|
||||
_IMPLICIT_CLOSE: dict[str, frozenset[str]] = {
|
||||
"p": frozenset(
|
||||
{
|
||||
"address",
|
||||
"article",
|
||||
"aside",
|
||||
"blockquote",
|
||||
"details",
|
||||
"div",
|
||||
"dl",
|
||||
"fieldset",
|
||||
"figcaption",
|
||||
"figure",
|
||||
"footer",
|
||||
"form",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"header",
|
||||
"hr",
|
||||
"main",
|
||||
"nav",
|
||||
"ol",
|
||||
"p",
|
||||
"pre",
|
||||
"section",
|
||||
"table",
|
||||
"ul",
|
||||
}
|
||||
),
|
||||
"li": frozenset({"li"}),
|
||||
"dt": frozenset({"dt", "dd"}),
|
||||
"dd": frozenset({"dt", "dd"}),
|
||||
"td": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
|
||||
"th": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
|
||||
"tr": frozenset({"tr", "tbody", "tfoot", "thead"}),
|
||||
"option": frozenset({"option", "optgroup"}),
|
||||
"thead": frozenset({"tbody", "tfoot"}),
|
||||
"tbody": frozenset({"tbody", "tfoot"}),
|
||||
}
|
||||
|
||||
# Attributes kept, per element. Everything else is discarded: inline styles and
|
||||
# layout attributes fight the EPUB stylesheet, and event handlers, tracking
|
||||
# attributes and class names have no meaning in a book.
|
||||
_KEPT_ATTRS: dict[str, frozenset[str]] = {
|
||||
"a": frozenset({"href", "title"}),
|
||||
"img": frozenset({"src", "alt", "title"}),
|
||||
"blockquote": frozenset({"cite"}),
|
||||
"q": frozenset({"cite"}),
|
||||
"time": frozenset({"datetime"}),
|
||||
"ins": frozenset({"datetime"}),
|
||||
"del": frozenset({"datetime"}),
|
||||
"abbr": frozenset({"title"}),
|
||||
"td": frozenset({"colspan", "rowspan"}),
|
||||
"th": frozenset({"colspan", "rowspan"}),
|
||||
"ol": frozenset({"start"}),
|
||||
}
|
||||
|
||||
# Empty structural elements worth removing once their contents are gone (for
|
||||
# example a wrapper whose only child was a rejected image). Table cells are
|
||||
# deliberately absent: an empty cell is layout, not noise.
|
||||
_DROPPABLE_WHEN_EMPTY = frozenset(
|
||||
{
|
||||
"p",
|
||||
"div",
|
||||
"span",
|
||||
"li",
|
||||
"ul",
|
||||
"ol",
|
||||
"dl",
|
||||
"blockquote",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"section",
|
||||
"article",
|
||||
}
|
||||
)
|
||||
|
||||
_LINK_SCHEMES = ("http", "https", "mailto")
|
||||
_IMAGE_SCHEMES = ("http", "https")
|
||||
|
||||
# XML 1.0 permits #x9, #xA, #xD, #x20-#xD7FF, #xE000-#xFFFD, #x10000-#x10FFFF.
|
||||
# Everything else -- NUL bytes, vertical tabs from mangled feeds, form feeds,
|
||||
# unpaired surrogates -- cannot be represented even as a character reference and
|
||||
# must be deleted before serialisation.
|
||||
_ILLEGAL_XML_RE = re.compile(
|
||||
"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]"
|
||||
)
|
||||
# Collapses runs of layout whitespace. A non-breaking space is deliberately not
|
||||
# matched: feeds use it for intentional spacing, and reflowable text keeps it.
|
||||
_WHITESPACE_RE = re.compile(r"[^\S\xa0]+")
|
||||
# Trims layout whitespace from the edges of a block. Python's str.strip() treats
|
||||
# a non-breaking space as whitespace and would eat one the author typed on
|
||||
# purpose, so the pattern excludes it here too.
|
||||
_BLOCK_TRIM_RE = re.compile(r"\A[^\S\xa0]+|[^\S\xa0]+\Z")
|
||||
_ZERO_WIDTH_RE = re.compile("[\u200b\u200c\u200d\u2060\ufeff]")
|
||||
# XML name rules for an id: starts with a letter or underscore, then a bounded run
|
||||
# of name characters. Anything else is not a usable anchor target.
|
||||
_ID_RE = re.compile(r"\A[A-Za-z_][A-Za-z0-9_.-]{0,63}\Z")
|
||||
_ID_ATTRS = frozenset({"id"})
|
||||
|
||||
# Wrappers that need the attribute filter relaxed for integer attributes.
|
||||
_INT_ATTR_LIMITS = {"colspan": (1, 100), "rowspan": (1, 100), "start": (-10000, 10000)}
|
||||
|
||||
|
||||
class _Element:
|
||||
"""One element node in the parsed tree."""
|
||||
|
||||
__slots__ = ("attrs", "children", "tag")
|
||||
|
||||
def __init__(self, tag: str) -> None:
|
||||
self.tag = tag
|
||||
self.attrs: dict[str, str] = {}
|
||||
self.children: list[_Element | _Text] = []
|
||||
|
||||
|
||||
class _Text:
|
||||
"""One text node in the parsed tree."""
|
||||
|
||||
__slots__ = ("text",)
|
||||
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = text
|
||||
|
||||
|
||||
class _TreeBuilder(HTMLParser):
|
||||
"""Build a normalised element tree from feed HTML.
|
||||
|
||||
The parser is tolerant by construction -- it never raises on malformed input,
|
||||
it only decides what to keep -- so this class is the single place where all
|
||||
of the "this feed is broken" handling lives.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# convert_charrefs decodes &/é/ to Unicode for us, so the
|
||||
# serialiser only ever has to escape what it writes itself.
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.root = _Element("#root")
|
||||
self._stack: list[_Element] = [self.root]
|
||||
# Names of open drop-with-content elements; while non-empty, all input is
|
||||
# discarded except the end tag that closes the innermost one.
|
||||
self._suppressed: list[str] = []
|
||||
|
||||
# -- parser callbacks -------------------------------------------------
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if self._suppressed:
|
||||
if tag in DROP_WITH_CONTENT and tag not in _HTML_VOID:
|
||||
self._suppressed.append(tag)
|
||||
return
|
||||
if tag in DROP_WITH_CONTENT:
|
||||
if tag not in _HTML_VOID:
|
||||
self._suppressed.append(tag)
|
||||
return
|
||||
self._close_implied(tag)
|
||||
if tag in _HTML_VOID and tag not in VOID_TAGS:
|
||||
# Void but not kept (<col>, <wbr>, <param>): it cannot have content,
|
||||
# so it must never open a subtree that would swallow the rest.
|
||||
# br/hr/img are void *and* kept, so they fall through to be emitted.
|
||||
return
|
||||
element = _Element(tag)
|
||||
element.attrs = self._filter_attrs(tag, attrs)
|
||||
self._stack[-1].children.append(element)
|
||||
if tag not in VOID_TAGS:
|
||||
self._stack.append(element)
|
||||
|
||||
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
# <br/> lands here. A self-closing element is complete: it is appended
|
||||
# and never pushed, so there is nothing left open. A self-closing
|
||||
# drop-with-content element has no content to discard and must not open a
|
||||
# suppression region, or the rest of the document would be swallowed.
|
||||
if self._suppressed or tag in DROP_WITH_CONTENT or tag in _HTML_VOID:
|
||||
return
|
||||
self._close_implied(tag)
|
||||
element = _Element(tag)
|
||||
element.attrs = self._filter_attrs(tag, attrs)
|
||||
self._stack[-1].children.append(element)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if self._suppressed:
|
||||
if tag == self._suppressed[-1]:
|
||||
self._suppressed.pop()
|
||||
return
|
||||
if tag in _HTML_VOID:
|
||||
return
|
||||
# Close down to the matching open element, tolerating unclosed tags in
|
||||
# between (this is exactly what browsers do).
|
||||
for index in range(len(self._stack) - 1, 0, -1):
|
||||
if self._stack[index].tag == tag:
|
||||
del self._stack[index:]
|
||||
return
|
||||
# Stray end tag with no matching open element: ignore it.
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._suppressed or not data:
|
||||
return
|
||||
self._stack[-1].children.append(_Text(data))
|
||||
|
||||
def handle_comment(self, data: str) -> None:
|
||||
return
|
||||
|
||||
def handle_decl(self, decl: str) -> None:
|
||||
return
|
||||
|
||||
def handle_pi(self, data: str) -> None:
|
||||
return
|
||||
|
||||
def unknown_decl(self, data: str) -> None:
|
||||
if self._suppressed:
|
||||
return
|
||||
# An HTML parser treats <![CDATA[...]]> as a bogus comment and discards
|
||||
# it, but a feed body that passed through an XML parser upstream can
|
||||
# legitimately contain one, and its payload is real content. The parser
|
||||
# hands over the section body with the closing ]] already stripped.
|
||||
marker = "CDATA["
|
||||
if data.startswith(marker):
|
||||
inner = data[len(marker) :]
|
||||
if inner:
|
||||
self._stack[-1].children.append(_Text(inner))
|
||||
|
||||
# -- helpers ---------------------------------------------------------
|
||||
|
||||
def _close_implied(self, tag: str) -> None:
|
||||
"""Close open elements that HTML would close implicitly."""
|
||||
while len(self._stack) > 1:
|
||||
conditions = _IMPLICIT_CLOSE.get(self._stack[-1].tag)
|
||||
if conditions is None or tag not in conditions:
|
||||
return
|
||||
self._stack.pop()
|
||||
|
||||
def _filter_attrs(self, tag: str, attrs: list[tuple[str, str | None]]) -> dict[str, str]:
|
||||
if tag not in KEPT_TAGS:
|
||||
# The element is unwrapped later, so its attributes are never used.
|
||||
return {}
|
||||
allowed = _KEPT_ATTRS.get(tag, frozenset())
|
||||
if tag not in VOID_TAGS:
|
||||
# Anchor targets survive so an in-book table of contents still jumps
|
||||
# to its section when the article is read offline.
|
||||
allowed = allowed | _ID_ATTRS
|
||||
kept: dict[str, str] = {}
|
||||
for raw_name, raw_value in attrs:
|
||||
name = raw_name.lower()
|
||||
if name not in allowed:
|
||||
continue
|
||||
value = html.unescape(raw_value or "") if raw_value is not None else ""
|
||||
value = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", value)).strip()
|
||||
if name == "id":
|
||||
if not _ID_RE.match(value):
|
||||
continue
|
||||
kept[name] = value
|
||||
continue
|
||||
if name in _INT_ATTR_LIMITS:
|
||||
bounds = _INT_ATTR_LIMITS[name]
|
||||
try:
|
||||
number = int(value)
|
||||
except ValueError:
|
||||
continue
|
||||
if not bounds[0] <= number <= bounds[1]:
|
||||
continue
|
||||
value = str(number)
|
||||
if not value:
|
||||
continue
|
||||
kept[name] = value
|
||||
return kept
|
||||
|
||||
|
||||
class Article:
|
||||
"""A parsed, normalised article body ready for EPUB serialisation."""
|
||||
|
||||
def __init__(self, tree: _Element, base_url: str, truncated: bool) -> None:
|
||||
self._tree = tree
|
||||
self._base_url = base_url
|
||||
self.truncated = truncated
|
||||
self._image_urls: list[str] | None = None
|
||||
|
||||
@property
|
||||
def image_urls(self) -> list[str]:
|
||||
"""Absolute http(s) image URLs, in document order, de-duplicated."""
|
||||
if self._image_urls is None:
|
||||
seen: list[str] = []
|
||||
known: set[str] = set()
|
||||
for element in _iter_elements(self._tree):
|
||||
if element.tag != "img":
|
||||
continue
|
||||
src = element.attrs.get("src", "")
|
||||
if src and src not in known:
|
||||
known.add(src)
|
||||
seen.append(src)
|
||||
self._image_urls = seen
|
||||
return self._image_urls
|
||||
|
||||
def text(self, limit: int = 600) -> str:
|
||||
"""Flattened plain text, for summaries and the search index."""
|
||||
parts: list[str] = []
|
||||
for element in _iter_text(self._tree):
|
||||
parts.append(element.text)
|
||||
collapsed = _WHITESPACE_RE.sub(" ", _ILLEGAL_XML_RE.sub("", "".join(parts))).strip()
|
||||
if len(collapsed) <= limit:
|
||||
return collapsed
|
||||
# Cut at a word boundary so a summary never ends mid-word.
|
||||
cut = collapsed[:limit].rsplit(" ", 1)[0]
|
||||
return (cut or collapsed[:limit]).rstrip() + "\u2026"
|
||||
|
||||
def to_xhtml(self, image_map: dict[str, str | None] | None = None) -> str:
|
||||
"""Serialise the body as an XHTML fragment.
|
||||
|
||||
``image_map`` maps an original absolute image URL to the local href to
|
||||
use in its place; a value of None drops that image (and its wrapper, if it
|
||||
becomes empty). Pass None to leave remote image URLs untouched -- the
|
||||
caller must then declare remote resources in the OPF manifest.
|
||||
|
||||
Applying a map rewrites the tree in place, so an Article is rendered
|
||||
once. The server builds a book and serialises it exactly once.
|
||||
"""
|
||||
if image_map is not None:
|
||||
_apply_image_map(self._tree, image_map)
|
||||
# A dropped image can leave an empty wrapper behind.
|
||||
_prune_empty(self._tree)
|
||||
body = _serialise_children(self._tree, inside_pre=False)
|
||||
if not body.strip():
|
||||
return ""
|
||||
return f'<div class="article-body">\n{body}\n</div>'
|
||||
|
||||
|
||||
class _Resolve:
|
||||
"""Decides which URLs survive normalisation, and where they point."""
|
||||
|
||||
def __init__(self, base_url: str) -> None:
|
||||
self.base_url = base_url
|
||||
|
||||
def link(self, raw: str) -> str | None:
|
||||
# A same-document reference is left alone so it resolves against the book
|
||||
# rather than against the web page the article came from. The matching id
|
||||
# was kept by the attribute filter.
|
||||
if raw.startswith("#"):
|
||||
return raw
|
||||
resolved = _absolutise(raw, self.base_url)
|
||||
if resolved is None:
|
||||
return None
|
||||
if urlsplit(resolved).scheme.lower() not in _LINK_SCHEMES:
|
||||
return None
|
||||
return resolved
|
||||
|
||||
def image(self, raw: str) -> str | None:
|
||||
resolved = _absolutise(raw, self.base_url)
|
||||
if resolved is None:
|
||||
return None
|
||||
if urlsplit(resolved).scheme.lower() not in _IMAGE_SCHEMES:
|
||||
return None
|
||||
return resolved
|
||||
|
||||
|
||||
def parse_article(raw_html: str, base_url: str = "", max_input_bytes: int = 0) -> Article:
|
||||
"""Parse a feed article body into a normalised tree.
|
||||
|
||||
``base_url`` is the article's own URL; relative links and images inside the
|
||||
body are resolved against it, which is the only way they keep working once
|
||||
the body is inside a book with a different base.
|
||||
"""
|
||||
text = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", raw_html or ""))
|
||||
truncated = False
|
||||
if max_input_bytes > 0 and len(text.encode("utf-8", "ignore")) > max_input_bytes:
|
||||
# Cut on a character boundary, not a byte boundary, and do not try to be
|
||||
# clever about tags: the parser tolerates a truncated tail.
|
||||
while len(text.encode("utf-8", "ignore")) > max_input_bytes:
|
||||
text = text[: int(len(text) * 0.98) or 1]
|
||||
truncated = True
|
||||
LOGGER.warning("article body truncated to %d bytes before parsing", max_input_bytes)
|
||||
|
||||
builder = _TreeBuilder()
|
||||
builder.feed(text)
|
||||
builder.close()
|
||||
tree = builder.root
|
||||
|
||||
if not _has_elements(tree):
|
||||
# A body with no markup at all (plain text feed) still needs block
|
||||
# structure, or the EPUB would contain a bare text node.
|
||||
paragraph = _Element("p")
|
||||
paragraph.children = list(tree.children)
|
||||
tree.children = [paragraph]
|
||||
|
||||
_resolve_urls(tree, _Resolve(base_url), set())
|
||||
_prune_empty(tree)
|
||||
return Article(tree, base_url, truncated)
|
||||
|
||||
|
||||
def _absolutise(raw: str, base_url: str) -> str | None:
|
||||
candidate = (raw or "").strip()
|
||||
if not candidate:
|
||||
return None
|
||||
# Control characters and raw whitespace inside a URL are always corruption.
|
||||
candidate = _ILLEGAL_XML_RE.sub("", candidate)
|
||||
if any(char.isspace() for char in candidate):
|
||||
candidate = candidate.split()[0]
|
||||
try:
|
||||
if base_url:
|
||||
return urljoin(base_url, candidate)
|
||||
return candidate
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_urls(element: _Element, resolve: _Resolve, seen_ids: set[str]) -> None:
|
||||
"""Validate URLs and de-duplicate anchor ids, in document order.
|
||||
|
||||
Runs once, at parse time, so everything downstream (the image collector, the
|
||||
fetcher, the OPF manifest) sees final absolute URLs and there is no second
|
||||
chance for a bad scheme to slip through.
|
||||
"""
|
||||
for child in element.children:
|
||||
if isinstance(child, _Text):
|
||||
continue
|
||||
_dedupe_id(child, seen_ids)
|
||||
if child.tag == "a":
|
||||
href = child.attrs.get("href")
|
||||
if href is not None:
|
||||
target = resolve.link(href)
|
||||
if target is None:
|
||||
child.attrs.pop("href", None)
|
||||
else:
|
||||
child.attrs["href"] = target
|
||||
elif child.tag == "img":
|
||||
src = child.attrs.get("src")
|
||||
target = resolve.image(src) if src else None
|
||||
if target is None:
|
||||
# Mark it for pruning: an image with no usable source is noise.
|
||||
child.tag = "#dropped-image"
|
||||
else:
|
||||
child.attrs["src"] = target
|
||||
elif child.tag in ("blockquote", "q"):
|
||||
cite = child.attrs.get("cite")
|
||||
if cite is not None:
|
||||
target = resolve.link(cite)
|
||||
if target is None:
|
||||
child.attrs.pop("cite", None)
|
||||
else:
|
||||
child.attrs["cite"] = target
|
||||
_resolve_urls(child, resolve, seen_ids)
|
||||
|
||||
|
||||
def _dedupe_id(element: _Element, seen_ids: set[str]) -> None:
|
||||
"""Make an element's id unique within the document.
|
||||
|
||||
Duplicate ids are invalid and would make the first anchor win, so a repeat is
|
||||
renamed rather than dropped -- dropping it would break the link pointing at
|
||||
it. One book holds one article, so a document-wide set is enough.
|
||||
"""
|
||||
anchor = element.attrs.get("id")
|
||||
if anchor is None:
|
||||
return
|
||||
if anchor not in seen_ids:
|
||||
seen_ids.add(anchor)
|
||||
return
|
||||
index = 2
|
||||
while f"{anchor}-{index}" in seen_ids:
|
||||
index += 1
|
||||
renamed = f"{anchor}-{index}"
|
||||
seen_ids.add(renamed)
|
||||
element.attrs["id"] = renamed
|
||||
|
||||
|
||||
def _apply_image_map(element: _Element, image_map: dict[str, str | None]) -> None:
|
||||
"""Point images at their local copies inside the book.
|
||||
|
||||
A URL mapped to None means the fetch failed or the image was refused, so the
|
||||
element is marked for pruning instead of being left pointing at the network.
|
||||
"""
|
||||
for child in element.children:
|
||||
if isinstance(child, _Text):
|
||||
continue
|
||||
if child.tag == "img":
|
||||
src = child.attrs.get("src")
|
||||
if src is not None and src in image_map:
|
||||
target = image_map[src]
|
||||
if target is None:
|
||||
child.tag = "#dropped-image"
|
||||
else:
|
||||
child.attrs["src"] = target
|
||||
_apply_image_map(child, image_map)
|
||||
|
||||
|
||||
def _prune_empty(node: _Element) -> None:
|
||||
"""Remove rejected images and the empty wrappers they leave behind."""
|
||||
kept: list[_Element | _Text] = []
|
||||
for child in node.children:
|
||||
if isinstance(child, _Element):
|
||||
if child.tag == "#dropped-image":
|
||||
continue
|
||||
_prune_empty(child)
|
||||
if _is_noise(child):
|
||||
continue
|
||||
kept.append(child)
|
||||
node.children = kept
|
||||
|
||||
|
||||
def _is_noise(element: _Element) -> bool:
|
||||
"""True for a droppable wrapper with no content left inside it.
|
||||
|
||||
Whitespace-only text does not count as content (``<p> </p>`` is noise),
|
||||
but a retained ``id`` does: it is the target of a link.
|
||||
"""
|
||||
if element.tag not in _DROPPABLE_WHEN_EMPTY or element.attrs:
|
||||
return False
|
||||
for child in element.children:
|
||||
if isinstance(child, _Element):
|
||||
return False
|
||||
if child.text.strip():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _has_elements(element: _Element) -> bool:
|
||||
"""True when the tree contains at least one element node."""
|
||||
return any(True for _ in _iter_elements(element))
|
||||
|
||||
|
||||
def _serialise_children(node: _Element, inside_pre: bool) -> str:
|
||||
return "".join(_serialise(child, inside_pre) for child in node.children)
|
||||
|
||||
|
||||
def _serialise(node: _Element | _Text, inside_pre: bool) -> str:
|
||||
if isinstance(node, _Text):
|
||||
if inside_pre:
|
||||
return escape_text(node.text)
|
||||
# Collapse whitespace outside <pre>: feed HTML arrives indented, and that
|
||||
# indentation is pure bytes in the book.
|
||||
return escape_text(_WHITESPACE_RE.sub(" ", node.text))
|
||||
if node.tag not in KEPT_TAGS:
|
||||
# Unwrapped elements keep their children but lose their own tag, so no
|
||||
# unknown element can appear in the output.
|
||||
return _serialise_children(node, inside_pre)
|
||||
|
||||
child_pre = inside_pre or node.tag == "pre"
|
||||
inner = _serialise_children(node, child_pre)
|
||||
if node.tag in ("p", "div", "li", "td", "th", "blockquote", "figcaption", "caption"):
|
||||
inner = _BLOCK_TRIM_RE.sub("", inner)
|
||||
attributes = "".join(
|
||||
f' {name}="{escape_attribute(value)}"' for name, value in _ordered_attrs(node)
|
||||
)
|
||||
if node.tag in VOID_TAGS:
|
||||
return f"<{node.tag}{attributes}/>"
|
||||
return f"<{node.tag}{attributes}>{inner}</{node.tag}>"
|
||||
|
||||
|
||||
def _ordered_attrs(element: _Element) -> list[tuple[str, str]]:
|
||||
"""Stable attribute order, with the most meaningful attribute first."""
|
||||
priority = {"id": 0, "href": 1, "src": 2, "alt": 3, "title": 4}
|
||||
return sorted(element.attrs.items(), key=lambda item: (priority.get(item[0], 9), item[0]))
|
||||
|
||||
|
||||
def escape_text(text: str) -> str:
|
||||
return text.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
|
||||
|
||||
def escape_attribute(value: str) -> str:
|
||||
# Attribute values are normalised by XML parsers anyway (tab/newline become
|
||||
# spaces), so do it explicitly rather than leaving it implicit.
|
||||
text = value.replace("\t", " ").replace("\n", " ").replace("\r", " ")
|
||||
return (
|
||||
text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)
|
||||
)
|
||||
|
||||
|
||||
def _iter_elements(node: _Element) -> Iterator[_Element]:
|
||||
"""Depth-first traversal over element nodes only."""
|
||||
for child in node.children:
|
||||
if isinstance(child, _Element):
|
||||
yield child
|
||||
yield from _iter_elements(child)
|
||||
|
||||
|
||||
def _iter_text(node: _Element) -> Iterator[_Text]:
|
||||
"""Depth-first traversal over text nodes only."""
|
||||
for child in node.children:
|
||||
if isinstance(child, _Text):
|
||||
yield child
|
||||
else:
|
||||
yield from _iter_text(child)
|
||||
Reference in New Issue
Block a user