Files
rss-to-opds/app/server.py
2026-11-15 18:15:03 -05:00

1076 lines
39 KiB
Python

"""HTTP layer: routing, the catalog, and EPUB delivery.
The URL space mirrors the shape of the account: a root navigation feed, then a
section (unread, read, starred), then a category, then a feed, then articles, and
finally a download. Every opaque identifier in a path is base64url so that feed
URLs and FreshRSS item ids -- which contain ``/``, ``?`` and ``:`` -- cannot break
a route.
Two behaviours are worth stating explicitly, because they are the ones a reader
notices:
*Links are absolute.* Readers navigate by following link hrefs rather than by
resolving relative paths, and some are strict about it, so every href is built
against a base URL: the configured one, or the request's own Host (plus
``X-Forwarded-*`` when the operator says a proxy is in front).
*Downloading marks the article read, and only a completed download does.* The
body is written to the socket first and the upstream call happens afterwards, so a
reader that disconnects halfway has not silently consumed the article. The call
is best-effort: if FreshRSS refuses it, the book still arrives and the failure is
logged.
Nothing is written to disk. The catalog is served from a short-lived in-memory
cache and each book is assembled in memory at the moment it is asked for.
"""
from __future__ import annotations
import json
import logging
import re
import time
import urllib.parse
from dataclasses import dataclass, field
from datetime import UTC, datetime
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Any
from app import auth, epub, freshrss, opds, urls
from app.cache import FailureLatch, TTLCache
from app.config import Config
from app.fetch import ImageFetcher
LOGGER = logging.getLogger(__name__)
ROOT_PATH = "/opds"
SECTIONS = ("unread", "read", "starred")
# A deep catalog is not worth unbounded upstream work; past this many items the
# page is empty and no next link is offered.
_MAX_OFFSET = 2000
@dataclass
class CatalogSnapshot:
"""The account's shape, resolved once per cache window."""
feeds: list[freshrss.Subscription]
categories: list[tuple[str, list[freshrss.Subscription]]]
uncategorised: list[freshrss.Subscription]
unread_counts: dict[str, int] = field(default_factory=dict)
fetched_at: datetime = field(default_factory=lambda: datetime.now(UTC))
def category_labels(self) -> list[str]:
return [label for label, _feeds in self.categories]
def feeds_in(self, label: str | None) -> list[freshrss.Subscription]:
"""Feeds in one category; ``None`` means the uncategorised group."""
if label is None:
return self.uncategorised
for candidate, feeds in self.categories:
if candidate == label:
return feeds
return []
def find_feed(self, stream_id: str) -> freshrss.Subscription | None:
for feed in self.feeds:
if feed.id == stream_id:
return feed
return None
def unread_for_stream(self, stream_id: str) -> int | None:
return self.unread_counts.get(stream_id)
@dataclass
class Page:
"""One page of items, plus whether a further page exists."""
items: list[freshrss.Item]
has_more: bool
@dataclass
class RequestContext:
"""Where this request arrived, so links can be absolute and correct."""
base_url: str
prefix: str = ""
def url(self, *parts: str) -> str:
"""An absolute URL for a path under this deployment's prefix."""
segments = [self.prefix.strip("/"), *(part.strip("/") for part in parts)]
path = "/" + "/".join(segment for segment in segments if segment)
return self.base_url + path
class Application:
"""The catalog logic, independent of the HTTP layer."""
def __init__(
self,
config: Config,
*,
client: freshrss.FreshRSSClient | None = None,
fetcher: ImageFetcher | None = None,
clock: Any = time.time,
sleep: Any = time.sleep,
) -> None:
self.config = config
self.client = client or freshrss.FreshRSSClient(config.freshrss, sleep=sleep)
self.fetcher = fetcher
if self.fetcher is None and config.epub.include_images:
self.fetcher = ImageFetcher(
allowed_hosts=config.epub.allowed_image_hosts,
blocked_hosts=config.epub.blocked_image_hosts,
max_image_bytes=config.epub.max_image_bytes,
max_total_bytes=config.epub.max_total_image_bytes,
timeout=config.epub.image_timeout_seconds,
)
self.authenticator = auth.Authenticator(config.auth)
self._clock = clock
self._cache = TTLCache(config.cache.max_entries)
# Optional upstream features that may simply not exist on this build.
self._degraded = FailureLatch(ttl_seconds=config.cache.subscriptions_ttl_seconds)
# -- catalog ---------------------------------------------------------
def snapshot(self) -> CatalogSnapshot:
"""The account's categories and feeds, cached for a short window."""
return self._cache.get_or_compute(
"subscriptions",
self.config.cache.subscriptions_ttl_seconds,
self._load_snapshot,
)
def _load_snapshot(self) -> CatalogSnapshot:
feeds = self.client.list_subscriptions()
counts: dict[str, int] = {}
if self.config.catalog.show_unread_counts and self._degraded.is_open("unread"):
try:
counts = self.client.unread_counts()
except freshrss.FreshRSSError as exc:
if self._degraded.trip("unread"):
LOGGER.warning(
"unread counts are unavailable (%s); continuing without them", exc
)
kept = [feed for feed in feeds if self._is_visible(feed)]
grouped: dict[str, list[freshrss.Subscription]] = {}
for feed in kept:
# A feed in several categories appears under each, matching FreshRSS.
for label in feed.categories or [freshrss.UNCATEGORISED]:
grouped.setdefault(label, []).append(feed)
categories = sorted(grouped.items(), key=lambda item: item[0].casefold())
uncategorised = grouped.pop(freshrss.UNCATEGORISED, [])
ordered = [
(label, grouped[label]) for label, _ in categories if label != freshrss.UNCATEGORISED
]
if self.config.catalog.hide_empty_categories:
ordered = [
(label, feeds_in)
for label, feeds_in in ordered
if self._has_items(feeds_in, counts)
]
return CatalogSnapshot(
feeds=kept,
categories=ordered,
uncategorised=uncategorised,
unread_counts=counts,
)
def _is_visible(self, feed: freshrss.Subscription) -> bool:
excluded_feeds = {name.casefold() for name in self.config.catalog.exclude_feeds}
if feed.title.casefold() in excluded_feeds:
return False
excluded_categories = {name.casefold() for name in self.config.catalog.exclude_categories}
return not excluded_categories.intersection(label.casefold() for label in feed.categories)
def _has_items(self, feeds: list[freshrss.Subscription], counts: dict[str, int]) -> bool:
if not counts:
# Without counts, emptiness cannot be judged cheaply; keep the category
# rather than hiding feeds that may well have articles.
return True
return any(counts.get(feed.id, 0) > 0 for feed in feeds)
def page(
self,
stream_id: str,
*,
page: int,
only_read: bool = False,
) -> Page:
"""One page of a stream, newest first.
Unread filtering happens upstream, which is exact and cheap. Read items
are filtered here instead, because only the unread exclusion is a
documented upstream parameter -- so that section is bounded by
``freshrss.read_window_items`` rather than by the whole history.
"""
per_page = self.config.freshrss.items_per_page
offset = max(0, (page - 1) * per_page)
if offset > _MAX_OFFSET:
return Page(items=[], has_more=False)
ceiling = self.config.freshrss.max_items_per_request
wanted = min(ceiling, offset + per_page + 1)
if only_read:
wanted = min(wanted, self.config.freshrss.read_window_items)
items = self._items(stream_id, count=wanted, exclude_read=not only_read)
if only_read:
items = [item for item in items if item.is_read]
window = items[offset : offset + per_page]
# One extra item was requested purely to answer "is there a next page?".
has_more = len(items) > offset + per_page
if not only_read and offset + per_page >= ceiling:
has_more = False
return Page(items=window, has_more=has_more)
def _items(self, stream_id: str, *, count: int, exclude_read: bool) -> list[freshrss.Item]:
key = f"stream:{stream_id}:{count}:{int(exclude_read)}"
return self._cache.get_or_compute(
key,
self.config.cache.stream_ttl_seconds,
lambda: self.client.stream_items(stream_id, count=count, exclude_read=exclude_read),
)
def item(self, stream_id: str, item_id: str) -> freshrss.Item | None:
"""Find one article, by id, within the stream it came from.
The download URL carries its origin stream precisely so this works on a
cold cache, after a restart, or when a reader kept a link from yesterday.
"""
key = f"item:{item_id}"
cached = self._cache.get(key)
if isinstance(cached, freshrss.Item):
return cached
for item in self._items(
stream_id, count=self.config.freshrss.max_items_per_request, exclude_read=False
):
if item.id == item_id:
self._cache.set(key, item, self.config.cache.item_ttl_seconds)
return item
return None
def search(self, term: str, limit: int = 200) -> list[freshrss.Item]:
"""Articles matching a term, over a bounded recent window.
A local filter over the recent reading list rather than an upstream search
call: the reader API's search behaviour varies between builds, and a bounded
window that always answers beats an endpoint that may not exist. The
window is what :attr:`FreshRSSConfig.read_window_items` bounds, so a very
old article may not be found.
"""
needle = term.casefold()
window = min(limit, self.config.freshrss.read_window_items)
items = self._items(freshrss.READING_LIST, count=window, exclude_read=False)
matches: list[freshrss.Item] = []
for item in items:
haystack = " ".join(
part for part in (item.title, item.author, item.feed_title, _summary(item)) if part
)
if needle in haystack.casefold():
matches.append(item)
return matches
def render_epub(self, item: freshrss.Item) -> bytes:
"""Build the book for one article, in memory."""
return epub.build_article_epub(
item,
epub_config=self.config.epub,
catalog_config=self.config.catalog,
fetcher=self.fetcher,
max_content_bytes=self.config.freshrss.max_content_bytes,
)
def mark_read(self, item: freshrss.Item) -> bool:
"""Mark an article read upstream, best effort."""
if not self.config.freshrss.mark_read:
return False
try:
self.client.mark_read([item.id])
except freshrss.FreshRSSError as exc:
LOGGER.warning("could not mark %s read upstream: %s", item.id, exc)
return False
# Streams now under-report, and the account snapshot holds the unread
# counts, so both are dropped: without the snapshot the catalog would
# advertise the old total while listing one article fewer. That costs one
# upstream call on the next browse, which is the right trade for a queue
# the reader is actively draining.
self._cache.invalidate("stream:")
self._cache.invalidate("item:")
self._cache.invalidate("subscriptions")
return True
def invalidate(self) -> None:
self._cache.invalidate()
def diagnostics(self) -> dict[str, Any]:
"""A small report for the health endpoint."""
reachable, detail = self.client.probe()
snapshot = None
if reachable:
try:
snapshot = self.snapshot()
except freshrss.FreshRSSError:
snapshot = None
return {
"upstream": {"reachable": reachable, "detail": detail},
"catalog": {
"categories": len(snapshot.categories) if snapshot else None,
"feeds": len(snapshot.feeds) if snapshot else None,
"unread_counts": bool(snapshot and snapshot.unread_counts),
},
"cache": self._cache.stats(),
"mark_read": self.config.freshrss.mark_read,
"images": self.config.epub.include_images,
}
# -- HTTP -----------------------------------------------------------------
def make_handler(application: Application) -> type[BaseHTTPRequestHandler]:
"""Build the request handler class bound to one application."""
router = _Router()
config = application.config
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "opds-from-rss"
# -- plumbing ----------------------------------------------------
def log_message(self, format: str, *args: object) -> None:
"""Route the access log through logging, at debug level."""
def log_error(self, format: str, *args: object) -> None:
LOGGER.warning("http error: %s", format % args)
def do_GET(self) -> None:
self._dispatch(send_body=True)
def do_HEAD(self) -> None:
self._dispatch(send_body=False)
def do_POST(self) -> None:
# The catalog is read-only; a write would be an SDK, not a browser.
self._respond(405, b"method not allowed", "text/plain")
def _dispatch(self, *, send_body: bool) -> None:
started = time.monotonic()
status = 500
try:
context = self._context()
route = self._route()
if route is None:
status = 404
self._respond(404, b"no such catalog entry\n", "text/plain")
return
status = route(application, self, context, send_body)
except auth.UnauthorizedError:
status = 401
self._respond(
401,
b"authentication required\n",
"text/plain",
extra={"WWW-Authenticate": application.authenticator.challenge},
)
except auth.RateLimitedError as exc:
status = 429
self._respond(
429,
b"too many failed attempts\n",
"text/plain",
extra={
"Retry-After": str(exc.retry_after),
"WWW-Authenticate": application.authenticator.challenge,
},
)
except freshrss.TransientError as exc:
status = 502
LOGGER.warning("upstream unavailable: %s", exc)
self._respond(
502,
b"the feed server is unavailable\n",
"text/plain",
extra={"Retry-After": "30"},
)
except freshrss.FreshRSSError as exc:
status = 502
LOGGER.error("upstream error: %s", exc)
self._respond(502, b"the feed server refused the request\n", "text/plain")
except epub.EpubTooLargeError as exc:
status = 500
LOGGER.error("refusing to build a book: %s", exc)
self._respond(500, b"that article is too large to convert\n", "text/plain")
except Exception:
status = 500
LOGGER.exception("unhandled error serving %s", self.path)
self._respond(500, b"internal error\n", "text/plain")
finally:
LOGGER.info(
"%s %s -> %s %.0fms",
self.command,
self.path,
status,
(time.monotonic() - started) * 1000,
)
def _client(self) -> str:
forwarded = self.headers.get("X-Forwarded-For")
if config.server.trust_proxy_headers and forwarded:
return forwarded.split(",")[0].strip()
return self.client_address[0]
def _context(self) -> RequestContext:
"""Work out this deployment's absolute base URL and mount prefix."""
prefix = ""
scheme = "http"
host = self.headers.get("Host") or ""
if config.server.trust_proxy_headers:
scheme = self.headers.get("X-Forwarded-Proto", scheme).split(",")[0].strip()
host = self.headers.get("X-Forwarded-Host", host).split(",")[0].strip()
prefix = self.headers.get("X-Forwarded-Prefix", "").strip()
if config.server.allowed_hosts and host.lower() not in config.server.allowed_hosts:
raise HostNotAllowedError(host)
if config.server.public_base_url:
public = urllib.parse.urlsplit(config.server.public_base_url)
return RequestContext(
base_url=f"{public.scheme}://{public.netloc}",
prefix=public.path.rstrip("/"),
)
return RequestContext(base_url=f"{scheme}://{host}", prefix=prefix)
def _route(self):
path = urllib.parse.urlsplit(self.path).path
prefix = config.server.public_base_url or ""
return router.match(path, known_prefix=prefix)
# -- responses ---------------------------------------------------
def _respond(
self,
status: int,
body: bytes,
content_type: str,
*,
extra: dict[str, str] | None = None,
send_body: bool = True,
) -> None:
try:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
# The catalog is a live view of a reading queue, so it is never
# cached by an intermediary; the in-memory cache protects upstream.
self.send_header("Cache-Control", "no-store")
self.send_header("X-Content-Type-Options", "nosniff")
for key, value in (extra or {}).items():
self.send_header(key, value)
self.end_headers()
if send_body and body:
self.wfile.write(body)
except (BrokenPipeError, ConnectionResetError):
# A reader that walks away mid-response is normal, not an error.
self.close_connection = True
def _send_xml(self, document: str, kind_type: str, send_body: bool) -> None:
self._respond(200, document.encode("utf-8"), kind_type, send_body=send_body)
return Handler
class HostNotAllowedError(Exception):
"""The request's Host header is not in the allowlist."""
def _json_bytes(payload: Any) -> bytes:
return json.dumps(payload, indent=2, sort_keys=True).encode("utf-8")
# -- routing -------------------------------------------------------------
class _Router:
"""Maps a request path onto a view function.
Routes are declared once, as patterns, rather than as a chain of string
comparisons, so the URL space is readable in one place.
"""
def __init__(self) -> None:
self._routes: list[tuple[re.Pattern[str], Any]] = [
(re.compile(r"^/$"), _view_root_redirect),
(re.compile(r"^/(healthz)$"), _view_health),
(re.compile(r"^/(opds)$"), _view_root),
(re.compile(r"^/(opds)/opensearch\.xml$"), _view_opensearch),
(re.compile(r"^/(opds)/search$"), _view_search),
(re.compile(r"^/(opds)/healthz$"), _view_diagnostics),
(re.compile(r"^/(opds)/starred$"), _view_starred),
(re.compile(r"^/(opds)/(unread|read)$"), _view_section),
(re.compile(r"^/(opds)/(unread|read)/all$"), _view_section_all),
(
re.compile(r"^/(opds)/(unread|read)/c/([A-Za-z0-9_-]+)$"),
_view_category,
),
(
re.compile(r"^/(opds)/(unread|read)/c/([A-Za-z0-9_-]+)/all$"),
_view_category_all,
),
(
re.compile(r"^/(opds)/(unread|read)/c/([A-Za-z0-9_-]+)/f/([A-Za-z0-9_-]+)$"),
_view_feed,
),
(
re.compile(r"^/(opds)/d/([A-Za-z0-9_-]+)/([A-Za-z0-9_-]+)/([^/]*)\.epub$"),
_view_download,
),
]
def match(self, path: str, known_prefix: str = ""):
"""Find the view for a path, tolerating a base path from config."""
for candidate in _candidate_paths(path, known_prefix):
for pattern, view in self._routes:
if pattern.match(candidate):
return _bind(view, pattern, candidate)
return None
def _candidate_paths(path: str, known_prefix: str) -> list[str]:
"""The path as given, plus the same path with a configured base stripped.
``public_base_url`` may include a path (``https://host/catalog``), in which
case the mount point has to come off before the routes are matched.
"""
candidates = [path]
if known_prefix:
stripped = urllib.parse.urlsplit(known_prefix).path.rstrip("/")
if stripped and path.startswith(stripped):
candidates.append(path[len(stripped) :] or "/")
return candidates
def _bind(view, pattern: re.Pattern[str], path: str):
"""Bind a matched view to its path, so the caller just invokes it."""
def dispatch(app: Application, handler, context: RequestContext, send_body: bool) -> int:
return _dispatch_view(view, pattern, path, app, handler, context, send_body)
return dispatch
def _dispatch_view(view, pattern, path, app, handler, context, send_body) -> int:
"""Run a view, requiring authentication unless it is the health check."""
match = pattern.match(path)
if match is None: # pragma: no cover - the router already matched
return 404
if view is not _view_root_redirect and view is not _view_health:
username = app.authenticator.authenticate(
handler.headers.get("Authorization"), handler._client()
)
LOGGER.debug("authenticated %s for %s", username, path)
arguments = match.groups()[1:]
return view(app, handler, context, send_body, *arguments)
# -- views ---------------------------------------------------------------
def _view_root_redirect(app, handler, context, send_body, *_rest) -> int:
"""Send the bare root to the catalog entry point."""
handler._respond(
302,
b"",
"text/plain",
extra={"Location": context.url(ROOT_PATH)},
send_body=False,
)
return 302
def _view_health(app, handler, context, send_body, *_rest) -> int:
"""Liveness only: no upstream call, no configuration, no credentials."""
handler._respond(200, _json_bytes({"status": "ok"}), "application/json")
return 200
def _view_diagnostics(app, handler, context, send_body, *_rest) -> int:
"""An authenticated report, for working out why a reader sees nothing."""
handler._respond(200, _json_bytes(app.diagnostics()), "application/json")
return 200
def _view_opensearch(app, handler, context, send_body, *_rest) -> int:
document = opds.opensearch_description(
search_url=context.url(ROOT_PATH, "search"),
short_name=config_title(app),
description=f"Search {config_title(app)}",
)
handler._send_xml(document, opds.OPENSEARCH_TYPE, send_body)
return 200
def config_title(app: Application) -> str:
return app.config.catalog.title
def _view_root(app, handler, context, send_body, *_rest) -> int:
"""The root navigation feed: the sections, and search."""
snapshot = app.snapshot()
now = datetime.now(UTC)
entries = [
opds.NavEntry(
title="Unread",
entry_id=urls.stable_urn("feed", "section:unread"),
href=context.url(ROOT_PATH, "unread"),
updated=now,
description=_describe(app, snapshot, "unread"),
)
]
if app.config.catalog.show_read_section:
entries.append(
opds.NavEntry(
title="Read",
entry_id=urls.stable_urn("feed", "section:read"),
href=context.url(ROOT_PATH, "read"),
updated=now,
description=f"{len(snapshot.feeds)} feeds, previously read articles",
)
)
if app.config.catalog.show_starred:
entries.append(
opds.NavEntry(
title="Starred",
entry_id=urls.stable_urn("feed", "section:starred"),
href=context.url(ROOT_PATH, "starred"),
updated=now,
description="Articles you starred in FreshRSS",
)
)
entries.append(
opds.NavEntry(
title="All unread",
entry_id=urls.stable_urn("feed", "all:unread"),
href=context.url(ROOT_PATH, "unread", "all"),
updated=now,
description="Every unread article, newest first",
)
)
document = opds.navigation_feed(
feed_id=urls.stable_urn("feed", "root"),
title=app.config.catalog.title,
updated=now,
links=opds.FeedLinks(
self_url=context.url(ROOT_PATH),
start_url=context.url(ROOT_PATH),
search_url=context.url(ROOT_PATH, "opensearch.xml"),
),
entries=entries,
author=app.config.catalog.author,
subtitle=f"{len(snapshot.categories)} categories, {len(snapshot.feeds)} feeds",
)
handler._send_xml(document, opds.NAVIGATION_TYPE, send_body)
return 200
def _describe(app: Application, snapshot: CatalogSnapshot, section: str) -> str:
if section != "unread" or not snapshot.unread_counts:
return f"{len(snapshot.feeds)} feeds"
total = sum(snapshot.unread_counts.get(feed.id, 0) for feed in snapshot.feeds)
return f"{total} unread articles"
def _view_section(app, handler, context, send_body, section: str) -> int:
"""A section's categories."""
snapshot = app.snapshot()
now = datetime.now(UTC)
entries = [
opds.NavEntry(
title="All feeds",
entry_id=urls.stable_urn("feed", f"{section}:all"),
href=context.url(ROOT_PATH, section, "all"),
updated=now,
description=_describe(app, snapshot, section),
)
]
for label, feeds in snapshot.categories:
entries.append(
opds.NavEntry(
title=label,
entry_id=urls.stable_urn("category", f"{section}:{label}"),
href=context.url(ROOT_PATH, section, "c", urls.b64url_encode(label)),
updated=now,
description=_describe_feeds(feeds, snapshot),
)
)
if snapshot.uncategorised:
entries.append(
opds.NavEntry(
title=freshrss.UNCATEGORISED,
entry_id=urls.stable_urn("category", f"{section}:uncategorised"),
href=context.url(
ROOT_PATH, section, "c", urls.b64url_encode(freshrss.UNCATEGORISED)
),
updated=now,
description=_describe_feeds(snapshot.uncategorised, snapshot),
)
)
document = opds.navigation_feed(
feed_id=urls.stable_urn("feed", f"section:{section}"),
title=f"{app.config.catalog.title} \u2014 {section.capitalize()}",
updated=now,
links=opds.FeedLinks(
self_url=context.url(ROOT_PATH, section),
start_url=context.url(ROOT_PATH),
search_url=context.url(ROOT_PATH, "opensearch.xml"),
),
entries=entries,
author=app.config.catalog.author,
)
handler._send_xml(document, opds.NAVIGATION_TYPE, send_body)
return 200
def _describe_feeds(feeds: list[freshrss.Subscription], snapshot: CatalogSnapshot) -> str:
if not snapshot.unread_counts:
return f"{len(feeds)} feeds"
unread = sum(snapshot.unread_counts.get(feed.id, 0) for feed in feeds)
return f"{len(feeds)} feeds, {unread} unread"
def _view_section_all(app, handler, context, send_body, section: str) -> int:
"""Every article in a section, newest first."""
page = _page_number(handler)
if section == "read":
result = app.page(freshrss.READING_LIST, page=page, only_read=True)
else:
result = app.page(freshrss.READING_LIST, page=page)
return _send_acquisition(
app,
handler,
context,
send_body,
title=f"All {section}",
feed_key=f"{section}:all",
stream_id=freshrss.READING_LIST,
section=section,
result=result,
page=page,
)
def _view_starred(app, handler, context, send_body, *_rest) -> int:
page = _page_number(handler)
result = app.page(freshrss.STARRED, page=page)
return _send_acquisition(
app,
handler,
context,
send_body,
title="Starred",
feed_key="starred",
stream_id=freshrss.STARRED,
section="starred",
result=result,
page=page,
)
def _view_category(app, handler, context, send_body, section: str, encoded: str) -> int:
"""A category's feeds."""
label = _decode(encoded, handler)
if label is None:
return 404
snapshot = app.snapshot()
feeds = snapshot.feeds_in(label)
now = datetime.now(UTC)
entries = [
opds.NavEntry(
title="All in this category",
entry_id=urls.stable_urn("feed", f"{section}:category:{label}:all"),
href=context.url(ROOT_PATH, section, "c", encoded, "all"),
updated=now,
description=_describe_feeds(feeds, snapshot),
)
]
for feed in feeds:
entries.append(
opds.NavEntry(
title=feed.title,
entry_id=urls.stable_urn("feed", f"{section}:{feed.id}"),
href=context.url(
ROOT_PATH, section, "c", encoded, "f", urls.b64url_encode(feed.id)
),
updated=now,
description=_describe_feed(feed, snapshot),
)
)
document = opds.navigation_feed(
feed_id=urls.stable_urn("feed", f"category:{section}:{label}"),
title=f"{app.config.catalog.title} \u2014 {label}",
updated=now,
links=opds.FeedLinks(
self_url=context.url(ROOT_PATH, section, "c", encoded),
start_url=context.url(ROOT_PATH),
search_url=context.url(ROOT_PATH, "opensearch.xml"),
),
entries=entries,
author=app.config.catalog.author,
subtitle=_describe_feeds(feeds, snapshot),
)
handler._send_xml(document, opds.NAVIGATION_TYPE, send_body)
return 200
def _describe_feed(feed: freshrss.Subscription, snapshot: CatalogSnapshot) -> str:
unread = snapshot.unread_for_stream(feed.id)
if unread is None:
return "Articles"
return f"{unread} unread"
def _view_category_all(app, handler, context, send_body, section: str, encoded: str) -> int:
label = _decode(encoded, handler)
if label is None:
return 404
page = _page_number(handler)
stream_id = freshrss.label_stream(label)
result = app.page(stream_id, page=page, only_read=(section == "read"))
return _send_acquisition(
app,
handler,
context,
send_body,
title=f"{label} \u2014 {section}",
feed_key=f"{section}:category:{label}",
stream_id=stream_id,
section=section,
result=result,
page=page,
)
def _view_feed(
app, handler, context, send_body, section: str, encoded_category: str, encoded_feed: str
) -> int:
feed_id = _decode(encoded_feed, handler)
if feed_id is None:
return 404
snapshot = app.snapshot()
feed = snapshot.find_feed(feed_id)
title = feed.title if feed else feed_id
page = _page_number(handler)
result = app.page(feed_id, page=page, only_read=(section == "read"))
return _send_acquisition(
app,
handler,
context,
send_body,
title=title,
feed_key=f"{section}:feed:{feed_id}",
stream_id=feed_id,
section=section,
result=result,
page=page,
)
def _view_search(app, handler, context, send_body, *_rest) -> int:
"""Search recent articles by title, author, feed and summary text.
A local filter over a bounded window rather than an upstream search call: the
reader API's search behaviour varies between builds, and a bounded window that
always answers beats an endpoint that may not exist.
"""
query = urllib.parse.parse_qs(urllib.parse.urlsplit(handler.path).query)
term = (query.get("q") or [""])[0].strip()
page = _page_number(handler)
if not term:
return _send_acquisition(
app,
handler,
context,
send_body,
title="Search",
feed_key="search:empty",
stream_id=freshrss.READING_LIST,
section="unread",
result=Page(items=[], has_more=False),
page=page,
terms=term,
)
matches = app.search(term)
per_page = app.config.freshrss.items_per_page
offset = max(0, (page - 1) * per_page)
window = matches[offset : offset + per_page]
has_more = len(matches) > offset + per_page
return _send_acquisition(
app,
handler,
context,
send_body,
title=f"Search: {term}",
feed_key=f"search:{term}",
stream_id=freshrss.READING_LIST,
section="unread",
result=Page(items=window, has_more=has_more),
page=page,
terms=term,
)
def _view_download(
app, handler, context, send_body, stream_encoded: str, item_encoded: str, slug: str
) -> int:
"""Build and deliver one article as an EPUB, then mark it read."""
stream_id = _decode(stream_encoded, handler)
item_id = _decode(item_encoded, handler)
if stream_id is None or item_id is None:
return 404
item = app.item(stream_id, item_id)
if item is None:
handler._respond(404, b"no such article\n", "text/plain")
return 404
body = app.render_epub(item)
filename = f"{urls.slugify(item.title, 60)}.epub"
handler._respond(
200,
body,
"application/epub+zip",
extra={
"Content-Disposition": urls.content_disposition(filename, f"{item.title}.epub"),
},
send_body=send_body,
)
if send_body:
# The book has been handed over, so honour the queue semantics now. A
# HEAD probe or a dropped connection must not consume the article.
app.mark_read(item)
return 200
def _send_acquisition(
app: Application,
handler,
context: RequestContext,
send_body: bool,
*,
title: str,
feed_key: str,
stream_id: str,
section: str,
result: Page,
page: int,
terms: str | None = None,
) -> int:
"""Render an acquisition feed, with entries pointing at their downloads."""
now = datetime.now(UTC)
entries = [
opds.AcquisitionEntry(
title=item.title,
entry_id=urls.stable_urn("item", item.id),
updated=item.updated_at,
download_href=_download_url(app, context, stream_id, item),
author=item.author or item.feed_title,
summary=_summary(item),
language=app.config.catalog.language,
alternate_href=item.url,
categories=_entry_categories(section, item),
)
for item in result.items
]
base = context.url(ROOT_PATH, section)
links = opds.FeedLinks(
self_url=_page_url(app, context, base, page, terms),
start_url=context.url(ROOT_PATH),
search_url=context.url(ROOT_PATH, "opensearch.xml"),
pages=opds.PageLinks(
first=_page_url(app, context, base, 1, terms),
previous=_page_url(app, context, base, page - 1, terms) if page > 1 else None,
next=_page_url(app, context, base, page + 1, terms) if result.has_more else None,
),
)
document = opds.acquisition_feed(
feed_id=urls.stable_urn("feed", feed_key),
title=f"{app.config.catalog.title} \u2014 {title}",
updated=now,
links=links,
entries=entries,
author=app.config.catalog.author,
subtitle=f"Page {page}" if page > 1 else None,
)
handler._send_xml(document, opds.ACQUISITION_TYPE, send_body)
return 200
def _page_url(
app: Application, context: RequestContext, base: str, page: int, terms: str | None
) -> str:
if page <= 1 and not terms:
return base
query = {}
if page > 1:
query["page"] = str(page)
if terms:
query["q"] = terms
return base + "?" + urllib.parse.urlencode(query)
def _download_url(
app: Application, context: RequestContext, stream_id: str, item: freshrss.Item
) -> str:
origin = item.feed_id or stream_id
return context.url(
ROOT_PATH,
"d",
urls.b64url_encode(origin),
urls.b64url_encode(item.id),
urls.slugify(item.title, 60) + ".epub",
)
def _entry_categories(section: str, item: freshrss.Item) -> tuple[tuple[str, str], ...]:
"""Category terms for an entry: the section it was listed under, then its feed."""
terms = [(section, section.capitalize())]
if item.feed_title:
terms.append((item.feed_title, item.feed_title))
return tuple(terms)
def _summary(item: freshrss.Item) -> str | None:
text = " ".join(item.content_html.split())
plain = re.sub(r"<[^>]+>", " ", text)
plain = " ".join(plain.split())
if not plain:
return item.feed_title or None
return plain[:400] + ("\u2026" if len(plain) > 400 else "")
def _page_number(handler) -> int:
query = urllib.parse.parse_qs(urllib.parse.urlsplit(handler.path).query)
raw = (query.get("page") or ["1"])[0]
try:
return max(1, min(1000, int(raw)))
except ValueError:
return 1
def _decode(encoded: str, handler) -> str | None:
"""Decode a path identifier, answering 404 rather than raising on junk."""
try:
return urls.b64url_decode(encoded)
except ValueError:
handler._respond(404, b"no such catalog entry\n", "text/plain")
return None
def build_server(config: Config) -> tuple[ThreadingHTTPServer, Application]:
"""Create the HTTP server, ready to serve_forever."""
application = Application(config)
handler = make_handler(application)
server = ThreadingHTTPServer((config.server.host, config.server.port), handler)
server.daemon_threads = True
return server, application