Files
rss-to-opds/tests/stub_freshrss.py
2026-11-15 18:15:03 -05:00

606 lines
23 KiB
Python

"""An in-process FreshRSS stand-in.
This exists so the client can be tested against the wire format a real FreshRSS
produces, rather than against a mock that would happily accept anything. The
response shapes here mirror the Google Reader API responses the firmware client
consumes (``lib/FreshRss/src/FreshRssSync.cpp``), including the parts that are
easy to get wrong: ``Auth=`` embedded in a multi-line ClientLogin body, category
labels nested inside each subscription, item payloads that put the body under
``content``/``summary``, and one ``i=`` parameter per id on ``edit-tag``.
It also records every request, so tests can assert on the exact query parameters
the client sent -- which is how the unread filter and the page size are verified.
The same class backs ``tools/fake-freshrss.py``, so the server can be smoke
tested by hand without pointing it at a real account.
"""
from __future__ import annotations
import json
import threading
import urllib.parse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Any
API_PREFIX = "/api/greader.php"
READ_TAG = "user/-/state/com.google/read"
STARRED_TAG = "user/-/state/com.google/starred"
READING_LIST = "user/-/state/com.google/reading-list"
def _tags(entry: Any) -> list[str]:
"""Category tags on an entry, tolerating a malformed payload.
Tests deliberately inject non-object entries to check that the client skips
them, so the stub's own filtering must survive them too.
"""
if not isinstance(entry, dict):
return []
tags = entry.get("categories")
if not isinstance(tags, list):
return []
return [tag for tag in tags if isinstance(tag, str)]
def _published(entry: Any) -> int:
"""Sort key that tolerates a malformed entry."""
if not isinstance(entry, dict):
return 0
value = entry.get("published")
return value if isinstance(value, int) else 0
def _with_read_state(entry: Any, read_ids: set[str]) -> Any:
"""The entry as the server would send it, with the read tag applied."""
if not isinstance(entry, dict) or entry.get("id") not in read_ids:
return entry
if READ_TAG in _tags(entry):
return entry
projected = dict(entry)
projected["categories"] = [*_tags(entry), READ_TAG]
return projected
def _is_tagged(entry: Any, tag: str, read_ids: set[str]) -> bool:
"""Whether an entry carries a tag, including tracked read state.
Marking an item read adds the read tag on the server, so an ``xt=read`` query
stops returning it. Without modelling that, a test could not tell whether
mark-read actually took effect.
"""
if tag in _tags(entry):
return True
return tag == READ_TAG and isinstance(entry, dict) and entry.get("id") in read_ids
def item_id(index: int) -> str:
"""A FreshRSS-shaped item id: the exact form that needs URL-safe encoding."""
return f"tag:google.com,2005:reader/item/{index:016x}"
class StubFreshRSS:
"""A scriptable FreshRSS with just enough behaviour to trust the client."""
def __init__(
self,
*,
username: str = "tyler",
password: str = "api-password",
page_limit: int = 1000,
bind_host: str = "127.0.0.1",
bind_port: int = 0,
public_host: str | None = None,
) -> None:
self.username = username
self.password = password
self.page_limit = page_limit
self._bind_host = bind_host
self._bind_port = bind_port
# The name other containers should use to reach this one. It defaults to
# the bind address, which is right for a local test and wrong inside a
# compose network, where the service name is the address.
self._public_host = public_host or bind_host
self.auth_token = "stub-auth-token"
self.write_token = "stub-write-token"
self.subscriptions: list[dict[str, Any]] = []
# Items are typed loosely on purpose: this class stands in for an upstream
# that can send anything, and tests rely on being able to inject malformed
# payloads to check that the client skips them.
self.items: dict[str, list[Any]] = {}
self.read: set[str] = set()
self.requests: list[dict[str, Any]] = []
self.supports_unread_counts = True
# When set, subscription list replies with 200 and a body that is not JSON,
# so the client's response handling can be tested against a proxy or an
# error page rather than against well-formed output.
self.corrupt_json = False
# Injected failures as (status, path fragment or None) pairs.
self._failures: list[tuple[int, str | None]] = []
self._lock = threading.Lock()
self._server: ThreadingHTTPServer | None = None
self._thread: threading.Thread | None = None
self.port = 0
# -- scripting -------------------------------------------------------
def add_feed(
self,
stream_id: str,
title: str,
categories: list[str] | None = None,
*,
html_url: str | None = None,
) -> str:
"""Register a feed under one or more category labels."""
self.subscriptions.append(
{
"id": stream_id,
"title": title,
"categories": [
{"id": f"user/-/label/{label}", "label": label} for label in (categories or [])
],
"htmlUrl": html_url or stream_id.removeprefix("feed/"),
"url": stream_id.removeprefix("feed/"),
"sortid": f"{len(self.subscriptions):08x}",
}
)
self.items.setdefault(stream_id, [])
return stream_id
def add_item(
self,
stream_id: str,
index: int,
title: str,
*,
content: str = "<p>body</p>",
summary: str | None = None,
published: int = 1_700_000_000,
author: str | None = "A. Writer",
url: str | None = None,
read: bool = False,
starred: bool = False,
categories: list[str] | None = None,
) -> str:
"""Add an article to a feed, returning its id."""
identifier = item_id(index)
tags = [READING_LIST, *(categories or [])]
if read:
tags.append(READ_TAG)
if starred:
tags.append(STARRED_TAG)
entry: dict[str, Any] = {
"id": identifier,
"title": title,
"published": published,
"updated": published,
"timestampUsec": str(published * 1_000_000),
"crawlTimeMsec": str(published * 1000),
"categories": tags,
"alternate": [
{
"href": url or f"https://example.com/articles/{index}",
"type": "text/html",
}
],
"origin": {
"streamId": stream_id,
"title": self._feed_title(stream_id),
"htmlUrl": stream_id.removeprefix("feed/"),
},
}
if author is not None:
entry["author"] = author
entry["content"] = {"content": content, "direction": "ltr"}
entry["summary"] = {
"content": summary if summary is not None else content,
"direction": "ltr",
}
self.items.setdefault(stream_id, []).append(entry)
if read:
self.read.add(identifier)
return identifier
def inject_raw_item(self, stream_id: str, payload: Any) -> None:
"""Push an arbitrary payload into a stream.
Upstream is not guaranteed to answer with well-formed objects, so tests
need a way to send junk through the real client path rather than trusting
that it would be handled. Guarded because handler threads read this.
"""
with self._lock:
self.items.setdefault(stream_id, []).append(payload)
def fail_next(self, status: int, count: int = 1, *, path: str | None = None) -> None:
"""Make the next ``count`` requests fail with ``status``.
``path`` narrows it to requests whose path contains that fragment, which
matters because the client logs in before anything else: an untargeted
injected failure would be spent on the login and never reach the call the
test is about. Guarded because it is called between requests while
handler threads are running.
"""
with self._lock:
self._failures.extend([(status, path)] * count)
def _feed_title(self, stream_id: str) -> str:
for feed in self.subscriptions:
if feed["id"] == stream_id:
return feed["title"]
return stream_id
# -- lifecycle -------------------------------------------------------
@property
def base_url(self) -> str:
return f"http://{self._public_host}:{self.port}"
@property
def api_url(self) -> str:
return self.base_url + API_PREFIX
def start(self) -> StubFreshRSS:
handler = _make_handler(self)
server = ThreadingHTTPServer((self._bind_host, self._bind_port), handler)
self._server = server
# Read the port back, because a request for port 0 means "any free port"
# and only the operating system knows which one that turned out to be.
self.port = server.server_address[1]
# A short poll interval keeps teardown cheap: shutdown() only returns once
# the serve loop notices the flag, and the default half second would be
# paid by every test that starts a stub.
self._thread = threading.Thread(
target=server.serve_forever,
kwargs={"poll_interval": 0.02},
daemon=True,
)
self._thread.start()
return self
def stop(self) -> None:
if self._server is not None:
self._server.shutdown()
self._server.server_close()
self._server = None
if self._thread is not None:
self._thread.join(timeout=5)
self._thread = None
def __enter__(self) -> StubFreshRSS:
return self.start()
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: object,
) -> None:
self.stop()
# -- request handling ------------------------------------------------
def record(self, entry: dict[str, Any]) -> None:
with self._lock:
self.requests.append(entry)
def next_failure(self, path: str) -> int | None:
"""The next injected failure that applies to ``path``, if any."""
with self._lock:
for index, (status, target) in enumerate(self._failures):
if target is None or target in path:
del self._failures[index]
return status
return None
def find_requests(self, fragment: str) -> list[dict[str, Any]]:
"""Every recorded request whose path contains ``fragment``.
A substring match rather than a suffix, because a stream path is followed
by the encoded stream id (``/stream/contents/feed%2F...``) and so never
ends where the interesting part of the route does.
"""
with self._lock:
return [entry for entry in self.requests if fragment in entry["path"]]
def mark_read(self, identifiers: list[str]) -> None:
with self._lock:
self.read.update(identifiers)
def clear_read(self) -> None:
"""Forget every read mark, for tests that re-run a marking step."""
with self._lock:
self.read.clear()
def read_snapshot(self) -> set[str]:
"""A copy of the read marks, taken under the lock."""
with self._lock:
return set(self.read)
def subscription_payload(self) -> dict[str, Any]:
return {"subscriptions": self.subscriptions}
def unread_count_payload(self) -> dict[str, Any]:
counts = []
for feed in self.subscriptions:
unread = sum(
1
for entry in self.items.get(feed["id"], [])
if isinstance(entry, dict) and entry.get("id") not in self.read
)
counts.append(
{
"id": feed["id"],
"count": unread,
"newestItemTimestampUsec": "1700000000000000",
}
)
return {"max": self.page_limit, "unreadcounts": counts}
def stream_payload(self, stream_id: str, query: dict[str, list[str]]) -> dict[str, Any]:
"""Items for a stream, honouring ``xt``, ``ot``, ``n`` and ``r``."""
read_ids = self.read_snapshot()
if stream_id == READING_LIST:
entries = [entry for group in self.items.values() for entry in group]
elif stream_id == STARRED_TAG:
entries = [
entry
for group in self.items.values()
for entry in group
if _is_tagged(entry, STARRED_TAG, read_ids)
]
elif stream_id.startswith("user/-/label/"):
label = stream_id.rsplit("/", 1)[-1]
labelled = {
feed["id"]
for feed in self.subscriptions
if any(category["label"] == label for category in feed["categories"])
}
entries = [
entry
for feed_id, group in self.items.items()
if feed_id in labelled
for entry in group
]
else:
entries = list(self.items.get(stream_id, []))
excluded = set(query.get("xt", []))
if excluded:
entries = [
entry
for entry in entries
if not any(_is_tagged(entry, tag, read_ids) for tag in excluded)
]
older_than = query.get("ot", [None])[0]
if older_than and older_than.isdigit():
entries = [
entry
for entry in entries
if not isinstance(entry, dict) or _published(entry) >= int(older_than)
]
descending = query.get("r", ["d"])[0] != "o"
entries.sort(key=_published, reverse=descending)
# Marking an item read adds the tag to the item itself upstream, so the
# returned payload has to carry it too -- otherwise a client could never
# observe a read mark it just made.
entries = [_with_read_state(entry, read_ids) for entry in entries]
limit = query.get("n", [None])[0]
if limit and limit.isdigit():
entries = entries[: int(limit)]
return {"id": stream_id, "updated": 1_700_000_000, "items": entries}
def _make_handler(stub: StubFreshRSS):
"""Build the request handler class bound to one stub instance."""
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "StubFreshRSS/1.0"
# The parameter must keep the base class's name for keyword compatibility,
# even though it shadows the builtin.
def log_message(self, format: str, *args: object) -> None:
"""Silence the default access log; tests assert on recorded requests."""
# -- plumbing ----------------------------------------------------
def _respond(self, status: int, body: bytes, content_type: str) -> None:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
if body and self.command != "HEAD":
self.wfile.write(body)
def _respond_json(self, payload: Any, status: int = 200) -> None:
self._respond(
status,
json.dumps(payload).encode("utf-8"),
"application/json; charset=utf-8",
)
def _record(self, body: str) -> tuple[str, dict[str, list[str]]]:
parsed = urllib.parse.urlsplit(self.path)
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
route = parsed.path
if route.startswith(API_PREFIX):
route = route[len(API_PREFIX) :]
stub.record(
{
"method": self.command,
"path": route,
"raw_path": self.path,
"query": query,
"body": body,
"authorization": self.headers.get("Authorization", ""),
}
)
return route, query
def _authorised(self) -> bool:
expected = f"GoogleLogin auth={stub.auth_token}"
return self.headers.get("Authorization", "") == expected
def _drain_body(self) -> str:
length = int(self.headers.get("Content-Length") or 0)
if not length:
return ""
return self.rfile.read(length).decode("utf-8", "replace")
# -- routes ------------------------------------------------------
def do_HEAD(self) -> None:
self.do_GET()
def do_GET(self) -> None:
route, query = self._record("")
failure = stub.next_failure(route)
if failure is not None:
self._respond_json({"error": "injected failure"}, status=failure)
return
if route == "/healthz":
# Unauthenticated on purpose: it only says the server answered, and
# it is what the demo compose stack waits on before starting the
# catalogue. Real FreshRSS has no such route.
self._respond_json({"status": "ok"})
return
if route == "/accounts/ClientLogin":
self._respond_json({"error": "ClientLogin is a POST"}, status=405)
return
if route == "/reader/api/0/subscription/list":
if not self._require_auth():
return
if stub.corrupt_json:
self._respond(200, b"<html>login page</html>", "text/html")
return
self._respond_json(stub.subscription_payload())
return
if route == "/reader/api/0/unread-count":
if not self._require_auth():
return
if not stub.supports_unread_counts:
self._respond_json({"error": "unsupported"}, status=404)
return
self._respond_json(stub.unread_count_payload())
return
if route == "/reader/api/0/token":
if not self._require_auth():
return
self._respond(200, stub.write_token.encode("utf-8"), "text/plain")
return
if route.startswith("/reader/api/0/stream/contents/"):
if not self._require_auth():
return
stream_id = urllib.parse.unquote(route[len("/reader/api/0/stream/contents/") :])
self._respond_json(stub.stream_payload(stream_id, query))
return
self._respond_json({"error": "not found"}, status=404)
def do_POST(self) -> None:
body = self._drain_body()
route, _query = self._record(body)
failure = stub.next_failure(route)
if failure is not None:
self._respond_json({"error": "injected failure"}, status=failure)
return
if route == "/accounts/ClientLogin":
self._handle_login(body)
return
if route == "/reader/api/0/edit-tag":
if not self._require_auth():
return
self._handle_edit_tag(body)
return
self._respond_json({"error": "not found"}, status=404)
def _require_auth(self) -> bool:
if self._authorised():
return True
self._respond_json({"error": "unauthorised"}, status=401)
return False
def _handle_login(self, body: str) -> None:
form = urllib.parse.parse_qs(body)
username = form.get("Email", [""])[0]
password = form.get("Passwd", [""])[0]
if username != stub.username or password != stub.password:
self._respond_json({"error": "BadAuthentication"}, status=403)
return
# The real endpoint returns three lines; only Auth matters to us, and a
# parser that grabs the first "=" would get SID instead.
payload = f"SID=stub-sid\nLSID=stub-lsid\nAuth={stub.auth_token}\n"
self._respond(200, payload.encode("utf-8"), "text/plain; charset=utf-8")
def _handle_edit_tag(self, body: str) -> None:
form = urllib.parse.parse_qs(body, keep_blank_values=True)
token = form.get("T", [""])[0]
if token != stub.write_token:
self._respond_json({"error": "Invalid token"}, status=401)
return
if form.get("a", [""])[0] != READ_TAG:
self._respond_json({"error": "unexpected tag"}, status=400)
return
stub.mark_read(form.get("i", []))
self._respond(200, b"OK", "text/plain; charset=utf-8")
return Handler
def build_demo_catalog(**stub_options: Any) -> StubFreshRSS:
"""A small but realistic account, used by tests and the manual smoke tool.
Three feeds across two categories, with a read item, a starred item, an item
in a feed that has no category, and content that exercises the sanitiser
(relative links, a script tag, an unclosed list, an entity).
Keyword options are forwarded to :class:`StubFreshRSS`, which is how
``tools/fake-freshrss.py`` serves this same account to a container --
``bind_host``, ``bind_port`` and ``public_host``, since a compose network needs
a wildcard bind, a fixed port and a service name in place of loopback. The
defaults are the loopback bind and ephemeral port the tests rely on.
"""
stub = StubFreshRSS(**stub_options)
tech = stub.add_feed("feed/https://example.com/tech/atom.xml", "Example Tech", ["Tech"])
news = stub.add_feed(
"feed/https://news.example.org/rss?source=daily",
"Daily News",
["News"],
)
lonely = stub.add_feed("feed/https://solo.example.net/feed.xml", "Solo Blog")
stub.add_item(
tech,
1,
"A Tech Post & Its Friends",
content=(
'<p>Intro with an <a href="/relative-link">in-page link</a>.</p>'
"<script>alert('nope')</script>"
"<ul><li>first<li>second</ul>"
'<p><img src="/images/diagram.png" alt="A diagram"></p>'
"<p>Entities: &amp; &lt;tag&gt; &nbsp; done.</p>"
),
published=1_700_000_100,
)
stub.add_item(tech, 2, "An Older Tech Post", published=1_699_000_000)
stub.add_item(
tech,
3,
"Already Read Post",
published=1_698_000_000,
read=True,
)
stub.add_item(news, 4, "Markets Dip Slightly", published=1_700_000_200, author=None)
stub.add_item(
news,
5,
"Starred Investigation",
published=1_700_000_300,
starred=True,
)
stub.add_item(lonely, 6, "Uncategorised Musings", published=1_700_000_050)
return stub