606 lines
23 KiB
Python
606 lines
23 KiB
Python
"""An in-process FreshRSS stand-in.
|
|
|
|
This exists so the client can be tested against the wire format a real FreshRSS
|
|
produces, rather than against a mock that would happily accept anything. The
|
|
response shapes here mirror the Google Reader API responses the firmware client
|
|
consumes (``lib/FreshRss/src/FreshRssSync.cpp``), including the parts that are
|
|
easy to get wrong: ``Auth=`` embedded in a multi-line ClientLogin body, category
|
|
labels nested inside each subscription, item payloads that put the body under
|
|
``content``/``summary``, and one ``i=`` parameter per id on ``edit-tag``.
|
|
|
|
It also records every request, so tests can assert on the exact query parameters
|
|
the client sent -- which is how the unread filter and the page size are verified.
|
|
|
|
The same class backs ``tools/fake-freshrss.py``, so the server can be smoke
|
|
tested by hand without pointing it at a real account.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import threading
|
|
import urllib.parse
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from typing import Any
|
|
|
|
API_PREFIX = "/api/greader.php"
|
|
|
|
READ_TAG = "user/-/state/com.google/read"
|
|
STARRED_TAG = "user/-/state/com.google/starred"
|
|
READING_LIST = "user/-/state/com.google/reading-list"
|
|
|
|
|
|
def _tags(entry: Any) -> list[str]:
|
|
"""Category tags on an entry, tolerating a malformed payload.
|
|
|
|
Tests deliberately inject non-object entries to check that the client skips
|
|
them, so the stub's own filtering must survive them too.
|
|
"""
|
|
if not isinstance(entry, dict):
|
|
return []
|
|
tags = entry.get("categories")
|
|
if not isinstance(tags, list):
|
|
return []
|
|
return [tag for tag in tags if isinstance(tag, str)]
|
|
|
|
|
|
def _published(entry: Any) -> int:
|
|
"""Sort key that tolerates a malformed entry."""
|
|
if not isinstance(entry, dict):
|
|
return 0
|
|
value = entry.get("published")
|
|
return value if isinstance(value, int) else 0
|
|
|
|
|
|
def _with_read_state(entry: Any, read_ids: set[str]) -> Any:
|
|
"""The entry as the server would send it, with the read tag applied."""
|
|
if not isinstance(entry, dict) or entry.get("id") not in read_ids:
|
|
return entry
|
|
if READ_TAG in _tags(entry):
|
|
return entry
|
|
projected = dict(entry)
|
|
projected["categories"] = [*_tags(entry), READ_TAG]
|
|
return projected
|
|
|
|
|
|
def _is_tagged(entry: Any, tag: str, read_ids: set[str]) -> bool:
|
|
"""Whether an entry carries a tag, including tracked read state.
|
|
|
|
Marking an item read adds the read tag on the server, so an ``xt=read`` query
|
|
stops returning it. Without modelling that, a test could not tell whether
|
|
mark-read actually took effect.
|
|
"""
|
|
if tag in _tags(entry):
|
|
return True
|
|
return tag == READ_TAG and isinstance(entry, dict) and entry.get("id") in read_ids
|
|
|
|
|
|
def item_id(index: int) -> str:
|
|
"""A FreshRSS-shaped item id: the exact form that needs URL-safe encoding."""
|
|
return f"tag:google.com,2005:reader/item/{index:016x}"
|
|
|
|
|
|
class StubFreshRSS:
|
|
"""A scriptable FreshRSS with just enough behaviour to trust the client."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
username: str = "tyler",
|
|
password: str = "api-password",
|
|
page_limit: int = 1000,
|
|
bind_host: str = "127.0.0.1",
|
|
bind_port: int = 0,
|
|
public_host: str | None = None,
|
|
) -> None:
|
|
self.username = username
|
|
self.password = password
|
|
self.page_limit = page_limit
|
|
self._bind_host = bind_host
|
|
self._bind_port = bind_port
|
|
# The name other containers should use to reach this one. It defaults to
|
|
# the bind address, which is right for a local test and wrong inside a
|
|
# compose network, where the service name is the address.
|
|
self._public_host = public_host or bind_host
|
|
self.auth_token = "stub-auth-token"
|
|
self.write_token = "stub-write-token"
|
|
self.subscriptions: list[dict[str, Any]] = []
|
|
# Items are typed loosely on purpose: this class stands in for an upstream
|
|
# that can send anything, and tests rely on being able to inject malformed
|
|
# payloads to check that the client skips them.
|
|
self.items: dict[str, list[Any]] = {}
|
|
self.read: set[str] = set()
|
|
self.requests: list[dict[str, Any]] = []
|
|
self.supports_unread_counts = True
|
|
# When set, subscription list replies with 200 and a body that is not JSON,
|
|
# so the client's response handling can be tested against a proxy or an
|
|
# error page rather than against well-formed output.
|
|
self.corrupt_json = False
|
|
# Injected failures as (status, path fragment or None) pairs.
|
|
self._failures: list[tuple[int, str | None]] = []
|
|
self._lock = threading.Lock()
|
|
self._server: ThreadingHTTPServer | None = None
|
|
self._thread: threading.Thread | None = None
|
|
self.port = 0
|
|
|
|
# -- scripting -------------------------------------------------------
|
|
|
|
def add_feed(
|
|
self,
|
|
stream_id: str,
|
|
title: str,
|
|
categories: list[str] | None = None,
|
|
*,
|
|
html_url: str | None = None,
|
|
) -> str:
|
|
"""Register a feed under one or more category labels."""
|
|
self.subscriptions.append(
|
|
{
|
|
"id": stream_id,
|
|
"title": title,
|
|
"categories": [
|
|
{"id": f"user/-/label/{label}", "label": label} for label in (categories or [])
|
|
],
|
|
"htmlUrl": html_url or stream_id.removeprefix("feed/"),
|
|
"url": stream_id.removeprefix("feed/"),
|
|
"sortid": f"{len(self.subscriptions):08x}",
|
|
}
|
|
)
|
|
self.items.setdefault(stream_id, [])
|
|
return stream_id
|
|
|
|
def add_item(
|
|
self,
|
|
stream_id: str,
|
|
index: int,
|
|
title: str,
|
|
*,
|
|
content: str = "<p>body</p>",
|
|
summary: str | None = None,
|
|
published: int = 1_700_000_000,
|
|
author: str | None = "A. Writer",
|
|
url: str | None = None,
|
|
read: bool = False,
|
|
starred: bool = False,
|
|
categories: list[str] | None = None,
|
|
) -> str:
|
|
"""Add an article to a feed, returning its id."""
|
|
identifier = item_id(index)
|
|
tags = [READING_LIST, *(categories or [])]
|
|
if read:
|
|
tags.append(READ_TAG)
|
|
if starred:
|
|
tags.append(STARRED_TAG)
|
|
entry: dict[str, Any] = {
|
|
"id": identifier,
|
|
"title": title,
|
|
"published": published,
|
|
"updated": published,
|
|
"timestampUsec": str(published * 1_000_000),
|
|
"crawlTimeMsec": str(published * 1000),
|
|
"categories": tags,
|
|
"alternate": [
|
|
{
|
|
"href": url or f"https://example.com/articles/{index}",
|
|
"type": "text/html",
|
|
}
|
|
],
|
|
"origin": {
|
|
"streamId": stream_id,
|
|
"title": self._feed_title(stream_id),
|
|
"htmlUrl": stream_id.removeprefix("feed/"),
|
|
},
|
|
}
|
|
if author is not None:
|
|
entry["author"] = author
|
|
entry["content"] = {"content": content, "direction": "ltr"}
|
|
entry["summary"] = {
|
|
"content": summary if summary is not None else content,
|
|
"direction": "ltr",
|
|
}
|
|
self.items.setdefault(stream_id, []).append(entry)
|
|
if read:
|
|
self.read.add(identifier)
|
|
return identifier
|
|
|
|
def inject_raw_item(self, stream_id: str, payload: Any) -> None:
|
|
"""Push an arbitrary payload into a stream.
|
|
|
|
Upstream is not guaranteed to answer with well-formed objects, so tests
|
|
need a way to send junk through the real client path rather than trusting
|
|
that it would be handled. Guarded because handler threads read this.
|
|
"""
|
|
with self._lock:
|
|
self.items.setdefault(stream_id, []).append(payload)
|
|
|
|
def fail_next(self, status: int, count: int = 1, *, path: str | None = None) -> None:
|
|
"""Make the next ``count`` requests fail with ``status``.
|
|
|
|
``path`` narrows it to requests whose path contains that fragment, which
|
|
matters because the client logs in before anything else: an untargeted
|
|
injected failure would be spent on the login and never reach the call the
|
|
test is about. Guarded because it is called between requests while
|
|
handler threads are running.
|
|
"""
|
|
with self._lock:
|
|
self._failures.extend([(status, path)] * count)
|
|
|
|
def _feed_title(self, stream_id: str) -> str:
|
|
for feed in self.subscriptions:
|
|
if feed["id"] == stream_id:
|
|
return feed["title"]
|
|
return stream_id
|
|
|
|
# -- lifecycle -------------------------------------------------------
|
|
|
|
@property
|
|
def base_url(self) -> str:
|
|
return f"http://{self._public_host}:{self.port}"
|
|
|
|
@property
|
|
def api_url(self) -> str:
|
|
return self.base_url + API_PREFIX
|
|
|
|
def start(self) -> StubFreshRSS:
|
|
handler = _make_handler(self)
|
|
server = ThreadingHTTPServer((self._bind_host, self._bind_port), handler)
|
|
self._server = server
|
|
# Read the port back, because a request for port 0 means "any free port"
|
|
# and only the operating system knows which one that turned out to be.
|
|
self.port = server.server_address[1]
|
|
# A short poll interval keeps teardown cheap: shutdown() only returns once
|
|
# the serve loop notices the flag, and the default half second would be
|
|
# paid by every test that starts a stub.
|
|
self._thread = threading.Thread(
|
|
target=server.serve_forever,
|
|
kwargs={"poll_interval": 0.02},
|
|
daemon=True,
|
|
)
|
|
self._thread.start()
|
|
return self
|
|
|
|
def stop(self) -> None:
|
|
if self._server is not None:
|
|
self._server.shutdown()
|
|
self._server.server_close()
|
|
self._server = None
|
|
if self._thread is not None:
|
|
self._thread.join(timeout=5)
|
|
self._thread = None
|
|
|
|
def __enter__(self) -> StubFreshRSS:
|
|
return self.start()
|
|
|
|
def __exit__(
|
|
self,
|
|
exc_type: type[BaseException] | None,
|
|
exc_value: BaseException | None,
|
|
traceback: object,
|
|
) -> None:
|
|
self.stop()
|
|
|
|
# -- request handling ------------------------------------------------
|
|
|
|
def record(self, entry: dict[str, Any]) -> None:
|
|
with self._lock:
|
|
self.requests.append(entry)
|
|
|
|
def next_failure(self, path: str) -> int | None:
|
|
"""The next injected failure that applies to ``path``, if any."""
|
|
with self._lock:
|
|
for index, (status, target) in enumerate(self._failures):
|
|
if target is None or target in path:
|
|
del self._failures[index]
|
|
return status
|
|
return None
|
|
|
|
def find_requests(self, fragment: str) -> list[dict[str, Any]]:
|
|
"""Every recorded request whose path contains ``fragment``.
|
|
|
|
A substring match rather than a suffix, because a stream path is followed
|
|
by the encoded stream id (``/stream/contents/feed%2F...``) and so never
|
|
ends where the interesting part of the route does.
|
|
"""
|
|
with self._lock:
|
|
return [entry for entry in self.requests if fragment in entry["path"]]
|
|
|
|
def mark_read(self, identifiers: list[str]) -> None:
|
|
with self._lock:
|
|
self.read.update(identifiers)
|
|
|
|
def clear_read(self) -> None:
|
|
"""Forget every read mark, for tests that re-run a marking step."""
|
|
with self._lock:
|
|
self.read.clear()
|
|
|
|
def read_snapshot(self) -> set[str]:
|
|
"""A copy of the read marks, taken under the lock."""
|
|
with self._lock:
|
|
return set(self.read)
|
|
|
|
def subscription_payload(self) -> dict[str, Any]:
|
|
return {"subscriptions": self.subscriptions}
|
|
|
|
def unread_count_payload(self) -> dict[str, Any]:
|
|
counts = []
|
|
for feed in self.subscriptions:
|
|
unread = sum(
|
|
1
|
|
for entry in self.items.get(feed["id"], [])
|
|
if isinstance(entry, dict) and entry.get("id") not in self.read
|
|
)
|
|
counts.append(
|
|
{
|
|
"id": feed["id"],
|
|
"count": unread,
|
|
"newestItemTimestampUsec": "1700000000000000",
|
|
}
|
|
)
|
|
return {"max": self.page_limit, "unreadcounts": counts}
|
|
|
|
def stream_payload(self, stream_id: str, query: dict[str, list[str]]) -> dict[str, Any]:
|
|
"""Items for a stream, honouring ``xt``, ``ot``, ``n`` and ``r``."""
|
|
read_ids = self.read_snapshot()
|
|
if stream_id == READING_LIST:
|
|
entries = [entry for group in self.items.values() for entry in group]
|
|
elif stream_id == STARRED_TAG:
|
|
entries = [
|
|
entry
|
|
for group in self.items.values()
|
|
for entry in group
|
|
if _is_tagged(entry, STARRED_TAG, read_ids)
|
|
]
|
|
elif stream_id.startswith("user/-/label/"):
|
|
label = stream_id.rsplit("/", 1)[-1]
|
|
labelled = {
|
|
feed["id"]
|
|
for feed in self.subscriptions
|
|
if any(category["label"] == label for category in feed["categories"])
|
|
}
|
|
entries = [
|
|
entry
|
|
for feed_id, group in self.items.items()
|
|
if feed_id in labelled
|
|
for entry in group
|
|
]
|
|
else:
|
|
entries = list(self.items.get(stream_id, []))
|
|
|
|
excluded = set(query.get("xt", []))
|
|
if excluded:
|
|
entries = [
|
|
entry
|
|
for entry in entries
|
|
if not any(_is_tagged(entry, tag, read_ids) for tag in excluded)
|
|
]
|
|
older_than = query.get("ot", [None])[0]
|
|
if older_than and older_than.isdigit():
|
|
entries = [
|
|
entry
|
|
for entry in entries
|
|
if not isinstance(entry, dict) or _published(entry) >= int(older_than)
|
|
]
|
|
descending = query.get("r", ["d"])[0] != "o"
|
|
entries.sort(key=_published, reverse=descending)
|
|
# Marking an item read adds the tag to the item itself upstream, so the
|
|
# returned payload has to carry it too -- otherwise a client could never
|
|
# observe a read mark it just made.
|
|
entries = [_with_read_state(entry, read_ids) for entry in entries]
|
|
limit = query.get("n", [None])[0]
|
|
if limit and limit.isdigit():
|
|
entries = entries[: int(limit)]
|
|
return {"id": stream_id, "updated": 1_700_000_000, "items": entries}
|
|
|
|
|
|
def _make_handler(stub: StubFreshRSS):
|
|
"""Build the request handler class bound to one stub instance."""
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
protocol_version = "HTTP/1.1"
|
|
server_version = "StubFreshRSS/1.0"
|
|
|
|
# The parameter must keep the base class's name for keyword compatibility,
|
|
# even though it shadows the builtin.
|
|
def log_message(self, format: str, *args: object) -> None:
|
|
"""Silence the default access log; tests assert on recorded requests."""
|
|
|
|
# -- plumbing ----------------------------------------------------
|
|
|
|
def _respond(self, status: int, body: bytes, content_type: str) -> None:
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", content_type)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
if body and self.command != "HEAD":
|
|
self.wfile.write(body)
|
|
|
|
def _respond_json(self, payload: Any, status: int = 200) -> None:
|
|
self._respond(
|
|
status,
|
|
json.dumps(payload).encode("utf-8"),
|
|
"application/json; charset=utf-8",
|
|
)
|
|
|
|
def _record(self, body: str) -> tuple[str, dict[str, list[str]]]:
|
|
parsed = urllib.parse.urlsplit(self.path)
|
|
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
|
|
route = parsed.path
|
|
if route.startswith(API_PREFIX):
|
|
route = route[len(API_PREFIX) :]
|
|
stub.record(
|
|
{
|
|
"method": self.command,
|
|
"path": route,
|
|
"raw_path": self.path,
|
|
"query": query,
|
|
"body": body,
|
|
"authorization": self.headers.get("Authorization", ""),
|
|
}
|
|
)
|
|
return route, query
|
|
|
|
def _authorised(self) -> bool:
|
|
expected = f"GoogleLogin auth={stub.auth_token}"
|
|
return self.headers.get("Authorization", "") == expected
|
|
|
|
def _drain_body(self) -> str:
|
|
length = int(self.headers.get("Content-Length") or 0)
|
|
if not length:
|
|
return ""
|
|
return self.rfile.read(length).decode("utf-8", "replace")
|
|
|
|
# -- routes ------------------------------------------------------
|
|
|
|
def do_HEAD(self) -> None:
|
|
self.do_GET()
|
|
|
|
def do_GET(self) -> None:
|
|
route, query = self._record("")
|
|
failure = stub.next_failure(route)
|
|
if failure is not None:
|
|
self._respond_json({"error": "injected failure"}, status=failure)
|
|
return
|
|
|
|
if route == "/healthz":
|
|
# Unauthenticated on purpose: it only says the server answered, and
|
|
# it is what the demo compose stack waits on before starting the
|
|
# catalogue. Real FreshRSS has no such route.
|
|
self._respond_json({"status": "ok"})
|
|
return
|
|
if route == "/accounts/ClientLogin":
|
|
self._respond_json({"error": "ClientLogin is a POST"}, status=405)
|
|
return
|
|
if route == "/reader/api/0/subscription/list":
|
|
if not self._require_auth():
|
|
return
|
|
if stub.corrupt_json:
|
|
self._respond(200, b"<html>login page</html>", "text/html")
|
|
return
|
|
self._respond_json(stub.subscription_payload())
|
|
return
|
|
if route == "/reader/api/0/unread-count":
|
|
if not self._require_auth():
|
|
return
|
|
if not stub.supports_unread_counts:
|
|
self._respond_json({"error": "unsupported"}, status=404)
|
|
return
|
|
self._respond_json(stub.unread_count_payload())
|
|
return
|
|
if route == "/reader/api/0/token":
|
|
if not self._require_auth():
|
|
return
|
|
self._respond(200, stub.write_token.encode("utf-8"), "text/plain")
|
|
return
|
|
if route.startswith("/reader/api/0/stream/contents/"):
|
|
if not self._require_auth():
|
|
return
|
|
stream_id = urllib.parse.unquote(route[len("/reader/api/0/stream/contents/") :])
|
|
self._respond_json(stub.stream_payload(stream_id, query))
|
|
return
|
|
self._respond_json({"error": "not found"}, status=404)
|
|
|
|
def do_POST(self) -> None:
|
|
body = self._drain_body()
|
|
route, _query = self._record(body)
|
|
failure = stub.next_failure(route)
|
|
if failure is not None:
|
|
self._respond_json({"error": "injected failure"}, status=failure)
|
|
return
|
|
|
|
if route == "/accounts/ClientLogin":
|
|
self._handle_login(body)
|
|
return
|
|
if route == "/reader/api/0/edit-tag":
|
|
if not self._require_auth():
|
|
return
|
|
self._handle_edit_tag(body)
|
|
return
|
|
self._respond_json({"error": "not found"}, status=404)
|
|
|
|
def _require_auth(self) -> bool:
|
|
if self._authorised():
|
|
return True
|
|
self._respond_json({"error": "unauthorised"}, status=401)
|
|
return False
|
|
|
|
def _handle_login(self, body: str) -> None:
|
|
form = urllib.parse.parse_qs(body)
|
|
username = form.get("Email", [""])[0]
|
|
password = form.get("Passwd", [""])[0]
|
|
if username != stub.username or password != stub.password:
|
|
self._respond_json({"error": "BadAuthentication"}, status=403)
|
|
return
|
|
# The real endpoint returns three lines; only Auth matters to us, and a
|
|
# parser that grabs the first "=" would get SID instead.
|
|
payload = f"SID=stub-sid\nLSID=stub-lsid\nAuth={stub.auth_token}\n"
|
|
self._respond(200, payload.encode("utf-8"), "text/plain; charset=utf-8")
|
|
|
|
def _handle_edit_tag(self, body: str) -> None:
|
|
form = urllib.parse.parse_qs(body, keep_blank_values=True)
|
|
token = form.get("T", [""])[0]
|
|
if token != stub.write_token:
|
|
self._respond_json({"error": "Invalid token"}, status=401)
|
|
return
|
|
if form.get("a", [""])[0] != READ_TAG:
|
|
self._respond_json({"error": "unexpected tag"}, status=400)
|
|
return
|
|
stub.mark_read(form.get("i", []))
|
|
self._respond(200, b"OK", "text/plain; charset=utf-8")
|
|
|
|
return Handler
|
|
|
|
|
|
def build_demo_catalog(**stub_options: Any) -> StubFreshRSS:
|
|
"""A small but realistic account, used by tests and the manual smoke tool.
|
|
|
|
Three feeds across two categories, with a read item, a starred item, an item
|
|
in a feed that has no category, and content that exercises the sanitiser
|
|
(relative links, a script tag, an unclosed list, an entity).
|
|
|
|
Keyword options are forwarded to :class:`StubFreshRSS`, which is how
|
|
``tools/fake-freshrss.py`` serves this same account to a container --
|
|
``bind_host``, ``bind_port`` and ``public_host``, since a compose network needs
|
|
a wildcard bind, a fixed port and a service name in place of loopback. The
|
|
defaults are the loopback bind and ephemeral port the tests rely on.
|
|
"""
|
|
stub = StubFreshRSS(**stub_options)
|
|
tech = stub.add_feed("feed/https://example.com/tech/atom.xml", "Example Tech", ["Tech"])
|
|
news = stub.add_feed(
|
|
"feed/https://news.example.org/rss?source=daily",
|
|
"Daily News",
|
|
["News"],
|
|
)
|
|
lonely = stub.add_feed("feed/https://solo.example.net/feed.xml", "Solo Blog")
|
|
|
|
stub.add_item(
|
|
tech,
|
|
1,
|
|
"A Tech Post & Its Friends",
|
|
content=(
|
|
'<p>Intro with an <a href="/relative-link">in-page link</a>.</p>'
|
|
"<script>alert('nope')</script>"
|
|
"<ul><li>first<li>second</ul>"
|
|
'<p><img src="/images/diagram.png" alt="A diagram"></p>'
|
|
"<p>Entities: & <tag> done.</p>"
|
|
),
|
|
published=1_700_000_100,
|
|
)
|
|
stub.add_item(tech, 2, "An Older Tech Post", published=1_699_000_000)
|
|
stub.add_item(
|
|
tech,
|
|
3,
|
|
"Already Read Post",
|
|
published=1_698_000_000,
|
|
read=True,
|
|
)
|
|
stub.add_item(news, 4, "Markets Dip Slightly", published=1_700_000_200, author=None)
|
|
stub.add_item(
|
|
news,
|
|
5,
|
|
"Starred Investigation",
|
|
published=1_700_000_300,
|
|
starred=True,
|
|
)
|
|
stub.add_item(lonely, 6, "Uncategorised Musings", published=1_700_000_050)
|
|
return stub
|