Initial commit

This commit is contained in:
2026-11-15 18:15:03 -05:00
commit b76aea2899
36 changed files with 9424 additions and 0 deletions

598
app/freshrss.py Normal file
View File

@@ -0,0 +1,598 @@
"""FreshRSS client over its Google Reader compatible API.
Only the endpoints the firmware client already proved out against a real server
are used (see ``lib/FreshRss/src/FreshRssSync.cpp`` in the reader tree):
* ``POST accounts/ClientLogin`` returns the ``Auth`` token that goes into an
``Authorization: GoogleLogin auth=...`` header.
* ``GET reader/api/0/subscription/list?output=json`` lists feeds, each with its
category labels.
* ``GET reader/api/0/stream/contents/<stream>?output=json&r=d&n=&xt=`` fetches
items. ``r=d`` is newest first, ``n`` bounds the count, and ``xt`` excludes
items carrying a tag -- which is how the unread queue is filtered server-side.
* ``GET reader/api/0/token`` returns the write token, and
``POST reader/api/0/edit-tag`` with ``T=`` and repeated ``i=`` marks items read.
Deliberately absent: anything that would need an undocumented parameter. Where a
filter is not expressible with these calls (read items), the caller filters
locally rather than assuming an endpoint behaves a certain way.
Note the two endpoints this module never touches: ``unread-count`` is optional
and its failure is tolerated, and nothing here fetches article bodies from the
open internet -- the body FreshRSS already stores is the source of truth.
"""
from __future__ import annotations
import json
import logging
import ssl
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
from app import config as app_config
LOGGER = logging.getLogger(__name__)
# Google Reader stream and tag identifiers.
READING_LIST = "user/-/state/com.google/reading-list"
STARRED = "user/-/state/com.google/starred"
READ_TAG = "user/-/state/com.google/read"
STARRED_TAG = "user/-/state/com.google/starred"
KEPT_UNREAD_TAG = "user/-/state/com.google/kept-unread"
USER_AGENT = "odps-from-rss/1.0 (+https://github.com/; OPDS 1.2 catalog)"
# Statuses worth retrying: the upstream is briefly unhappy, not wrong.
_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504, 507, 509})
_AUTH_STATUSES = frozenset({401, 403})
_MISSING_STATUSES = frozenset({400, 404})
class FreshRSSError(Exception):
"""An upstream call failed."""
def __init__(self, message: str, *, status: int | None = None) -> None:
super().__init__(message)
self.status = status
class AuthenticationError(FreshRSSError):
"""The API password or username was rejected."""
class TransientError(FreshRSSError):
"""The upstream could not be reached, or answered with a retryable status."""
class NotFoundError(FreshRSSError):
"""The requested stream or item does not exist upstream."""
@dataclass
class Subscription:
"""One feed in the FreshRSS account."""
id: str
title: str
categories: list[str] = field(default_factory=list)
html_url: str | None = None
unread: int | None = None
@property
def primary_category(self) -> str:
"""The label this feed is shelved under, or a stable placeholder."""
return self.categories[0] if self.categories else UNCATEGORISED
@dataclass
class Item:
"""One article."""
id: str
title: str
published: int
updated: int
feed_id: str
feed_title: str
content_html: str = ""
author: str | None = None
url: str | None = None
feed_url: str | None = None
categories: list[str] = field(default_factory=list)
@property
def is_read(self) -> bool:
"""True when the item is read upstream.
FreshRSS keeps both the read and the kept-unread tag on an item that was
read and then explicitly marked unread again, and there the explicit
signal is the one that reflects what the reader wants.
"""
if KEPT_UNREAD_TAG in self.categories:
return False
return READ_TAG in self.categories
@property
def is_starred(self) -> bool:
return STARRED_TAG in self.categories
@property
def published_at(self) -> datetime:
"""Publication time as an aware UTC datetime.
Falls back to the epoch rather than raising: a feed with a broken date is
not a reason to fail the whole catalog.
"""
stamp = self.published or self.updated or 0
try:
return datetime.fromtimestamp(stamp, tz=UTC)
except (OverflowError, OSError, ValueError):
return datetime(1970, 1, 1, tzinfo=UTC)
@property
def updated_at(self) -> datetime:
stamp = self.updated or self.published or 0
try:
return datetime.fromtimestamp(stamp, tz=UTC)
except (OverflowError, OSError, ValueError):
return self.published_at
UNCATEGORISED = "Uncategorised"
def label_stream(label: str) -> str:
"""Stream id for a category label."""
return f"user/-/label/{label}"
class FreshRSSClient:
"""A small client for one FreshRSS account.
Instances are safe to share between threads: the only mutable state is the
cached session token, which is guarded by a lock.
"""
def __init__(
self,
config: app_config.FreshRSSConfig,
*,
clock: Any = time.time,
sleep: Any = time.sleep,
) -> None:
self._config = config
self._clock = clock
# Injectable so tests can exercise retry logic without waiting it out.
self._sleep = sleep
self._auth_token: str | None = None
self._write_token: str | None = None
self._write_token_fetched_at = 0.0
self._token_lock = threading.Lock()
self._tls_context = _build_tls_context(config)
self._opener = urllib.request.build_opener(
urllib.request.HTTPSHandler(context=self._tls_context)
)
# -- public API ------------------------------------------------------
def list_subscriptions(self) -> list[Subscription]:
"""Every feed in the account, with its category labels."""
payload = self._get_json("/reader/api/0/subscription/list")
raw_subscriptions = payload.get("subscriptions")
if not isinstance(raw_subscriptions, list):
raise FreshRSSError("subscription list had no subscriptions array")
subscriptions: list[Subscription] = []
for entry in raw_subscriptions:
parsed = _parse_subscription(entry)
if parsed is not None:
subscriptions.append(parsed)
subscriptions.sort(key=lambda feed: feed.title.casefold())
return subscriptions
def unread_counts(self) -> dict[str, int]:
"""Unread counts keyed by stream id. Optional upstream feature.
Raises whatever the client raises, so the caller can latch the failure and
carry on without counts -- the catalog is perfectly usable without them.
"""
payload = self._get_json("/reader/api/0/unread-count", {"all": "true"})
counts: dict[str, int] = {}
for entry in payload.get("unreadcounts") or []:
if not isinstance(entry, dict):
continue
stream_id = entry.get("id")
count = entry.get("count")
if isinstance(stream_id, str) and isinstance(count, int):
counts[stream_id] = count
return counts
def stream_items(
self,
stream_id: str,
*,
count: int,
exclude_read: bool = False,
older_than: int | None = None,
) -> list[Item]:
"""Fetch items from one stream, newest first.
A stream that has disappeared upstream (``400``/``404``) is reported as an
empty result rather than an error: a feed you unsubscribed from should not
break the catalog that still lists it.
"""
params = {
"output": "json",
"r": "d",
"n": str(max(1, count)),
# Cache killer: asks FreshRSS not to serve a cached response.
"ck": str(int(self._clock())),
}
if exclude_read:
params["xt"] = READ_TAG
if older_than:
params["ot"] = str(int(older_than))
path = "/reader/api/0/stream/contents/" + urllib.parse.quote(stream_id, safe="")
try:
payload = self._get_json(path, params)
except NotFoundError:
LOGGER.info("stream %s is gone upstream; treating it as empty", stream_id)
return []
raw_items = payload.get("items")
if not isinstance(raw_items, list):
return []
items = [item for item in (_parse_item(entry) for entry in raw_items) if item]
return items
def mark_read(self, item_ids: list[str]) -> int:
"""Mark items read upstream. Returns the number of ids sent.
Idempotent: adding the read tag to an already-read item is a no-op
upstream, so a repeated download of the same book is harmless.
"""
wanted = [item_id for item_id in item_ids if item_id]
if not wanted:
return 0
token = self._write_token_value()
form: list[tuple[str, str]] = [
("T", token),
("a", READ_TAG),
]
form.extend(("i", item_id) for item_id in wanted)
# edit-tag needs both the session header and the write token: the header
# authenticates the caller, the token authorises the write.
try:
self._request(
"POST",
"/reader/api/0/edit-tag",
body=urllib.parse.urlencode(form),
authenticated=True,
)
except AuthenticationError:
# Either the session or the write token outlived its validity; drop
# both, fetch fresh ones, and retry the write once.
self._forget_tokens()
form[0] = ("T", self._write_token_value())
self._request(
"POST",
"/reader/api/0/edit-tag",
body=urllib.parse.urlencode(form),
authenticated=True,
)
return len(wanted)
def probe(self) -> tuple[bool, str]:
"""Check credentials and reachability, for the diagnostics endpoint."""
try:
subscriptions = self.list_subscriptions()
except FreshRSSError as exc:
return False, str(exc)
return True, f"{len(subscriptions)} feeds"
def close(self) -> None:
"""Release upstream resources (connections are per-request)."""
self._forget_tokens()
# -- transport -------------------------------------------------------
def _forget_tokens(self) -> None:
with self._token_lock:
self._auth_token = None
self._write_token = None
self._write_token_fetched_at = 0.0
def _auth_header(self) -> dict[str, str]:
with self._token_lock:
token = self._auth_token
if token is None:
token = self._login()
with self._token_lock:
self._auth_token = token
return {"Authorization": f"GoogleLogin auth={token}"}
def _write_token_value(self) -> str:
with self._token_lock:
token = self._write_token
fetched_at = self._write_token_fetched_at
# Refresh hourly: the token is cheap to fetch and this bounds how long a
# stale one can survive a password change upstream.
if token and self._clock() - fetched_at < 3600:
return token
response = self._request("GET", "/reader/api/0/token", authenticated=True)
token = response.decode("utf-8", "replace").strip()
if not token:
raise FreshRSSError("upstream returned an empty write token")
with self._token_lock:
self._write_token = token
self._write_token_fetched_at = self._clock()
return token
def _login(self) -> str:
"""Exchange the API password for an Auth token."""
form = urllib.parse.urlencode(
{"Email": self._config.username, "Passwd": self._config.api_password}
)
response = self._request("POST", "/accounts/ClientLogin", body=form)
text = response.decode("utf-8", "replace")
for line in text.splitlines():
if line.startswith("Auth="):
value = line[len("Auth=") :].strip()
if value:
LOGGER.debug("authenticated with FreshRSS as %s", self._config.username)
return value
raise AuthenticationError("FreshRSS rejected the API password")
def _get_json(self, path: str, params: dict[str, str] | None = None) -> dict[str, Any]:
query = dict(params or {})
query.setdefault("output", "json")
response = self._request("GET", path, query=query, authenticated=True)
try:
payload = json.loads(response.decode("utf-8", "replace"))
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
raise FreshRSSError(f"upstream sent a response that is not JSON ({exc})") from exc
if not isinstance(payload, dict):
raise FreshRSSError("upstream sent a JSON value that is not an object")
return payload
def _request(
self,
method: str,
path: str,
*,
query: dict[str, str] | None = None,
body: str | None = None,
authenticated: bool = False,
) -> bytes:
"""Perform one upstream request, retrying transient failures."""
url = self._config.api_base + path
if query:
url = f"{url}?{urllib.parse.urlencode(query)}"
data = body.encode("utf-8") if body is not None else None
attempts = max(1, self._config.retries + 1)
last_error: Exception | None = None
for attempt in range(attempts):
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
if body is not None:
headers["Content-Type"] = "application/x-www-form-urlencoded"
if authenticated:
headers.update(self._auth_header())
request = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
with self._opener.open(request, timeout=self._config.timeout_seconds) as reply:
return reply.read()
except urllib.error.HTTPError as exc:
status = exc.code
try:
detail = _read_error_body(exc)
finally:
# The error object owns the response socket. Leaving it open
# leaks one connection for every failed upstream request.
exc.close()
if status in _AUTH_STATUSES:
raise AuthenticationError(
f"FreshRSS refused the request ({status}); check the API password",
status=status,
) from exc
if status in _MISSING_STATUSES:
raise NotFoundError(
f"upstream has no such resource ({status})", status=status
) from exc
if status in _RETRY_STATUSES:
# A retryable status stays transient even once the attempts run
# out, so the caller can tell "upstream is unwell" from
# "upstream says this request is wrong".
error = TransientError(f"upstream returned {status}{detail}", status=status)
if attempt + 1 < attempts:
last_error = error
self._backoff(attempt, status)
continue
raise error from exc
raise FreshRSSError(f"upstream returned {status}{detail}", status=status) from exc
except urllib.error.URLError as exc:
last_error = TransientError(f"cannot reach FreshRSS: {exc.reason}")
if attempt + 1 < attempts:
self._backoff(attempt, None)
continue
raise last_error from exc
except TimeoutError as exc:
last_error = TransientError("the upstream request timed out")
if attempt + 1 < attempts:
self._backoff(attempt, None)
continue
raise last_error from exc
except ssl.SSLError as exc:
raise FreshRSSError(f"TLS failure talking to FreshRSS: {exc}") from exc
# Only reachable if the loop exhausted its attempts without raising.
raise last_error or TransientError("the upstream request failed")
def _backoff(self, attempt: int, status: int | None) -> None:
delay = min(2.0, 0.2 * (2**attempt))
LOGGER.warning(
"upstream %s, retrying in %.1fs%s",
f"returned {status}" if status else "unreachable",
delay,
f" (attempt {attempt + 1})" if attempt else "",
)
self._sleep(delay)
def _build_tls_context(config: app_config.FreshRSSConfig) -> ssl.SSLContext:
"""Build the TLS context for upstream calls.
An operator pointing at a LAN instance with a private CA sets ``ca_file``;
disabling verification is possible but logged loudly, because it silently
removes the protection against a machine-in-the-middle reading the API
password.
"""
if not config.verify_tls:
LOGGER.warning(
"freshrss.verify_tls is false: TLS certificates are not verified. "
"The API password and article contents are exposed to anyone able to "
"intercept the connection."
)
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
return context
context = ssl.create_default_context()
if config.ca_file:
context.load_verify_locations(cafile=config.ca_file)
return context
def _read_error_body(error: urllib.error.HTTPError) -> str:
"""Include a short, sanitised excerpt of an error body in the message."""
try:
raw = error.read(512)
except Exception:
# A diagnostic excerpt must never mask the error it is describing.
return ""
text = raw.decode("utf-8", "replace").strip().replace("\n", " ")
if not text:
return ""
return f": {text[:200]}"
def _text(value: Any) -> str:
"""A trimmed string from upstream JSON, or an empty string.
Upstream is an arbitrary FreshRSS build, so every field is treated as
untrusted input of unknown type rather than as the shape the docs suggest.
"""
return value.strip() if isinstance(value, str) else ""
def _as_int(value: Any) -> int:
"""Best-effort integer from upstream JSON, tolerating strings."""
if isinstance(value, bool):
return 0
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
if isinstance(value, str):
try:
return int(value)
except ValueError:
return 0
return 0
def _parse_subscription(entry: Any) -> Subscription | None:
if not isinstance(entry, dict):
return None
stream_id = entry.get("id")
if not isinstance(stream_id, str) or not stream_id:
return None
categories: list[str] = []
for category in entry.get("categories") or []:
if not isinstance(category, dict):
continue
label = _text(category.get("label"))
if label and label not in categories:
categories.append(label)
return Subscription(
id=stream_id,
title=_text(entry.get("title")) or stream_id,
categories=categories,
html_url=_text(entry.get("htmlUrl")) or None,
)
def _parse_item(entry: Any) -> Item | None:
if not isinstance(entry, dict):
return None
item_id = entry.get("id")
if not isinstance(item_id, str) or not item_id:
return None
raw_origin = entry.get("origin")
origin: dict[str, Any] = raw_origin if isinstance(raw_origin, dict) else {}
feed_id = _text(origin.get("streamId"))
return Item(
id=item_id,
title=_text(entry.get("title")) or "(untitled)",
published=_as_int(entry.get("published")) or _timestamp_usec(entry),
updated=_as_int(entry.get("updated")) or _timestamp_usec(entry),
feed_id=feed_id,
feed_title=_text(origin.get("title")) or feed_id,
content_html=_extract_content(entry),
author=_text(entry.get("author")) or None,
url=_extract_url(entry, origin),
feed_url=_text(origin.get("htmlUrl")) or None,
categories=[tag for tag in (entry.get("categories") or []) if isinstance(tag, str)],
)
def _timestamp_usec(entry: dict[str, Any]) -> int:
"""Derive seconds from Google Reader's microsecond timestamp, when present."""
raw = entry.get("timestampUsec")
if isinstance(raw, str) and raw.isdigit():
return int(raw) // 1_000_000
if isinstance(raw, int):
return raw // 1_000_000
return 0
def _extract_content(entry: dict[str, Any]) -> str:
"""Prefer the full body over the summary.
FreshRSS stores whichever it has; ``content`` is the article body when the
feed provides one, and ``summary`` is the fallback.
"""
for key in ("content", "summary"):
block = entry.get(key)
if isinstance(block, str) and block.strip():
return block
if isinstance(block, dict):
inner = block.get("content")
if isinstance(inner, str) and inner.strip():
return inner
return ""
def _extract_url(entry: dict[str, Any], origin: dict[str, Any]) -> str | None:
alternates = entry.get("alternate")
if isinstance(alternates, list):
for candidate in alternates:
if not isinstance(candidate, dict):
continue
href = candidate.get("href")
if not isinstance(href, str) or not href:
continue
kind = candidate.get("type")
if kind in (None, "text/html"):
return href
for candidate in alternates:
if isinstance(candidate, dict) and isinstance(candidate.get("href"), str):
return candidate["href"]
html_url = origin.get("htmlUrl")
return html_url if isinstance(html_url, str) and html_url else None