"""FreshRSS client over its Google Reader compatible API. Only the endpoints the firmware client already proved out against a real server are used (see ``lib/FreshRss/src/FreshRssSync.cpp`` in the reader tree): * ``POST accounts/ClientLogin`` returns the ``Auth`` token that goes into an ``Authorization: GoogleLogin auth=...`` header. * ``GET reader/api/0/subscription/list?output=json`` lists feeds, each with its category labels. * ``GET reader/api/0/stream/contents/?output=json&r=d&n=&xt=`` fetches items. ``r=d`` is newest first, ``n`` bounds the count, and ``xt`` excludes items carrying a tag -- which is how the unread queue is filtered server-side. * ``GET reader/api/0/token`` returns the write token, and ``POST reader/api/0/edit-tag`` with ``T=`` and repeated ``i=`` marks items read. Deliberately absent: anything that would need an undocumented parameter. Where a filter is not expressible with these calls (read items), the caller filters locally rather than assuming an endpoint behaves a certain way. Note the two endpoints this module never touches: ``unread-count`` is optional and its failure is tolerated, and nothing here fetches article bodies from the open internet -- the body FreshRSS already stores is the source of truth. """ from __future__ import annotations import json import logging import ssl import threading import time import urllib.error import urllib.parse import urllib.request from dataclasses import dataclass, field from datetime import UTC, datetime from typing import Any from app import config as app_config LOGGER = logging.getLogger(__name__) # Google Reader stream and tag identifiers. READING_LIST = "user/-/state/com.google/reading-list" STARRED = "user/-/state/com.google/starred" READ_TAG = "user/-/state/com.google/read" STARRED_TAG = "user/-/state/com.google/starred" KEPT_UNREAD_TAG = "user/-/state/com.google/kept-unread" USER_AGENT = "odps-from-rss/1.0 (+https://github.com/; OPDS 1.2 catalog)" # Statuses worth retrying: the upstream is briefly unhappy, not wrong. _RETRY_STATUSES = frozenset({429, 500, 502, 503, 504, 507, 509}) _AUTH_STATUSES = frozenset({401, 403}) _MISSING_STATUSES = frozenset({400, 404}) class FreshRSSError(Exception): """An upstream call failed.""" def __init__(self, message: str, *, status: int | None = None) -> None: super().__init__(message) self.status = status class AuthenticationError(FreshRSSError): """The API password or username was rejected.""" class TransientError(FreshRSSError): """The upstream could not be reached, or answered with a retryable status.""" class NotFoundError(FreshRSSError): """The requested stream or item does not exist upstream.""" @dataclass class Subscription: """One feed in the FreshRSS account.""" id: str title: str categories: list[str] = field(default_factory=list) html_url: str | None = None unread: int | None = None @property def primary_category(self) -> str: """The label this feed is shelved under, or a stable placeholder.""" return self.categories[0] if self.categories else UNCATEGORISED @dataclass class Item: """One article.""" id: str title: str published: int updated: int feed_id: str feed_title: str content_html: str = "" author: str | None = None url: str | None = None feed_url: str | None = None categories: list[str] = field(default_factory=list) @property def is_read(self) -> bool: """True when the item is read upstream. FreshRSS keeps both the read and the kept-unread tag on an item that was read and then explicitly marked unread again, and there the explicit signal is the one that reflects what the reader wants. """ if KEPT_UNREAD_TAG in self.categories: return False return READ_TAG in self.categories @property def is_starred(self) -> bool: return STARRED_TAG in self.categories @property def published_at(self) -> datetime: """Publication time as an aware UTC datetime. Falls back to the epoch rather than raising: a feed with a broken date is not a reason to fail the whole catalog. """ stamp = self.published or self.updated or 0 try: return datetime.fromtimestamp(stamp, tz=UTC) except (OverflowError, OSError, ValueError): return datetime(1970, 1, 1, tzinfo=UTC) @property def updated_at(self) -> datetime: stamp = self.updated or self.published or 0 try: return datetime.fromtimestamp(stamp, tz=UTC) except (OverflowError, OSError, ValueError): return self.published_at UNCATEGORISED = "Uncategorised" def label_stream(label: str) -> str: """Stream id for a category label.""" return f"user/-/label/{label}" class FreshRSSClient: """A small client for one FreshRSS account. Instances are safe to share between threads: the only mutable state is the cached session token, which is guarded by a lock. """ def __init__( self, config: app_config.FreshRSSConfig, *, clock: Any = time.time, sleep: Any = time.sleep, ) -> None: self._config = config self._clock = clock # Injectable so tests can exercise retry logic without waiting it out. self._sleep = sleep self._auth_token: str | None = None self._write_token: str | None = None self._write_token_fetched_at = 0.0 self._token_lock = threading.Lock() self._tls_context = _build_tls_context(config) self._opener = urllib.request.build_opener( urllib.request.HTTPSHandler(context=self._tls_context) ) # -- public API ------------------------------------------------------ def list_subscriptions(self) -> list[Subscription]: """Every feed in the account, with its category labels.""" payload = self._get_json("/reader/api/0/subscription/list") raw_subscriptions = payload.get("subscriptions") if not isinstance(raw_subscriptions, list): raise FreshRSSError("subscription list had no subscriptions array") subscriptions: list[Subscription] = [] for entry in raw_subscriptions: parsed = _parse_subscription(entry) if parsed is not None: subscriptions.append(parsed) subscriptions.sort(key=lambda feed: feed.title.casefold()) return subscriptions def unread_counts(self) -> dict[str, int]: """Unread counts keyed by stream id. Optional upstream feature. Raises whatever the client raises, so the caller can latch the failure and carry on without counts -- the catalog is perfectly usable without them. """ payload = self._get_json("/reader/api/0/unread-count", {"all": "true"}) counts: dict[str, int] = {} for entry in payload.get("unreadcounts") or []: if not isinstance(entry, dict): continue stream_id = entry.get("id") count = entry.get("count") if isinstance(stream_id, str) and isinstance(count, int): counts[stream_id] = count return counts def stream_items( self, stream_id: str, *, count: int, exclude_read: bool = False, older_than: int | None = None, ) -> list[Item]: """Fetch items from one stream, newest first. A stream that has disappeared upstream (``400``/``404``) is reported as an empty result rather than an error: a feed you unsubscribed from should not break the catalog that still lists it. """ params = { "output": "json", "r": "d", "n": str(max(1, count)), # Cache killer: asks FreshRSS not to serve a cached response. "ck": str(int(self._clock())), } if exclude_read: params["xt"] = READ_TAG if older_than: params["ot"] = str(int(older_than)) path = "/reader/api/0/stream/contents/" + urllib.parse.quote(stream_id, safe="") try: payload = self._get_json(path, params) except NotFoundError: LOGGER.info("stream %s is gone upstream; treating it as empty", stream_id) return [] raw_items = payload.get("items") if not isinstance(raw_items, list): return [] items = [item for item in (_parse_item(entry) for entry in raw_items) if item] return items def mark_read(self, item_ids: list[str]) -> int: """Mark items read upstream. Returns the number of ids sent. Idempotent: adding the read tag to an already-read item is a no-op upstream, so a repeated download of the same book is harmless. """ wanted = [item_id for item_id in item_ids if item_id] if not wanted: return 0 token = self._write_token_value() form: list[tuple[str, str]] = [ ("T", token), ("a", READ_TAG), ] form.extend(("i", item_id) for item_id in wanted) # edit-tag needs both the session header and the write token: the header # authenticates the caller, the token authorises the write. try: self._request( "POST", "/reader/api/0/edit-tag", body=urllib.parse.urlencode(form), authenticated=True, ) except AuthenticationError: # Either the session or the write token outlived its validity; drop # both, fetch fresh ones, and retry the write once. self._forget_tokens() form[0] = ("T", self._write_token_value()) self._request( "POST", "/reader/api/0/edit-tag", body=urllib.parse.urlencode(form), authenticated=True, ) return len(wanted) def probe(self) -> tuple[bool, str]: """Check credentials and reachability, for the diagnostics endpoint.""" try: subscriptions = self.list_subscriptions() except FreshRSSError as exc: return False, str(exc) return True, f"{len(subscriptions)} feeds" def close(self) -> None: """Release upstream resources (connections are per-request).""" self._forget_tokens() # -- transport ------------------------------------------------------- def _forget_tokens(self) -> None: with self._token_lock: self._auth_token = None self._write_token = None self._write_token_fetched_at = 0.0 def _auth_header(self) -> dict[str, str]: with self._token_lock: token = self._auth_token if token is None: token = self._login() with self._token_lock: self._auth_token = token return {"Authorization": f"GoogleLogin auth={token}"} def _write_token_value(self) -> str: with self._token_lock: token = self._write_token fetched_at = self._write_token_fetched_at # Refresh hourly: the token is cheap to fetch and this bounds how long a # stale one can survive a password change upstream. if token and self._clock() - fetched_at < 3600: return token response = self._request("GET", "/reader/api/0/token", authenticated=True) token = response.decode("utf-8", "replace").strip() if not token: raise FreshRSSError("upstream returned an empty write token") with self._token_lock: self._write_token = token self._write_token_fetched_at = self._clock() return token def _login(self) -> str: """Exchange the API password for an Auth token.""" form = urllib.parse.urlencode( {"Email": self._config.username, "Passwd": self._config.api_password} ) response = self._request("POST", "/accounts/ClientLogin", body=form) text = response.decode("utf-8", "replace") for line in text.splitlines(): if line.startswith("Auth="): value = line[len("Auth=") :].strip() if value: LOGGER.debug("authenticated with FreshRSS as %s", self._config.username) return value raise AuthenticationError("FreshRSS rejected the API password") def _get_json(self, path: str, params: dict[str, str] | None = None) -> dict[str, Any]: query = dict(params or {}) query.setdefault("output", "json") response = self._request("GET", path, query=query, authenticated=True) try: payload = json.loads(response.decode("utf-8", "replace")) except (json.JSONDecodeError, UnicodeDecodeError) as exc: raise FreshRSSError(f"upstream sent a response that is not JSON ({exc})") from exc if not isinstance(payload, dict): raise FreshRSSError("upstream sent a JSON value that is not an object") return payload def _request( self, method: str, path: str, *, query: dict[str, str] | None = None, body: str | None = None, authenticated: bool = False, ) -> bytes: """Perform one upstream request, retrying transient failures.""" url = self._config.api_base + path if query: url = f"{url}?{urllib.parse.urlencode(query)}" data = body.encode("utf-8") if body is not None else None attempts = max(1, self._config.retries + 1) last_error: Exception | None = None for attempt in range(attempts): headers = {"User-Agent": USER_AGENT, "Accept": "application/json"} if body is not None: headers["Content-Type"] = "application/x-www-form-urlencoded" if authenticated: headers.update(self._auth_header()) request = urllib.request.Request(url, data=data, headers=headers, method=method) try: with self._opener.open(request, timeout=self._config.timeout_seconds) as reply: return reply.read() except urllib.error.HTTPError as exc: status = exc.code try: detail = _read_error_body(exc) finally: # The error object owns the response socket. Leaving it open # leaks one connection for every failed upstream request. exc.close() if status in _AUTH_STATUSES: raise AuthenticationError( f"FreshRSS refused the request ({status}); check the API password", status=status, ) from exc if status in _MISSING_STATUSES: raise NotFoundError( f"upstream has no such resource ({status})", status=status ) from exc if status in _RETRY_STATUSES: # A retryable status stays transient even once the attempts run # out, so the caller can tell "upstream is unwell" from # "upstream says this request is wrong". error = TransientError(f"upstream returned {status}{detail}", status=status) if attempt + 1 < attempts: last_error = error self._backoff(attempt, status) continue raise error from exc raise FreshRSSError(f"upstream returned {status}{detail}", status=status) from exc except urllib.error.URLError as exc: last_error = TransientError(f"cannot reach FreshRSS: {exc.reason}") if attempt + 1 < attempts: self._backoff(attempt, None) continue raise last_error from exc except TimeoutError as exc: last_error = TransientError("the upstream request timed out") if attempt + 1 < attempts: self._backoff(attempt, None) continue raise last_error from exc except ssl.SSLError as exc: raise FreshRSSError(f"TLS failure talking to FreshRSS: {exc}") from exc # Only reachable if the loop exhausted its attempts without raising. raise last_error or TransientError("the upstream request failed") def _backoff(self, attempt: int, status: int | None) -> None: delay = min(2.0, 0.2 * (2**attempt)) LOGGER.warning( "upstream %s, retrying in %.1fs%s", f"returned {status}" if status else "unreachable", delay, f" (attempt {attempt + 1})" if attempt else "", ) self._sleep(delay) def _build_tls_context(config: app_config.FreshRSSConfig) -> ssl.SSLContext: """Build the TLS context for upstream calls. An operator pointing at a LAN instance with a private CA sets ``ca_file``; disabling verification is possible but logged loudly, because it silently removes the protection against a machine-in-the-middle reading the API password. """ if not config.verify_tls: LOGGER.warning( "freshrss.verify_tls is false: TLS certificates are not verified. " "The API password and article contents are exposed to anyone able to " "intercept the connection." ) context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE return context context = ssl.create_default_context() if config.ca_file: context.load_verify_locations(cafile=config.ca_file) return context def _read_error_body(error: urllib.error.HTTPError) -> str: """Include a short, sanitised excerpt of an error body in the message.""" try: raw = error.read(512) except Exception: # A diagnostic excerpt must never mask the error it is describing. return "" text = raw.decode("utf-8", "replace").strip().replace("\n", " ") if not text: return "" return f": {text[:200]}" def _text(value: Any) -> str: """A trimmed string from upstream JSON, or an empty string. Upstream is an arbitrary FreshRSS build, so every field is treated as untrusted input of unknown type rather than as the shape the docs suggest. """ return value.strip() if isinstance(value, str) else "" def _as_int(value: Any) -> int: """Best-effort integer from upstream JSON, tolerating strings.""" if isinstance(value, bool): return 0 if isinstance(value, int): return value if isinstance(value, float): return int(value) if isinstance(value, str): try: return int(value) except ValueError: return 0 return 0 def _parse_subscription(entry: Any) -> Subscription | None: if not isinstance(entry, dict): return None stream_id = entry.get("id") if not isinstance(stream_id, str) or not stream_id: return None categories: list[str] = [] for category in entry.get("categories") or []: if not isinstance(category, dict): continue label = _text(category.get("label")) if label and label not in categories: categories.append(label) return Subscription( id=stream_id, title=_text(entry.get("title")) or stream_id, categories=categories, html_url=_text(entry.get("htmlUrl")) or None, ) def _parse_item(entry: Any) -> Item | None: if not isinstance(entry, dict): return None item_id = entry.get("id") if not isinstance(item_id, str) or not item_id: return None raw_origin = entry.get("origin") origin: dict[str, Any] = raw_origin if isinstance(raw_origin, dict) else {} feed_id = _text(origin.get("streamId")) return Item( id=item_id, title=_text(entry.get("title")) or "(untitled)", published=_as_int(entry.get("published")) or _timestamp_usec(entry), updated=_as_int(entry.get("updated")) or _timestamp_usec(entry), feed_id=feed_id, feed_title=_text(origin.get("title")) or feed_id, content_html=_extract_content(entry), author=_text(entry.get("author")) or None, url=_extract_url(entry, origin), feed_url=_text(origin.get("htmlUrl")) or None, categories=[tag for tag in (entry.get("categories") or []) if isinstance(tag, str)], ) def _timestamp_usec(entry: dict[str, Any]) -> int: """Derive seconds from Google Reader's microsecond timestamp, when present.""" raw = entry.get("timestampUsec") if isinstance(raw, str) and raw.isdigit(): return int(raw) // 1_000_000 if isinstance(raw, int): return raw // 1_000_000 return 0 def _extract_content(entry: dict[str, Any]) -> str: """Prefer the full body over the summary. FreshRSS stores whichever it has; ``content`` is the article body when the feed provides one, and ``summary`` is the fallback. """ for key in ("content", "summary"): block = entry.get(key) if isinstance(block, str) and block.strip(): return block if isinstance(block, dict): inner = block.get("content") if isinstance(inner, str) and inner.strip(): return inner return "" def _extract_url(entry: dict[str, Any], origin: dict[str, Any]) -> str | None: alternates = entry.get("alternate") if isinstance(alternates, list): for candidate in alternates: if not isinstance(candidate, dict): continue href = candidate.get("href") if not isinstance(href, str) or not href: continue kind = candidate.get("type") if kind in (None, "text/html"): return href for candidate in alternates: if isinstance(candidate, dict) and isinstance(candidate.get("href"), str): return candidate["href"] html_url = origin.get("htmlUrl") return html_url if isinstance(html_url, str) and html_url else None