Files
rss-to-opds/tests/test_server.py
2026-11-15 18:15:03 -05:00

544 lines
24 KiB
Python

"""End-to-end tests: a real server, talking to a real (stand-in) FreshRSS.
These run the whole stack rather than units of it: the configuration is loaded
through its validation path, the HTTP server serves on a socket, the FreshRSS
client speaks HTTP to the stub, and the EPUB that comes back is opened and
checked. The claim that matters most -- that downloading a book marks the
matching article read upstream, and only a completed download does -- can only be
verified here, by looking at what the stub received.
Image embedding is turned off in this configuration so the suite stays offline;
image handling has its own tests with a stand-in fetcher.
"""
from __future__ import annotations
import base64
import json
import tempfile
import threading
import time
import unittest
import urllib.error
import urllib.parse
import urllib.request
import zipfile
from io import BytesIO
from pathlib import Path
from app import server as app_server
from app.config import load_config, password_digest
from app.epub import ARTICLE_PATH, OPF_PATH
from tests.stub_freshrss import READING_LIST, build_demo_catalog, item_id
from tests.support import parse_xml
class _NoRedirects(urllib.request.HTTPRedirectHandler):
"""Surface a redirect as its own status instead of following it.
urlopen follows redirects transparently, so a test that asserts on a 302 would
otherwise be looking at the status of whatever the redirect pointed at.
"""
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
USERNAME = "reader"
PASSWORD = "reader-password"
TECH_FEED = "feed/https://example.com/tech/atom.xml"
class ServerTestCase(unittest.TestCase):
"""Starts a stub FreshRSS and the real server for each test."""
include_images = False
def setUp(self):
self.stub = build_demo_catalog().start()
self.addCleanup(self.stub.stop)
# The demo account ships one already-read article, so assertions about
# marking read are written against the change, not the absolute state.
self.baseline_read = set(self.stub.read)
self._dir = tempfile.TemporaryDirectory()
self.addCleanup(self._dir.cleanup)
self._no_redirects = urllib.request.build_opener(_NoRedirects)
self.config = self._write_config()
self.httpd, self.application = app_server.build_server(self.config)
self.port = self.httpd.server_address[1]
self.thread = threading.Thread(
target=self.httpd.serve_forever, kwargs={"poll_interval": 0.02}, daemon=True
)
self.thread.start()
self.addCleanup(self._stop_server)
def _stop_server(self):
self.httpd.shutdown()
self.httpd.server_close()
self.thread.join(timeout=5)
@property
def base(self) -> str:
return f"http://127.0.0.1:{self.port}"
def _write_config(self):
document = {
"server": {"host": "127.0.0.1", "port": 0, "log_level": "critical"},
"freshrss": {
"url": self.stub.api_url,
"username": self.stub.username,
"api_password": self.stub.password,
"items_per_page": 2,
"timeout_seconds": 5,
"retries": 0,
},
"catalog": {"title": "Test Catalog", "author": "Tester"},
"epub": {"include_images": self.include_images},
"auth": {
"users": [{"username": USERNAME, "password_sha256": password_digest(PASSWORD)}]
},
"cache": {"subscriptions_ttl_seconds": 1, "stream_ttl_seconds": 1},
}
path = Path(self._dir.name) / "config.json"
path.write_text(json.dumps(document), encoding="utf-8")
return load_config(str(path))
# -- HTTP helpers ----------------------------------------------------
def request(self, path, *, auth=True, method="GET", headers=None, follow_redirects=True):
request = urllib.request.Request(self.base + path, method=method)
if auth:
token = base64.b64encode(f"{USERNAME}:{PASSWORD}".encode()).decode()
request.add_header("Authorization", f"Basic {token}")
for key, value in (headers or {}).items():
request.add_header(key, value)
opener = urllib.request.urlopen if follow_redirects else self._no_redirects.open
try:
with opener(request, timeout=10) as response:
return response.status, dict(response.headers), response.read()
except urllib.error.HTTPError as error:
# With redirects suppressed, a 3xx arrives here, which is the point.
# The error object owns the response socket, so it is closed rather
# than left for the garbage collector.
try:
body = error.read()
finally:
error.close()
return error.code, dict(error.headers), body
def wait_for_mark_read(self, expected: set[str], timeout: float = 5.0) -> set[str]:
"""Wait for a mark-read to land upstream (returns what landed).
The server marks an article read only after the body has been delivered,
so from the client's side that upstream effect is asynchronous: a test
asserting immediately would be racing the handler thread. It is
deliberately not the other way round -- marking before delivery would
consume articles whose transfer failed.
"""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
landed = self.stub.read - self.baseline_read
if landed >= expected:
return landed
time.sleep(0.02)
return self.stub.read - self.baseline_read
def get_xml(self, path, *, auth=True):
status, headers, body = self.request(path, auth=auth)
self.assertEqual(status, 200, body[:400])
node = parse_xml(body)
return node, headers
def entries(self, node):
return {entry.must_find("title").all_text(): entry for entry in node.must_find_all("entry")}
def follow(self, entry, rel: str) -> str:
for link in entry.must_find_all("link"):
if link.attr("rel") == rel:
return link.must_attr("href")
raise AssertionError(f"no {rel!r} link in entry")
def acquisition_path(self, section="unread", label="Tech", feed_title="Example Tech") -> str:
"""Walk the catalog down to one feed's acquisition feed.
Shared by the suite rather than owned by one test class: the download,
pagination, search and serialisation tests all start from a feed.
"""
node, _ = self.get_xml(f"/opds/{section}")
category = self.follow(self.entries(node)[label], "subsection")
node, _ = self.get_xml(urllib.parse.urlsplit(category).path)
return urllib.parse.urlsplit(self.follow(self.entries(node)[feed_title], "subsection")).path
def download_path(self, title="A Tech Post & Its Friends", section="unread") -> str:
"""The EPUB acquisition URL for one article."""
node, _ = self.get_xml(self.acquisition_path(section=section))
entry = self.entries(node)[title]
return urllib.parse.urlsplit(
self.follow(entry, "http://opds-spec.org/acquisition/open-access")
).path
class AuthenticationTests(ServerTestCase):
def test_catalog_requires_credentials(self):
status, headers, _body = self.request("/opds", auth=False)
self.assertEqual(status, 401)
self.assertIn("Basic", headers.get("WWW-Authenticate", ""))
self.assertIn("Test" if False else "OPDS", headers.get("WWW-Authenticate", ""))
def test_wrong_password_is_rejected(self):
token = base64.b64encode(f"{USERNAME}:wrong".encode()).decode()
status, _headers, _body = self.request(
"/opds", auth=False, headers={"Authorization": f"Basic {token}"}
)
self.assertEqual(status, 401)
def test_health_needs_no_credentials_and_touches_nothing(self):
status, _headers, body = self.request("/healthz", auth=False)
self.assertEqual(status, 200)
self.assertEqual(json.loads(body), {"status": "ok"})
self.assertEqual(self.stub.requests, [])
def test_malformed_authorization_header_is_rejected(self):
for value in ("Basic", "Bearer xyz", "Basic !!!not-base64", "Basic " + "aGk="):
with self.subTest(value=value):
status, _headers, _body = self.request(
"/opds", auth=False, headers={"Authorization": value}
)
self.assertEqual(status, 401)
def test_diagnostics_requires_credentials(self):
self.assertEqual(self.request("/opds/healthz", auth=False)[0], 401)
def test_diagnostics_reports_upstream_state(self):
status, _headers, body = self.request("/opds/healthz")
self.assertEqual(status, 200)
payload = json.loads(body)
self.assertTrue(payload["upstream"]["reachable"])
self.assertEqual(payload["catalog"]["feeds"], 3)
self.assertTrue(payload["mark_read"])
class NavigationTests(ServerTestCase):
def test_bare_root_redirects_to_the_catalog(self):
# Redirects must be suppressed, or this would see the 401 that /opds
# answers an unauthenticated follower with.
status, headers, _body = self.request("/", auth=False, follow_redirects=False)
self.assertEqual(status, 302)
self.assertTrue(headers["Location"].endswith("/opds"))
def test_root_lists_the_sections(self):
node, headers = self.get_xml("/opds")
self.assertEqual(node.must_find("title").all_text(), "Test Catalog")
titles = set(self.entries(node))
self.assertIn("Unread", titles)
self.assertIn("Read", titles)
self.assertIn("Starred", titles)
def test_navigation_feed_declares_its_profile(self):
_node, headers = self.get_xml("/opds")
self.assertEqual(
headers["Content-Type"],
"application/atom+xml;profile=opds-catalog;kind=navigation",
)
def test_root_offers_a_search_link(self):
node, _headers = self.get_xml("/opds")
rels = {link.attr("rel") for link in node.must_find_all("link")}
self.assertIn("search", rels)
self.assertIn("self", rels)
def test_section_lists_categories_with_unread_counts(self):
node, _headers = self.get_xml("/opds/unread")
entries = self.entries(node)
self.assertIn("Tech", entries)
self.assertIn("News", entries)
self.assertIn("Uncategorised", entries)
self.assertIn("unread", entries["Tech"].must_find("content").all_text())
def test_category_lists_its_feeds(self):
category = self.follow(self.entries(self.get_xml("/opds/unread")[0])["Tech"], "subsection")
node, _headers = self.get_xml(urllib.parse.urlsplit(category).path)
titles = set(self.entries(node))
self.assertIn("Example Tech", titles)
self.assertIn("All in this category", titles)
def test_unknown_path_is_404(self):
self.assertEqual(self.request("/opds/nope")[0], 404)
def test_undecodable_identifier_is_404(self):
self.assertEqual(self.request("/opds/unread/c/!!!not-base64")[0], 404)
def test_post_is_not_allowed(self):
self.assertEqual(self.request("/opds", method="POST")[0], 405)
def test_opensearch_descriptor_is_served(self):
node, headers = self.get_xml("/opds/opensearch.xml")
self.assertEqual(headers["Content-Type"], "application/opensearchdescription+xml")
url = node.must_find("Url")
self.assertIn("searchTerms", url.must_attr("template"))
class AcquisitionTests(ServerTestCase):
def test_acquisition_feed_lists_articles(self):
node, headers = self.get_xml(self.acquisition_path())
self.assertEqual(
headers["Content-Type"],
"application/atom+xml;profile=opds-catalog;kind=acquisition",
)
titles = set(self.entries(node))
self.assertIn("A Tech Post & Its Friends", titles)
def test_read_articles_are_absent_from_the_unread_feed(self):
titles = set(self.entries(self.get_xml(self.acquisition_path())[0]))
self.assertNotIn("Already Read Post", titles)
def test_read_section_contains_read_articles(self):
path = self.acquisition_path(section="read")
titles = set(self.entries(self.get_xml(path)[0]))
self.assertIn("Already Read Post", titles)
def test_entry_offers_an_epub_acquisition_link(self):
node, _headers = self.get_xml(self.acquisition_path())
entry = self.entries(node)["A Tech Post & Its Friends"]
href = self.follow(entry, "http://opds-spec.org/acquisition/open-access")
self.assertTrue(href.endswith(".epub"))
types = {link.attr("type") for link in entry.must_find_all("link")}
self.assertIn("application/epub+zip", types)
# The generic relation is offered too, because readers disagree.
self.assertIn(
"http://opds-spec.org/acquisition",
{link.attr("rel") for link in entry.must_find_all("link")},
)
def test_entry_links_back_to_the_original_article(self):
node, _headers = self.get_xml(self.acquisition_path())
entry = self.entries(node)["A Tech Post & Its Friends"]
self.assertTrue(self.follow(entry, "alternate").startswith("https://"))
def test_entry_carries_its_feed_as_a_category(self):
node, _headers = self.get_xml(self.acquisition_path())
entry = self.entries(node)["A Tech Post & Its Friends"]
labels = {category.attr("label") for category in entry.must_find_all("category")}
self.assertIn("Example Tech", labels)
def test_articles_are_newest_first(self):
node, _headers = self.get_xml(self.acquisition_path())
stamps = [entry.must_find("updated").all_text() for entry in node.must_find_all("entry")]
self.assertEqual(stamps, sorted(stamps, reverse=True))
def test_pagination_link_appears_when_there_is_more(self):
# The test configuration serves two items per page, and the reading list
# holds five unread articles.
node, _headers = self.get_xml("/opds/unread/all")
rels = {link.attr("rel") for link in node.must_find_all("link")}
self.assertIn("next", rels)
self.assertIn("first", rels)
self.assertNotIn("previous", rels)
def test_second_page_is_reachable_and_different(self):
path = "/opds/unread/all"
first, _ = self.get_xml(path)
second, _ = self.get_xml(path + "?page=2")
first_ids = {entry.must_find("id").all_text() for entry in first.must_find_all("entry")}
second_ids = {entry.must_find("id").all_text() for entry in second.must_find_all("entry")}
self.assertTrue(second_ids)
self.assertFalse(first_ids & second_ids)
def test_starred_section_lists_starred_articles(self):
titles = set(self.entries(self.get_xml("/opds/starred")[0]))
self.assertIn("Starred Investigation", titles)
def test_search_finds_articles_in_the_window(self):
node, _headers = self.get_xml("/opds/search?q=Tech+Post")
titles = set(self.entries(node))
self.assertIn("A Tech Post & Its Friends", titles)
self.assertNotIn("Markets Dip Slightly", titles)
def test_empty_search_returns_an_empty_feed(self):
# A search with no term is a valid, empty acquisition feed rather than an
# error, so a reader that sends one sees an empty list.
node, _headers = self.get_xml("/opds/search?q=")
self.assertEqual(node.find_all("entry"), [])
def test_bad_page_number_falls_back_to_the_first_page(self):
node, _headers = self.get_xml(self.acquisition_path() + "?page=nonsense")
self.assertTrue(node.must_find_all("entry"))
class DownloadTests(ServerTestCase):
"""The claim under test: a completed download is what marks an article read."""
def test_download_returns_an_epub(self):
status, headers, body = self.request(self.download_path())
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "application/epub+zip")
disposition = headers["Content-Disposition"]
self.assertTrue(disposition.startswith("attachment;"))
# Both spellings are sent: a plain name every reader understands, and an
# RFC 5987 one so a non-ASCII title survives.
self.assertIn('filename="a-tech-post-its-friends.epub"', disposition)
self.assertIn("filename*=UTF-8''", disposition)
def test_downloaded_file_is_a_valid_epub(self):
_status, _headers, body = self.request(self.download_path())
archive = zipfile.ZipFile(BytesIO(body))
self.addCleanup(archive.close)
self.assertEqual(archive.namelist()[0], "mimetype")
self.assertEqual(archive.read("mimetype"), b"application/epub+zip")
self.assertEqual(archive.getinfo("mimetype").compress_type, zipfile.ZIP_STORED)
opf = parse_xml(archive.read(OPF_PATH))
self.assertEqual(opf.must_find("title").all_text(), "A Tech Post & Its Friends")
# The body went through the sanitiser on the way in.
article = archive.read(ARTICLE_PATH).decode("utf-8")
self.assertNotIn("alert(", article)
self.assertIn("Entities: &amp;", article)
def test_download_marks_the_article_read_upstream(self):
self.request(self.download_path())
self.assertEqual(self.wait_for_mark_read({item_id(1)}), {item_id(1)})
edits = self.stub.find_requests("/reader/api/0/edit-tag")
self.assertEqual(len(edits), 1)
self.assertIn("i=tag%3Agoogle.com", edits[0]["body"])
def test_downloads_mark_only_their_own_article(self):
self.request(self.download_path())
self.assertEqual(self.wait_for_mark_read({item_id(1)}), {item_id(1)})
def test_head_does_not_mark_read(self):
# A reader probing the URL, or a CDN, must not consume the article.
status, headers, body = self.request(self.download_path(), method="HEAD")
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "application/epub+zip")
self.assertEqual(body, b"")
# Deterministic rather than a race: a HEAD never reaches the mark-read.
self.assertEqual(self.stub.read - self.baseline_read, set())
self.assertEqual(self.stub.find_requests("/reader/api/0/edit-tag"), [])
def test_missing_article_is_404(self):
stream = base64.urlsafe_b64encode(TECH_FEED.encode()).decode().rstrip("=")
absent = (
base64.urlsafe_b64encode(b"tag:google.com,2005:reader/item/nope").decode().rstrip("=")
)
status, _headers, _body = self.request(f"/opds/d/{stream}/{absent}/nope.epub")
self.assertEqual(status, 404)
def test_article_id_is_encoded_in_the_url_and_survives(self):
# FreshRSS ids contain ':' and '/', so the URL must carry them encoded.
path = self.download_path()
self.assertNotIn("tag:google.com", path)
self.assertEqual(path.count("/"), 5)
def test_marked_read_article_leaves_the_unread_queue(self):
path = self.download_path()
self.request(path)
self.wait_for_mark_read({item_id(1)})
# The stream cache is invalidated by the mark-read, so the next browse
# reflects it without waiting for a TTL.
titles = set(self.entries(self.get_xml(self.acquisition_path())[0]))
self.assertNotIn("A Tech Post & Its Friends", titles)
def test_marked_read_article_appears_in_the_read_section(self):
self.request(self.download_path())
self.wait_for_mark_read({item_id(1)})
path = self.acquisition_path(section="read")
titles = set(self.entries(self.get_xml(path)[0]))
self.assertIn("A Tech Post & Its Friends", titles)
def test_mark_read_can_be_disabled(self):
self.application.config.freshrss.mark_read = False
self.request(self.download_path())
self.assertEqual(self.stub.read - self.baseline_read, set())
self.assertEqual(self.stub.find_requests("/reader/api/0/edit-tag"), [])
def test_advertised_unread_count_refreshes_after_a_download(self):
"""The count must not lag the queue it describes.
Unread counts come from the cached account snapshot, so a mark-read has to
drop that cache too -- otherwise a reader downloads an article and is still
told the old total, with one fewer entry underneath it.
"""
def advertised_unread() -> int:
node, _ = self.get_xml("/opds/unread")
for entry in node.must_find_all("entry"):
if entry.must_find("title").all_text() == "All feeds":
return int(entry.must_find("content").all_text().split()[0])
raise AssertionError("no 'All feeds' entry in the unread catalog")
before = advertised_unread()
self.request(self.download_path())
self.wait_for_mark_read({item_id(1)})
self.assertEqual(advertised_unread(), before - 1)
def test_second_download_is_still_served_after_the_queue_moved_on(self):
path = self.download_path()
self.request(path)
status, _headers, body = self.request(path)
self.assertEqual(status, 200)
self.assertTrue(body.startswith(b"PK"))
class UpstreamFailureTests(ServerTestCase):
def test_upstream_outage_is_a_bad_gateway(self):
self.stub.fail_next(503, count=10, path="/subscription/list")
status, _headers, _body = self.request("/opds")
self.assertEqual(status, 502)
def test_catalog_recovers_once_upstream_returns(self):
# Exactly one failure, so the next attempt succeeds.
self.stub.fail_next(503, path="/subscription/list")
self.assertEqual(self.request("/opds")[0], 502)
self.application.invalidate()
self.assertEqual(self.request("/opds")[0], 200)
def test_unread_counts_missing_is_not_fatal(self):
# The feature is optional; the catalog must still browse without it.
self.stub.supports_unread_counts = False
self.application.invalidate()
status, _headers, body = self.request("/opds/unread")
self.assertEqual(status, 200)
node = parse_xml(body)
titles = {entry.must_find("title").all_text() for entry in node.must_find_all("entry")}
self.assertIn("Tech", titles)
def test_unknown_host_is_refused_when_an_allowlist_is_set(self):
self.application.config.server.allowed_hosts = ["catalog.example.com"]
status, _headers, _body = self.request("/opds", auth=False)
self.assertEqual(status, 500)
def test_public_base_url_controls_the_links(self):
self.application.config.server.public_base_url = "https://catalog.example.com/books"
node, _headers = self.get_xml("/opds")
hrefs = {link.must_attr("href") for link in node.must_find_all("link")}
self.assertTrue(
any(href.startswith("https://catalog.example.com/books/opds") for href in hrefs)
)
class SerializationTests(ServerTestCase):
"""Responses must be correct XML even when titles contain markup."""
def test_titles_with_markup_are_escaped(self):
self.stub.subscriptions[0]["title"] = 'Feed & <Friends> "quoted"'
self.application.invalidate()
category = urllib.parse.urlsplit(
self.follow(self.entries(self.get_xml("/opds/unread")[0])["Tech"], "subsection")
).path
node, _headers = self.get_xml(category)
titles = {entry.must_find("title").all_text() for entry in node.must_find_all("entry")}
self.assertIn('Feed & <Friends> "quoted"', titles)
def test_feed_ids_are_stable_across_requests(self):
first, _ = self.get_xml("/opds/unread")
second, _ = self.get_xml("/opds/unread")
self.assertEqual(first.must_find("id").all_text(), second.must_find("id").all_text())
def test_entry_ids_are_unique_within_a_feed(self):
node, _headers = self.get_xml(self.acquisition_path())
ids = [entry.must_find("id").all_text() for entry in node.must_find_all("entry")]
self.assertEqual(len(ids), len(set(ids)))
if __name__ == "__main__":
unittest.main()