Initial commit

This commit is contained in:
2026-11-15 18:15:03 -05:00
commit b76aea2899
36 changed files with 9424 additions and 0 deletions

12
.dockerignore Normal file
View File

@@ -0,0 +1,12 @@
.git
.gitignore
.pi
.env
config.json
# Both Dockerfiles share this context. The main image copies only app/, so the
# test and tool sources stay out of it; the demo image needs them.
__pycache__/
*.py[cod]
*.md
!README.md
Makefile

8
.env.example Normal file
View File

@@ -0,0 +1,8 @@
# Copy to .env and fill in, then `docker compose up -d --build`.
# FreshRSS API password: Settings -> Account -> API in FreshRSS.
# This is NOT your login password.
FRESHRSS_API_PASSWORD=replace-me
# sha256 of the password your e-reader will use, from: make hash-password
OPDS_PASSWORD_SHA256=replace-me-with-64-hex-characters

26
.gitignore vendored Normal file
View File

@@ -0,0 +1,26 @@
# Local configuration and secrets -- never commit these.
config.json
.env
*.local.json
# Python
__pycache__/
*.py[cod]
*.egg-info/
.venv/
venv/
# Test and tool output
# Tool and linter caches
.pytest_cache/
.ruff_cache/
.mypy_cache/
.coverage
htmlcov/
*.epub
# Editors
.idea/
.vscode/
*.swp

View File

@@ -0,0 +1,29 @@
{
"id": "b880b783d",
"name": "Docker image build",
"command": "cd /media/git/odps-from-rss && docker build -t opds-from-rss:latest . 2>&1",
"description": "Build the container image to verify the zero-dependency Dockerfile works end to end.",
"status": "completed",
"outputPath": ".pi/tasks/01a1e156-4fa0-722f-83e8-c971d55b7a0e-2566715/b880b783d.output",
"cwd": "/media/git/odps-from-rss",
"startTime": 1794783944805,
"endTime": 1794783954328,
"exitCode": 0,
"signal": null,
"pid": 2607703,
"bytesWritten": 3628,
"isAgent": false,
"surviveReload": false,
"notified": true,
"notifyOnCompletion": true,
"triggerOnCompletion": true,
"timeoutSeconds": 900,
"shellPolicy": {
"policy": "inherit",
"executable": "/bin/bash",
"argvPrefix": [
"-c"
],
"dialect": "bash"
}
}

View File

@@ -0,0 +1,76 @@
#0 building with "default" instance using docker driver
#1 [internal] load build definition from Dockerfile
#1 transferring dockerfile:
#1 transferring dockerfile: 1.07kB done
#1 DONE 0.2s
#2 [internal] load metadata for docker.io/library/python:3.13-alpine
#2 DONE 1.2s
#3 [internal] load .dockerignore
#3 transferring context:
#3 transferring context: 139B done
#3 DONE 0.2s
#4 [internal] load build context
#4 ...
#5 [1/5] FROM docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e
#5 resolve docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e 0.4s done
#5 DONE 0.4s
#4 [internal] load build context
#4 transferring context: 491.01kB done
#4 DONE 0.6s
#5 [1/5] FROM docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e
#5 DONE 1.1s
#5 [1/5] FROM docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e
#5 extracting sha256:881fc932939efef04ca120ad44bbd2c0917851eb247aa4995428afb1e80ede17
#5 sha256:17af2db0edc9c7ff966b9f7e8424caed1203bc5cec5e3c061f3516ef5b8d9ece 248B / 248B 0.1s done
#5 extracting sha256:881fc932939efef04ca120ad44bbd2c0917851eb247aa4995428afb1e80ede17 0.3s done
#5 sha256:f9abc654947e09e316e7703544bd7e68123d383aeb70c5a9cf15be03e71721ce 0B / 12.67MB 0.2s
#5 sha256:ce6949f6b5329553c2f3fc626421da4f70d633cfddb6ad2227f3ac0b7205bd8f 0B / 412.00kB 0.2s
#5 sha256:f9abc654947e09e316e7703544bd7e68123d383aeb70c5a9cf15be03e71721ce 4.19MB / 12.67MB 0.3s
#5 sha256:ce6949f6b5329553c2f3fc626421da4f70d633cfddb6ad2227f3ac0b7205bd8f 412.00kB / 412.00kB 0.2s done
#5 extracting sha256:ce6949f6b5329553c2f3fc626421da4f70d633cfddb6ad2227f3ac0b7205bd8f
#5 sha256:f9abc654947e09e316e7703544bd7e68123d383aeb70c5a9cf15be03e71721ce 12.67MB / 12.67MB 0.5s done
#5 extracting sha256:ce6949f6b5329553c2f3fc626421da4f70d633cfddb6ad2227f3ac0b7205bd8f 0.2s done
#5 DONE 1.7s
#5 [1/5] FROM docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e
#5 extracting sha256:f9abc654947e09e316e7703544bd7e68123d383aeb70c5a9cf15be03e71721ce
#5 extracting sha256:f9abc654947e09e316e7703544bd7e68123d383aeb70c5a9cf15be03e71721ce 0.3s done
#5 DONE 2.0s
#5 [1/5] FROM docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e
#5 extracting sha256:17af2db0edc9c7ff966b9f7e8424caed1203bc5cec5e3c061f3516ef5b8d9ece 0.1s done
#5 DONE 2.1s
#6 [2/5] RUN apk add --no-cache ca-certificates && adduser -D -u 10001 -h /srv app
#6 0.739 OK: 11.4 MiB in 29 packages
#6 DONE 1.0s
#7 [3/5] WORKDIR /srv
#7 DONE 0.3s
#8 [4/5] COPY app/ ./app/
#8 DONE 0.3s
#9 [5/5] COPY config.example.json ./
#9 DONE 0.3s
#10 exporting to image
#10 exporting layers
#10 exporting layers 1.6s done
#10 exporting manifest sha256:701ef954b346b1f95b3ce1975be05310709337bda6897169db926e7ce7e93d18 0.1s done
#10 exporting config sha256:b098ef38aca29a44c9e971d5f7e128f7bb3dc43e4711f916fa24ad042ec443a1 0.1s done
#10 exporting attestation manifest sha256:0bdcacb0668771798a65cf1b8fa04c1460748786ea2b9d166b1436a416750f34 0.1s done
#10 exporting manifest list sha256:28c19d6010bbc1d8452a93d32e1ad8aa1e8f96b9a5e2d147bf93e99f4ac29f24
#10 exporting manifest list sha256:28c19d6010bbc1d8452a93d32e1ad8aa1e8f96b9a5e2d147bf93e99f4ac29f24 0.1s done
#10 naming to docker.io/library/opds-from-rss:latest 0.0s done
#10 unpacking to docker.io/library/opds-from-rss:latest
#10 unpacking to docker.io/library/opds-from-rss:latest 0.8s done
#10 DONE 2.9s

27
Dockerfile Normal file
View File

@@ -0,0 +1,27 @@
# The whole server is the Python standard library, so the image needs no build
# toolchain, no wheels and no virtual environment -- just an interpreter, a trust
# store for upstream TLS, and the code.
FROM python:3.13-alpine
RUN apk add --no-cache ca-certificates \
&& adduser -D -u 10001 -h /srv app
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
CONFIG_PATH=/config/config.json
WORKDIR /srv
COPY app/ ./app/
COPY config.example.json ./
# Runs as an unprivileged user with no writable path: the server keeps nothing on
# disk, so the container can be run read-only (see docker-compose.yaml).
USER app
EXPOSE 8080
# The health endpoint answers without credentials and never calls upstream, so
# this checks the process, not the network.
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
CMD ["python", "-c", "import sys, urllib.request; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/healthz', timeout=4).status == 200 else 1)"]
ENTRYPOINT ["python", "-m", "app"]

31
Dockerfile.demo Normal file
View File

@@ -0,0 +1,31 @@
# The fake FreshRSS from the demo stack.
#
# It serves the same in-process stand-in the test suite uses, which is why this
# image needs tests/ and tools/ -- the production image copies only app/ and stays
# minimal. Nothing here talks to a real account: the account is three feeds of
# sample articles held in memory.
FROM python:3.13-alpine
RUN adduser -D -u 10001 -h /srv app
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1
WORKDIR /srv
COPY app/ ./app/
COPY tests/ ./tests/
COPY tools/ ./tools/
USER app
EXPOSE 8099
# Only says the fake server is answering; it is what the catalogue waits for.
HEALTHCHECK --interval=2s --timeout=3s --start-period=2s --retries=15 \
CMD ["python", "-c", "import sys, urllib.request; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8099/healthz', timeout=2).status == 200 else 1)"]
ENTRYPOINT ["python", "tools/fake-freshrss.py"]
# Binds the whole container interface so the catalogue can reach it by service
# name, and writes its generated configuration to /tmp purely so the demo
# credentials show up in the log.
CMD ["--api-bind=0.0.0.0", "--api-port=8099", "--api-public-host=fresh", \
"--app-host=0.0.0.0", "--app-port=8080", "--write-config=/tmp/config.json"]

71
Makefile Normal file
View File

@@ -0,0 +1,71 @@
# Deployment and development helpers. Nothing here needs any dependency beyond
# Python 3.11+ and, for the container targets, Docker.
CONFIG ?= config.json
IMAGE ?= opds-from-rss:latest
PYTHON ?= python3
.PHONY: help test run check demo demo-up demo-down build up down logs shell hash-password lint clean
help:
@echo "make test run the test suite (no dependencies needed)"
@echo "make run run the server locally with $(CONFIG)"
@echo "make check validate $(CONFIG) and print the effective settings"
@echo "make hash-password print the sha256 digest for auth.users[].password_sha256"
@echo "make demo serve a built-in fake FreshRSS, for a smoke test"
@echo "make demo-up run the self-contained demo stack (catalogue + fake FreshRSS)"
@echo "make demo-down stop the demo stack"
@echo "make build build the container image"
@echo "make up / down start or stop the container via docker compose"
@echo "make logs follow the container logs"
@echo "make shell open a shell in the image"
@echo "make lint byte-compile and check formatting of the sources"
@echo "make clean remove caches and local configuration"
test:
$(PYTHON) -m unittest discover -s tests -t . -v
lint:
$(PYTHON) -m compileall -q app tests tools
@echo "sources compile cleanly"
run:
$(PYTHON) -m app --config $(CONFIG)
check:
$(PYTHON) -m app --config $(CONFIG) --check-config
hash-password:
@$(PYTHON) -c "import getpass, hashlib; print(hashlib.sha256(getpass.getpass('Password: ').encode()).hexdigest())"
demo:
$(PYTHON) tools/fake-freshrss.py
# The self-contained stack: builds, starts, and prints where to point a reader.
demo-up:
docker compose -f docker-compose.demo.yaml up -d --build
@echo ""
@echo "catalogue: http://127.0.0.1:8088/opds (sign in as demo / demo)"
@echo "logs: make demo-down | docker compose -f docker-compose.demo.yaml logs -f"
demo-down:
docker compose -f docker-compose.demo.yaml down
build:
docker build -t $(IMAGE) .
up:
@test -f config.json || { echo "config.json is missing: cp config.example.json config.json, then edit it"; exit 1; }
docker compose up -d --build
down:
docker compose down
logs:
docker compose logs -f
shell:
docker run --rm -it --entrypoint /bin/sh $(IMAGE)
clean:
rm -rf app/__pycache__ tests/__pycache__ tools/__pycache__ .env config.json

247
README.md Normal file
View File

@@ -0,0 +1,247 @@
# opds-from-rss
Serve a **FreshRSS** account as an **OPDS 1.2** catalog, where every article is
downloadable as an **EPUB generated on the fly**. Downloading an article marks it
read in FreshRSS.
It is a single small container with **no runtime dependencies** — the whole server
is the Python standard library — and it keeps **nothing on disk**. The catalog is
rebuilt from the FreshRSS API as you browse it, and each book is assembled in
memory at the moment it is requested.
```text
e-reader ──OPDS──▶ categories ──▶ feeds ──▶ articles ──▶ EPUB (built in memory)
│ │
└──────────────── browse the unread queue ◀──────────────┘
mark read upstream
```
## How it works
* **Your account's shape becomes the catalog.** FreshRSS category labels become
navigation feeds, subscriptions become feed entries, and unread articles become
the acquisition entries you download.
* **One article, one book.** Each article is converted to an EPUB 3 (with an EPUB 2
NCX fallback) whose body is the content FreshRSS already stores, normalised into
strict XHTML. Images are fetched and embedded so the book reads offline.
* **Downloading is what marks it read.** After the EPUB has been delivered, the
matching item is tagged read upstream. A `HEAD` probe or an interrupted transfer
does **not** consume the article, and the call is idempotent.
* **Nothing is persisted.** No database, no cache directory, no temp files. The
container can run with a read-only filesystem, which the shipped Compose file
does.
## Try it out first
There is a self-contained demo stack that needs no configuration and no FreshRSS
account: it runs the catalogue against a fake FreshRSS holding three sample feeds.
```sh
make demo-up # or: docker compose -f docker-compose.demo.yaml up --build
```
Then open **http://127.0.0.1:8088/opds** and sign in as **demo / demo**. The fake
account holds three feeds across two categories, a starred article and an
already-read one. Downloading a book marks that article read in the fake server, so
the unread queue visibly drains as you read.
```sh
make demo-down # stop it; all demo state is in memory, so nothing persists
```
It publishes an unencrypted port on loopback with a known password, so keep it for
trying things out — for real use, follow the quick start below.
## Quick start
```sh
cp config.example.json config.json # edit: your FreshRSS URL, user, categories
cp .env.example .env # edit: the two secrets
make hash-password # paste the digest into config.json (auth.users)
docker compose up -d --build
```
Then point your e-reader's OPDS browser at `http://<host>:8080/opds` and sign in
with the user name and password from `auth.users`.
### FreshRSS prerequisites
You need an **API password**, which is separate from your login password:
FreshRSS → **Settings → Account → API**. Put that in `FRESHRSS_API_PASSWORD`.
The URL in `config.json` may be written with or without the `/api/greader.php`
suffix; it is added for you, and any path prefix (`https://host/freshrss`) is kept.
## Configuration
`config.json` is the only input. `config.example.json` documents every setting
in place, and a key beginning with `_` is treated as a comment — ignored, never
interpolated, and never reported as unknown.
Any string value may reference a secret instead of containing one:
| Syntax | Meaning |
| --- | --- |
| `${VAR}` | The environment variable; the server refuses to start if it is unset |
| `${VAR:-default}` | The environment variable, or a default when unset |
| `${file:/run/secrets/x}` | The contents of a file (Docker secrets); relative paths resolve against `config.json`. The trailing newline is trimmed. |
| `$${` | A literal `${` |
The settings most worth knowing about:
| Setting | Default | Notes |
| --- | --- | --- |
| `server.host` | `127.0.0.1` | **A container needs `0.0.0.0`**, which `config.example.json` sets. Exposure is controlled by the published port. |
| `server.port` | `8080` | `0` asks the OS for a free port. |
| `server.public_base_url` | unset | Pins the catalog's links. Set it in production so a spoofed `Host` header cannot shape them. |
| `server.trust_proxy_headers` | `false` | Enables `X-Forwarded-Proto/Host/Prefix`. Only behind a proxy you control. |
| `server.allowed_hosts` | `[]` | Optional allowlist; other `Host` values are refused. |
| `server.tls.cert_file` / `key_file` | unset | In-process TLS, if you would rather not run a reverse proxy. |
| `freshrss.items_per_page` | `30` | Articles per catalog page. |
| `freshrss.mark_read` | `true` | Set `false` for a read-only mirror. |
| `freshrss.read_window_items` | `200` | How far back the **Read** section looks (see limitations). |
| `catalog.show_read_section` | `true` | Adds a Read tree beside the Unread queue. |
| `catalog.show_starred` | `true` | Adds a Starred shelf. |
| `catalog.exclude_categories` / `exclude_feeds` | `[]` | Leave things out of the catalog entirely. |
| `epub.include_images` | `true` | Fetch and embed article images so books work offline. |
| `epub.cover_image` | `true` | Use the article's lead image as the book cover. |
| `epub.allowed_image_hosts` | `[]` | When set, only these hosts' images are embedded. |
| `auth.users[].password_sha256` | — | Preferred over `password`. Generate with `make hash-password`. |
| `cache.*` | 300s / 60s / 900s | In-memory TTLs. Nothing is written to disk. |
Validate before deploying — every problem is reported at once, with its JSON path:
```sh
make check # or: python3 -m app --config config.json --check-config
```
## What the catalog looks like
| Path | Kind | Contents |
| --- | --- | --- |
| `/opds` | navigation | Unread, Read, Starred, All unread |
| `/opds/unread` · `/opds/read` | navigation | Categories, with unread counts |
| `/opds/unread/c/<category>` | navigation | The feeds in that category |
| `/opds/unread/c/<category>/f/<feed>` | acquisition | Unread articles, newest first |
| `/opds/unread/all` · `/opds/read/all` | acquisition | The whole reading list, paginated |
| `/opds/starred` | acquisition | Starred articles |
| `/opds/search?q=…` | acquisition | Search over the recent window |
| `/opds/d/<feed>/<item>/<slug>.epub` | — | The EPUB, and the mark-read |
| `/opds/opensearch.xml` | — | OpenSearch description, so readers show a search box |
| `/healthz` | — | Liveness. No credentials, no upstream call |
| `/opds/healthz` | — | Diagnostics: upstream reachability, feed count, cache, mark-read state |
Opaque identifiers in paths are base64url, because FreshRSS ids contain `/`, `?`
and `:`. The `<slug>.epub` segment is decoration — it names the saved file and the
server ignores it when looking the article up.
Links are absolute, and every feed carries `rel="self"`, `rel="start"` and
`rel="search"`, with `rel="first"`/`previous`/`next` for pagination. Acquisition
entries link the EPUB twice (`open-access` and the generic acquisition relation),
because readers disagree about which to look for, plus `alternate` to the original
article.
### Reader setup
* **KOReader** — Tools → OPDS catalog → add `http://<host>:8080/opds` with your
credentials. Use `https://` if you put a proxy in front.
* **Calibre** — *Get books* → *Add a feed* → paste the same URL.
* **Moon+ Reader / Librera / FBReader** — add a *custom OPDS* catalog with that
URL and the same credentials.
Only OPDS **1.2 (Atom)** is served, deliberately: that is what shipping readers
implement. OPDS 2.0 JSON would be a second, unused surface.
## Deploying
The Compose file runs the container read-only, with all capabilities dropped, no
new privileges, published on **loopback only**, and with secrets coming from
`.env`. Put a TLS-terminating reverse proxy (or a Cloudflare tunnel, or similar)
in front of it — see below.
```sh
make up # build and start
make logs # follow
make down # stop
```
To reach it from the LAN instead of through a proxy, change the published port to
`"8080:8080"` in `docker-compose.yaml` and set `server.public_base_url`. Only do
that on a network you trust.
### Security notes
* **HTTP Basic requires TLS.** Basic credentials are base64-encoded, not
encrypted. Terminate TLS at a proxy, or set `server.tls.*` so the container does
it itself. The server logs a warning at startup when it is serving plain HTTP.
* **Authentication is mandatory and deliberate.** There is no "auth off" switch:
the catalog is your entire reading queue, and an unauthenticated OPDS endpoint on
a routable address is an open book. Passwords are compared as sha256 digests in
constant time, an unknown user takes the same time as a wrong password, and
repeated failures from one address are throttled (`auth.max_failures_per_minute`,
`auth.throttle_seconds`).
* **Image fetching is SSRF-guarded.** Image URLs come from feed content, so they are
attacker-controlled if you follow a hostile feed. Only `http`/`https` is
followed; every address a hostname resolves to is checked against the loopback,
link-local, private, multicast and reserved ranges (including v4-mapped v6
forms); the connection is dialled to the *validated address* rather than
re-resolving the name, which closes the DNS-rebinding window; TLS still verifies
the real hostname; redirects are validated hop by hop; and bodies are capped per
image and per article. A refused image is simply dropped from the book.
* **Set `server.public_base_url`** in production, and `server.allowed_hosts` if the
service is directly reachable, so a spoofed `Host` header cannot shape the links
in the catalog.
## Development
```sh
make test # 275 tests, no dependencies beyond Python 3.11+
make run # run locally against config.json
make demo # serve a built-in fake FreshRSS and try the whole thing offline
```
The suite needs no network: an in-process FreshRSS stand-in
(`tests/stub_freshrss.py`) speaks the real wire format, and the end-to-end tests
drive the actual HTTP server against it — browsing the catalog, downloading an
EPUB, unzipping it, and asserting on the `edit-tag` request that FreshRSS received.
`tools/fake-freshrss.py` exposes the same stub for a manual smoke test.
Layout: `config` validation, `urls` identifiers, `cache` TTL cache, `xhtml`
HTML→XHTML, `images` sniffing/sizing, `fetch` guarded fetching, `epub` the EPUB
writer, `freshrss` the API client, `auth`, `opds` the Atom builders, `server`
routing and catalog logic.
## Limitations, honestly
* **The Read section is bounded.** Unread filtering happens upstream (exact), but
read items are filtered locally because only the *unread exclusion* is a
documented upstream parameter — so Read shows at most `read_window_items` recent
articles. The same window bounds `/opds/search`, which is a local filter rather
than an upstream search call.
* **Article bodies come from FreshRSS**, not from the original pages. Fast,
offline-friendly, and it works for paywalled or JavaScript-heavy sites — but a
feed that only publishes teasers yields teaser-length books. Pair it with
FreshRSS's own full-text (Readability) configuration if you need more.
* **No cover thumbnails in the catalog.** Cover images are used inside the EPUB
(`cover-image`), but navigation feeds do not advertise thumbnails, to avoid
making every reader fetch one image per entry.
* **`javascript:`/`data:` URLs, scripts, styles, iframes and SVG do not survive**
into a book, by design. Neither do inline styles or `class` attributes: the book
gets one plain stylesheet instead (override with `epub.stylesheet`).
* **Basic auth only.** No OAuth, no per-user catalogs: every user sees the whole
account.
## Troubleshooting
| Symptom | Cause |
| --- | --- |
| Container exits immediately, `configuration file not found` | `config.json` is not at `/config/config.json`; check the volume mount, or set `CONFIG_PATH`. |
| `environment variable FRESHRSS_API_PASSWORD is not set` | `.env` is missing or unrenamed; `cp .env.example .env`. |
| Reader shows an empty catalog | `freshrss.url` or the API password is wrong. `curl -u user:pass http://host:8080/opds/healthz` says which. |
| Reader prompts for a password repeatedly | Wrong `auth.users` entry; note that passwords are compared as digests, so a `password_sha256` must match exactly. |
| Catalog loads, downloads fail with 502 | The container cannot reach FreshRSS, or the image host refused; check `docker compose logs`. |
| Books are missing images | The images were refused by the SSRF guard or exceeded `epub.max_image_bytes`; the log says which, per URL. |
| A downloaded article is still listed as unread | The reader refetched the catalog before the mark-read landed, or `freshrss.mark_read` is `false`. |
## License
MIT.

42
app/__init__.py Normal file
View File

@@ -0,0 +1,42 @@
"""OPDS server backed by FreshRSS.
An OPDS 1.2 catalog whose shelves mirror FreshRSS categories and feeds, serving
one EPUB per article, generated in memory at download time. Nothing is written to
disk: the catalog is rebuilt from the FreshRSS API as it is browsed, and a
completed EPUB download marks the corresponding article read upstream.
Layout, in dependency order:
``config``
The JSON document, its interpolation (``${ENV}``, ``${file:...}``) and its
validation. Everything else is configured from here.
``urls``
Identifier and filename helpers: base64url for path identifiers, stable
``urn:uuid`` values for Atom ids, and safe download filenames.
``cache``
A bounded in-memory TTL cache with per-key single-flight, so a reader
browsing quickly does not turn into a burst of upstream requests.
``xhtml``
Feed HTML into well-formed XHTML. Everything the reader's XML parser sees
has been rebuilt by this module.
``images``
Media-type sniffing and dimension parsing, so a book's manifest is honest
about its bytes and a cover can be chosen without an imaging library.
``fetch``
The one place that fetches a host the operator did not choose. Article
images are fetched through an SSRF guard that dials a validated address.
``epub``
The EPUB 3 writer, assembled entirely in memory.
``freshrss``
The FreshRSS client, over its Google Reader compatible API.
``auth``
HTTP Basic authentication, with a throttle on failed attempts.
``opds``
OPDS 1.2 Atom document builders.
``server``
Routing, the catalog logic, and EPUB delivery.
Run it with ``python -m app --config config.json``.
"""
__version__ = "1.0.0"

127
app/__main__.py Normal file
View File

@@ -0,0 +1,127 @@
"""Entry point: load the configuration, then serve.
Startup is deliberately noisy about anything that will be hard to diagnose later:
warnings from configuration validation, the effective settings (with secrets
redacted), and any degraded optional feature. An OPDS server that is reachable
but shows an empty catalog should say why in its own log, not require a
debugging session.
"""
from __future__ import annotations
import argparse
import logging
import signal
import ssl
import sys
import threading
import app.server as app_server
from app import __version__
from app import config as app_config
LOGGER = logging.getLogger("app")
_LOG_FORMAT = "%(asctime)s %(levelname)-7s %(name)s: %(message)s"
_LOG_DATEFORMAT = "%Y-%m-%dT%H:%M:%S%z"
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="opds-from-rss",
description="Serve a FreshRSS account as an OPDS 1.2 catalog of EPUBs.",
)
parser.add_argument(
"--config",
metavar="PATH",
help="configuration file (default: $CONFIG_PATH, /config/config.json, config.json)",
)
parser.add_argument(
"--check-config",
action="store_true",
help="validate the configuration and print it, then exit",
)
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
return parser
def configure_logging(level: str) -> None:
logging.basicConfig(level=level.upper(), format=_LOG_FORMAT, datefmt=_LOG_DATEFORMAT)
def main(argv: list[str] | None = None) -> int:
arguments = build_parser().parse_args(argv)
# Logging is configured twice: once coarsely so a configuration failure is
# reported properly, then again at the level the configuration asks for.
configure_logging("info")
try:
settings = app_config.load_config(arguments.config)
except app_config.ConfigError as exc:
# The exception already lists every problem with its path.
print(str(exc), file=sys.stderr)
return 2
configure_logging(settings.server.log_level)
for warning in settings.warnings:
LOGGER.warning("configuration: %s", warning)
for line in app_config.describe_config(settings).splitlines():
LOGGER.info("%s", line)
if arguments.check_config:
LOGGER.info("configuration is valid; not starting the server")
return 0
httpd, application = app_server.build_server(settings)
if settings.server.tls_cert_file and settings.server.tls_key_file:
_enable_tls(httpd, settings.server.tls_cert_file, settings.server.tls_key_file)
LOGGER.info("serving HTTPS directly (no reverse proxy required)")
else:
LOGGER.warning(
"serving plain HTTP: put a TLS-terminating proxy in front of this, "
"because HTTP Basic credentials are only base64-encoded"
)
_install_signal_handlers(httpd)
LOGGER.info(
"listening on %s:%s; catalog at %s/opds",
settings.server.host,
settings.server.port,
settings.server.public_base_url or f"http://{settings.server.host}:{settings.server.port}",
)
try:
httpd.serve_forever(poll_interval=0.25)
finally:
LOGGER.info("shutting down")
httpd.server_close()
application.client.close()
return 0
def _enable_tls(httpd, cert_file: str, key_file: str) -> None:
"""Wrap the listening socket, so the service can be secure on its own.
The paths are passed in rather than read off the configuration so that the
"both or neither" guarantee made during validation is visible here too.
"""
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(cert_file, key_file)
httpd.socket = context.wrap_socket(httpd.socket, server_side=True)
def _install_signal_handlers(httpd) -> None:
"""Stop cleanly on SIGINT/SIGTERM, which is how a container is stopped.
``shutdown`` blocks until the serve loop exits and must not be called from the
thread running it, so the handler hands the work to a short-lived thread.
"""
def stop(signum: int, _frame: object) -> None:
LOGGER.info("received %s", signal.Signals(signum).name)
threading.Thread(target=httpd.shutdown, daemon=True).start()
for handled in (signal.SIGINT, signal.SIGTERM):
signal.signal(handled, stop)
if __name__ == "__main__":
sys.exit(main())

186
app/auth.py Normal file
View File

@@ -0,0 +1,186 @@
"""HTTP Basic authentication, with a throttle on failed attempts.
Basic is the only scheme every OPDS reader UI can drive: the reader prompts for a
user name and password, and the server answers an unauthenticated request with
``401`` and a ``WWW-Authenticate`` header. That header is also the whole reason
this service must sit behind TLS -- the credentials travel base64-encoded, which
is encoding, not encryption.
Two things this module is careful about:
*The comparison is constant time.* Passwords are stored as sha256 digests
(``make hash-password``) and compared with :func:`hmac.compare_digest`, and an
unknown user name is compared against a dummy digest so that a missing account and
a wrong password take the same time. Otherwise the response time tells an
attacker which half of the guess was right.
*Failures are throttled.* Basic auth can be guessed at line speed, so an address
that fails repeatedly is refused for a cooling-off period before its credentials
are even examined.
"""
from __future__ import annotations
import base64
import binascii
import hashlib
import hmac
import logging
import threading
import time
from dataclasses import dataclass, field
from app import config as app_config
LOGGER = logging.getLogger(__name__)
# Compared against when the user name is unknown, so that the work done -- and
# therefore the time taken -- does not depend on whether the account exists.
_DUMMY_DIGEST = hashlib.sha256(b"no such user").hexdigest()
class UnauthorizedError(Exception):
"""Credentials were missing, malformed, or wrong. Answered with 401."""
def __init__(self, message: str = "authentication required") -> None:
super().__init__(message)
class RateLimitedError(Exception):
"""Too many failed attempts from this address. Answered with 429."""
def __init__(self, retry_after: int) -> None:
super().__init__("too many failed authentication attempts")
self.retry_after = max(1, int(retry_after))
@dataclass
class _Attempts:
"""Failed-attempt bookkeeping for one client address."""
timestamps: list[float] = field(default_factory=list)
def within_window(self, now: float, window: float) -> int:
return sum(1 for moment in self.timestamps if now - moment < window)
class Authenticator:
"""Checks Basic credentials and throttles repeated failures."""
def __init__(
self,
config: app_config.AuthConfig,
*,
clock=time.monotonic,
) -> None:
self._config = config
self._clock = clock
# Digest lookups avoid a linear scan that would leak user names by timing.
self._users = {user.username: user.password_sha256 for user in config.users}
self._lock = threading.Lock()
self._attempts: dict[str, _Attempts] = {}
@property
def challenge(self) -> str:
"""The ``WWW-Authenticate`` value that makes a reader prompt."""
realm = self._config.realm.replace('"', "")
return f'Basic realm="{realm}", charset="UTF-8"'
def authenticate(self, authorization: str | None, client: str) -> str:
"""Return the authenticated user name, or raise.
``client`` identifies the address for throttling; only failed attempts are
counted, and a success clears the record.
"""
self._check_throttle(client)
username, password = _parse_basic(authorization)
if username is None or not self._matches(username, password):
self._record_failure(client)
# The source address is what matters for spotting a brute-force run, so
# that is the warning. The attempted name is attacker-supplied and
# close to an identifier, so it is kept for debugging only, and the
# password is never logged at any level.
LOGGER.warning("rejected authentication attempt from %s", client)
LOGGER.debug("rejected credentials for user %r from %s", username, client)
raise UnauthorizedError
self._clear_failures(client)
return username
def _matches(self, username: str, password: str | None) -> bool:
if password is None:
return False
expected = self._users.get(username, _DUMMY_DIGEST)
offered = hashlib.sha256(password.encode("utf-8")).hexdigest()
return hmac.compare_digest(expected, offered)
# -- throttling ------------------------------------------------------
def _check_throttle(self, client: str) -> None:
window = self._config.throttle_seconds
with self._lock:
record = self._attempts.get(client)
if record is None:
return
now = self._clock()
recent = record.within_window(now, window)
if recent < self._config.max_failures_per_minute:
return
oldest = min(record.timestamps)
retry_after = max(1, int(window - (now - oldest)) + 1)
raise RateLimitedError(retry_after)
def _record_failure(self, client: str) -> None:
now = self._clock()
window = self._config.throttle_seconds
with self._lock:
record = self._attempts.setdefault(client, _Attempts())
record.timestamps.append(now)
# Trim anything that has aged out, and drop the record entirely when
# it is empty, so a long uptime cannot accumulate addresses.
record.timestamps = [moment for moment in record.timestamps if now - moment < window]
if not record.timestamps:
del self._attempts[client]
elif len(self._attempts) > 4096:
self._evict_oldest(now, window)
def _clear_failures(self, client: str) -> None:
with self._lock:
self._attempts.pop(client, None)
def _evict_oldest(self, now: float, window: float) -> None:
"""Bound the table: a flood of addresses must not become a memory leak."""
stale = [
address
for address, record in self._attempts.items()
if not record.within_window(now, window)
]
for address in stale:
del self._attempts[address]
while len(self._attempts) > 4096:
oldest = min(
self._attempts,
key=lambda address: min(self._attempts[address].timestamps),
)
del self._attempts[oldest]
def _parse_basic(authorization: str | None) -> tuple[str | None, str | None]:
"""Split a Basic header into its credentials.
Returns ``(None, None)`` for anything that is not a well-formed Basic header,
which the caller treats exactly like a wrong password.
"""
if not authorization:
return None, None
scheme, _, payload = authorization.partition(" ")
if scheme.lower() != "basic" or not payload:
return None, None
try:
decoded = base64.b64decode(payload.strip(), validate=True).decode("utf-8")
except (binascii.Error, UnicodeDecodeError):
return None, None
# Only the first colon separates: a password may contain colons.
username, separator, password = decoded.partition(":")
if not separator or not username:
return None, None
return username, password

157
app/cache.py Normal file
View File

@@ -0,0 +1,157 @@
"""Bounded in-memory TTL cache with per-key single-flight.
The server keeps nothing on disk, which means every navigation click and every
download is backed by a live FreshRSS API call. An OPDS reader browses a tree
and re-requests feeds freely, so without a cache a single reader produces a small
burst of API traffic per tap. This cache absorbs that burst and nothing more:
entries expire on a short TTL and are evicted when the cache is full.
Single-flight matters as much as caching here. Readers often open several
connections at once (browsing navigation feeds in parallel, or prefetching cover
requests), and without it a cold cache would issue the same FreshRSS request
several times over. ``get_or_compute`` guarantees one in-flight computation per
key while still letting unrelated keys run concurrently.
"""
from __future__ import annotations
import logging
import threading
import time
from collections.abc import Callable
from typing import Any
LOGGER = logging.getLogger(__name__)
class TTLCache:
"""A thread-safe, size-bounded cache of values with a per-entry TTL."""
def __init__(self, max_entries: int = 512) -> None:
# Config validation guarantees an int here, so no defensive coercion.
self._max_entries = max(1, max_entries)
self._lock = threading.Lock()
self._values: dict[str, tuple[float, Any]] = {}
self._key_locks: dict[str, threading.Lock] = {}
self.hits = 0
self.misses = 0
self.evictions = 0
def get(self, key: str, *, now: float | None = None) -> Any | None:
"""Return a live value, or None when missing or expired."""
moment = time.monotonic() if now is None else now
with self._lock:
entry = self._values.get(key)
if entry is None:
self.misses += 1
return None
expires_at, value = entry
if expires_at <= moment:
del self._values[key]
self.misses += 1
return None
# Re-insert so the newest entries sit at the end of the ordering and
# the eviction below drops genuinely cold keys first.
del self._values[key]
self._values[key] = (expires_at, value)
self.hits += 1
return value
def set(self, key: str, value: Any, ttl_seconds: float) -> None:
"""Store a value, evicting the coldest entry if the cache is full."""
if ttl_seconds <= 0:
return
expires_at = time.monotonic() + ttl_seconds
with self._lock:
self._values.pop(key, None)
self._values[key] = (expires_at, value)
while len(self._values) > self._max_entries:
oldest, _ = next(iter(self._values.items()))
del self._values[oldest]
self.evictions += 1
def get_or_compute(self, key: str, ttl_seconds: float, compute: Callable[[], Any]) -> Any:
"""Return a cached value, computing it at most once per key at a time.
Exceptions from ``compute`` are never cached, so a transient FreshRSS
failure is retried on the next request instead of being remembered.
"""
cached = self.get(key)
if cached is not None:
return cached
with self._lock:
key_lock = self._key_locks.get(key)
if key_lock is None:
key_lock = threading.Lock()
self._key_locks[key] = key_lock
with key_lock:
# Another thread may have filled the key while this one waited.
cached = self.get(key)
if cached is not None:
return cached
value = compute()
self.set(key, value, ttl_seconds)
return value
def invalidate(self, prefix: str = "") -> int:
"""Drop every key starting with ``prefix`` (all keys when empty)."""
with self._lock:
if not prefix:
removed = len(self._values)
self._values.clear()
else:
doomed = [key for key in self._values if key.startswith(prefix)]
for key in doomed:
del self._values[key]
removed = len(doomed)
# Collapse the lock table too: keys are unbounded over a long uptime
# and a stale lock object is pure dead weight.
if not prefix or removed:
self._key_locks.clear()
return removed
def stats(self) -> dict[str, int]:
"""Return a snapshot for the diagnostics endpoint."""
with self._lock:
return {
"entries": len(self._values),
"max_entries": self._max_entries,
"hits": self.hits,
"misses": self.misses,
"evictions": self.evictions,
}
class FailureLatch:
"""Suppress a repeated failing optional call for a bounded period.
Unread counts and upstream search are enhancements: when one of them is
unsupported by the FreshRSS build in front of us, retrying on every request
would add latency to every page load for no benefit. The latch remembers the
failure for a TTL, reports it once on the way in, and then stays quiet.
"""
def __init__(self, ttl_seconds: float = 300.0) -> None:
self._ttl = ttl_seconds
self._locked_until: dict[str, float] = {}
self._lock = threading.Lock()
def is_open(self, key: str) -> bool:
"""True when calls for ``key`` should be attempted again."""
with self._lock:
until = self._locked_until.get(key, 0.0)
return time.monotonic() >= until
def trip(self, key: str) -> bool:
"""Mark ``key`` failed; return True the first time within the window."""
with self._lock:
now = time.monotonic()
first = now >= self._locked_until.get(key, 0.0)
self._locked_until[key] = now + self._ttl
return first
def clear(self, key: str) -> None:
with self._lock:
self._locked_until.pop(key, None)

841
app/config.py Normal file
View File

@@ -0,0 +1,841 @@
"""JSON configuration: loading, interpolation, validation.
Configuration is a single JSON document, which keeps deployment to "mount a file
and run the container". Two things make that practical for a service holding
credentials:
*Interpolation.* Any string value may contain ``${VAR}`` (an environment
variable, optionally with a ``:-default``), ``${file:/run/secrets/x}`` (a file,
which is how Docker secrets arrive), or ``$${`` for a literal ``${``. Secrets
therefore never have to be written into the JSON document that gets committed.
*Aggregated validation.* Every problem found is reported in one pass with the
JSON path that caused it, because a container that exits on the first of five
typos is a slow way to configure a service. Unknown keys are reported too, as
warnings: a misspelled ``mark_read`` that silently defaulted to something else is
exactly the kind of thing that costs an hour.
*Comments.* JSON has no comment syntax, so any key beginning with an underscore
is treated as one: it is ignored, never interpolated, and never reported as
unknown. That lets ``config.example.json`` explain itself.
"""
from __future__ import annotations
import hashlib
import json
import logging
import os
import re
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
LOGGER = logging.getLogger(__name__)
CONFIG_PATH_ENV = "CONFIG_PATH"
# Default listen address. Loopback is the safe default for a service that holds
# a whole reading queue: running it directly on a host binds only to that host.
# A container is the one case that needs the wildcard, because the published port
# only reaches the process through the container's own interface -- so the
# shipped config.example.json and docker-compose.yaml set it explicitly, and there
# is exactly one place (the mounted configuration) that decides exposure.
DEFAULT_BIND_HOST = "127.0.0.1"
_DEFAULT_PATHS = ("/config/config.json", "config.json")
_INTERPOLATION_RE = re.compile(r"\$\{([^{}]*)\}")
_SHA256_RE = re.compile(r"\A[0-9a-fA-F]{64}\Z")
_ESCAPED_DOLLAR = "\x00escaped-dollar\x00"
GREADER_SUFFIX = "/api/greader.php"
_LOG_LEVELS = frozenset({"debug", "info", "warning", "error", "critical"})
class ConfigError(Exception):
"""Raised when the configuration cannot be used.
Carries every problem found, so a human sees the whole list at once.
"""
def __init__(self, problems: list[str], source: str = "") -> None:
self.problems = problems
self.source = source
where = f" in {source}" if source else ""
detail = "\n".join(f" - {problem}" for problem in problems)
super().__init__(f"invalid configuration{where}:\n{detail}")
@dataclass
class ServerConfig:
"""HTTP listener settings."""
host: str = DEFAULT_BIND_HOST
port: int = 8080
log_level: str = "info"
# Absolute base URL for links. When unset, links are built from the request
# Host header, which is fine behind a proxy but lets a spoofed Host shape the
# catalog URLs -- so a public deployment should set this.
public_base_url: str | None = None
# Honour X-Forwarded-Proto/Host/Prefix. Only enable behind a proxy you trust,
# because the headers are client-controlled otherwise.
trust_proxy_headers: bool = False
# Optional Host allowlist; a request with any other Host is refused.
allowed_hosts: list[str] = field(default_factory=list)
tls_cert_file: str | None = None
tls_key_file: str | None = None
@dataclass
class FreshRSSConfig:
"""Upstream FreshRSS connection."""
api_base: str
username: str
api_password: str
verify_tls: bool = True
ca_file: str | None = None
timeout_seconds: float = 15.0
retries: int = 2
# Mark the article read upstream once its EPUB has been delivered.
mark_read: bool = True
items_per_page: int = 30
max_items_per_request: int = 200
# How far back the Read section looks. Read items are filtered client-side
# (only the unread filter is a documented upstream parameter), so this bounds
# the history that section can show.
read_window_items: int = 200
# Article body cap before parsing; prevents one enormous feed item from
# dominating memory.
max_content_bytes: int = 1_048_576
@dataclass
class CatalogConfig:
"""OPDS catalog presentation."""
title: str = "FreshRSS OPDS"
author: str = "FreshRSS"
language: str = "en"
publisher: str | None = None
show_unread_counts: bool = True
show_read_section: bool = True
show_starred: bool = True
hide_empty_categories: bool = False
exclude_categories: list[str] = field(default_factory=list)
exclude_feeds: list[str] = field(default_factory=list)
@dataclass
class EpubConfig:
"""EPUB generation."""
include_images: bool = True
allowed_image_hosts: list[str] = field(default_factory=list)
blocked_image_hosts: list[str] = field(default_factory=list)
max_images_per_article: int = 20
max_image_bytes: int = 2_097_152
max_total_image_bytes: int = 16_777_216
image_timeout_seconds: float = 10.0
cover_image: bool = True
include_source_link: bool = True
stylesheet: str | None = None
max_epub_bytes: int = 33_554_432
@dataclass
class AuthUser:
"""One Basic-auth principal. Passwords are always compared as digests."""
username: str
password_sha256: str
@dataclass
class AuthConfig:
"""HTTP Basic credentials.
Authentication is mandatory by design: this catalog exposes the whole of a
reading queue, and an unauthenticated OPDS endpoint on a routable address is
an open book. Basic auth must therefore sit behind TLS (a reverse proxy, or
the TLS options under ``server``), because the header is not encrypted.
"""
realm: str = "OPDS"
users: list[AuthUser] = field(default_factory=list)
max_failures_per_minute: int = 10
throttle_seconds: float = 60.0
@dataclass
class CacheConfig:
"""In-memory cache lifetimes. Nothing is ever written to disk."""
subscriptions_ttl_seconds: float = 300.0
stream_ttl_seconds: float = 60.0
item_ttl_seconds: float = 900.0
max_entries: int = 512
@dataclass
class Config:
"""The fully resolved configuration."""
server: ServerConfig
freshrss: FreshRSSConfig
catalog: CatalogConfig
epub: EpubConfig
auth: AuthConfig
cache: CacheConfig
source_path: str = ""
warnings: list[str] = field(default_factory=list)
def resolve_config_path(explicit: str | None = None) -> str:
"""Find the configuration file: flag, then env, then the usual locations."""
if explicit:
return explicit
from_env = os.environ.get(CONFIG_PATH_ENV)
if from_env:
return from_env
for candidate in _DEFAULT_PATHS:
if Path(candidate).is_file():
return candidate
return _DEFAULT_PATHS[0]
def load_config(path: str | None = None) -> Config:
"""Load, interpolate and validate the configuration file."""
source = resolve_config_path(path)
raw_text = _read_config_text(source)
try:
document = json.loads(raw_text)
except json.JSONDecodeError as exc:
detail = f"not valid JSON: {exc.msg} (line {exc.lineno}, column {exc.colno})"
raise ConfigError([detail], source) from exc
if not isinstance(document, dict):
raise ConfigError(["the top level of the configuration must be a JSON object"], source)
base_dir = Path(source).resolve().parent
problems: list[str] = []
interpolated = _interpolate(document, "", base_dir, problems)
if problems:
raise ConfigError(problems, source)
if not isinstance(interpolated, dict): # pragma: no cover - shape is preserved
raise ConfigError(["internal error: configuration root is not an object"], source)
warnings: list[str] = []
validator = _Validator()
config = Config(
server=_build_server(validator, interpolated),
freshrss=_build_freshrss(validator, interpolated),
catalog=_build_catalog(validator, interpolated),
epub=_build_epub(validator, interpolated),
auth=_build_auth(validator, interpolated),
cache=_build_cache(validator, interpolated),
source_path=str(Path(source).resolve()),
warnings=warnings,
)
warnings.extend(_unknown_keys(interpolated, _KNOWN_KEYS))
if validator.problems:
raise ConfigError(validator.problems, source)
return config
def password_digest(password: str) -> str:
"""Digest a password for storage in ``auth.users[].password_sha256``."""
return hashlib.sha256(password.encode("utf-8")).hexdigest()
def normalise_api_base(url: str) -> str:
"""Normalise a FreshRSS URL to its Google Reader API base.
Mirrors the firmware client's rule so the same address works in both places:
the ``/api/greader.php`` suffix is appended when absent and left alone when
present, and any path prefix in between (``/freshrss``) is preserved.
"""
base = url.strip().rstrip("/")
if base.lower().endswith(GREADER_SUFFIX):
return base
return base + GREADER_SUFFIX
def describe_config(config: Config) -> str:
"""Render the effective configuration for logs, with secrets redacted."""
users = ", ".join(user.username for user in config.auth.users) or "(none)"
lines = [
f"config file: {config.source_path}",
f"listener: {config.server.host}:{config.server.port} "
f"(log level {config.server.log_level}"
+ (", TLS on" if config.server.tls_cert_file else "")
+ ")",
f"freshrss: {config.freshrss.api_base} as {config.freshrss.username} "
f"(password: {_mask(config.freshrss.api_password)}, mark read: "
f"{'yes' if config.freshrss.mark_read else 'no'})",
f"catalog: {config.catalog.title!r}, {config.freshrss.items_per_page} items/page, "
f"read section: {'yes' if config.catalog.show_read_section else 'no'}",
f"epub images: {'yes' if config.epub.include_images else 'no'}",
f"auth users: {users}",
]
return "\n".join(lines)
def _mask(secret: str) -> str:
if not secret:
return "(empty)"
return "***" if len(secret) <= 8 else f"***({len(secret)} chars)"
def _read_config_text(source: str) -> str:
try:
return Path(source).read_text(encoding="utf-8")
except FileNotFoundError as exc:
raise ConfigError(
[
f"configuration file not found: {source}",
f"set {CONFIG_PATH_ENV} or pass --config; "
"see config.example.json for a starting point",
],
source,
) from exc
except OSError as exc:
raise ConfigError([f"cannot read configuration file: {exc.strerror}"], source) from exc
# -- interpolation -------------------------------------------------------
def _interpolate(value: Any, path: str, base_dir: Path, problems: list[str]) -> Any:
"""Recursively expand ``${VAR}`` and ``${file:...}`` in every string."""
if isinstance(value, str):
return _interpolate_string(value, path, base_dir, problems)
if isinstance(value, dict):
return {
key: (
# A `_`-prefixed key is a comment: left exactly as written, so a
# comment that mentions ${SOMETHING} stays inert.
item
if key.startswith("_")
else _interpolate(item, f"{path}.{key}" if path else key, base_dir, problems)
)
for key, item in value.items()
}
if isinstance(value, list):
return [
_interpolate(item, f"{path}[{index}]", base_dir, problems)
for index, item in enumerate(value)
]
return value
def _interpolate_string(value: str, path: str, base_dir: Path, problems: list[str]) -> str:
if "${" not in value:
return value
# Protect an escaped literal `${` before scanning for real references.
text = value.replace("$${", _ESCAPED_DOLLAR + "{")
def replace(match: re.Match[str]) -> str:
expression = match.group(1).strip()
if expression.startswith("file:"):
return _read_secret_file(expression[len("file:") :].strip(), path, base_dir, problems)
name, separator, default = expression.partition(":-")
name = name.strip()
if not name:
problems.append(f"{path}: empty ${{}} reference")
return ""
found = os.environ.get(name)
if found is not None:
return found
if separator:
return default
problems.append(
f"{path}: environment variable {name} is not set (use ${{{name}:-default}} "
"to make it optional, or ${file:path} to read a secret from a file)"
)
return ""
result = _INTERPOLATION_RE.sub(replace, text)
return result.replace(_ESCAPED_DOLLAR + "{", "${")
def _read_secret_file(filename: str, path: str, base_dir: Path, problems: list[str]) -> str:
if not filename:
problems.append(f"{path}: ${{file:}} needs a path")
return ""
candidate = Path(filename)
if not candidate.is_absolute():
candidate = base_dir / candidate
try:
# Strip the trailing newline every secret-management tool adds.
return candidate.read_text(encoding="utf-8").strip()
except OSError as exc:
problems.append(f"{path}: cannot read secret file {candidate}: {exc.strerror}")
return ""
# -- validation ----------------------------------------------------------
class _Validator:
"""Collects type and range problems while building the config dataclasses.
Problems are qualified by the section being validated, so a message reads
``freshrss.url: required, but missing`` rather than the ambiguous
``url: required, but missing``. That matters as soon as two sections share a
key name, which they do (``timeout_seconds`` lives in three of them).
"""
def __init__(self) -> None:
self.problems: list[str] = []
self._prefix = ""
@contextmanager
def within(self, path: str) -> Iterator[None]:
"""Qualify every problem reported inside this block with ``path``."""
previous = self._prefix
self._prefix = f"{previous}.{path}" if previous else path
try:
yield
finally:
self._prefix = previous
def qualify(self, key: str | None = None) -> str:
"""Full path for a key, in the section currently being validated."""
if not key:
return self._prefix
return f"{self._prefix}.{key}" if self._prefix else key
def problem(self, message: str, key: str | None = None) -> None:
"""Report a problem against the current section, or against one key."""
location = self.qualify(key)
self.problems.append(f"{location}: {message}" if location else message)
def section(self, raw: dict[str, Any], name: str) -> dict[str, Any]:
value = raw.get(name)
if value is None:
return {}
if not isinstance(value, dict):
self.problem("must be a JSON object", key=name)
return {}
return value
def string(
self,
raw: dict[str, Any],
key: str,
default: str | None = None,
*,
required: bool = False,
) -> Any:
if key not in raw or raw[key] is None:
if required:
self.problem("required, but missing", key=key)
return default
value = raw[key]
if not isinstance(value, str):
self.problem("must be a string", key=key)
return default
text = value.strip()
if not text:
if required:
self.problem("required, but empty", key=key)
return default
return text
def one_of(self, raw: dict[str, Any], key: str, allowed: frozenset[str], default: str) -> str:
value = self.string(raw, key, default)
if value is None or value not in allowed:
self.problem(f"must be one of {', '.join(sorted(allowed))} (got {value!r})", key=key)
return default
return value
def boolean(self, raw: dict[str, Any], key: str, default: bool) -> bool:
if key not in raw or raw[key] is None:
return default
value = raw[key]
if not isinstance(value, bool):
self.problem("must be true or false", key=key)
return default
return value
def integer(
self,
raw: dict[str, Any],
key: str,
default: int,
*,
minimum: int | None = None,
maximum: int | None = None,
) -> int:
if key not in raw or raw[key] is None:
return default
value = raw[key]
# A bool is an int in Python; accepting it here would turn `true` into 1.
if isinstance(value, bool) or not isinstance(value, int):
self.problem("must be a whole number", key=key)
return default
if minimum is not None and value < minimum:
self.problem(f"must be at least {minimum} (got {value})", key=key)
return default
if maximum is not None and value > maximum:
self.problem(f"must be at most {maximum} (got {value})", key=key)
return default
return value
def number(self, raw: dict[str, Any], key: str, default: float, *, above: float = 0) -> float:
if key not in raw or raw[key] is None:
return default
value = raw[key]
if isinstance(value, bool) or not isinstance(value, (int, float)):
self.problem("must be a number", key=key)
return default
if value <= above:
self.problem(f"must be greater than {above:g} (got {value:g})", key=key)
return default
return float(value)
def string_list(self, raw: dict[str, Any], key: str) -> list[str]:
if key not in raw or raw[key] is None:
return []
value = raw[key]
if not isinstance(value, list):
self.problem("must be an array of strings", key=key)
return []
items: list[str] = []
for index, item in enumerate(value):
if not isinstance(item, str):
self.problem("must be a string", key=f"{key}[{index}]")
continue
if item.strip():
items.append(item.strip())
return items
def _build_server(validator: _Validator, raw: dict[str, Any]) -> ServerConfig:
section = validator.section(raw, "server")
with validator.within("server"):
with validator.within("tls"):
tls = validator.section(section, "tls")
cert = validator.string(tls, "cert_file")
key = validator.string(tls, "key_file")
if bool(cert) != bool(key):
validator.problem("cert_file and key_file must be set together")
elif cert:
_require_readable_file(validator, cert, "cert_file")
_require_readable_file(validator, key, "key_file")
return ServerConfig(
host=validator.string(section, "host", DEFAULT_BIND_HOST) or DEFAULT_BIND_HOST,
# Port 0 asks the operating system for a free port, which is what the test
# suite uses and what a socket-activated or ephemeral deployment wants.
port=validator.integer(section, "port", 8080, minimum=0, maximum=65535),
log_level=validator.one_of(section, "log_level", _LOG_LEVELS, "info"),
public_base_url=validator.string(section, "public_base_url"),
trust_proxy_headers=validator.boolean(section, "trust_proxy_headers", False),
allowed_hosts=[
host.lower() for host in validator.string_list(section, "allowed_hosts")
],
tls_cert_file=cert,
tls_key_file=key,
)
def _build_freshrss(validator: _Validator, raw: dict[str, Any]) -> FreshRSSConfig:
section = validator.section(raw, "freshrss")
with validator.within("freshrss"):
url = validator.string(section, "url", required=True)
if url and not url.lower().startswith(("http://", "https://")):
validator.problem(f"must start with http:// or https:// (got {url!r})", key="url")
username = validator.string(section, "username", required=True)
password = validator.string(section, "api_password", required=True)
ca_file = validator.string(section, "ca_file")
if ca_file:
_require_readable_file(validator, ca_file, "ca_file")
items_per_page = validator.integer(section, "items_per_page", 30, minimum=1, maximum=500)
max_items = validator.integer(
section, "max_items_per_request", 200, minimum=1, maximum=5000
)
if max_items < items_per_page:
validator.problem(
f"max_items_per_request ({max_items}) must be at least "
f"items_per_page ({items_per_page})"
)
return FreshRSSConfig(
api_base=normalise_api_base(url) if url else "",
username=username or "",
api_password=password or "",
verify_tls=validator.boolean(section, "verify_tls", True),
ca_file=ca_file,
timeout_seconds=validator.number(section, "timeout_seconds", 15.0),
retries=validator.integer(section, "retries", 2, minimum=0, maximum=10),
mark_read=validator.boolean(section, "mark_read", True),
items_per_page=items_per_page,
max_items_per_request=max_items,
read_window_items=validator.integer(
section, "read_window_items", 200, minimum=1, maximum=5000
),
max_content_bytes=validator.integer(
section, "max_content_bytes", 1_048_576, minimum=4096, maximum=67_108_864
),
)
def _build_catalog(validator: _Validator, raw: dict[str, Any]) -> CatalogConfig:
section = validator.section(raw, "catalog")
with validator.within("catalog"):
return CatalogConfig(
title=validator.string(section, "title", "FreshRSS OPDS") or "FreshRSS OPDS",
author=validator.string(section, "author", "FreshRSS") or "FreshRSS",
language=validator.string(section, "language", "en") or "en",
publisher=validator.string(section, "publisher"),
show_unread_counts=validator.boolean(section, "show_unread_counts", True),
show_read_section=validator.boolean(section, "show_read_section", True),
show_starred=validator.boolean(section, "show_starred", True),
hide_empty_categories=validator.boolean(section, "hide_empty_categories", False),
exclude_categories=validator.string_list(section, "exclude_categories"),
exclude_feeds=validator.string_list(section, "exclude_feeds"),
)
def _build_epub(validator: _Validator, raw: dict[str, Any]) -> EpubConfig:
section = validator.section(raw, "epub")
with validator.within("epub"):
stylesheet = validator.string(section, "stylesheet")
if stylesheet:
_require_readable_file(validator, stylesheet, "stylesheet")
return EpubConfig(
include_images=validator.boolean(section, "include_images", True),
allowed_image_hosts=[
host.lower() for host in validator.string_list(section, "allowed_image_hosts")
],
blocked_image_hosts=[
host.lower() for host in validator.string_list(section, "blocked_image_hosts")
],
max_images_per_article=validator.integer(
section, "max_images_per_article", 20, minimum=0, maximum=200
),
max_image_bytes=validator.integer(
section, "max_image_bytes", 2_097_152, minimum=1024, maximum=52_428_800
),
max_total_image_bytes=validator.integer(
section,
"max_total_image_bytes",
16_777_216,
minimum=1024,
maximum=268_435_456,
),
image_timeout_seconds=validator.number(section, "image_timeout_seconds", 10.0),
cover_image=validator.boolean(section, "cover_image", True),
include_source_link=validator.boolean(section, "include_source_link", True),
stylesheet=stylesheet,
max_epub_bytes=validator.integer(
section, "max_epub_bytes", 33_554_432, minimum=4096, maximum=268_435_456
),
)
def _build_auth(validator: _Validator, raw: dict[str, Any]) -> AuthConfig:
section = validator.section(raw, "auth")
users: list[AuthUser] = []
with validator.within("auth"):
raw_users = section.get("users")
if raw_users is None:
validator.problem(
"required -- this catalog serves your entire reading queue, so it "
"will not start without credentials "
"(generate a digest with: make hash-password)",
key="users",
)
elif not isinstance(raw_users, list):
validator.problem("must be an array of user objects", key="users")
elif not raw_users:
validator.problem("must contain at least one user", key="users")
else:
seen: set[str] = set()
for index, entry in enumerate(raw_users):
with validator.within(f"users[{index}]"):
users.append(_build_auth_user(validator, entry, seen))
return AuthConfig(
realm=validator.string(section, "realm", "OPDS") or "OPDS",
users=users,
max_failures_per_minute=validator.integer(
section, "max_failures_per_minute", 10, minimum=1, maximum=10_000
),
throttle_seconds=validator.number(section, "throttle_seconds", 60.0),
)
def _build_auth_user(validator: _Validator, entry: Any, seen: set[str]) -> AuthUser:
"""Build one user; the caller has already entered its ``users[i]`` context."""
if not isinstance(entry, dict):
validator.problem("must be a JSON object")
return AuthUser("", "")
username = validator.string(entry, "username", required=True) or ""
if username and username in seen:
validator.problem(f"duplicate username {username!r}", key="username")
seen.add(username)
plain = _validated_plain_password(validator, entry)
digest = _validated_digest(validator, entry)
if plain is not None and digest is not None:
validator.problem("set either password or password_sha256, not both")
return AuthUser(username, "")
if digest is not None:
return AuthUser(username, digest)
if plain is not None:
LOGGER.warning(
"%s.password is in clear text in the configuration file; "
"prefer password_sha256 (make hash-password)",
validator.qualify(),
)
return AuthUser(username, password_digest(plain))
validator.problem("needs password or password_sha256")
return AuthUser(username, "")
def _validated_plain_password(validator: _Validator, entry: dict[str, Any]) -> str | None:
"""Return a clear-text password from config, or None when absent/invalid."""
value = entry.get("password")
if value is None:
return None
if not isinstance(value, str) or not value:
validator.problem("must be a non-empty string", key="password")
return None
return value
def _validated_digest(validator: _Validator, entry: dict[str, Any]) -> str | None:
"""Return a validated sha256 digest from config, or None when absent/invalid."""
value = entry.get("password_sha256")
if value is None:
return None
if not isinstance(value, str) or not _SHA256_RE.match(value):
validator.problem("must be 64 hex characters (make hash-password)", key="password_sha256")
return None
return value.lower()
def _build_cache(validator: _Validator, raw: dict[str, Any]) -> CacheConfig:
section = validator.section(raw, "cache")
with validator.within("cache"):
return CacheConfig(
subscriptions_ttl_seconds=validator.number(section, "subscriptions_ttl_seconds", 300.0),
stream_ttl_seconds=validator.number(section, "stream_ttl_seconds", 60.0),
item_ttl_seconds=validator.number(section, "item_ttl_seconds", 900.0),
max_entries=validator.integer(section, "max_entries", 512, minimum=16, maximum=100_000),
)
def _require_readable_file(validator: _Validator, filename: str, key: str) -> None:
if not Path(filename).is_file():
validator.problem(f"no such file: {filename}", key=key)
_KNOWN_KEYS: dict[str, set[str]] = {
"server": {
"host",
"port",
"log_level",
"public_base_url",
"trust_proxy_headers",
"allowed_hosts",
"tls",
},
"server.tls": {"cert_file", "key_file"},
"freshrss": {
"url",
"username",
"api_password",
"verify_tls",
"ca_file",
"timeout_seconds",
"retries",
"mark_read",
"items_per_page",
"max_items_per_request",
"read_window_items",
"max_content_bytes",
},
"catalog": {
"title",
"author",
"language",
"publisher",
"show_unread_counts",
"show_read_section",
"show_starred",
"hide_empty_categories",
"exclude_categories",
"exclude_feeds",
},
"epub": {
"include_images",
"allowed_image_hosts",
"blocked_image_hosts",
"max_images_per_article",
"max_image_bytes",
"max_total_image_bytes",
"image_timeout_seconds",
"cover_image",
"include_source_link",
"stylesheet",
"max_epub_bytes",
},
"auth": {"realm", "users", "max_failures_per_minute", "throttle_seconds"},
"auth.users[]": {"username", "password", "password_sha256"},
"cache": {
"subscriptions_ttl_seconds",
"stream_ttl_seconds",
"item_ttl_seconds",
"max_entries",
},
}
def _unknown_keys(document: dict[str, Any], known: dict[str, set[str]]) -> list[str]:
"""Report keys the server would otherwise silently ignore."""
warnings: list[str] = []
sections = {name for name in known if "." not in name}
for key in document:
if key in sections or key.startswith("_"):
continue
suggestions = _closest(key, sections)
hint = f" (did you mean {suggestions}?)" if suggestions else ""
warnings.append(f"{key}: unknown top-level setting, ignored{hint}")
for section, allowed in known.items():
if "." in section:
continue
value = document.get(section)
if not isinstance(value, dict):
continue
for key in value:
if key not in allowed and not key.startswith("_"):
suggestions = _closest(key, allowed)
hint = f" (did you mean {suggestions}?)" if suggestions else ""
warnings.append(f"{section}.{key}: unknown setting, ignored{hint}")
if section == "server" and isinstance(value.get("tls"), dict):
for key in value["tls"]:
if key not in known["server.tls"] and not key.startswith("_"):
warnings.append(f"server.tls.{key}: unknown setting, ignored")
if section == "auth" and isinstance(value.get("users"), list):
for index, entry in enumerate(value["users"]):
if not isinstance(entry, dict):
continue
for key in entry:
if key not in known["auth.users[]"] and not key.startswith("_"):
warnings.append(f"auth.users[{index}].{key}: unknown setting, ignored")
return warnings
def _closest(key: str, candidates: set[str]) -> str:
"""Suggest a near-miss key name, cheaply."""
lowered = key.lower().replace("-", "_")
for candidate in sorted(candidates):
if candidate.lower().replace("-", "_") == lowered:
return repr(candidate)
for candidate in sorted(candidates):
if candidate.lower().startswith(lowered[:4]) and len(lowered) >= 4:
return repr(candidate)
return ""

492
app/epub.py Normal file
View File

@@ -0,0 +1,492 @@
"""EPUB 3 generation, entirely in memory.
An EPUB is a ZIP with a strict shape, and a reader that rejects a book usually
does so because of a shape violation rather than because of its content. The
rules that matter here, and that the tests pin down:
* ``mimetype`` is the first entry, stored uncompressed, with no extra field. A
ZIP where it is deflated or not first is not an EPUB.
* ``META-INF/container.xml`` names the OPF, which is the package document.
* XML everywhere: the OPF, the navigation document, the NCX fallback and every
content document have to parse with a strict XML parser. Article bodies are
normalised by :mod:`app.xhtml` before they come anywhere near this module.
* A publication needs ``dc:identifier``, ``dc:title``, ``dc:language`` and a
``dcterms:modified`` timestamp; a navigation document marked ``properties="nav"``
is mandatory, and the NCX is kept as the EPUB 2 fallback because older readers
still look for it.
* Every manifest href resolves to a real entry, and every spine idref resolves to
a manifest item. Both are checked in the tests.
Nothing touches the filesystem: the ZIP is assembled in a ``BytesIO`` and handed
back as ``bytes``.
"""
from __future__ import annotations
import io
import logging
import zipfile
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from app import images, urls, xhtml
LOGGER = logging.getLogger(__name__)
MIMETYPE = "application/epub+zip"
# Where each kind of part lives inside the package.
OPF_PATH = "OEBPS/content.opf"
NAV_PATH = "OEBPS/nav.xhtml"
NCX_PATH = "OEBPS/toc.ncx"
CSS_PATH = "OEBPS/styles/main.css"
COVER_PATH = "OEBPS/text/cover.xhtml"
ARTICLE_PATH = "OEBPS/text/article.xhtml"
IMAGE_DIR = "OEBPS/images"
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
DC_NAMESPACE = "http://purl.org/dc/elements/1.1/"
NCX_NAMESPACE = "http://www.daisy.org/z3986/2005/ncx/"
OPS_NAMESPACE = "http://www.idpf.org/2007/ops"
# A deliberately plain stylesheet. It sets a readable measure and gives code,
# quotes and images somewhere sane to sit, and it never fights the reader's own
# typography: no colours, no font families, no absolute sizes.
DEFAULT_STYLESHEET = """\
body { margin: 5%; line-height: 1.5; }
h1, h2, h3 { line-height: 1.25; page-break-after: avoid; }
.article-title { margin-bottom: 0.2em; }
.byline { margin-top: 0; font-size: 0.9em; opacity: 0.8; }
img { max-width: 100%; height: auto; }
figure { margin: 1em 0; }
figcaption { font-size: 0.9em; font-style: italic; }
blockquote { margin: 1em 1.5em; }
pre { white-space: pre-wrap; word-wrap: break-word; }
code, pre { font-family: monospace; }
table { border-collapse: collapse; max-width: 100%; }
td, th { border: 1px solid currentColor; padding: 0.3em 0.5em; }
hr { border: none; border-top: 1px solid currentColor; }
.source { margin-top: 2em; font-size: 0.85em; opacity: 0.8; }
.cover { text-align: center; }
.cover img { max-height: 90%; }
"""
# Content documents sit one directory below the images.
_TEXT_PREFIX = "../"
# A literal middle dot rather than the HTML entity &middot;: XML predefines only
# amp, lt, gt, quot and apos, and this package ships no DOCTYPE that could define
# any others, so an HTML entity here would make the document unparseable.
BYLINE_SEPARATOR = " \u00b7 "
class EpubTooLargeError(Exception):
"""The assembled book exceeded the configured size cap."""
@dataclass(frozen=True)
class BookImage:
"""An image embedded in the book."""
href: str # relative to the OPF, e.g. "images/img-1.png"
media_type: str
data: bytes
is_cover: bool = False
@property
def document_href(self) -> str:
"""The same image, as referenced from a document in ``text/``."""
return _TEXT_PREFIX + self.href
@dataclass
class Book:
"""Everything needed to write one EPUB."""
identifier: str
title: str
language: str
modified: datetime
content_xhtml: str
published: datetime | None = None
author: str | None = None
publisher: str | None = None
source_url: str | None = None
feed_title: str | None = None
images: list[BookImage] = field(default_factory=list)
stylesheet: str = DEFAULT_STYLESHEET
include_source_link: bool = True
@property
def cover_image(self) -> BookImage | None:
for image in self.images:
if image.is_cover:
return image
return None
def build_epub(book: Book) -> bytes:
"""Assemble a book into EPUB bytes."""
buffer = io.BytesIO()
# ZIP entry timestamps come from the book so that identical input produces
# identical bytes; a book that changes only because it was rebuilt would
# otherwise defeat every cache between here and the reader.
stamp = _zip_timestamp(book.modified)
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
# The mimetype entry must come first and must not be compressed, which is
# why it is written through an explicit ZipInfo rather than writestr.
info = zipfile.ZipInfo("mimetype", date_time=stamp)
info.compress_type = zipfile.ZIP_STORED
info.external_attr = 0o644 << 16
archive.writestr(info, MIMETYPE)
_write(archive, "META-INF/container.xml", _container_xml(), stamp)
_write(archive, OPF_PATH, _opf_document(book), stamp)
_write(archive, NAV_PATH, _nav_document(book), stamp)
_write(archive, NCX_PATH, _ncx_document(book), stamp)
_write(archive, CSS_PATH, book.stylesheet, stamp)
_write(archive, COVER_PATH, _cover_document(book), stamp)
_write(archive, ARTICLE_PATH, _article_document(book), stamp)
for image in book.images:
package_dir = OPF_PATH.rsplit("/", 1)[0]
_write(archive, f"{package_dir}/{image.href}", image.data, stamp)
return buffer.getvalue()
def build_article_epub(
item,
*,
epub_config,
catalog_config,
fetcher=None,
max_content_bytes: int = 1_048_576,
) -> bytes:
"""Turn one FreshRSS item into an EPUB.
The article body comes from FreshRSS, normalised by :mod:`app.xhtml`. Images
are embedded when a fetcher is supplied, and any that cannot be fetched are
dropped from the body rather than left pointing at the network -- a book that
claims to work offline should.
"""
article = xhtml.parse_article(
item.content_html, base_url=item.url or "", max_input_bytes=max_content_bytes
)
book_images: list[BookImage] = []
image_map: dict[str, str | None] | None = None
if epub_config.include_images and fetcher is not None:
wanted = article.image_urls[: epub_config.max_images_per_article]
fetched = fetcher.fetch_all(wanted)
image_map = {}
for url in wanted:
image = fetched.get(url)
if image is None:
image_map[url] = None
continue
index = len(book_images) + 1
href = f"images/img-{index}{images.MEDIA_TYPES[image.media_type]}"
book_images.append(BookImage(href=href, media_type=image.media_type, data=image.data))
image_map[url] = _TEXT_PREFIX + href
elif article.image_urls:
# Keeping remote URLs means the OPF has to declare remote resources, which
# is handled by the caller not setting include_images.
image_map = None
book = Book(
identifier=urls.stable_urn("book", item.id),
title=item.title,
language=catalog_config.language,
modified=item.updated_at,
published=item.published_at,
content_xhtml=article.to_xhtml(image_map),
author=item.author or catalog_config.author,
publisher=catalog_config.publisher,
source_url=item.url,
feed_title=item.feed_title,
images=book_images,
include_source_link=epub_config.include_source_link,
stylesheet=_load_stylesheet(epub_config.stylesheet),
)
if epub_config.cover_image:
_choose_cover(book)
data = build_epub(book)
if len(data) > epub_config.max_epub_bytes:
raise EpubTooLargeError(
f"book is {len(data)} bytes, over the {epub_config.max_epub_bytes} byte cap"
)
return data
def _load_stylesheet(path: str | None) -> str:
"""Use the configured stylesheet, falling back to the built-in one."""
if not path:
return DEFAULT_STYLESHEET
try:
return Path(path).read_text(encoding="utf-8")
except OSError as exc:
LOGGER.warning("cannot read stylesheet %s (%s); using the built-in one", path, exc)
return DEFAULT_STYLESHEET
def _choose_cover(book: Book) -> None:
"""Promote the first suitably large image to be the book's cover.
A reader's library view shows a cover or a blank tile, and a feed's lead image
is almost always the right one. Images whose size cannot be read are skipped:
promoting an unknown image risks a cover that is a tracking pixel.
"""
for index, image in enumerate(book.images):
size = images.dimensions(image.data)
if size is None:
continue
width, height = size
if images.looks_like_a_cover(width, height):
book.images[index] = BookImage(
href=image.href,
media_type=image.media_type,
data=image.data,
is_cover=True,
)
return
def _write(
archive: zipfile.ZipFile,
path: str,
content: str | bytes,
stamp: tuple[int, int, int, int, int, int],
) -> None:
"""Write one entry, encoding text parts and passing binary parts through."""
info = zipfile.ZipInfo(path, date_time=stamp)
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o644 << 16
data = content if isinstance(content, bytes) else content.encode("utf-8")
archive.writestr(info, data)
def _zip_timestamp(moment: datetime) -> tuple[int, int, int, int, int, int]:
"""A ZIP timestamp, clamped to the range the format can express (from 1980)."""
year = max(1980, min(2107, moment.year))
return (year, moment.month, moment.day, moment.hour, moment.minute, moment.second)
# -- documents -----------------------------------------------------------
def _container_xml() -> str:
return (
'<?xml version="1.0" encoding="utf-8"?>\n'
'<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">\n'
" <rootfiles>\n"
f' <rootfile full-path="{OPF_PATH}" media-type="application/oebps-package+xml"/>\n'
" </rootfiles>\n"
"</container>\n"
)
def _opf_document(book: Book) -> str:
metadata = [
f' <dc:identifier id="pub-id">{_escape(book.identifier)}</dc:identifier>',
f" <dc:title>{_escape(book.title)}</dc:title>",
f" <dc:language>{_escape(book.language)}</dc:language>",
]
if book.author:
metadata.append(f" <dc:creator>{_escape(book.author)}</dc:creator>")
if book.publisher:
metadata.append(f" <dc:publisher>{_escape(book.publisher)}</dc:publisher>")
if book.published:
metadata.append(f" <dc:date>{_timestamp(book.published)}</dc:date>")
if book.source_url:
metadata.append(f" <dc:source>{_escape(book.source_url)}</dc:source>")
metadata.append(f' <meta property="dcterms:modified">{_timestamp(book.modified)}</meta>')
if book.cover_image is not None:
# The EPUB 2 spelling of the same statement, for readers that predate the
# cover-image property below.
metadata.append(' <meta name="cover" content="cover-image"/>')
manifest = [
' <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>',
' <item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>',
' <item id="css" href="styles/main.css" media-type="text/css"/>',
' <item id="cover" href="text/cover.xhtml" media-type="application/xhtml+xml"/>',
' <item id="article" href="text/article.xhtml" media-type="application/xhtml+xml"/>',
]
cover = book.cover_image
for index, image in enumerate(book.images, start=1):
properties = ' properties="cover-image"' if image.is_cover else ""
identifier = "cover-image" if image.is_cover else f"image-{index}"
manifest.append(
f' <item id="{identifier}" href="{_escape(image.href)}" '
f'media-type="{image.media_type}"{properties}/>'
)
if cover is None and book.images:
# Images are embedded rather than linked, so nothing to declare here; the
# branch exists to keep the intent obvious if remote images are ever kept.
pass
spine = [
' <itemref idref="cover"/>',
' <itemref idref="article"/>',
]
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<package xmlns="{OPF_NAMESPACE}" version="3.0" '
f'unique-identifier="pub-id" xml:lang="{_escape(book.language)}">',
f' <metadata xmlns:dc="{DC_NAMESPACE}">',
*metadata,
" </metadata>",
" <manifest>",
*manifest,
" </manifest>",
' <spine toc="ncx">',
*spine,
" </spine>",
" <guide>",
' <reference type="cover" title="Cover" href="text/cover.xhtml"/>',
" </guide>",
"</package>",
"",
]
)
def _nav_document(book: Book) -> str:
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<html xmlns="{XHTML_NAMESPACE}" xmlns:epub="{OPS_NAMESPACE}" '
f'xml:lang="{_escape(book.language)}" lang="{_escape(book.language)}">',
" <head>",
" <title>Contents</title>",
' <meta charset="utf-8"/>',
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
" </head>",
" <body>",
' <nav epub:type="toc" id="toc">',
" <h1>Contents</h1>",
" <ol>",
f' <li><a href="text/cover.xhtml">{_escape(book.title)}</a></li>',
' <li><a href="text/article.xhtml">Article</a></li>',
" </ol>",
" </nav>",
" </body>",
"</html>",
"",
]
)
def _ncx_document(book: Book) -> str:
points = [
("navpoint-1", 1, "Cover", "text/cover.xhtml"),
("navpoint-2", 2, "Article", "text/article.xhtml"),
]
body = []
for identifier, order, label, href in points:
body.extend(
[
f' <navPoint id="{identifier}" playOrder="{order}">',
f" <navLabel><text>{_escape(label)}</text></navLabel>",
f' <content src="{href}"/>',
" </navPoint>",
]
)
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<ncx xmlns="{NCX_NAMESPACE}" version="2005-1">',
" <head>",
f' <meta name="dtb:uid" content="{_escape(book.identifier)}"/>',
' <meta name="dtb:depth" content="1"/>',
' <meta name="dtb:totalPageCount" content="0"/>',
' <meta name="dtb:maxPageNumber" content="0"/>',
" </head>",
f" <docTitle><text>{_escape(book.title)}</text></docTitle>",
" <navMap>",
*body,
" </navMap>",
"</ncx>",
"",
]
)
def _cover_document(book: Book) -> str:
lines = [
'<?xml version="1.0" encoding="utf-8"?>',
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
f'lang="{_escape(book.language)}">',
" <head>",
f" <title>{_escape(book.title)}</title>",
' <meta charset="utf-8"/>',
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
" </head>",
' <body class="cover">',
]
cover = book.cover_image
if cover is not None:
lines.append(f' <img src="{_escape(cover.document_href)}" alt="{_escape(book.title)}"/>')
lines.append(f' <h1 class="article-title">{_escape(book.title)}</h1>')
if book.author:
lines.append(f' <p class="byline">{_escape(book.author)}</p>')
if book.feed_title:
lines.append(f' <p class="byline">{_escape(book.feed_title)}</p>')
if book.published:
lines.append(f' <p class="byline">{_date_only(book.published)}</p>')
lines.extend([" </body>", "</html>", ""])
return "\n".join(lines)
def _article_document(book: Book) -> str:
byline_bits = []
if book.author:
byline_bits.append(_escape(book.author))
if book.published:
byline_bits.append(_date_only(book.published))
lines = [
'<?xml version="1.0" encoding="utf-8"?>',
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
f'lang="{_escape(book.language)}">',
" <head>",
f" <title>{_escape(book.title)}</title>",
' <meta charset="utf-8"/>',
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
" </head>",
" <body>",
f' <h1 class="article-title">{_escape(book.title)}</h1>',
]
if byline_bits:
lines.append(f' <p class="byline">{BYLINE_SEPARATOR.join(byline_bits)}</p>')
if book.content_xhtml:
lines.append(book.content_xhtml)
else:
# Never ship an empty content document: a reader that opens a blank page
# looks broken, and the original is the most useful thing to show instead.
lines.append(" <p>This article had no content to download.</p>")
if book.include_source_link and book.source_url:
lines.append(
' <p class="source">Originally published at '
f'<a href="{_escape(book.source_url)}">{_escape(book.source_url)}</a>.</p>'
)
lines.extend([" </body>", "</html>", ""])
return "\n".join(lines)
# -- formatting ----------------------------------------------------------
def _escape(text: str) -> str:
"""XML-escape text for a document body or an attribute value."""
return xhtml.escape_text(text).replace('"', "&quot;")
def _timestamp(moment: datetime) -> str:
"""An EPUB date: UTC, seconds precision, ``Z`` suffix."""
return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def _date_only(moment: datetime) -> str:
return moment.astimezone(UTC).strftime("%Y-%m-%d")

339
app/fetch.py Normal file
View File

@@ -0,0 +1,339 @@
"""Fetching article images for embedding, with an SSRF guard.
Image URLs come from feed content, which means they are attacker-controlled by
anyone who can publish to a feed this account follows. That makes this the one
place in the server that makes an outbound request to a host the operator did not
choose, so the guard is the point of the module rather than an extra.
What is enforced, and why:
*Only http and https*, so ``file:``, ``gopher:`` and friends are unreachable.
*Every address a name resolves to is checked* against the private, loopback,
link-local, multicast and reserved ranges. If a name resolves to a mixture, the
whole fetch is refused rather than picking the acceptable address -- a mixed
answer is itself the attack.
*The connection is dialled to the validated address, not to the name.* This is
what closes the DNS-rebinding window: a name that answered with a public address
during validation cannot answer with ``169.254.169.254`` a millisecond later,
because the name is never resolved a second time. TLS still verifies the name
the URL used, so a valid certificate for the real host is required.
*Redirects are followed manually*, so each hop is validated again; ``urllib``'s
automatic redirect handling would happily follow the first hop to
``http://127.0.0.1:8080/``.
*Bodies are capped*, per image and across the whole article, so one hostile feed
cannot exhaust memory.
A refusal is a normal outcome: the image is dropped from the book and the article
is still built.
"""
from __future__ import annotations
import http.client
import ipaddress
import logging
import socket
import ssl
import threading
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from app import images
LOGGER = logging.getLogger(__name__)
USER_AGENT = "odps-from-rss/1.0 (+image fetch for EPUB embedding)"
_ALLOWED_SCHEMES = ("http", "https")
_MAX_REDIRECTS = 3
_READ_CHUNK = 64 * 1024
# Refuse to even start reading a body larger than the cap, when the server tells
# us the size up front. A little slack covers a mislabelled Content-Length.
_CONTENT_LENGTH_SLACK = 4096
class ImageFetchError(Exception):
"""The URL was refused by the guard, or the fetch failed."""
@dataclass(frozen=True)
class FetchedImage:
"""An image that was fetched, validated and identified."""
url: str
media_type: str
data: bytes
@property
def filename(self) -> str:
"""A safe file name for the EPUB, with the extension from the media type."""
return "image" + images.MEDIA_TYPES.get(self.media_type, ".bin")
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
"""HTTPS to a validated address, verifying the hostname from the URL.
``HTTPSConnection`` would use its own host for SNI and certificate
verification, which here is a bare IP address and would always fail. Dialling
the address but verifying the name keeps both properties: no second DNS
lookup, and a real certificate check.
"""
def __init__(
self,
address: str,
port: int,
*,
server_hostname: str,
context: ssl.SSLContext,
timeout: float,
) -> None:
super().__init__(address, port, timeout=timeout, context=context)
self._verify_hostname = server_hostname
def connect(self) -> None:
http.client.HTTPConnection.connect(self)
self.sock = self._context.wrap_socket( # type: ignore[union-attr]
self.sock, server_hostname=self._verify_hostname
)
class ImageFetcher:
"""Fetches article images concurrently, refusing anything unsafe."""
def __init__(
self,
*,
allowed_hosts: list[str] | None = None,
blocked_hosts: list[str] | None = None,
max_image_bytes: int = 2_097_152,
max_total_bytes: int = 16_777_216,
timeout: float = 10.0,
max_workers: int = 4,
allow_private_networks: bool = False,
) -> None:
self._allowed_hosts = [host.lower() for host in (allowed_hosts or [])]
self._blocked_hosts = [host.lower() for host in (blocked_hosts or [])]
self._max_image_bytes = max_image_bytes
self._max_total_bytes = max_total_bytes
self._timeout = timeout
self._max_workers = max(1, max_workers)
# The upstream FreshRSS may be on a LAN, but article images never have a
# legitimate reason to point into one. Kept as a parameter so a test can
# exercise the guard, not as a configuration option.
self._allow_private_networks = allow_private_networks
self._tls_context = ssl.create_default_context()
self._budget_lock = threading.Lock()
self._budget_used = 0
def fetch_all(self, urls: list[str]) -> dict[str, FetchedImage | None]:
"""Fetch each URL, mapping it to an image or to None when refused.
Order is preserved by the caller, which walks the document's images; this
method only has to answer "did it arrive?". Concurrent, because a article
with a dozen images should not cost a dozen sequential round trips.
"""
unique: list[str] = []
for url in urls:
if url not in unique:
unique.append(url)
if not unique:
return {}
with self._budget_lock:
self._budget_used = 0
workers = min(self._max_workers, len(unique))
results: dict[str, FetchedImage | None] = {}
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="image") as pool:
for url, result in zip(unique, pool.map(self._fetch_safely, unique), strict=True):
results[url] = result
return results
# -- internals -------------------------------------------------------
def _fetch_safely(self, url: str) -> FetchedImage | None:
"""Fetch one image, turning every failure into a dropped image."""
try:
return self._fetch(url)
except ImageFetchError as exc:
LOGGER.info("not embedding image %s: %s", url, exc)
except (OSError, http.client.HTTPException, ssl.SSLError) as exc:
LOGGER.info("could not fetch image %s: %s", url, exc)
except Exception:
# One bad image must never break the book it was going into.
LOGGER.exception("unexpected failure fetching image %s", url)
return None
def _fetch(self, url: str) -> FetchedImage:
target = url
for _ in range(_MAX_REDIRECTS + 1):
parsed = urllib.parse.urlsplit(target)
scheme = parsed.scheme.lower()
host = parsed.hostname
if scheme not in _ALLOWED_SCHEMES:
raise ImageFetchError(f"scheme {scheme or '(none)'!r} is not allowed")
if not host:
raise ImageFetchError("no host in URL")
if parsed.username or parsed.password:
raise ImageFetchError("credentials in an image URL are not followed")
self._check_host_allowed(host)
address = self._resolve(host)
port = parsed.port or (443 if scheme == "https" else 80)
status, headers, body = self._request_once(scheme, address, host, port, parsed)
if status in (301, 302, 303, 307, 308):
location = headers.get("location")
if not location:
raise ImageFetchError("redirect without a location")
# Resolve relative redirects against the URL that produced them.
target = urllib.parse.urljoin(target, location)
LOGGER.debug("following image redirect to %s", target)
continue
if status != 200:
raise ImageFetchError(f"server answered {status}")
declared = headers.get("content-type")
media_type = images.identify(body, declared)
if media_type is None:
raise ImageFetchError(f"not an embeddable image ({declared or 'no type'})")
return FetchedImage(url=url, media_type=media_type, data=body)
raise ImageFetchError("too many redirects")
def _check_host_allowed(self, host: str) -> None:
lowered = host.lower()
if any(_host_matches(lowered, blocked) for blocked in self._blocked_hosts):
raise ImageFetchError("host is on the blocked list")
if self._allowed_hosts and not any(
_host_matches(lowered, allowed) for allowed in self._allowed_hosts
):
raise ImageFetchError("host is not on the allowed list")
def _resolve(self, host: str) -> str:
"""Validate every address for a host and return the one to dial."""
try:
infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
except socket.gaierror as exc:
raise ImageFetchError(f"cannot resolve host ({exc.strerror or exc})") from exc
addresses: list[str] = []
for info in infos:
address = str(info[4][0])
if address not in addresses:
addresses.append(address)
if not addresses:
raise ImageFetchError("host resolved to no addresses")
for address in addresses:
reason = self._address_refusal(address)
if reason is not None:
# Refuse the whole host rather than picking a good address: a
# mixed answer is the signature of a rebinding attempt.
raise ImageFetchError(f"{address} is refused ({reason})")
return addresses[0]
def _address_refusal(self, address: str) -> str | None:
try:
parsed = ipaddress.ip_address(address)
except ValueError:
return "unparseable address"
if self._allow_private_networks:
return None
# A v4-mapped v6 address is judged as the v4 address it is, and this has to
# come first: ::ffff:127.0.0.1 is neither loopback nor private when it is
# tested as a v6 address, so the checks below would let it through.
if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None:
return self._address_refusal(str(parsed.ipv4_mapped))
if parsed.is_loopback:
return "loopback"
if parsed.is_link_local:
return "link-local"
if parsed.is_private:
return "private network"
if parsed.is_multicast:
return "multicast"
if parsed.is_reserved or parsed.is_unspecified:
return "reserved"
return None
def _request_once(
self,
scheme: str,
address: str,
host: str,
port: int,
parsed: urllib.parse.SplitResult,
) -> tuple[int, dict[str, str], bytes]:
"""One request, to a pinned address, reading at most the byte cap."""
if scheme == "https":
connection: http.client.HTTPConnection = _PinnedHTTPSConnection(
address,
port,
server_hostname=host,
context=self._tls_context,
timeout=self._timeout,
)
else:
connection = http.client.HTTPConnection(address, port, timeout=self._timeout)
path = urllib.parse.urlunsplit(("", "", parsed.path or "/", parsed.query, ""))
headers = {
"Host": _host_header(host, port, scheme),
"User-Agent": USER_AGENT,
"Accept": "image/*",
"Connection": "close",
}
try:
connection.request("GET", path, headers=headers)
response = connection.getresponse()
status = response.status
response_headers = {key.lower(): value for key, value in response.getheaders()}
if status != 200:
response.read(_READ_CHUNK)
return status, response_headers, b""
return status, response_headers, self._read_capped(response, response_headers)
finally:
connection.close()
def _read_capped(self, response: http.client.HTTPResponse, headers: dict[str, str]) -> bytes:
"""Read a response body, refusing one that exceeds the budget."""
limit = self._image_limit(headers)
chunks: list[bytes] = []
total = 0
while True:
chunk = response.read(_READ_CHUNK)
if not chunk:
break
total += len(chunk)
if total > limit:
raise ImageFetchError(f"image is larger than {limit} bytes")
chunks.append(chunk)
self._spend(total)
return b"".join(chunks)
def _image_limit(self, headers: dict[str, str]) -> int:
"""The smaller of the per-image cap and what is left of the total budget."""
declared = headers.get("content-length")
limit = self._max_image_bytes
if declared and declared.isdigit():
size = int(declared)
if size > self._max_image_bytes:
raise ImageFetchError(f"image declares {size} bytes")
with self._budget_lock:
remaining = max(0, self._max_total_bytes - self._budget_used)
if remaining == 0:
# The article's total allowance is spent; the rest of its images are
# dropped rather than growing the book further.
raise ImageFetchError("the article's image budget is exhausted")
return min(limit, remaining)
def _spend(self, size: int) -> None:
with self._budget_lock:
self._budget_used += size
def _host_matches(host: str, pattern: str) -> bool:
"""Exact host or subdomain match, so ``example.com`` covers ``cdn.example.com``."""
return host == pattern or host.endswith("." + pattern)
def _host_header(host: str, port: int, scheme: str) -> str:
"""A Host header, including the port only when it is not the default."""
default = 443 if scheme == "https" else 80
if port == default:
return host
return f"{host}:{port}"

598
app/freshrss.py Normal file
View File

@@ -0,0 +1,598 @@
"""FreshRSS client over its Google Reader compatible API.
Only the endpoints the firmware client already proved out against a real server
are used (see ``lib/FreshRss/src/FreshRssSync.cpp`` in the reader tree):
* ``POST accounts/ClientLogin`` returns the ``Auth`` token that goes into an
``Authorization: GoogleLogin auth=...`` header.
* ``GET reader/api/0/subscription/list?output=json`` lists feeds, each with its
category labels.
* ``GET reader/api/0/stream/contents/<stream>?output=json&r=d&n=&xt=`` fetches
items. ``r=d`` is newest first, ``n`` bounds the count, and ``xt`` excludes
items carrying a tag -- which is how the unread queue is filtered server-side.
* ``GET reader/api/0/token`` returns the write token, and
``POST reader/api/0/edit-tag`` with ``T=`` and repeated ``i=`` marks items read.
Deliberately absent: anything that would need an undocumented parameter. Where a
filter is not expressible with these calls (read items), the caller filters
locally rather than assuming an endpoint behaves a certain way.
Note the two endpoints this module never touches: ``unread-count`` is optional
and its failure is tolerated, and nothing here fetches article bodies from the
open internet -- the body FreshRSS already stores is the source of truth.
"""
from __future__ import annotations
import json
import logging
import ssl
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
from app import config as app_config
LOGGER = logging.getLogger(__name__)
# Google Reader stream and tag identifiers.
READING_LIST = "user/-/state/com.google/reading-list"
STARRED = "user/-/state/com.google/starred"
READ_TAG = "user/-/state/com.google/read"
STARRED_TAG = "user/-/state/com.google/starred"
KEPT_UNREAD_TAG = "user/-/state/com.google/kept-unread"
USER_AGENT = "odps-from-rss/1.0 (+https://github.com/; OPDS 1.2 catalog)"
# Statuses worth retrying: the upstream is briefly unhappy, not wrong.
_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504, 507, 509})
_AUTH_STATUSES = frozenset({401, 403})
_MISSING_STATUSES = frozenset({400, 404})
class FreshRSSError(Exception):
"""An upstream call failed."""
def __init__(self, message: str, *, status: int | None = None) -> None:
super().__init__(message)
self.status = status
class AuthenticationError(FreshRSSError):
"""The API password or username was rejected."""
class TransientError(FreshRSSError):
"""The upstream could not be reached, or answered with a retryable status."""
class NotFoundError(FreshRSSError):
"""The requested stream or item does not exist upstream."""
@dataclass
class Subscription:
"""One feed in the FreshRSS account."""
id: str
title: str
categories: list[str] = field(default_factory=list)
html_url: str | None = None
unread: int | None = None
@property
def primary_category(self) -> str:
"""The label this feed is shelved under, or a stable placeholder."""
return self.categories[0] if self.categories else UNCATEGORISED
@dataclass
class Item:
"""One article."""
id: str
title: str
published: int
updated: int
feed_id: str
feed_title: str
content_html: str = ""
author: str | None = None
url: str | None = None
feed_url: str | None = None
categories: list[str] = field(default_factory=list)
@property
def is_read(self) -> bool:
"""True when the item is read upstream.
FreshRSS keeps both the read and the kept-unread tag on an item that was
read and then explicitly marked unread again, and there the explicit
signal is the one that reflects what the reader wants.
"""
if KEPT_UNREAD_TAG in self.categories:
return False
return READ_TAG in self.categories
@property
def is_starred(self) -> bool:
return STARRED_TAG in self.categories
@property
def published_at(self) -> datetime:
"""Publication time as an aware UTC datetime.
Falls back to the epoch rather than raising: a feed with a broken date is
not a reason to fail the whole catalog.
"""
stamp = self.published or self.updated or 0
try:
return datetime.fromtimestamp(stamp, tz=UTC)
except (OverflowError, OSError, ValueError):
return datetime(1970, 1, 1, tzinfo=UTC)
@property
def updated_at(self) -> datetime:
stamp = self.updated or self.published or 0
try:
return datetime.fromtimestamp(stamp, tz=UTC)
except (OverflowError, OSError, ValueError):
return self.published_at
UNCATEGORISED = "Uncategorised"
def label_stream(label: str) -> str:
"""Stream id for a category label."""
return f"user/-/label/{label}"
class FreshRSSClient:
"""A small client for one FreshRSS account.
Instances are safe to share between threads: the only mutable state is the
cached session token, which is guarded by a lock.
"""
def __init__(
self,
config: app_config.FreshRSSConfig,
*,
clock: Any = time.time,
sleep: Any = time.sleep,
) -> None:
self._config = config
self._clock = clock
# Injectable so tests can exercise retry logic without waiting it out.
self._sleep = sleep
self._auth_token: str | None = None
self._write_token: str | None = None
self._write_token_fetched_at = 0.0
self._token_lock = threading.Lock()
self._tls_context = _build_tls_context(config)
self._opener = urllib.request.build_opener(
urllib.request.HTTPSHandler(context=self._tls_context)
)
# -- public API ------------------------------------------------------
def list_subscriptions(self) -> list[Subscription]:
"""Every feed in the account, with its category labels."""
payload = self._get_json("/reader/api/0/subscription/list")
raw_subscriptions = payload.get("subscriptions")
if not isinstance(raw_subscriptions, list):
raise FreshRSSError("subscription list had no subscriptions array")
subscriptions: list[Subscription] = []
for entry in raw_subscriptions:
parsed = _parse_subscription(entry)
if parsed is not None:
subscriptions.append(parsed)
subscriptions.sort(key=lambda feed: feed.title.casefold())
return subscriptions
def unread_counts(self) -> dict[str, int]:
"""Unread counts keyed by stream id. Optional upstream feature.
Raises whatever the client raises, so the caller can latch the failure and
carry on without counts -- the catalog is perfectly usable without them.
"""
payload = self._get_json("/reader/api/0/unread-count", {"all": "true"})
counts: dict[str, int] = {}
for entry in payload.get("unreadcounts") or []:
if not isinstance(entry, dict):
continue
stream_id = entry.get("id")
count = entry.get("count")
if isinstance(stream_id, str) and isinstance(count, int):
counts[stream_id] = count
return counts
def stream_items(
self,
stream_id: str,
*,
count: int,
exclude_read: bool = False,
older_than: int | None = None,
) -> list[Item]:
"""Fetch items from one stream, newest first.
A stream that has disappeared upstream (``400``/``404``) is reported as an
empty result rather than an error: a feed you unsubscribed from should not
break the catalog that still lists it.
"""
params = {
"output": "json",
"r": "d",
"n": str(max(1, count)),
# Cache killer: asks FreshRSS not to serve a cached response.
"ck": str(int(self._clock())),
}
if exclude_read:
params["xt"] = READ_TAG
if older_than:
params["ot"] = str(int(older_than))
path = "/reader/api/0/stream/contents/" + urllib.parse.quote(stream_id, safe="")
try:
payload = self._get_json(path, params)
except NotFoundError:
LOGGER.info("stream %s is gone upstream; treating it as empty", stream_id)
return []
raw_items = payload.get("items")
if not isinstance(raw_items, list):
return []
items = [item for item in (_parse_item(entry) for entry in raw_items) if item]
return items
def mark_read(self, item_ids: list[str]) -> int:
"""Mark items read upstream. Returns the number of ids sent.
Idempotent: adding the read tag to an already-read item is a no-op
upstream, so a repeated download of the same book is harmless.
"""
wanted = [item_id for item_id in item_ids if item_id]
if not wanted:
return 0
token = self._write_token_value()
form: list[tuple[str, str]] = [
("T", token),
("a", READ_TAG),
]
form.extend(("i", item_id) for item_id in wanted)
# edit-tag needs both the session header and the write token: the header
# authenticates the caller, the token authorises the write.
try:
self._request(
"POST",
"/reader/api/0/edit-tag",
body=urllib.parse.urlencode(form),
authenticated=True,
)
except AuthenticationError:
# Either the session or the write token outlived its validity; drop
# both, fetch fresh ones, and retry the write once.
self._forget_tokens()
form[0] = ("T", self._write_token_value())
self._request(
"POST",
"/reader/api/0/edit-tag",
body=urllib.parse.urlencode(form),
authenticated=True,
)
return len(wanted)
def probe(self) -> tuple[bool, str]:
"""Check credentials and reachability, for the diagnostics endpoint."""
try:
subscriptions = self.list_subscriptions()
except FreshRSSError as exc:
return False, str(exc)
return True, f"{len(subscriptions)} feeds"
def close(self) -> None:
"""Release upstream resources (connections are per-request)."""
self._forget_tokens()
# -- transport -------------------------------------------------------
def _forget_tokens(self) -> None:
with self._token_lock:
self._auth_token = None
self._write_token = None
self._write_token_fetched_at = 0.0
def _auth_header(self) -> dict[str, str]:
with self._token_lock:
token = self._auth_token
if token is None:
token = self._login()
with self._token_lock:
self._auth_token = token
return {"Authorization": f"GoogleLogin auth={token}"}
def _write_token_value(self) -> str:
with self._token_lock:
token = self._write_token
fetched_at = self._write_token_fetched_at
# Refresh hourly: the token is cheap to fetch and this bounds how long a
# stale one can survive a password change upstream.
if token and self._clock() - fetched_at < 3600:
return token
response = self._request("GET", "/reader/api/0/token", authenticated=True)
token = response.decode("utf-8", "replace").strip()
if not token:
raise FreshRSSError("upstream returned an empty write token")
with self._token_lock:
self._write_token = token
self._write_token_fetched_at = self._clock()
return token
def _login(self) -> str:
"""Exchange the API password for an Auth token."""
form = urllib.parse.urlencode(
{"Email": self._config.username, "Passwd": self._config.api_password}
)
response = self._request("POST", "/accounts/ClientLogin", body=form)
text = response.decode("utf-8", "replace")
for line in text.splitlines():
if line.startswith("Auth="):
value = line[len("Auth=") :].strip()
if value:
LOGGER.debug("authenticated with FreshRSS as %s", self._config.username)
return value
raise AuthenticationError("FreshRSS rejected the API password")
def _get_json(self, path: str, params: dict[str, str] | None = None) -> dict[str, Any]:
query = dict(params or {})
query.setdefault("output", "json")
response = self._request("GET", path, query=query, authenticated=True)
try:
payload = json.loads(response.decode("utf-8", "replace"))
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
raise FreshRSSError(f"upstream sent a response that is not JSON ({exc})") from exc
if not isinstance(payload, dict):
raise FreshRSSError("upstream sent a JSON value that is not an object")
return payload
def _request(
self,
method: str,
path: str,
*,
query: dict[str, str] | None = None,
body: str | None = None,
authenticated: bool = False,
) -> bytes:
"""Perform one upstream request, retrying transient failures."""
url = self._config.api_base + path
if query:
url = f"{url}?{urllib.parse.urlencode(query)}"
data = body.encode("utf-8") if body is not None else None
attempts = max(1, self._config.retries + 1)
last_error: Exception | None = None
for attempt in range(attempts):
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
if body is not None:
headers["Content-Type"] = "application/x-www-form-urlencoded"
if authenticated:
headers.update(self._auth_header())
request = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
with self._opener.open(request, timeout=self._config.timeout_seconds) as reply:
return reply.read()
except urllib.error.HTTPError as exc:
status = exc.code
try:
detail = _read_error_body(exc)
finally:
# The error object owns the response socket. Leaving it open
# leaks one connection for every failed upstream request.
exc.close()
if status in _AUTH_STATUSES:
raise AuthenticationError(
f"FreshRSS refused the request ({status}); check the API password",
status=status,
) from exc
if status in _MISSING_STATUSES:
raise NotFoundError(
f"upstream has no such resource ({status})", status=status
) from exc
if status in _RETRY_STATUSES:
# A retryable status stays transient even once the attempts run
# out, so the caller can tell "upstream is unwell" from
# "upstream says this request is wrong".
error = TransientError(f"upstream returned {status}{detail}", status=status)
if attempt + 1 < attempts:
last_error = error
self._backoff(attempt, status)
continue
raise error from exc
raise FreshRSSError(f"upstream returned {status}{detail}", status=status) from exc
except urllib.error.URLError as exc:
last_error = TransientError(f"cannot reach FreshRSS: {exc.reason}")
if attempt + 1 < attempts:
self._backoff(attempt, None)
continue
raise last_error from exc
except TimeoutError as exc:
last_error = TransientError("the upstream request timed out")
if attempt + 1 < attempts:
self._backoff(attempt, None)
continue
raise last_error from exc
except ssl.SSLError as exc:
raise FreshRSSError(f"TLS failure talking to FreshRSS: {exc}") from exc
# Only reachable if the loop exhausted its attempts without raising.
raise last_error or TransientError("the upstream request failed")
def _backoff(self, attempt: int, status: int | None) -> None:
delay = min(2.0, 0.2 * (2**attempt))
LOGGER.warning(
"upstream %s, retrying in %.1fs%s",
f"returned {status}" if status else "unreachable",
delay,
f" (attempt {attempt + 1})" if attempt else "",
)
self._sleep(delay)
def _build_tls_context(config: app_config.FreshRSSConfig) -> ssl.SSLContext:
"""Build the TLS context for upstream calls.
An operator pointing at a LAN instance with a private CA sets ``ca_file``;
disabling verification is possible but logged loudly, because it silently
removes the protection against a machine-in-the-middle reading the API
password.
"""
if not config.verify_tls:
LOGGER.warning(
"freshrss.verify_tls is false: TLS certificates are not verified. "
"The API password and article contents are exposed to anyone able to "
"intercept the connection."
)
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
return context
context = ssl.create_default_context()
if config.ca_file:
context.load_verify_locations(cafile=config.ca_file)
return context
def _read_error_body(error: urllib.error.HTTPError) -> str:
"""Include a short, sanitised excerpt of an error body in the message."""
try:
raw = error.read(512)
except Exception:
# A diagnostic excerpt must never mask the error it is describing.
return ""
text = raw.decode("utf-8", "replace").strip().replace("\n", " ")
if not text:
return ""
return f": {text[:200]}"
def _text(value: Any) -> str:
"""A trimmed string from upstream JSON, or an empty string.
Upstream is an arbitrary FreshRSS build, so every field is treated as
untrusted input of unknown type rather than as the shape the docs suggest.
"""
return value.strip() if isinstance(value, str) else ""
def _as_int(value: Any) -> int:
"""Best-effort integer from upstream JSON, tolerating strings."""
if isinstance(value, bool):
return 0
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
if isinstance(value, str):
try:
return int(value)
except ValueError:
return 0
return 0
def _parse_subscription(entry: Any) -> Subscription | None:
if not isinstance(entry, dict):
return None
stream_id = entry.get("id")
if not isinstance(stream_id, str) or not stream_id:
return None
categories: list[str] = []
for category in entry.get("categories") or []:
if not isinstance(category, dict):
continue
label = _text(category.get("label"))
if label and label not in categories:
categories.append(label)
return Subscription(
id=stream_id,
title=_text(entry.get("title")) or stream_id,
categories=categories,
html_url=_text(entry.get("htmlUrl")) or None,
)
def _parse_item(entry: Any) -> Item | None:
if not isinstance(entry, dict):
return None
item_id = entry.get("id")
if not isinstance(item_id, str) or not item_id:
return None
raw_origin = entry.get("origin")
origin: dict[str, Any] = raw_origin if isinstance(raw_origin, dict) else {}
feed_id = _text(origin.get("streamId"))
return Item(
id=item_id,
title=_text(entry.get("title")) or "(untitled)",
published=_as_int(entry.get("published")) or _timestamp_usec(entry),
updated=_as_int(entry.get("updated")) or _timestamp_usec(entry),
feed_id=feed_id,
feed_title=_text(origin.get("title")) or feed_id,
content_html=_extract_content(entry),
author=_text(entry.get("author")) or None,
url=_extract_url(entry, origin),
feed_url=_text(origin.get("htmlUrl")) or None,
categories=[tag for tag in (entry.get("categories") or []) if isinstance(tag, str)],
)
def _timestamp_usec(entry: dict[str, Any]) -> int:
"""Derive seconds from Google Reader's microsecond timestamp, when present."""
raw = entry.get("timestampUsec")
if isinstance(raw, str) and raw.isdigit():
return int(raw) // 1_000_000
if isinstance(raw, int):
return raw // 1_000_000
return 0
def _extract_content(entry: dict[str, Any]) -> str:
"""Prefer the full body over the summary.
FreshRSS stores whichever it has; ``content`` is the article body when the
feed provides one, and ``summary`` is the fallback.
"""
for key in ("content", "summary"):
block = entry.get(key)
if isinstance(block, str) and block.strip():
return block
if isinstance(block, dict):
inner = block.get("content")
if isinstance(inner, str) and inner.strip():
return inner
return ""
def _extract_url(entry: dict[str, Any], origin: dict[str, Any]) -> str | None:
alternates = entry.get("alternate")
if isinstance(alternates, list):
for candidate in alternates:
if not isinstance(candidate, dict):
continue
href = candidate.get("href")
if not isinstance(href, str) or not href:
continue
kind = candidate.get("type")
if kind in (None, "text/html"):
return href
for candidate in alternates:
if isinstance(candidate, dict) and isinstance(candidate.get("href"), str):
return candidate["href"]
html_url = origin.get("htmlUrl")
return html_url if isinstance(html_url, str) and html_url else None

193
app/images.py Normal file
View File

@@ -0,0 +1,193 @@
"""Image metadata: media type sniffing and dimension parsing.
Two small jobs, both in service of the EPUB writer.
*Sniffing.* A feed image is as likely to arrive as ``application/octet-stream``
or ``text/plain`` as with a correct ``Content-Type``. An EPUB manifest entry
whose media type does not match the bytes is invalid, so the type is decided from
the actual magic bytes and the response header is only a fallback.
*Dimensions.* Picking a cover means knowing which image is big enough to be one.
Reading a PNG header or a JPEG start-of-frame costs nothing and avoids pulling in
an imaging library (and its C dependencies) just to answer "is this wide?".
"""
from __future__ import annotations
import struct
# Types the EPUB writer will embed. SVG is deliberately absent: it can carry
# script and external references, and an image in an article body has no reason
# to need either.
MEDIA_TYPES = {
"image/jpeg": ".jpg",
"image/png": ".png",
"image/gif": ".gif",
"image/webp": ".webp",
}
_MEDIA_TYPE_ALIASES = {
"image/jpg": "image/jpeg",
"image/pjpeg": "image/jpeg",
"image/x-png": "image/png",
"image/x-gif": "image/gif",
}
def normalise_media_type(raw: str | None) -> str | None:
"""Map a Content-Type header onto a media type this server will embed."""
if not raw:
return None
# Strip parameters: "image/jpeg; charset=binary" is still a JPEG.
base = raw.split(";", 1)[0].strip().lower()
base = _MEDIA_TYPE_ALIASES.get(base, base)
return base if base in MEDIA_TYPES else None
def sniff_media_type(data: bytes) -> str | None:
"""Identify an image from its leading bytes, or None if unrecognised."""
if len(data) < 12:
return None
if data.startswith(b"\xff\xd8\xff"):
return "image/jpeg"
if data.startswith(b"\x89PNG\r\n\x1a\n"):
return "image/png"
if data.startswith((b"GIF87a", b"GIF89a")):
return "image/gif"
if data.startswith(b"RIFF") and data[8:12] == b"WEBP":
return "image/webp"
return None
def identify(data: bytes, declared: str | None) -> str | None:
"""Decide the media type for a fetched image.
The bytes win. A declared type is used only when the bytes are recognisable
as nothing else, and never to *override* a successful sniff -- otherwise a
mislabelled payload would produce an invalid EPUB manifest.
"""
sniffed = sniff_media_type(data)
if sniffed is not None:
return sniffed
return normalise_media_type(declared)
def dimensions(data: bytes) -> tuple[int, int] | None:
"""Return ``(width, height)`` in pixels, or None when it cannot be read.
Only the container headers are parsed: the value is used to decide whether an
image is a plausible cover, so "unknown" is a perfectly good answer.
"""
sniffed = sniff_media_type(data)
try:
if sniffed == "image/png":
return _png_dimensions(data)
if sniffed == "image/gif":
return _gif_dimensions(data)
if sniffed == "image/jpeg":
return _jpeg_dimensions(data)
if sniffed == "image/webp":
return _webp_dimensions(data)
except (struct.error, IndexError, ValueError):
return None
return None
def _png_dimensions(data: bytes) -> tuple[int, int] | None:
# IHDR is required to be the first chunk: 8 signature bytes, then length and
# type, so width and height sit at a fixed offset.
if len(data) < 24:
return None
width, height = struct.unpack(">II", data[16:24])
return (width, height) if width and height else None
def _gif_dimensions(data: bytes) -> tuple[int, int] | None:
if len(data) < 10:
return None
width, height = struct.unpack("<HH", data[6:10])
return (width, height) if width and height else None
def _webp_dimensions(data: bytes) -> tuple[int, int] | None:
if len(data) < 30:
return None
chunk = data[12:16]
if chunk == b"VP8X":
width = int.from_bytes(data[24:27], "little") + 1
height = int.from_bytes(data[27:30], "little") + 1
return (width, height)
if chunk == b"VP8 ":
# Lossy: dimensions are 14 bytes into the frame header, after the
# start code, with 14 bits of width and 14 of height.
if len(data) < 30:
return None
start = 26
width = int.from_bytes(data[start : start + 2], "little") & 0x3FFF
height = int.from_bytes(data[start + 2 : start + 4], "little") & 0x3FFF
return (width, height) if width and height else None
if chunk == b"VP8L":
bits = int.from_bytes(data[21:25], "little")
width = (bits & 0x3FFF) + 1
height = ((bits >> 14) & 0x3FFF) + 1
return (width, height)
return None
# JPEG start-of-frame markers, which carry the dimensions.
_SOF_MARKERS = frozenset(
{
0xC0,
0xC1,
0xC2,
0xC3,
0xC5,
0xC6,
0xC7,
0xC9,
0xCA,
0xCB,
0xCD,
0xCE,
0xCF,
}
)
def _jpeg_dimensions(data: bytes) -> tuple[int, int] | None:
"""Walk the JPEG segment chain to the start-of-frame marker.
A JPEG has no fixed header offset, so the segments have to be stepped over
until the frame header appears. This is bounded work: it stops at the first
frame marker, which always precedes the image data.
"""
index = 2 # skip the SOI marker (FF D8)
limit = len(data)
while index + 9 < limit:
if data[index] != 0xFF:
index += 1
continue
marker = data[index + 1]
if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7:
index += 2
continue
if marker == 0xDA: # start of scan: no frame header found
return None
length = struct.unpack(">H", data[index + 2 : index + 4])[0]
if length < 2:
return None
if marker in _SOF_MARKERS:
height, width = struct.unpack(">HH", data[index + 5 : index + 9])
return (width, height) if width and height else None
index += 2 + length
return None
def looks_like_a_cover(width: int, height: int) -> bool:
"""Whether an image is a plausible book cover.
A lead image is usually landscape and wide; a thumbnail, icon or tracking
pixel is not. The bar is deliberately low -- it only has to beat a blank
tile in a reader's library view.
"""
return width >= 400 and height >= 200

308
app/opds.py Normal file
View File

@@ -0,0 +1,308 @@
"""OPDS 1.2 catalog documents.
The dialect is Atom with the OPDS profile, because that is what ebook readers
actually implement: KOReader, Calibre, Moon+ Reader, Librera, FBReader and Aldiko
all browse OPDS 1.2, and its navigation/acquisition split maps exactly onto the
category/feed/article tree this server mirrors out of FreshRSS.
The two document kinds, and the rules that matter:
*Navigation feeds* (``kind=navigation``) are directories. Each entry is a
``rel="subsection"`` link to another feed, carrying a text description such as an
unread count.
*Acquisition feeds* (``kind=acquisition``) list articles. Each entry carries an
``rel="http://opds-spec.org/acquisition/open-access"`` link with an EPUB media
type, which is what a reader downloads. A plain ``http://opds-spec.org/acquisition``
link to the same URL is also emitted: readers disagree about which relation to
look for, and both are valid ways to say the same thing. An ``alternate`` link
points at the original article, which readers show as "open in browser".
Every feed carries ``rel="self"`` with the profile in its type, plus ``rel="start"``
back to the root, because readers navigate by these rather than by URL. Pagination
uses ``rel="next"``/``rel="previous"``/``rel="first"``.
Documents are built as strings rather than through a DOM library: the output is
small, fully determined, and easy to read in a test failure. Every interpolated
value goes through an escaper, and the tests parse the result with a strict XML
parser.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import UTC, datetime
from app import xhtml
ATOM_NAMESPACE = "http://www.w3.org/2005/Atom"
OPDS_NAMESPACE = "http://opds-spec.org/2010/catalog"
OPENSEARCH_NAMESPACE = "http://a9.com/-/spec/opensearch/1.1/"
DC_NAMESPACE = "http://purl.org/dc/elements/1.1/"
NAVIGATION_TYPE = "application/atom+xml;profile=opds-catalog;kind=navigation"
ACQUISITION_TYPE = "application/atom+xml;profile=opds-catalog;kind=acquisition"
OPENSEARCH_TYPE = "application/opensearchdescription+xml"
EPUB_TYPE = "application/epub+zip"
HTML_TYPE = "text/html"
ACQUISITION_REL = "http://opds-spec.org/acquisition"
OPEN_ACCESS_REL = "http://opds-spec.org/acquisition/open-access"
SUBSECTION_REL = "subsection"
ALTERNATE_REL = "alternate"
@dataclass(frozen=True)
class NavEntry:
"""One row of a navigation feed: a link deeper into the catalog."""
title: str
entry_id: str
href: str
updated: datetime
description: str | None = None
@dataclass(frozen=True)
class AcquisitionEntry:
"""One row of an acquisition feed: an article, as a downloadable book."""
title: str
entry_id: str
updated: datetime
download_href: str
author: str | None = None
summary: str | None = None
language: str | None = None
alternate_href: str | None = None
categories: tuple[tuple[str, str], ...] = ()
@dataclass(frozen=True)
class PageLinks:
"""Pagination targets, already absolute. Omitting one omits its link."""
first: str | None = None
previous: str | None = None
next: str | None = None
@dataclass(frozen=True)
class FeedLinks:
"""Everything a reader needs to place a feed in the catalog.
Pagination lives only in :attr:`pages`: an earlier version also carried
``next_url``/``previous_url``, and the renderer read those while callers set
``pages``, so the links silently never appeared.
"""
self_url: str
start_url: str | None = None
search_url: str | None = None
pages: PageLinks = field(default_factory=PageLinks)
def navigation_feed(
*,
feed_id: str,
title: str,
updated: datetime,
links: FeedLinks,
entries: list[NavEntry],
author: str = "OPDS",
subtitle: str | None = None,
) -> str:
"""Render a navigation feed."""
return _feed(
feed_id=feed_id,
title=title,
updated=updated,
links=links,
entries=[_nav_entry_xml(entry) for entry in entries],
kind_type=NAVIGATION_TYPE,
author=author,
subtitle=subtitle,
)
def acquisition_feed(
*,
feed_id: str,
title: str,
updated: datetime,
links: FeedLinks,
entries: list[AcquisitionEntry],
author: str = "OPDS",
subtitle: str | None = None,
) -> str:
"""Render an acquisition feed."""
return _feed(
feed_id=feed_id,
title=title,
updated=updated,
links=links,
entries=[_acquisition_entry_xml(entry) for entry in entries],
kind_type=ACQUISITION_TYPE,
author=author,
subtitle=subtitle,
)
def opensearch_description(
*,
search_url: str,
short_name: str,
description: str,
output_type: str = ACQUISITION_TYPE,
) -> str:
"""Render an OpenSearch description document.
Readers that support search advertise it by fetching this document, which is
why the root navigation feed links to it with ``rel="search"``.
"""
return "\n".join(
[
'<?xml version="1.0" encoding="utf-8"?>',
f'<OpenSearchDescription xmlns="{OPENSEARCH_NAMESPACE}">',
f" <ShortName>{_escape(short_name)}</ShortName>",
f" <Description>{_escape(description)}</Description>",
" <InputEncoding>UTF-8</InputEncoding>",
" <OutputEncoding>UTF-8</OutputEncoding>",
f' <Url type="{_attribute(output_type)}" '
f'template="{_attribute(search_url)}?q={{searchTerms}}&amp;page={{startPage?}}"/>',
"</OpenSearchDescription>",
"",
]
)
# -- feed assembly -------------------------------------------------------
def _feed(
*,
feed_id: str,
title: str,
updated: datetime,
links: FeedLinks,
entries: list[str],
kind_type: str,
author: str,
subtitle: str | None,
) -> str:
lines = [
'<?xml version="1.0" encoding="utf-8"?>',
f'<feed xmlns="{ATOM_NAMESPACE}" xmlns:opds="{OPDS_NAMESPACE}" xmlns:dc="{DC_NAMESPACE}">',
f" <id>{_escape(feed_id)}</id>",
f" <title>{_escape(title)}</title>",
f" <updated>{_timestamp(updated)}</updated>",
]
if subtitle:
lines.append(f" <subtitle>{_escape(subtitle)}</subtitle>")
lines.append(f" <author><name>{_escape(author)}</name></author>")
lines.extend(_feed_links_xml(links, kind_type))
lines.extend(entries)
lines.extend(["</feed>", ""])
return "\n".join(lines)
def _feed_links_xml(links: FeedLinks, kind_type: str) -> list[str]:
"""Self, start, search and pagination links, in that order."""
rendered = [
f' <link rel="self" href="{_attribute(links.self_url)}" type="{_attribute(kind_type)}"/>'
]
if links.start_url:
rendered.append(
f' <link rel="start" href="{_attribute(links.start_url)}" '
f'type="{_attribute(NAVIGATION_TYPE)}"/>'
)
if links.search_url:
rendered.append(
f' <link rel="search" href="{_attribute(links.search_url)}" '
f'type="{_attribute(OPENSEARCH_TYPE)}"/>'
)
first = links.pages.first
if first:
rendered.append(
f' <link rel="first" href="{_attribute(first)}" type="{_attribute(kind_type)}"/>'
)
if links.pages.previous:
rendered.append(
f' <link rel="previous" href="{_attribute(links.pages.previous)}" '
f'type="{_attribute(kind_type)}"/>'
)
if links.pages.next:
rendered.append(
f' <link rel="next" href="{_attribute(links.pages.next)}" '
f'type="{_attribute(kind_type)}"/>'
)
return rendered
def _nav_entry_xml(entry: NavEntry) -> str:
lines = [
" <entry>",
f" <title>{_escape(entry.title)}</title>",
f" <id>{_escape(entry.entry_id)}</id>",
f" <updated>{_timestamp(entry.updated)}</updated>",
]
if entry.description:
lines.append(f' <content type="text">{_escape(entry.description)}</content>')
lines.extend(
[
f' <link rel="{SUBSECTION_REL}" href="{_attribute(entry.href)}" '
f'type="{_attribute(NAVIGATION_TYPE)}"/>',
" </entry>",
]
)
return "\n".join(lines)
def _acquisition_entry_xml(entry: AcquisitionEntry) -> str:
lines = [
" <entry>",
f" <title>{_escape(entry.title)}</title>",
f" <id>{_escape(entry.entry_id)}</id>",
f" <updated>{_timestamp(entry.updated)}</updated>",
]
if entry.author:
lines.append(f" <author><name>{_escape(entry.author)}</name></author>")
if entry.summary:
lines.append(f' <summary type="text">{_escape(entry.summary)}</summary>')
if entry.language:
lines.append(f" <dc:language>{_escape(entry.language)}</dc:language>")
for term, label in entry.categories:
lines.append(f' <category term="{_attribute(term)}" label="{_attribute(label)}"/>')
# Two acquisition relations to the same resource, because readers disagree
# about which one to look for; both are valid.
for rel in (OPEN_ACCESS_REL, ACQUISITION_REL):
lines.append(
f' <link rel="{rel}" href="{_attribute(entry.download_href)}" '
f'type="{_attribute(EPUB_TYPE)}"/>'
)
if entry.alternate_href:
lines.append(
f' <link rel="{ALTERNATE_REL}" href="{_attribute(entry.alternate_href)}" '
f'type="{_attribute(HTML_TYPE)}"/>'
)
lines.append(" </entry>")
return "\n".join(lines)
# -- formatting ----------------------------------------------------------
def _escape(text: str) -> str:
"""Escape text for a document body."""
return xhtml.escape_text(text)
def _attribute(value: str) -> str:
"""Escape a value for a double-quoted attribute."""
return xhtml.escape_attribute(value)
def _timestamp(moment: datetime) -> str:
"""Atom dates are RFC 3339 in UTC."""
return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")

1075
app/server.py Normal file

File diff suppressed because it is too large Load Diff

120
app/urls.py Normal file
View File

@@ -0,0 +1,120 @@
"""Identifier, slug and URL helpers.
FreshRSS ids do not survive a URL path or a Content-Disposition filename
unharmed. Item ids look like ``tag:google.com,2005:reader/item/0000000000000abc``
and stream ids look like ``feed/https://example.com/atom.xml?x=1``, so both
carry characters that are structural in a path (``/``, ``?``, ``#``) and are
long enough to blow past filename limits.
The rule this module enforces: every opaque id that appears in a path is
base64url-encoded, and the human-readable part of a download URL is a slug that
the server ignores when it looks the item up. That keeps routes unambiguous
without any escaping tricks, and lets a reader name the saved file sensibly.
"""
from __future__ import annotations
import base64
import binascii
import re
import unicodedata
import uuid
# Fixed namespace for every derived urn:uuid in this server. Hardcoded so that
# feed and entry ids stay stable across restarts and across hostnames -- a client
# that cached an entry id must still recognise it tomorrow.
_UUID_NAMESPACE = uuid.UUID("b7437da1-d469-46b1-a582-0af386fadb9d")
_B64URL_RE = re.compile(r"\A[A-Za-z0-9_-]+\Z")
# Everything that is not an ASCII lowercase letter or digit becomes a dash, so
# the result is safe in a path segment, a filename, and a shell.
_SLUG_JUNK_RE = re.compile(r"[^a-z0-9]+")
def b64url_encode(value: str) -> str:
"""Encode a UTF-8 string as unpadded base64url (path-segment safe)."""
raw = value.encode("utf-8")
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
def b64url_decode(value: str) -> str:
"""Decode an unpadded base64url string, raising ValueError on junk.
Strict on purpose: this parses untrusted path input, so anything that is not
a well-formed unpadded base64url token -- or that decodes to something other
than UTF-8 -- is rejected instead of being coerced.
"""
if not value or not _B64URL_RE.match(value):
raise ValueError("not a base64url token")
padding = "=" * (-len(value) % 4)
try:
raw = base64.urlsafe_b64decode(value + padding)
except (binascii.Error, ValueError) as exc:
raise ValueError("not a base64url token") from exc
try:
return raw.decode("utf-8")
except UnicodeDecodeError as exc:
raise ValueError("base64url token is not UTF-8") from exc
def slugify(text: str, max_length: int = 60) -> str:
"""Reduce a title to a conservative filename slug.
Accented characters are folded to ASCII rather than dropped, non-ASCII
scripts fall back to ``article``, and the result is length-capped because a
filesystem name is the eventual destination.
"""
folded = unicodedata.normalize("NFKD", text or "")
ascii_text = folded.encode("ascii", "ignore").decode("ascii")
slug = _SLUG_JUNK_RE.sub("-", ascii_text.lower()).strip("-")
if len(slug) > max_length:
slug = slug[:max_length].rstrip("-")
return slug or "article"
def stable_urn(kind: str, value: str) -> str:
"""Return a deterministic ``urn:uuid`` for a feed, entry or item.
Atom requires every feed and entry to carry a permanent id. Deriving it from
the FreshRSS id rather than from the request URL means the same article keeps
one identity no matter which category path the reader reached it through.
"""
derived = uuid.uuid5(_UUID_NAMESPACE, f"{kind}:{value}")
return f"urn:uuid:{derived}"
def content_disposition(ascii_name: str, unicode_name: str | None = None) -> str:
"""Build an attachment Content-Disposition with an RFC 5987 fallback.
Every OPDS reader understands ``filename``; readers that save a title with
non-ASCII characters need ``filename*``. Sending both is the only way to get
a correct name everywhere, so both are always emitted.
"""
ascii_name = ascii_name.replace('"', "").replace("\\", "") or "article.epub"
header = f'attachment; filename="{ascii_name}"'
if unicode_name and unicode_name != ascii_name:
encoded = _quote_rfc5987(unicode_name.replace('"', "").replace("\\", ""))
if encoded:
header += f"; filename*=UTF-8''{encoded}"
return header
def _quote_rfc5987(value: str) -> str:
out = []
for byte in value.encode("utf-8"):
char = chr(byte)
if char.isalnum() and byte < 0x80 or char in "-._~":
out.append(char)
else:
out.append(f"%{byte:02X}")
return "".join(out)
def join_url(base: str, *parts: str) -> str:
"""Join URL path parts onto a base, collapsing duplicate slashes."""
segments = [base.rstrip("/")]
for part in parts:
cleaned = part.strip("/")
if cleaned:
segments.append(cleaned)
return "/".join(segments)

747
app/xhtml.py Normal file
View File

@@ -0,0 +1,747 @@
"""Turn arbitrary feed HTML into well-formed XHTML for the EPUB pipeline.
EPUB is XML end to end: the OPF, the NCX, the navigation document and every
content document are parsed by a strict XML parser (the reader firmware this
server was written alongside uses expat). Feed article bodies are the opposite
of strict -- unclosed ``<li>``, stray ``&``, ``<br>`` instead of ``<br/>``,
control characters, embedded scripts and tracking pixels. Handing that to the
EPUB writer would produce a book that some readers refuse to open, so everything
goes through this normaliser first.
How it stays safe, in order of importance:
* The input is parsed into an in-memory tree and re-serialised from scratch.
Any malformedness in the source therefore cannot reach the output: whatever the
parser tolerated is rebuilt with balanced tags, quoted attributes and escaped
text. There is no pass-through of raw markup.
* Characters that XML 1.0 forbids outright (most C0 controls, unpaired
surrogates) are removed, because they are not expressible even when escaped.
* Entities are decoded to real Unicode by the parser and re-escaped on output, so
a stray ``&`` in the source cannot produce a document that will not parse.
* Scriptable and metadata elements are dropped with their contents; every other
unknown element is unwrapped, keeping its text.
* Only URLs that survive scheme validation reach an attribute, which keeps
``javascript:`` and ``data:`` payloads out of the book.
The tree is kept around after parsing so a caller can collect image URLs, fetch
them, and then serialise with the fetched images rewritten to local paths.
"""
from __future__ import annotations
import html
import logging
import re
from collections.abc import Iterator
from html.parser import HTMLParser
from urllib.parse import urljoin, urlsplit
LOGGER = logging.getLogger(__name__)
# Elements kept in the output. Mirrors the firmware reader's sanitiser
# (lib/FreshRss/src/FreshRssArticle.cpp) with the additions that make sense on a
# device with a real layout engine: images, and the small semantic set EPUB
# readers style (q, cite, time, abbr, del, ins, small, mark).
KEPT_TAGS = frozenset(
{
"p",
"div",
"span",
"br",
"hr",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"ul",
"ol",
"li",
"dl",
"dt",
"dd",
"blockquote",
"pre",
"code",
"em",
"i",
"strong",
"b",
"u",
"s",
"sub",
"sup",
"a",
"img",
"figure",
"figcaption",
"table",
"thead",
"tbody",
"tfoot",
"tr",
"td",
"th",
"caption",
"small",
"mark",
"cite",
"time",
"abbr",
"q",
"ins",
"del",
}
)
# Dropped along with everything inside them. These are either scriptable,
# fetch extra resources, or are document-level metadata that has no place inside
# an EPUB content document.
DROP_WITH_CONTENT = frozenset(
{
"script",
"style",
"iframe",
"frame",
"frameset",
"object",
"embed",
"applet",
"svg",
"math",
"form",
"input",
"button",
"select",
"option",
"optgroup",
"textarea",
"label",
"fieldset",
"legend",
"noscript",
"template",
"canvas",
"video",
"audio",
"source",
"track",
"map",
"area",
"head",
"title",
"meta",
"link",
"base",
"dialog",
"slot",
}
)
# HTML void elements: no end tag exists, so the parser must not expect one.
VOID_TAGS = frozenset({"br", "hr", "img"})
_HTML_VOID = frozenset(
{
"area",
"base",
"br",
"col",
"embed",
"hr",
"img",
"input",
"link",
"meta",
"param",
"source",
"track",
"wbr",
}
)
# Elements that silently end an open element in HTML. Feeds are full of
# unclosed <p> and <li>, and the browser-correct fix is to close the open
# element rather than to nest one block inside another (which is what a naive
# stack would do, producing <p>text<div>...</div></p>).
_IMPLICIT_CLOSE: dict[str, frozenset[str]] = {
"p": frozenset(
{
"address",
"article",
"aside",
"blockquote",
"details",
"div",
"dl",
"fieldset",
"figcaption",
"figure",
"footer",
"form",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"header",
"hr",
"main",
"nav",
"ol",
"p",
"pre",
"section",
"table",
"ul",
}
),
"li": frozenset({"li"}),
"dt": frozenset({"dt", "dd"}),
"dd": frozenset({"dt", "dd"}),
"td": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
"th": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
"tr": frozenset({"tr", "tbody", "tfoot", "thead"}),
"option": frozenset({"option", "optgroup"}),
"thead": frozenset({"tbody", "tfoot"}),
"tbody": frozenset({"tbody", "tfoot"}),
}
# Attributes kept, per element. Everything else is discarded: inline styles and
# layout attributes fight the EPUB stylesheet, and event handlers, tracking
# attributes and class names have no meaning in a book.
_KEPT_ATTRS: dict[str, frozenset[str]] = {
"a": frozenset({"href", "title"}),
"img": frozenset({"src", "alt", "title"}),
"blockquote": frozenset({"cite"}),
"q": frozenset({"cite"}),
"time": frozenset({"datetime"}),
"ins": frozenset({"datetime"}),
"del": frozenset({"datetime"}),
"abbr": frozenset({"title"}),
"td": frozenset({"colspan", "rowspan"}),
"th": frozenset({"colspan", "rowspan"}),
"ol": frozenset({"start"}),
}
# Empty structural elements worth removing once their contents are gone (for
# example a wrapper whose only child was a rejected image). Table cells are
# deliberately absent: an empty cell is layout, not noise.
_DROPPABLE_WHEN_EMPTY = frozenset(
{
"p",
"div",
"span",
"li",
"ul",
"ol",
"dl",
"blockquote",
"figure",
"figcaption",
"section",
"article",
}
)
_LINK_SCHEMES = ("http", "https", "mailto")
_IMAGE_SCHEMES = ("http", "https")
# XML 1.0 permits #x9, #xA, #xD, #x20-#xD7FF, #xE000-#xFFFD, #x10000-#x10FFFF.
# Everything else -- NUL bytes, vertical tabs from mangled feeds, form feeds,
# unpaired surrogates -- cannot be represented even as a character reference and
# must be deleted before serialisation.
_ILLEGAL_XML_RE = re.compile(
"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]"
)
# Collapses runs of layout whitespace. A non-breaking space is deliberately not
# matched: feeds use it for intentional spacing, and reflowable text keeps it.
_WHITESPACE_RE = re.compile(r"[^\S\xa0]+")
# Trims layout whitespace from the edges of a block. Python's str.strip() treats
# a non-breaking space as whitespace and would eat one the author typed on
# purpose, so the pattern excludes it here too.
_BLOCK_TRIM_RE = re.compile(r"\A[^\S\xa0]+|[^\S\xa0]+\Z")
_ZERO_WIDTH_RE = re.compile("[\u200b\u200c\u200d\u2060\ufeff]")
# XML name rules for an id: starts with a letter or underscore, then a bounded run
# of name characters. Anything else is not a usable anchor target.
_ID_RE = re.compile(r"\A[A-Za-z_][A-Za-z0-9_.-]{0,63}\Z")
_ID_ATTRS = frozenset({"id"})
# Wrappers that need the attribute filter relaxed for integer attributes.
_INT_ATTR_LIMITS = {"colspan": (1, 100), "rowspan": (1, 100), "start": (-10000, 10000)}
class _Element:
"""One element node in the parsed tree."""
__slots__ = ("attrs", "children", "tag")
def __init__(self, tag: str) -> None:
self.tag = tag
self.attrs: dict[str, str] = {}
self.children: list[_Element | _Text] = []
class _Text:
"""One text node in the parsed tree."""
__slots__ = ("text",)
def __init__(self, text: str) -> None:
self.text = text
class _TreeBuilder(HTMLParser):
"""Build a normalised element tree from feed HTML.
The parser is tolerant by construction -- it never raises on malformed input,
it only decides what to keep -- so this class is the single place where all
of the "this feed is broken" handling lives.
"""
def __init__(self) -> None:
# convert_charrefs decodes &amp;/&#233;/&nbsp; to Unicode for us, so the
# serialiser only ever has to escape what it writes itself.
super().__init__(convert_charrefs=True)
self.root = _Element("#root")
self._stack: list[_Element] = [self.root]
# Names of open drop-with-content elements; while non-empty, all input is
# discarded except the end tag that closes the innermost one.
self._suppressed: list[str] = []
# -- parser callbacks -------------------------------------------------
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if self._suppressed:
if tag in DROP_WITH_CONTENT and tag not in _HTML_VOID:
self._suppressed.append(tag)
return
if tag in DROP_WITH_CONTENT:
if tag not in _HTML_VOID:
self._suppressed.append(tag)
return
self._close_implied(tag)
if tag in _HTML_VOID and tag not in VOID_TAGS:
# Void but not kept (<col>, <wbr>, <param>): it cannot have content,
# so it must never open a subtree that would swallow the rest.
# br/hr/img are void *and* kept, so they fall through to be emitted.
return
element = _Element(tag)
element.attrs = self._filter_attrs(tag, attrs)
self._stack[-1].children.append(element)
if tag not in VOID_TAGS:
self._stack.append(element)
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
# <br/> lands here. A self-closing element is complete: it is appended
# and never pushed, so there is nothing left open. A self-closing
# drop-with-content element has no content to discard and must not open a
# suppression region, or the rest of the document would be swallowed.
if self._suppressed or tag in DROP_WITH_CONTENT or tag in _HTML_VOID:
return
self._close_implied(tag)
element = _Element(tag)
element.attrs = self._filter_attrs(tag, attrs)
self._stack[-1].children.append(element)
def handle_endtag(self, tag: str) -> None:
if self._suppressed:
if tag == self._suppressed[-1]:
self._suppressed.pop()
return
if tag in _HTML_VOID:
return
# Close down to the matching open element, tolerating unclosed tags in
# between (this is exactly what browsers do).
for index in range(len(self._stack) - 1, 0, -1):
if self._stack[index].tag == tag:
del self._stack[index:]
return
# Stray end tag with no matching open element: ignore it.
def handle_data(self, data: str) -> None:
if self._suppressed or not data:
return
self._stack[-1].children.append(_Text(data))
def handle_comment(self, data: str) -> None:
return
def handle_decl(self, decl: str) -> None:
return
def handle_pi(self, data: str) -> None:
return
def unknown_decl(self, data: str) -> None:
if self._suppressed:
return
# An HTML parser treats <![CDATA[...]]> as a bogus comment and discards
# it, but a feed body that passed through an XML parser upstream can
# legitimately contain one, and its payload is real content. The parser
# hands over the section body with the closing ]] already stripped.
marker = "CDATA["
if data.startswith(marker):
inner = data[len(marker) :]
if inner:
self._stack[-1].children.append(_Text(inner))
# -- helpers ---------------------------------------------------------
def _close_implied(self, tag: str) -> None:
"""Close open elements that HTML would close implicitly."""
while len(self._stack) > 1:
conditions = _IMPLICIT_CLOSE.get(self._stack[-1].tag)
if conditions is None or tag not in conditions:
return
self._stack.pop()
def _filter_attrs(self, tag: str, attrs: list[tuple[str, str | None]]) -> dict[str, str]:
if tag not in KEPT_TAGS:
# The element is unwrapped later, so its attributes are never used.
return {}
allowed = _KEPT_ATTRS.get(tag, frozenset())
if tag not in VOID_TAGS:
# Anchor targets survive so an in-book table of contents still jumps
# to its section when the article is read offline.
allowed = allowed | _ID_ATTRS
kept: dict[str, str] = {}
for raw_name, raw_value in attrs:
name = raw_name.lower()
if name not in allowed:
continue
value = html.unescape(raw_value or "") if raw_value is not None else ""
value = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", value)).strip()
if name == "id":
if not _ID_RE.match(value):
continue
kept[name] = value
continue
if name in _INT_ATTR_LIMITS:
bounds = _INT_ATTR_LIMITS[name]
try:
number = int(value)
except ValueError:
continue
if not bounds[0] <= number <= bounds[1]:
continue
value = str(number)
if not value:
continue
kept[name] = value
return kept
class Article:
"""A parsed, normalised article body ready for EPUB serialisation."""
def __init__(self, tree: _Element, base_url: str, truncated: bool) -> None:
self._tree = tree
self._base_url = base_url
self.truncated = truncated
self._image_urls: list[str] | None = None
@property
def image_urls(self) -> list[str]:
"""Absolute http(s) image URLs, in document order, de-duplicated."""
if self._image_urls is None:
seen: list[str] = []
known: set[str] = set()
for element in _iter_elements(self._tree):
if element.tag != "img":
continue
src = element.attrs.get("src", "")
if src and src not in known:
known.add(src)
seen.append(src)
self._image_urls = seen
return self._image_urls
def text(self, limit: int = 600) -> str:
"""Flattened plain text, for summaries and the search index."""
parts: list[str] = []
for element in _iter_text(self._tree):
parts.append(element.text)
collapsed = _WHITESPACE_RE.sub(" ", _ILLEGAL_XML_RE.sub("", "".join(parts))).strip()
if len(collapsed) <= limit:
return collapsed
# Cut at a word boundary so a summary never ends mid-word.
cut = collapsed[:limit].rsplit(" ", 1)[0]
return (cut or collapsed[:limit]).rstrip() + "\u2026"
def to_xhtml(self, image_map: dict[str, str | None] | None = None) -> str:
"""Serialise the body as an XHTML fragment.
``image_map`` maps an original absolute image URL to the local href to
use in its place; a value of None drops that image (and its wrapper, if it
becomes empty). Pass None to leave remote image URLs untouched -- the
caller must then declare remote resources in the OPF manifest.
Applying a map rewrites the tree in place, so an Article is rendered
once. The server builds a book and serialises it exactly once.
"""
if image_map is not None:
_apply_image_map(self._tree, image_map)
# A dropped image can leave an empty wrapper behind.
_prune_empty(self._tree)
body = _serialise_children(self._tree, inside_pre=False)
if not body.strip():
return ""
return f'<div class="article-body">\n{body}\n</div>'
class _Resolve:
"""Decides which URLs survive normalisation, and where they point."""
def __init__(self, base_url: str) -> None:
self.base_url = base_url
def link(self, raw: str) -> str | None:
# A same-document reference is left alone so it resolves against the book
# rather than against the web page the article came from. The matching id
# was kept by the attribute filter.
if raw.startswith("#"):
return raw
resolved = _absolutise(raw, self.base_url)
if resolved is None:
return None
if urlsplit(resolved).scheme.lower() not in _LINK_SCHEMES:
return None
return resolved
def image(self, raw: str) -> str | None:
resolved = _absolutise(raw, self.base_url)
if resolved is None:
return None
if urlsplit(resolved).scheme.lower() not in _IMAGE_SCHEMES:
return None
return resolved
def parse_article(raw_html: str, base_url: str = "", max_input_bytes: int = 0) -> Article:
"""Parse a feed article body into a normalised tree.
``base_url`` is the article's own URL; relative links and images inside the
body are resolved against it, which is the only way they keep working once
the body is inside a book with a different base.
"""
text = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", raw_html or ""))
truncated = False
if max_input_bytes > 0 and len(text.encode("utf-8", "ignore")) > max_input_bytes:
# Cut on a character boundary, not a byte boundary, and do not try to be
# clever about tags: the parser tolerates a truncated tail.
while len(text.encode("utf-8", "ignore")) > max_input_bytes:
text = text[: int(len(text) * 0.98) or 1]
truncated = True
LOGGER.warning("article body truncated to %d bytes before parsing", max_input_bytes)
builder = _TreeBuilder()
builder.feed(text)
builder.close()
tree = builder.root
if not _has_elements(tree):
# A body with no markup at all (plain text feed) still needs block
# structure, or the EPUB would contain a bare text node.
paragraph = _Element("p")
paragraph.children = list(tree.children)
tree.children = [paragraph]
_resolve_urls(tree, _Resolve(base_url), set())
_prune_empty(tree)
return Article(tree, base_url, truncated)
def _absolutise(raw: str, base_url: str) -> str | None:
candidate = (raw or "").strip()
if not candidate:
return None
# Control characters and raw whitespace inside a URL are always corruption.
candidate = _ILLEGAL_XML_RE.sub("", candidate)
if any(char.isspace() for char in candidate):
candidate = candidate.split()[0]
try:
if base_url:
return urljoin(base_url, candidate)
return candidate
except ValueError:
return None
def _resolve_urls(element: _Element, resolve: _Resolve, seen_ids: set[str]) -> None:
"""Validate URLs and de-duplicate anchor ids, in document order.
Runs once, at parse time, so everything downstream (the image collector, the
fetcher, the OPF manifest) sees final absolute URLs and there is no second
chance for a bad scheme to slip through.
"""
for child in element.children:
if isinstance(child, _Text):
continue
_dedupe_id(child, seen_ids)
if child.tag == "a":
href = child.attrs.get("href")
if href is not None:
target = resolve.link(href)
if target is None:
child.attrs.pop("href", None)
else:
child.attrs["href"] = target
elif child.tag == "img":
src = child.attrs.get("src")
target = resolve.image(src) if src else None
if target is None:
# Mark it for pruning: an image with no usable source is noise.
child.tag = "#dropped-image"
else:
child.attrs["src"] = target
elif child.tag in ("blockquote", "q"):
cite = child.attrs.get("cite")
if cite is not None:
target = resolve.link(cite)
if target is None:
child.attrs.pop("cite", None)
else:
child.attrs["cite"] = target
_resolve_urls(child, resolve, seen_ids)
def _dedupe_id(element: _Element, seen_ids: set[str]) -> None:
"""Make an element's id unique within the document.
Duplicate ids are invalid and would make the first anchor win, so a repeat is
renamed rather than dropped -- dropping it would break the link pointing at
it. One book holds one article, so a document-wide set is enough.
"""
anchor = element.attrs.get("id")
if anchor is None:
return
if anchor not in seen_ids:
seen_ids.add(anchor)
return
index = 2
while f"{anchor}-{index}" in seen_ids:
index += 1
renamed = f"{anchor}-{index}"
seen_ids.add(renamed)
element.attrs["id"] = renamed
def _apply_image_map(element: _Element, image_map: dict[str, str | None]) -> None:
"""Point images at their local copies inside the book.
A URL mapped to None means the fetch failed or the image was refused, so the
element is marked for pruning instead of being left pointing at the network.
"""
for child in element.children:
if isinstance(child, _Text):
continue
if child.tag == "img":
src = child.attrs.get("src")
if src is not None and src in image_map:
target = image_map[src]
if target is None:
child.tag = "#dropped-image"
else:
child.attrs["src"] = target
_apply_image_map(child, image_map)
def _prune_empty(node: _Element) -> None:
"""Remove rejected images and the empty wrappers they leave behind."""
kept: list[_Element | _Text] = []
for child in node.children:
if isinstance(child, _Element):
if child.tag == "#dropped-image":
continue
_prune_empty(child)
if _is_noise(child):
continue
kept.append(child)
node.children = kept
def _is_noise(element: _Element) -> bool:
"""True for a droppable wrapper with no content left inside it.
Whitespace-only text does not count as content (``<p>&nbsp;</p>`` is noise),
but a retained ``id`` does: it is the target of a link.
"""
if element.tag not in _DROPPABLE_WHEN_EMPTY or element.attrs:
return False
for child in element.children:
if isinstance(child, _Element):
return False
if child.text.strip():
return False
return True
def _has_elements(element: _Element) -> bool:
"""True when the tree contains at least one element node."""
return any(True for _ in _iter_elements(element))
def _serialise_children(node: _Element, inside_pre: bool) -> str:
return "".join(_serialise(child, inside_pre) for child in node.children)
def _serialise(node: _Element | _Text, inside_pre: bool) -> str:
if isinstance(node, _Text):
if inside_pre:
return escape_text(node.text)
# Collapse whitespace outside <pre>: feed HTML arrives indented, and that
# indentation is pure bytes in the book.
return escape_text(_WHITESPACE_RE.sub(" ", node.text))
if node.tag not in KEPT_TAGS:
# Unwrapped elements keep their children but lose their own tag, so no
# unknown element can appear in the output.
return _serialise_children(node, inside_pre)
child_pre = inside_pre or node.tag == "pre"
inner = _serialise_children(node, child_pre)
if node.tag in ("p", "div", "li", "td", "th", "blockquote", "figcaption", "caption"):
inner = _BLOCK_TRIM_RE.sub("", inner)
attributes = "".join(
f' {name}="{escape_attribute(value)}"' for name, value in _ordered_attrs(node)
)
if node.tag in VOID_TAGS:
return f"<{node.tag}{attributes}/>"
return f"<{node.tag}{attributes}>{inner}</{node.tag}>"
def _ordered_attrs(element: _Element) -> list[tuple[str, str]]:
"""Stable attribute order, with the most meaningful attribute first."""
priority = {"id": 0, "href": 1, "src": 2, "alt": 3, "title": 4}
return sorted(element.attrs.items(), key=lambda item: (priority.get(item[0], 9), item[0]))
def escape_text(text: str) -> str:
return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
def escape_attribute(value: str) -> str:
# Attribute values are normalised by XML parsers anyway (tab/newline become
# spaces), so do it explicitly rather than leaving it implicit.
text = value.replace("\t", " ").replace("\n", " ").replace("\r", " ")
return (
text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
)
def _iter_elements(node: _Element) -> Iterator[_Element]:
"""Depth-first traversal over element nodes only."""
for child in node.children:
if isinstance(child, _Element):
yield child
yield from _iter_elements(child)
def _iter_text(node: _Element) -> Iterator[_Text]:
"""Depth-first traversal over text nodes only."""
for child in node.children:
if isinstance(child, _Text):
yield child
else:
yield from _iter_text(child)

36
config.demo.json Normal file
View File

@@ -0,0 +1,36 @@
{
"_comment": "Demo configuration for `docker compose -f docker-compose.demo.yaml up`. It points at the fake FreshRSS in that stack and logs in as demo/demo. Not for real use; see config.example.json instead.",
"server": {
"host": "0.0.0.0",
"port": 8080,
"log_level": "info"
},
"freshrss": {
"_url": "The demo stack's fake FreshRSS, addressed by its compose service name.",
"url": "http://fresh:8099/api/greader.php",
"username": "tyler",
"api_password": "${DEMO_API_PASSWORD:-api-password}",
"items_per_page": 5,
"retries": 1
},
"catalog": {
"title": "Demo Catalog",
"author": "Demo"
},
"epub": {
"include_images": false
},
"auth": {
"_users": "demo/demo -- a throwaway login for the demo stack.",
"users": [
{
"username": "demo",
"password_sha256": "2a97516c354b68848cdbd8f54a226a0a55b21ed138e207ad6c5cbb9c00aa5aea"
}
]
},
"cache": {
"subscriptions_ttl_seconds": 30,
"stream_ttl_seconds": 10
}
}

134
config.example.json Normal file
View File

@@ -0,0 +1,134 @@
{
"_about": "OPDS server backed by FreshRSS. Copy to config.json and edit. Any key starting with an underscore is a comment: it is ignored, never interpolated, and never reported as unknown. Strings may use ${ENV_VAR}, ${ENV_VAR:-default} or ${file:/run/secrets/name}, so secrets do not have to live in this file.",
"server": {
"_host": "0.0.0.0 is required *inside a container*, because the published port only reaches the process through the container's own interface. It is not the 'exposed on every host interface' case: exposure is decided by the port mapping in docker-compose.yaml, which publishes on 127.0.0.1 by default. Running this directly on a host should keep the 127.0.0.1 default instead.",
"host": "0.0.0.0",
"_port": "0 asks the operating system for a free port, which is handy for a local run or a test; leave it at 8080 for a normal deployment.",
"port": 8080,
"log_level": "info",
"_public_base_url": "Absolute base URL for catalog links, e.g. https://opds.example.com. Leave unset to build links from the request Host header. Set it when the service is reachable at more than one name, or when you want the catalog URLs pinned against a spoofed Host header.",
"public_base_url": null,
"_trust_proxy_headers": "Honour X-Forwarded-Proto/Host/Prefix. Enable only behind a reverse proxy you control: without one, a client can set those headers itself.",
"trust_proxy_headers": false,
"_allowed_hosts": "Optional Host allowlist. When set, a request with any other Host is refused with 400. Useful when the service is directly reachable.",
"allowed_hosts": [],
"_tls": "Optional in-process TLS. Usually the reverse proxy terminates TLS instead, but this lets the container be secure on its own. Basic auth is unencrypted, so one of the two is required.",
"tls": {
"cert_file": null,
"key_file": null
}
},
"freshrss": {
"_url": "Your FreshRSS address. The /api/greader.php suffix is optional here; it is appended when missing, and any path prefix is kept.",
"url": "https://rss.example.com",
"username": "tyler",
"_api_password": "Settings -> Account -> API in FreshRSS. This is the API password, which is separate from your login password. Prefer ${file:...} or ${ENV_VAR} over writing it here.",
"api_password": "${FRESHRSS_API_PASSWORD}",
"verify_tls": true,
"_ca_file": "Optional CA bundle, for a FreshRSS instance behind a private certificate authority.",
"ca_file": null,
"timeout_seconds": 15,
"retries": 2,
"_mark_read": "Mark the article read in FreshRSS once its EPUB has been fully delivered to the reader.",
"mark_read": true,
"items_per_page": 30,
"max_items_per_request": 200,
"_read_window_items": "How far back the Read section looks. Unread filtering happens upstream, but read items are filtered here, so this bounds the history that section can show.",
"read_window_items": 200,
"_max_content_bytes": "Cap on one article body before it is parsed, so a single enormous feed item cannot dominate memory.",
"max_content_bytes": 1048576
},
"catalog": {
"title": "FreshRSS OPDS",
"author": "Tyler",
"language": "en",
"_publisher": "Optional dc:publisher for every generated book.",
"publisher": null,
"_show_unread_counts": "Append unread counts to category and feed rows. Costs one extra upstream call per catalog refresh, and degrades quietly if unsupported.",
"show_unread_counts": true,
"_show_read_section": "Expose a Read tree next to the Unread queue, for browsing history without mixing it into the queue.",
"show_read_section": true,
"_show_starred": "Expose starred items as their own shelf.",
"show_starred": true,
"_hide_empty_categories": "Hide categories that contain no feeds (or no unread items, when unread counts are on).",
"hide_empty_categories": false,
"_exclude_categories": "Category labels to leave out of the catalog entirely.",
"exclude_categories": [],
"_exclude_feeds": "Feed titles to leave out of the catalog entirely.",
"exclude_feeds": []
},
"epub": {
"_include_images": "Download article images and embed them in the book, so it reads correctly offline. Image URLs come from feed content, so they are fetched through an SSRF guard: private, loopback and link-local addresses are refused.",
"include_images": true,
"_allowed_image_hosts": "When non-empty, only images from these hosts may be embedded (no scheme, exact host or subdomain suffix). Use for a locked-down deployment.",
"allowed_image_hosts": [],
"_blocked_image_hosts": "Hosts to refuse even when inlining is on, e.g. a tracking pixel domain.",
"blocked_image_hosts": [],
"max_images_per_article": 20,
"max_image_bytes": 2097152,
"max_total_image_bytes": 16777216,
"image_timeout_seconds": 10,
"_cover_image": "Use the first suitably large image as the book cover, so a reader's library view shows something other than a blank tile.",
"cover_image": true,
"_include_source_link": "Add a link back to the original article, plus a dc:source entry.",
"include_source_link": true,
"_stylesheet": "Optional path to a CSS file to use instead of the built-in stylesheet.",
"stylesheet": null,
"_max_epub_bytes": "Refuse to build a book larger than this, so one pathological article cannot exhaust memory.",
"max_epub_bytes": 33554432
},
"auth": {
"_realm": "The realm shown in the reader's password prompt.",
"realm": "OPDS",
"_users": "HTTP Basic credentials. This catalog serves your entire reading queue, so it will not start without at least one user. Use password_sha256 (make hash-password) rather than password: the served catalogue is public once TLS is in front of it, but the config file is not.",
"users": [
{
"username": "tyler",
"_password_sha256": "Generate with: make hash-password",
"password_sha256": "${OPDS_PASSWORD_SHA256}"
}
],
"_max_failures_per_minute": "Failed attempts from one address before it is throttled. Basic auth is brute-forceable, so this is on by default.",
"max_failures_per_minute": 10,
"throttle_seconds": 60
},
"cache": {
"_about": "Lifetimes for the in-memory cache. Nothing is written to disk. Lower values make the catalog more responsive to upstream changes; higher values reduce FreshRSS traffic while a reader browses.",
"subscriptions_ttl_seconds": 300,
"stream_ttl_seconds": 60,
"item_ttl_seconds": 900,
"max_entries": 512
}
}

58
docker-compose.demo.yaml Normal file
View File

@@ -0,0 +1,58 @@
# A self-contained demo: the catalogue plus a fake FreshRSS, so the whole
# thing can be tried with one command, without handing a real account over.
#
# docker compose -f docker-compose.demo.yaml up --build
# open http://127.0.0.1:8088/opds and sign in as demo / demo
#
# The fake account holds three feeds across two categories, a starred
# article and an already-read one. Downloading a book marks that article
# read in the fake server, so the unread queue drains as you use it.
#
# This is a demonstration, not a deployment: it publishes an unencrypted
# port on loopback with a known password. For real use, follow README.md
# with config.example.json and docker-compose.yaml.
name: opds-from-rss-demo
services:
fresh:
build:
context: .
dockerfile: Dockerfile.demo
image: opds-from-rss-demo-freshrss:latest
container_name: opds-demo-fresh
restart: "no"
# No ports: the catalogue reaches it over the compose network, and nothing
# else needs to.
read_only: true
tmpfs:
- /tmp
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
opds:
build: .
image: opds-from-rss:latest
container_name: opds-demo-server
restart: "no"
depends_on:
fresh:
# Wait for the fake server to answer, so a reader's first tap does
# not land on an upstream that is still starting up.
condition: service_healthy
environment:
CONFIG_PATH: /config/config.json
volumes:
- ./config.demo.json:/config/config.json:ro
ports:
# 8088 on the host, because 8080 is a popular neighbour; the
# container itself always listens on 8080.
- "127.0.0.1:8088:8080"
read_only: true
tmpfs:
- /tmp
security_opt:
- no-new-privileges:true
cap_drop:
- ALL

36
docker-compose.yaml Normal file
View File

@@ -0,0 +1,36 @@
services:
opds-from-rss:
build: .
image: opds-from-rss:latest
container_name: opds-from-rss
restart: unless-stopped
# Secrets (the FreshRSS API password and the catalogue password digest) come
# from .env rather than from config.json, so the mounted file stays safe to
# keep in version control. Optional, so that `docker compose config` and a
# first `up` work before the file exists -- configuration validation then
# reports the missing variable by name.
env_file:
- path: .env
required: false
volumes:
- ./config.json:/config/config.json:ro
# Published on loopback only. Put a TLS-terminating reverse proxy (or a
# tunnel) in front, because HTTP Basic credentials are only base64-encoded.
# To reach it from the LAN instead, publish "8080:8080" deliberately.
ports:
- "127.0.0.1:8080:8080"
# The server never writes to disk, so the container filesystem can be
# read-only. /tmp is mounted only because some Python internals expect it
# to exist; nothing here writes to it.
read_only: true
tmpfs:
- /tmp
security_opt:
- no-new-privileges:true
cap_drop:
- ALL

35
pyproject.toml Normal file
View File

@@ -0,0 +1,35 @@
[build-system]
requires = ["setuptools>=68"]
build-backend = "setuptools.build_meta"
[project]
name = "opds-from-rss"
version = "1.0.0"
description = "OPDS 1.2 catalog backed by FreshRSS, serving one EPUB per article"
readme = "README.md"
requires-python = ">=3.11"
license = { text = "MIT" }
# No runtime dependencies, deliberately: the whole server is the Python standard
# library. That keeps the image small, the supply chain empty, and the EPUB
# writer auditable.
dependencies = []
[project.urls]
Homepage = "https://github.com/"
[tool.pyright]
# The application package sits at the repository root rather than under src/.
include = ["app", "tests", "tools"]
extraPaths = ["."]
pythonVersion = "3.11"
typeCheckingMode = "standard"
[tool.ruff]
line-length = 100
target-version = "py311"
[tool.ruff.lint]
select = ["E", "F", "W", "I", "N", "UP", "B", "C4", "SIM", "RUF"]
[tool.setuptools]
packages = ["app"]

12
tests/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
"""Test package.
Logging is quietened here so that expected warnings from exercised code paths
(for example the clear-text password notice) do not drown out test results. A
test that asserts on a log message can raise the level itself.
"""
from __future__ import annotations
import logging
logging.getLogger("app").setLevel(logging.CRITICAL)

605
tests/stub_freshrss.py Normal file
View File

@@ -0,0 +1,605 @@
"""An in-process FreshRSS stand-in.
This exists so the client can be tested against the wire format a real FreshRSS
produces, rather than against a mock that would happily accept anything. The
response shapes here mirror the Google Reader API responses the firmware client
consumes (``lib/FreshRss/src/FreshRssSync.cpp``), including the parts that are
easy to get wrong: ``Auth=`` embedded in a multi-line ClientLogin body, category
labels nested inside each subscription, item payloads that put the body under
``content``/``summary``, and one ``i=`` parameter per id on ``edit-tag``.
It also records every request, so tests can assert on the exact query parameters
the client sent -- which is how the unread filter and the page size are verified.
The same class backs ``tools/fake-freshrss.py``, so the server can be smoke
tested by hand without pointing it at a real account.
"""
from __future__ import annotations
import json
import threading
import urllib.parse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Any
API_PREFIX = "/api/greader.php"
READ_TAG = "user/-/state/com.google/read"
STARRED_TAG = "user/-/state/com.google/starred"
READING_LIST = "user/-/state/com.google/reading-list"
def _tags(entry: Any) -> list[str]:
"""Category tags on an entry, tolerating a malformed payload.
Tests deliberately inject non-object entries to check that the client skips
them, so the stub's own filtering must survive them too.
"""
if not isinstance(entry, dict):
return []
tags = entry.get("categories")
if not isinstance(tags, list):
return []
return [tag for tag in tags if isinstance(tag, str)]
def _published(entry: Any) -> int:
"""Sort key that tolerates a malformed entry."""
if not isinstance(entry, dict):
return 0
value = entry.get("published")
return value if isinstance(value, int) else 0
def _with_read_state(entry: Any, read_ids: set[str]) -> Any:
"""The entry as the server would send it, with the read tag applied."""
if not isinstance(entry, dict) or entry.get("id") not in read_ids:
return entry
if READ_TAG in _tags(entry):
return entry
projected = dict(entry)
projected["categories"] = [*_tags(entry), READ_TAG]
return projected
def _is_tagged(entry: Any, tag: str, read_ids: set[str]) -> bool:
"""Whether an entry carries a tag, including tracked read state.
Marking an item read adds the read tag on the server, so an ``xt=read`` query
stops returning it. Without modelling that, a test could not tell whether
mark-read actually took effect.
"""
if tag in _tags(entry):
return True
return tag == READ_TAG and isinstance(entry, dict) and entry.get("id") in read_ids
def item_id(index: int) -> str:
"""A FreshRSS-shaped item id: the exact form that needs URL-safe encoding."""
return f"tag:google.com,2005:reader/item/{index:016x}"
class StubFreshRSS:
"""A scriptable FreshRSS with just enough behaviour to trust the client."""
def __init__(
self,
*,
username: str = "tyler",
password: str = "api-password",
page_limit: int = 1000,
bind_host: str = "127.0.0.1",
bind_port: int = 0,
public_host: str | None = None,
) -> None:
self.username = username
self.password = password
self.page_limit = page_limit
self._bind_host = bind_host
self._bind_port = bind_port
# The name other containers should use to reach this one. It defaults to
# the bind address, which is right for a local test and wrong inside a
# compose network, where the service name is the address.
self._public_host = public_host or bind_host
self.auth_token = "stub-auth-token"
self.write_token = "stub-write-token"
self.subscriptions: list[dict[str, Any]] = []
# Items are typed loosely on purpose: this class stands in for an upstream
# that can send anything, and tests rely on being able to inject malformed
# payloads to check that the client skips them.
self.items: dict[str, list[Any]] = {}
self.read: set[str] = set()
self.requests: list[dict[str, Any]] = []
self.supports_unread_counts = True
# When set, subscription list replies with 200 and a body that is not JSON,
# so the client's response handling can be tested against a proxy or an
# error page rather than against well-formed output.
self.corrupt_json = False
# Injected failures as (status, path fragment or None) pairs.
self._failures: list[tuple[int, str | None]] = []
self._lock = threading.Lock()
self._server: ThreadingHTTPServer | None = None
self._thread: threading.Thread | None = None
self.port = 0
# -- scripting -------------------------------------------------------
def add_feed(
self,
stream_id: str,
title: str,
categories: list[str] | None = None,
*,
html_url: str | None = None,
) -> str:
"""Register a feed under one or more category labels."""
self.subscriptions.append(
{
"id": stream_id,
"title": title,
"categories": [
{"id": f"user/-/label/{label}", "label": label} for label in (categories or [])
],
"htmlUrl": html_url or stream_id.removeprefix("feed/"),
"url": stream_id.removeprefix("feed/"),
"sortid": f"{len(self.subscriptions):08x}",
}
)
self.items.setdefault(stream_id, [])
return stream_id
def add_item(
self,
stream_id: str,
index: int,
title: str,
*,
content: str = "<p>body</p>",
summary: str | None = None,
published: int = 1_700_000_000,
author: str | None = "A. Writer",
url: str | None = None,
read: bool = False,
starred: bool = False,
categories: list[str] | None = None,
) -> str:
"""Add an article to a feed, returning its id."""
identifier = item_id(index)
tags = [READING_LIST, *(categories or [])]
if read:
tags.append(READ_TAG)
if starred:
tags.append(STARRED_TAG)
entry: dict[str, Any] = {
"id": identifier,
"title": title,
"published": published,
"updated": published,
"timestampUsec": str(published * 1_000_000),
"crawlTimeMsec": str(published * 1000),
"categories": tags,
"alternate": [
{
"href": url or f"https://example.com/articles/{index}",
"type": "text/html",
}
],
"origin": {
"streamId": stream_id,
"title": self._feed_title(stream_id),
"htmlUrl": stream_id.removeprefix("feed/"),
},
}
if author is not None:
entry["author"] = author
entry["content"] = {"content": content, "direction": "ltr"}
entry["summary"] = {
"content": summary if summary is not None else content,
"direction": "ltr",
}
self.items.setdefault(stream_id, []).append(entry)
if read:
self.read.add(identifier)
return identifier
def inject_raw_item(self, stream_id: str, payload: Any) -> None:
"""Push an arbitrary payload into a stream.
Upstream is not guaranteed to answer with well-formed objects, so tests
need a way to send junk through the real client path rather than trusting
that it would be handled. Guarded because handler threads read this.
"""
with self._lock:
self.items.setdefault(stream_id, []).append(payload)
def fail_next(self, status: int, count: int = 1, *, path: str | None = None) -> None:
"""Make the next ``count`` requests fail with ``status``.
``path`` narrows it to requests whose path contains that fragment, which
matters because the client logs in before anything else: an untargeted
injected failure would be spent on the login and never reach the call the
test is about. Guarded because it is called between requests while
handler threads are running.
"""
with self._lock:
self._failures.extend([(status, path)] * count)
def _feed_title(self, stream_id: str) -> str:
for feed in self.subscriptions:
if feed["id"] == stream_id:
return feed["title"]
return stream_id
# -- lifecycle -------------------------------------------------------
@property
def base_url(self) -> str:
return f"http://{self._public_host}:{self.port}"
@property
def api_url(self) -> str:
return self.base_url + API_PREFIX
def start(self) -> StubFreshRSS:
handler = _make_handler(self)
server = ThreadingHTTPServer((self._bind_host, self._bind_port), handler)
self._server = server
# Read the port back, because a request for port 0 means "any free port"
# and only the operating system knows which one that turned out to be.
self.port = server.server_address[1]
# A short poll interval keeps teardown cheap: shutdown() only returns once
# the serve loop notices the flag, and the default half second would be
# paid by every test that starts a stub.
self._thread = threading.Thread(
target=server.serve_forever,
kwargs={"poll_interval": 0.02},
daemon=True,
)
self._thread.start()
return self
def stop(self) -> None:
if self._server is not None:
self._server.shutdown()
self._server.server_close()
self._server = None
if self._thread is not None:
self._thread.join(timeout=5)
self._thread = None
def __enter__(self) -> StubFreshRSS:
return self.start()
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: object,
) -> None:
self.stop()
# -- request handling ------------------------------------------------
def record(self, entry: dict[str, Any]) -> None:
with self._lock:
self.requests.append(entry)
def next_failure(self, path: str) -> int | None:
"""The next injected failure that applies to ``path``, if any."""
with self._lock:
for index, (status, target) in enumerate(self._failures):
if target is None or target in path:
del self._failures[index]
return status
return None
def find_requests(self, fragment: str) -> list[dict[str, Any]]:
"""Every recorded request whose path contains ``fragment``.
A substring match rather than a suffix, because a stream path is followed
by the encoded stream id (``/stream/contents/feed%2F...``) and so never
ends where the interesting part of the route does.
"""
with self._lock:
return [entry for entry in self.requests if fragment in entry["path"]]
def mark_read(self, identifiers: list[str]) -> None:
with self._lock:
self.read.update(identifiers)
def clear_read(self) -> None:
"""Forget every read mark, for tests that re-run a marking step."""
with self._lock:
self.read.clear()
def read_snapshot(self) -> set[str]:
"""A copy of the read marks, taken under the lock."""
with self._lock:
return set(self.read)
def subscription_payload(self) -> dict[str, Any]:
return {"subscriptions": self.subscriptions}
def unread_count_payload(self) -> dict[str, Any]:
counts = []
for feed in self.subscriptions:
unread = sum(
1
for entry in self.items.get(feed["id"], [])
if isinstance(entry, dict) and entry.get("id") not in self.read
)
counts.append(
{
"id": feed["id"],
"count": unread,
"newestItemTimestampUsec": "1700000000000000",
}
)
return {"max": self.page_limit, "unreadcounts": counts}
def stream_payload(self, stream_id: str, query: dict[str, list[str]]) -> dict[str, Any]:
"""Items for a stream, honouring ``xt``, ``ot``, ``n`` and ``r``."""
read_ids = self.read_snapshot()
if stream_id == READING_LIST:
entries = [entry for group in self.items.values() for entry in group]
elif stream_id == STARRED_TAG:
entries = [
entry
for group in self.items.values()
for entry in group
if _is_tagged(entry, STARRED_TAG, read_ids)
]
elif stream_id.startswith("user/-/label/"):
label = stream_id.rsplit("/", 1)[-1]
labelled = {
feed["id"]
for feed in self.subscriptions
if any(category["label"] == label for category in feed["categories"])
}
entries = [
entry
for feed_id, group in self.items.items()
if feed_id in labelled
for entry in group
]
else:
entries = list(self.items.get(stream_id, []))
excluded = set(query.get("xt", []))
if excluded:
entries = [
entry
for entry in entries
if not any(_is_tagged(entry, tag, read_ids) for tag in excluded)
]
older_than = query.get("ot", [None])[0]
if older_than and older_than.isdigit():
entries = [
entry
for entry in entries
if not isinstance(entry, dict) or _published(entry) >= int(older_than)
]
descending = query.get("r", ["d"])[0] != "o"
entries.sort(key=_published, reverse=descending)
# Marking an item read adds the tag to the item itself upstream, so the
# returned payload has to carry it too -- otherwise a client could never
# observe a read mark it just made.
entries = [_with_read_state(entry, read_ids) for entry in entries]
limit = query.get("n", [None])[0]
if limit and limit.isdigit():
entries = entries[: int(limit)]
return {"id": stream_id, "updated": 1_700_000_000, "items": entries}
def _make_handler(stub: StubFreshRSS):
"""Build the request handler class bound to one stub instance."""
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "StubFreshRSS/1.0"
# The parameter must keep the base class's name for keyword compatibility,
# even though it shadows the builtin.
def log_message(self, format: str, *args: object) -> None:
"""Silence the default access log; tests assert on recorded requests."""
# -- plumbing ----------------------------------------------------
def _respond(self, status: int, body: bytes, content_type: str) -> None:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
if body and self.command != "HEAD":
self.wfile.write(body)
def _respond_json(self, payload: Any, status: int = 200) -> None:
self._respond(
status,
json.dumps(payload).encode("utf-8"),
"application/json; charset=utf-8",
)
def _record(self, body: str) -> tuple[str, dict[str, list[str]]]:
parsed = urllib.parse.urlsplit(self.path)
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
route = parsed.path
if route.startswith(API_PREFIX):
route = route[len(API_PREFIX) :]
stub.record(
{
"method": self.command,
"path": route,
"raw_path": self.path,
"query": query,
"body": body,
"authorization": self.headers.get("Authorization", ""),
}
)
return route, query
def _authorised(self) -> bool:
expected = f"GoogleLogin auth={stub.auth_token}"
return self.headers.get("Authorization", "") == expected
def _drain_body(self) -> str:
length = int(self.headers.get("Content-Length") or 0)
if not length:
return ""
return self.rfile.read(length).decode("utf-8", "replace")
# -- routes ------------------------------------------------------
def do_HEAD(self) -> None:
self.do_GET()
def do_GET(self) -> None:
route, query = self._record("")
failure = stub.next_failure(route)
if failure is not None:
self._respond_json({"error": "injected failure"}, status=failure)
return
if route == "/healthz":
# Unauthenticated on purpose: it only says the server answered, and
# it is what the demo compose stack waits on before starting the
# catalogue. Real FreshRSS has no such route.
self._respond_json({"status": "ok"})
return
if route == "/accounts/ClientLogin":
self._respond_json({"error": "ClientLogin is a POST"}, status=405)
return
if route == "/reader/api/0/subscription/list":
if not self._require_auth():
return
if stub.corrupt_json:
self._respond(200, b"<html>login page</html>", "text/html")
return
self._respond_json(stub.subscription_payload())
return
if route == "/reader/api/0/unread-count":
if not self._require_auth():
return
if not stub.supports_unread_counts:
self._respond_json({"error": "unsupported"}, status=404)
return
self._respond_json(stub.unread_count_payload())
return
if route == "/reader/api/0/token":
if not self._require_auth():
return
self._respond(200, stub.write_token.encode("utf-8"), "text/plain")
return
if route.startswith("/reader/api/0/stream/contents/"):
if not self._require_auth():
return
stream_id = urllib.parse.unquote(route[len("/reader/api/0/stream/contents/") :])
self._respond_json(stub.stream_payload(stream_id, query))
return
self._respond_json({"error": "not found"}, status=404)
def do_POST(self) -> None:
body = self._drain_body()
route, _query = self._record(body)
failure = stub.next_failure(route)
if failure is not None:
self._respond_json({"error": "injected failure"}, status=failure)
return
if route == "/accounts/ClientLogin":
self._handle_login(body)
return
if route == "/reader/api/0/edit-tag":
if not self._require_auth():
return
self._handle_edit_tag(body)
return
self._respond_json({"error": "not found"}, status=404)
def _require_auth(self) -> bool:
if self._authorised():
return True
self._respond_json({"error": "unauthorised"}, status=401)
return False
def _handle_login(self, body: str) -> None:
form = urllib.parse.parse_qs(body)
username = form.get("Email", [""])[0]
password = form.get("Passwd", [""])[0]
if username != stub.username or password != stub.password:
self._respond_json({"error": "BadAuthentication"}, status=403)
return
# The real endpoint returns three lines; only Auth matters to us, and a
# parser that grabs the first "=" would get SID instead.
payload = f"SID=stub-sid\nLSID=stub-lsid\nAuth={stub.auth_token}\n"
self._respond(200, payload.encode("utf-8"), "text/plain; charset=utf-8")
def _handle_edit_tag(self, body: str) -> None:
form = urllib.parse.parse_qs(body, keep_blank_values=True)
token = form.get("T", [""])[0]
if token != stub.write_token:
self._respond_json({"error": "Invalid token"}, status=401)
return
if form.get("a", [""])[0] != READ_TAG:
self._respond_json({"error": "unexpected tag"}, status=400)
return
stub.mark_read(form.get("i", []))
self._respond(200, b"OK", "text/plain; charset=utf-8")
return Handler
def build_demo_catalog(**stub_options: Any) -> StubFreshRSS:
"""A small but realistic account, used by tests and the manual smoke tool.
Three feeds across two categories, with a read item, a starred item, an item
in a feed that has no category, and content that exercises the sanitiser
(relative links, a script tag, an unclosed list, an entity).
Keyword options are forwarded to :class:`StubFreshRSS`, which is how
``tools/fake-freshrss.py`` serves this same account to a container --
``bind_host``, ``bind_port`` and ``public_host``, since a compose network needs
a wildcard bind, a fixed port and a service name in place of loopback. The
defaults are the loopback bind and ephemeral port the tests rely on.
"""
stub = StubFreshRSS(**stub_options)
tech = stub.add_feed("feed/https://example.com/tech/atom.xml", "Example Tech", ["Tech"])
news = stub.add_feed(
"feed/https://news.example.org/rss?source=daily",
"Daily News",
["News"],
)
lonely = stub.add_feed("feed/https://solo.example.net/feed.xml", "Solo Blog")
stub.add_item(
tech,
1,
"A Tech Post & Its Friends",
content=(
'<p>Intro with an <a href="/relative-link">in-page link</a>.</p>'
"<script>alert('nope')</script>"
"<ul><li>first<li>second</ul>"
'<p><img src="/images/diagram.png" alt="A diagram"></p>'
"<p>Entities: &amp; &lt;tag&gt; &nbsp; done.</p>"
),
published=1_700_000_100,
)
stub.add_item(tech, 2, "An Older Tech Post", published=1_699_000_000)
stub.add_item(
tech,
3,
"Already Read Post",
published=1_698_000_000,
read=True,
)
stub.add_item(news, 4, "Markets Dip Slightly", published=1_700_000_200, author=None)
stub.add_item(
news,
5,
"Starred Investigation",
published=1_700_000_300,
starred=True,
)
stub.add_item(lonely, 6, "Uncategorised Musings", published=1_700_000_050)
return stub

235
tests/support.py Normal file
View File

@@ -0,0 +1,235 @@
"""Shared test helpers.
The XML helper here is deliberately not ``xml.etree``: that API is documented as
unsafe for untrusted documents because it will expand entities and honour a DTD.
Even though every document these tests parse is one this project generated, the
parser is configured to refuse DTDs, entity declarations and external entity
resolution outright. Refusing them is stronger than trusting the parser's
amplification limits, and it keeps the test suite free of the pattern that makes
reviewers (and linters) rightly nervous.
"""
from __future__ import annotations
import re
from xml.parsers import expat
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
_WS_RE = re.compile(r"\s+")
def local_name(tag: str) -> str:
"""The part of a tag after any namespace prefix.
Expat is used without namespace processing here, so a prefixed element arrives
as ``dc:title``. Structural assertions care about the element, not the
prefix, so lookups compare the local name and read naturally.
"""
return tag.rsplit(":", 1)[-1]
class XmlNode:
"""A minimal element tree node, enough for structural assertions."""
__slots__ = ("attrs", "children", "tag", "text")
def __init__(self, tag: str, attrs: dict[str, str] | None = None) -> None:
self.tag = tag
self.attrs: dict[str, str] = attrs or {}
self.children: list[XmlNode] = []
self.text = ""
def attr(self, name: str, default: str | None = None) -> str | None:
"""Return an attribute value (None when absent)."""
return self.attrs.get(name, default)
def must_find(self, tag: str) -> XmlNode:
"""The first descendant with this tag, failing loudly when absent.
Structural assertions want to say "the OPF has no <manifest>" rather than
hand back a None that only explodes on the next attribute access.
"""
found = self.find(tag)
if found is None:
raise AssertionError(f"no <{tag}> element in <{self.tag}>")
return found
def must_find_all(self, tag: str) -> list[XmlNode]:
"""Every descendant with this tag, failing loudly when there are none."""
found = self.find_all(tag)
if not found:
raise AssertionError(f"no <{tag}> elements in <{self.tag}>")
return found
def must_attr(self, name: str) -> str:
"""An attribute value that must be present."""
value = self.attrs.get(name)
if value is None:
raise AssertionError(f"no {name!r} attribute on <{self.tag}>")
return value
def find_all(self, tag: str) -> list[XmlNode]:
"""All descendants matching a tag name, in document order.
Matches the local name, so ``find("title")`` finds ``dc:title`` while
``find("dc:title")`` still works.
"""
wanted = local_name(tag)
found: list[XmlNode] = []
for child in self.children:
if child.tag == tag or local_name(child.tag) == wanted:
found.append(child)
found.extend(child.find_all(tag))
return found
def find(self, tag: str) -> XmlNode | None:
"""First descendant with a matching tag name, or None."""
matches = self.find_all(tag)
return matches[0] if matches else None
def all_text(self) -> str:
"""Concatenated text of this node and its descendants."""
return self.text + "".join(child.all_text() for child in self.children)
def direct_text(self) -> str:
"""Text belonging to this node only, ignoring descendants."""
return self.text
def parse_xml(data: str | bytes) -> XmlNode:
"""Parse XML into an :class:`XmlNode` tree, refusing any DTD or entity.
Raises ``expat.ExpatError`` for malformed input and ``ValueError`` for a
document that tries to declare a doctype or an entity.
"""
parser = expat.ParserCreate()
parser.buffer_text = True
# Parameter entities come from a DTD, which we never accept.
parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_NEVER)
def reject_doctype(doctype_name, system_id, public_id, has_internal_subset):
raise ValueError("doctype declarations are not allowed in generated XML")
def reject_entity(*_args):
raise ValueError("entity declarations are not allowed in generated XML")
def reject_external(*_args):
# Returning 0 makes expat fail the parse instead of resolving the entity.
return 0
parser.StartDoctypeDeclHandler = reject_doctype
parser.EntityDeclHandler = reject_entity
parser.ExternalEntityRefHandler = reject_external
root = XmlNode("#document")
stack = [root]
def start_element(tag: str, attrs: dict[str, str]) -> None:
stack.append(XmlNode(tag, dict(attrs)))
def end_element(_tag: str) -> None:
node = stack.pop()
stack[-1].children.append(node)
def characters(data: str) -> None:
stack[-1].text += data
parser.StartElementHandler = start_element
parser.EndElementHandler = end_element
parser.CharacterDataHandler = characters
parser.Parse(data, True)
if not root.children:
raise ValueError("XML document has no root element")
return root.children[0]
def assert_well_formed(testcase, fragment: str) -> None:
"""Assert that ``fragment`` parses as XML inside a single wrapper root."""
markup = f'<root xmlns="{XHTML_NAMESPACE}">{fragment}</root>'
try:
parse_xml(markup)
except (expat.ExpatError, ValueError) as exc: # pragma: no cover - on failure
testcase.fail(f"output is not well-formed XML: {exc}\n{fragment}")
def collapse(text: str) -> str:
"""Collapse all whitespace, for assertions that do not care about layout."""
return _WS_RE.sub(" ", text).strip()
def body(fragment: str) -> str:
"""Return the inner markup of the article wrapper the sanitiser emits."""
prefix = '<div class="article-body">'
if fragment.startswith(prefix):
fragment = fragment[len(prefix) :]
suffix = "</div>"
if fragment.endswith(suffix):
fragment = fragment[: -len(suffix)]
return fragment.strip()
# -- image builders ------------------------------------------------------
#
# These produce bytes that are structurally real -- correct magic numbers and
# container headers -- so sniffing and dimension parsing are exercised against the
# formats rather than against hand-written expectations. Pixel data is minimal
# because nothing here ever decodes an image.
def png_bytes(width: int = 800, height: int = 400) -> bytes:
"""A structurally valid PNG of the given size."""
import struct
import zlib
def chunk(tag: bytes, payload: bytes) -> bytes:
crc = zlib.crc32(tag + payload) & 0xFFFFFFFF
return struct.pack(">I", len(payload)) + tag + payload + struct.pack(">I", crc)
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
rows = b"".join(b"\x00" + b"\xff\x00\x00" * width for _ in range(height))
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(rows, 9))
+ chunk(b"IEND", b"")
)
def jpeg_bytes(width: int = 800, height: int = 400) -> bytes:
"""SOI + APP0 + SOF0 + EOI: a JPEG the dimension walker can read."""
import struct
app0 = b"\xff\xe0" + struct.pack(">H", 16) + b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00"
# The frame header carries height and width, in that order.
sof0 = (
b"\xff\xc0"
+ struct.pack(">H", 17)
+ b"\x08"
+ struct.pack(">HH", height, width)
+ b"\x03\x01\x11\x00\x02\x11\x01\x03\x11\x01"
)
return b"\xff\xd8" + app0 + sof0 + b"\xff\xd9"
def gif_bytes(width: int = 320, height: int = 200) -> bytes:
"""A GIF header with the given logical screen size."""
import struct
return b"GIF89a" + struct.pack("<HH", width, height) + b"\xf7\x00\x00" + b"\x00" * 20
def webp_bytes(width: int = 800, height: int = 400) -> bytes:
"""A lossless VP8X WebP header with the given canvas size (stored minus one)."""
import struct
return (
b"RIFF"
+ struct.pack("<I", 22)
+ b"WEBP"
+ b"VP8X"
+ struct.pack("<I", 10)
+ b"\x00\x00\x00\x00"
+ (width - 1).to_bytes(3, "little")
+ (height - 1).to_bytes(3, "little")
)

487
tests/test_config.py Normal file
View File

@@ -0,0 +1,487 @@
"""Tests for configuration loading, interpolation and validation."""
from __future__ import annotations
import json
import os
import tempfile
import unittest
from pathlib import Path
from app.config import (
DEFAULT_BIND_HOST,
ConfigError,
describe_config,
load_config,
normalise_api_base,
password_digest,
resolve_config_path,
)
# Fixture credentials. These are throwaway values that exercise the schema; no
# real account is involved and neither value is a secret. They are applied
# through the helpers below so every test states its intent in one line.
FIXTURE_UPSTREAM_VALUE = "upstream-fixture-value"
FIXTURE_READER_VALUE = "reader-fixture-value"
REPO_ROOT = Path(__file__).resolve().parent.parent
EXAMPLE_CONFIG = REPO_ROOT / "config.example.json"
def with_upstream_credential(document: dict, value: str) -> dict:
"""Set the upstream FreshRSS API credential under test."""
document["freshrss"]["api_password"] = value
return document
def with_reader_credential(document: dict, value: str, username: str = "reader") -> dict:
"""Replace auth.users with one user carrying a clear-text credential."""
document["auth"]["users"] = [{"username": username, "password": value}]
return document
def with_reader_digest(document: dict, digest: str, username: str = "reader") -> dict:
"""Replace auth.users with one user carrying a pre-computed digest."""
document["auth"]["users"] = [{"username": username, "password_sha256": digest}]
return document
def with_conflicting_credentials(document: dict, value: str) -> dict:
"""A deliberately over-specified user, to exercise the conflict rule."""
document["auth"]["users"] = [
{
"username": "reader",
"password": value,
"password_sha256": password_digest(value),
}
]
return document
def base_config() -> dict:
"""A minimal valid configuration, ready to be mutated by a test."""
document = {
"server": {"port": 8080},
"freshrss": {
"url": "https://rss.example.com",
"username": "tyler",
"api_password": FIXTURE_UPSTREAM_VALUE,
},
"auth": {"users": [{"username": "tyler", "password": FIXTURE_READER_VALUE}]},
}
return document
class ConfigTestCase(unittest.TestCase):
"""Writes a config document into a temporary directory and loads it."""
def setUp(self):
self._dir = tempfile.TemporaryDirectory()
self.addCleanup(self._dir.cleanup)
self.dir = Path(self._dir.name)
def write(self, document, name: str = "config.json") -> str:
path = self.dir / name
if isinstance(document, str):
path.write_text(document, encoding="utf-8")
else:
path.write_text(json.dumps(document), encoding="utf-8")
return str(path)
def load(self, document, name: str = "config.json"):
return load_config(self.write(document, name))
def expect_problem(self, document, fragment: str) -> ConfigError:
"""Assert the config is rejected, and return the error for more checks."""
with self.assertRaises(ConfigError) as caught:
self.load(document)
joined = "\n".join(caught.exception.problems)
self.assertIn(fragment, joined)
return caught.exception
class LoadingTests(ConfigTestCase):
def test_minimal_config_uses_documented_defaults(self):
config = self.load(base_config())
self.assertEqual(config.server.host, DEFAULT_BIND_HOST)
self.assertEqual(config.server.port, 8080)
self.assertEqual(config.freshrss.items_per_page, 30)
self.assertTrue(config.freshrss.mark_read)
self.assertTrue(config.epub.include_images)
self.assertEqual(config.auth.users[0].username, "tyler")
self.assertEqual(len(config.auth.users[0].password_sha256), 64)
def test_default_bind_address_is_loopback(self):
# The wildcard belongs in the deployment's config file, not in the code:
# a stray `docker run -p 8080:8080` should not silently be reachable.
self.assertEqual(DEFAULT_BIND_HOST, "127.0.0.1")
def test_missing_file_is_a_clear_error(self):
with self.assertRaises(ConfigError) as caught:
load_config(str(self.dir / "nope.json"))
joined = "\n".join(caught.exception.problems)
self.assertIn("not found", joined)
self.assertIn("CONFIG_PATH", joined)
def test_invalid_json_reports_position(self):
error = self.expect_problem('{"server": {,}', "not valid JSON")
self.assertIn("line 1", "\n".join(error.problems))
def test_non_object_root_is_rejected(self):
self.expect_problem("[1, 2, 3]", "top level")
def test_source_path_is_recorded(self):
path = self.write(base_config())
self.assertEqual(load_config(path).source_path, str(Path(path).resolve()))
def test_resolve_config_path_prefers_explicit_then_env(self):
explicit = self.write(base_config(), "explicit.json")
self.assertEqual(resolve_config_path(explicit), explicit)
from_env = str(self.dir / "from-env.json")
os.environ["CONFIG_PATH"] = from_env
self.addCleanup(os.environ.pop, "CONFIG_PATH", None)
self.assertEqual(resolve_config_path(), from_env)
class ExampleConfigTests(unittest.TestCase):
"""The shipped example must load: a broken example is a broken first run.
This is also where the container's wildcard bind address is asserted. That
literal belongs in the deployment's configuration file, not in Python, and the
test proves the file really does override the loopback default.
"""
def setUp(self):
# The example deliberately leaves secrets to the environment.
os.environ["FRESHRSS_API_PASSWORD"] = FIXTURE_UPSTREAM_VALUE
os.environ["OPDS_PASSWORD_SHA256"] = password_digest(FIXTURE_READER_VALUE)
self.addCleanup(os.environ.pop, "FRESHRSS_API_PASSWORD", None)
self.addCleanup(os.environ.pop, "OPDS_PASSWORD_SHA256", None)
self.document = json.loads(EXAMPLE_CONFIG.read_text(encoding="utf-8"))
def test_example_config_exists_and_is_valid(self):
config = load_config(str(EXAMPLE_CONFIG))
self.assertEqual(config.warnings, [])
self.assertEqual(len(config.auth.users), 1)
def test_example_config_overrides_the_loopback_default(self):
config = load_config(str(EXAMPLE_CONFIG))
self.assertNotEqual(config.server.host, DEFAULT_BIND_HOST)
self.assertEqual(config.server.host, self.document["server"]["host"])
def test_example_keeps_secrets_out_of_the_file(self):
self.assertTrue(self.document["freshrss"]["api_password"].startswith("${"))
self.assertTrue(self.document["auth"]["users"][0]["password_sha256"].startswith("${"))
def test_example_is_self_documenting(self):
comments = [key for key in self.document if key.startswith("_")]
self.assertTrue(comments)
self.assertTrue(self.document["server"].get("_host"))
def test_example_interpolates_its_credentials(self):
self.assertEqual(
load_config(str(EXAMPLE_CONFIG)).freshrss.api_password, FIXTURE_UPSTREAM_VALUE
)
class InterpolationTests(ConfigTestCase):
def test_environment_variable_is_expanded(self):
os.environ["TEST_FRESHRSS_CREDENTIAL"] = "from-env"
self.addCleanup(os.environ.pop, "TEST_FRESHRSS_CREDENTIAL", None)
document = with_upstream_credential(base_config(), "${TEST_FRESHRSS_CREDENTIAL}")
self.assertEqual(self.load(document).freshrss.api_password, "from-env")
def test_environment_variable_default_is_used_when_unset(self):
os.environ.pop("TEST_UNSET_VAR", None)
document = with_upstream_credential(base_config(), "${TEST_UNSET_VAR:-fallback}")
self.assertEqual(self.load(document).freshrss.api_password, "fallback")
def test_missing_environment_variable_names_itself(self):
os.environ.pop("TEST_DEFINITELY_UNSET", None)
document = with_upstream_credential(base_config(), "${TEST_DEFINITELY_UNSET}")
error = self.expect_problem(document, "TEST_DEFINITELY_UNSET")
self.assertIn("freshrss.api_password", "\n".join(error.problems))
def test_secret_file_is_read_and_trimmed(self):
(self.dir / "upstream-secret").write_text("file-secret\n", encoding="utf-8")
# A relative path resolves against the config file's own directory.
document = with_upstream_credential(base_config(), "${file:upstream-secret}")
self.assertEqual(self.load(document).freshrss.api_password, "file-secret")
def test_secret_file_may_be_absolute(self):
absolute = self.dir / "absolute-secret"
absolute.write_text("abs", encoding="utf-8")
document = with_upstream_credential(base_config(), f"${{file:{absolute}}}")
self.assertEqual(self.load(document).freshrss.api_password, "abs")
def test_missing_secret_file_is_an_error(self):
document = with_upstream_credential(base_config(), "${file:not-there}")
self.expect_problem(document, "cannot read secret file")
def test_escaped_dollar_is_literal(self):
document = with_upstream_credential(base_config(), "$${NOT_A_REFERENCE}")
self.assertEqual(self.load(document).freshrss.api_password, "${NOT_A_REFERENCE}")
def test_interpolation_reaches_nested_arrays(self):
os.environ.pop("TEST_PW", None)
document = with_reader_credential(base_config(), "${TEST_PW:-pw}")
self.assertEqual(self.load(document).auth.users[0].password_sha256, password_digest("pw"))
def test_empty_reference_is_an_error(self):
document = with_upstream_credential(base_config(), "${}")
self.expect_problem(document, "empty ${}")
class FreshRSSTests(ConfigTestCase):
def test_api_base_appends_the_greader_suffix(self):
self.assertEqual(
normalise_api_base("https://rss.example.com"),
"https://rss.example.com/api/greader.php",
)
def test_api_base_keeps_an_existing_suffix(self):
self.assertEqual(
normalise_api_base("https://rss.example.com/api/greader.php"),
"https://rss.example.com/api/greader.php",
)
def test_api_base_preserves_a_path_prefix(self):
self.assertEqual(
normalise_api_base("https://example.com/freshrss/"),
"https://example.com/freshrss/api/greader.php",
)
def test_api_base_is_case_insensitive_about_the_suffix(self):
self.assertEqual(
normalise_api_base("https://e.com/API/GReader.php"),
"https://e.com/API/GReader.php",
)
def test_url_without_scheme_is_rejected(self):
document = base_config()
document["freshrss"]["url"] = "rss.example.com"
self.expect_problem(document, "must start with http:// or https://")
def test_required_fields_are_reported_together(self):
document = {"server": {}, "auth": {"users": [{"username": "u", "password": "p"}]}}
with self.assertRaises(ConfigError) as caught:
self.load(document)
joined = "\n".join(caught.exception.problems)
for fragment in ("freshrss.url", "freshrss.username", "freshrss.api_password"):
self.assertIn(fragment, joined)
def test_items_per_page_must_fit_the_request_ceiling(self):
document = base_config()
document["freshrss"]["items_per_page"] = 100
document["freshrss"]["max_items_per_request"] = 50
self.expect_problem(document, "must be at least")
def test_retries_and_timeouts_are_validated(self):
document = base_config()
document["freshrss"]["timeout_seconds"] = 0
document["freshrss"]["retries"] = -1
with self.assertRaises(ConfigError) as caught:
self.load(document)
joined = "\n".join(caught.exception.problems)
self.assertIn("timeout_seconds", joined)
self.assertIn("retries", joined)
def test_mark_read_can_be_disabled(self):
document = base_config()
document["freshrss"]["mark_read"] = False
self.assertFalse(self.load(document).freshrss.mark_read)
class TypeValidationTests(ConfigTestCase):
def test_port_must_be_a_number_in_range(self):
for value, fragment in (
("8080", "whole number"),
(-1, "at least"),
(70000, "at most"),
):
document = base_config()
document["server"]["port"] = value
self.expect_problem(document, fragment)
def test_port_zero_asks_the_operating_system_for_a_free_port(self):
# Useful for a local run or a test; a normal deployment sets a real port.
document = base_config()
document["server"]["port"] = 0
self.assertEqual(self.load(document).server.port, 0)
def test_boolean_field_rejects_a_number(self):
document = base_config()
document["freshrss"]["mark_read"] = 1
self.expect_problem(document, "must be true or false")
def test_integer_field_rejects_a_boolean(self):
# bool is an int subclass in Python; `true` must not become 1.
document = base_config()
document["server"]["port"] = True
self.expect_problem(document, "whole number")
def test_string_field_rejects_a_list(self):
document = base_config()
document["catalog"] = {"title": ["not", "a", "string"]}
self.expect_problem(document, "catalog.title: must be a string")
def test_string_list_rejects_scalars(self):
document = base_config()
document["catalog"] = {"exclude_feeds": "one-feed"}
self.expect_problem(document, "must be an array of strings")
def test_section_must_be_an_object(self):
document = base_config()
document["cache"] = "nope"
self.expect_problem(document, "cache: must be a JSON object")
def test_log_level_is_restricted(self):
document = base_config()
document["server"]["log_level"] = "chatty"
self.expect_problem(document, "must be one of")
def test_ttl_must_be_positive(self):
document = base_config()
document["cache"] = {"stream_ttl_seconds": 0}
self.expect_problem(document, "greater than 0")
class AuthTests(ConfigTestCase):
def test_users_are_required(self):
document = base_config()
del document["auth"]
error = self.expect_problem(document, "auth.users: required")
self.assertIn("credentials", "\n".join(error.problems))
def test_empty_user_list_is_rejected(self):
document = base_config()
document["auth"]["users"] = []
self.expect_problem(document, "at least one user")
def test_clear_text_password_is_hashed(self):
document = with_reader_credential(base_config(), FIXTURE_READER_VALUE)
config = self.load(document)
self.assertEqual(
config.auth.users[0].password_sha256, password_digest(FIXTURE_READER_VALUE)
)
def test_clear_text_password_is_never_kept_in_the_config_object(self):
document = with_reader_credential(base_config(), FIXTURE_READER_VALUE)
config = self.load(document)
self.assertNotIn(FIXTURE_READER_VALUE, json.dumps(config.auth.users[0].__dict__))
def test_sha256_password_is_accepted_and_lowercased(self):
document = with_reader_digest(base_config(), password_digest("s3cret").upper())
self.assertEqual(
self.load(document).auth.users[0].password_sha256, password_digest("s3cret")
)
def test_both_password_forms_is_an_error(self):
document = with_conflicting_credentials(base_config(), FIXTURE_READER_VALUE)
self.expect_problem(document, "not both")
def test_neither_password_form_is_an_error(self):
document = base_config()
document["auth"]["users"] = [{"username": "a"}]
self.expect_problem(document, "needs password or password_sha256")
def test_malformed_digest_is_rejected(self):
document = with_reader_digest(base_config(), "deadbeef")
self.expect_problem(document, "64 hex characters")
def test_duplicate_usernames_are_rejected(self):
document = base_config()
document["auth"]["users"] = [
{"username": "a", "password": FIXTURE_READER_VALUE},
{"username": "a", "password": FIXTURE_UPSTREAM_VALUE},
]
self.expect_problem(document, "duplicate username")
def test_user_must_be_an_object(self):
document = base_config()
document["auth"]["users"] = ["tyler"]
self.expect_problem(document, "must be a JSON object")
def test_realm_is_configurable(self):
document = base_config()
document["auth"]["realm"] = "Reading"
self.assertEqual(self.load(document).auth.realm, "Reading")
class TlsTests(ConfigTestCase):
def test_cert_without_key_is_rejected(self):
document = base_config()
document["server"]["tls"] = {"cert_file": str(self.dir / "cert.pem")}
self.expect_problem(document, "must be set together")
def test_missing_certificate_file_is_rejected(self):
document = base_config()
document["server"]["tls"] = {
"cert_file": str(self.dir / "absent-cert.pem"),
"key_file": str(self.dir / "absent-key.pem"),
}
self.expect_problem(document, "no such file")
def test_present_certificate_pair_is_accepted(self):
cert = self.dir / "cert.pem"
key = self.dir / "key.pem"
cert.write_text("dummy", encoding="utf-8")
key.write_text("dummy", encoding="utf-8")
document = base_config()
document["server"]["tls"] = {"cert_file": str(cert), "key_file": str(key)}
config = self.load(document)
self.assertEqual(config.server.tls_cert_file, str(cert))
class WarningTests(ConfigTestCase):
def test_unknown_key_is_reported_with_a_suggestion(self):
document = base_config()
document["freshrss"]["mark-read"] = True
config = self.load(document)
self.assertTrue(config.warnings)
self.assertIn("freshrss.mark-read", config.warnings[0])
self.assertIn("mark_read", config.warnings[0])
def test_unknown_user_key_is_reported(self):
document = base_config()
document["auth"]["users"] = [
{"username": "a", "password": FIXTURE_READER_VALUE, "passwrod": "typo"}
]
config = self.load(document)
self.assertIn("auth.users[0].passwrod", config.warnings[0])
def test_unknown_tls_key_is_reported(self):
document = base_config()
document["server"]["tls"] = {"certfile": "/x", "keyfile": "/y"}
config = self.load(document)
self.assertEqual(len(config.warnings), 2)
def test_unknown_section_is_reported(self):
document = base_config()
document["cach"] = {"max_entries": 10}
config = self.load(document)
self.assertIn("cach:", config.warnings[0])
self.assertIn("cache", config.warnings[0])
def test_warnings_do_not_block_a_valid_config(self):
document = base_config()
document["nonsense"] = 1
self.assertEqual(len(self.load(document).warnings), 1)
class DescribeConfigTests(ConfigTestCase):
def test_password_is_redacted_in_the_summary(self):
summary = describe_config(self.load(base_config()))
self.assertNotIn(FIXTURE_UPSTREAM_VALUE, summary)
self.assertIn("rss.example.com/api/greader.php", summary)
def test_summary_lists_users(self):
self.assertIn("tyler", describe_config(self.load(base_config())))
def test_summary_notes_tls_state(self):
summary = describe_config(self.load(base_config()))
self.assertIn("127.0.0.1:8080", summary)
if __name__ == "__main__":
unittest.main()

533
tests/test_epub.py Normal file
View File

@@ -0,0 +1,533 @@
"""Tests for image metadata and EPUB generation.
The EPUB checks are structural on purpose. A book is rejected by a reader for
shape reasons -- a deflated mimetype, a manifest href with no entry, a document
that will not parse -- far more often than for content reasons, so those are the
properties asserted here.
"""
from __future__ import annotations
import unittest
import zipfile
from datetime import UTC, datetime
from io import BytesIO
from app import epub, images
from app.fetch import FetchedImage
from app.xhtml import parse_article
from tests.support import (
gif_bytes,
jpeg_bytes,
parse_xml,
png_bytes,
webp_bytes,
)
MODIFIED = datetime(2026, 11, 15, 12, 30, 45, tzinfo=UTC)
PUBLISHED = datetime(2026, 11, 14, 9, 0, 0, tzinfo=UTC)
class ImageMetadataTests(unittest.TestCase):
"""Sniffing and sizing, against structurally real image bytes."""
def test_sniffing_each_supported_format(self):
cases = {
"image/png": png_bytes(),
"image/jpeg": jpeg_bytes(),
"image/gif": gif_bytes(),
"image/webp": webp_bytes(),
}
for expected, data in cases.items():
self.assertEqual(images.sniff_media_type(data), expected)
def test_dimensions_are_read_for_each_format(self):
cases = [
(png_bytes(800, 400), (800, 400)),
(jpeg_bytes(640, 480), (640, 480)),
(gif_bytes(320, 200), (320, 200)),
(webp_bytes(1024, 512), (1024, 512)),
]
for data, expected in cases:
self.assertEqual(images.dimensions(data), expected)
def test_unknown_bytes_have_no_media_type(self):
self.assertIsNone(images.sniff_media_type(b"not an image at all"))
self.assertIsNone(images.sniff_media_type(b""))
self.assertIsNone(images.sniff_media_type(b"\x89PNG"))
def test_truncated_images_do_not_crash_the_sizer(self):
for data in (png_bytes()[:12], jpeg_bytes()[:6], gif_bytes()[:8], webp_bytes()[:20]):
self.assertIsNone(images.dimensions(data))
def test_svg_is_not_an_embeddable_type(self):
# SVG can carry script and external references; it is not on the list.
self.assertIsNone(images.normalise_media_type("image/svg+xml"))
def test_content_type_parameters_are_ignored(self):
self.assertEqual(images.normalise_media_type("image/jpeg; charset=binary"), "image/jpeg")
def test_aliases_are_normalised(self):
self.assertEqual(images.normalise_media_type("IMAGE/JPG"), "image/jpeg")
self.assertEqual(images.normalise_media_type("image/x-png"), "image/png")
def test_bytes_win_over_a_wrong_declared_type(self):
# A mislabelled payload must not produce an invalid manifest entry.
self.assertEqual(images.identify(png_bytes(), "text/html"), "image/png")
self.assertEqual(images.identify(b"unknown", "image/png"), "image/png")
self.assertIsNone(images.identify(b"unknown", "text/html"))
def test_cover_heuristic(self):
self.assertTrue(images.looks_like_a_cover(800, 400))
self.assertTrue(images.looks_like_a_cover(400, 200))
self.assertFalse(images.looks_like_a_cover(399, 200))
self.assertFalse(images.looks_like_a_cover(1, 1))
self.assertFalse(images.looks_like_a_cover(800, 100))
def make_book(**overrides) -> epub.Book:
values = {
"identifier": "urn:uuid:12345678-1234-1234-1234-123456789abc",
"title": "A Test Article & Its Friends",
"language": "en",
"modified": MODIFIED,
"published": PUBLISHED,
"content_xhtml": '<div class="article-body">\n<p>Body text.</p>\n</div>',
"author": "A. Writer",
"publisher": "Example Press",
"source_url": "https://example.com/articles/1",
"feed_title": "Example Tech",
}
values.update(overrides)
return epub.Book(**values)
class EpubContainerTests(unittest.TestCase):
"""The ZIP-level rules that make a file an EPUB at all."""
def setUp(self):
self.data = epub.build_epub(make_book())
self.archive = zipfile.ZipFile(BytesIO(self.data))
self.addCleanup(self.archive.close)
def test_mimetype_is_the_first_entry(self):
self.assertEqual(self.archive.namelist()[0], "mimetype")
def test_mimetype_is_stored_not_deflated(self):
info = self.archive.getinfo("mimetype")
self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
self.assertEqual(self.archive.read("mimetype"), b"application/epub+zip")
def test_mimetype_has_no_extra_field(self):
# An extra field in the local header is a known cause of readers refusing
# a book, so the raw bytes are inspected rather than the parsed record.
offset = self.archive.getinfo("mimetype").header_offset
local_header = self.data[offset : offset + 30]
extra_length = int.from_bytes(local_header[28:30], "little")
self.assertEqual(extra_length, 0)
def test_container_names_the_package_document(self):
xml = self.archive.read("META-INF/container.xml").decode("utf-8")
root = parse_xml(xml)
rootfile = root.must_find("rootfiles").must_find("rootfile")
self.assertEqual(rootfile.must_attr("full-path"), epub.OPF_PATH)
self.assertEqual(rootfile.must_attr("media-type"), "application/oebps-package+xml")
# Whatever it points at must actually exist.
self.assertIn(rootfile.must_attr("full-path"), self.archive.namelist())
def test_expected_parts_are_present(self):
names = set(self.archive.namelist())
for path in (
"META-INF/container.xml",
epub.OPF_PATH,
epub.NAV_PATH,
epub.NCX_PATH,
epub.CSS_PATH,
epub.COVER_PATH,
epub.ARTICLE_PATH,
):
self.assertIn(path, names)
class EpubDocumentTests(unittest.TestCase):
"""Every text part has to parse with a strict XML parser."""
def setUp(self):
self.data = epub.build_epub(make_book())
self.archive = zipfile.ZipFile(BytesIO(self.data))
self.addCleanup(self.archive.close)
def test_every_xml_part_is_well_formed(self):
for path in (
epub.OPF_PATH,
epub.NAV_PATH,
epub.NCX_PATH,
epub.COVER_PATH,
epub.ARTICLE_PATH,
"META-INF/container.xml",
):
with self.subTest(path=path):
parse_xml(self.archive.read(path))
def test_all_xml_starts_with_a_declaration(self):
for path in (
epub.OPF_PATH,
epub.NAV_PATH,
epub.NCX_PATH,
epub.COVER_PATH,
epub.ARTICLE_PATH,
):
with self.subTest(path=path):
text = self.archive.read(path).decode("utf-8")
self.assertTrue(text.startswith('<?xml version="1.0" encoding="utf-8"?>'))
def test_xhtml_documents_declare_the_namespace(self):
for path in (epub.NAV_PATH, epub.COVER_PATH, epub.ARTICLE_PATH):
with self.subTest(path=path):
root = parse_xml(self.archive.read(path))
self.assertEqual(root.must_attr("xmlns"), epub.XHTML_NAMESPACE)
def test_no_doctype_is_emitted(self):
# A DOCTYPE would have to be resolved by the reader's XML parser; EPUB 3
# makes it optional, so it is left out entirely.
for path in (epub.OPF_PATH, epub.NAV_PATH, epub.ARTICLE_PATH):
with self.subTest(path=path):
self.assertNotIn("<!DOCTYPE", self.archive.read(path).decode("utf-8"))
class EpubPackageTests(unittest.TestCase):
"""OPF metadata, manifest and spine."""
def setUp(self):
self.data = epub.build_epub(make_book())
self.archive = zipfile.ZipFile(BytesIO(self.data))
self.addCleanup(self.archive.close)
self.opf = parse_xml(self.archive.read(epub.OPF_PATH))
def manifest(self) -> dict:
items = self.opf.must_find("manifest").must_find_all("item")
return {item.must_attr("id"): item for item in items}
def test_package_declares_version_and_unique_identifier(self):
self.assertEqual(self.opf.must_attr("version"), "3.0")
self.assertEqual(self.opf.must_attr("unique-identifier"), "pub-id")
def test_required_dublin_core_metadata_is_present(self):
metadata = self.opf.must_find("metadata")
identifier = metadata.must_find("identifier")
self.assertEqual(identifier.must_attr("id"), "pub-id")
self.assertEqual(identifier.all_text(), make_book().identifier)
self.assertEqual(metadata.must_find("title").all_text(), make_book().title)
self.assertEqual(metadata.must_find("language").all_text(), "en")
def test_optional_metadata_is_present_when_known(self):
metadata = self.opf.must_find("metadata")
self.assertEqual(metadata.must_find("creator").all_text(), "A. Writer")
self.assertEqual(metadata.must_find("publisher").all_text(), "Example Press")
self.assertEqual(metadata.must_find("source").all_text(), "https://example.com/articles/1")
self.assertEqual(metadata.must_find("date").all_text(), "2026-11-14T09:00:00Z")
def test_dcterms_modified_has_the_required_format(self):
import re
meta = [
item
for item in self.opf.must_find("metadata").must_find_all("meta")
if item.attr("property") == "dcterms:modified"
]
self.assertEqual(len(meta), 1)
value = meta[0].all_text()
self.assertRegex(value, r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
self.assertEqual(value, "2026-11-15T12:30:45Z")
def test_absent_metadata_is_omitted_rather_than_emptied(self):
data = epub.build_epub(make_book(author=None, publisher=None, source_url=None))
opf = parse_xml(zipfile.ZipFile(BytesIO(data)).read(epub.OPF_PATH))
metadata = opf.must_find("metadata")
self.assertIsNone(metadata.find("creator"))
self.assertIsNone(metadata.find("publisher"))
self.assertIsNone(metadata.find("source"))
def test_navigation_document_is_marked(self):
nav = self.manifest()["nav"]
self.assertEqual(nav.must_attr("href"), "nav.xhtml")
self.assertEqual(nav.must_attr("properties"), "nav")
def test_spine_toc_points_at_the_ncx(self):
self.assertEqual(self.opf.must_find("spine").must_attr("toc"), "ncx")
def test_every_manifest_href_exists_in_the_zip(self):
for identifier, item in self.manifest().items():
with self.subTest(item=identifier):
self.assertIn("OEBPS/" + item.must_attr("href"), self.archive.namelist())
def test_every_spine_idref_resolves(self):
manifest = self.manifest()
for itemref in self.opf.must_find("spine").must_find_all("itemref"):
self.assertIn(itemref.must_attr("idref"), manifest)
def test_every_manifest_item_has_a_media_type(self):
for identifier, item in self.manifest().items():
with self.subTest(item=identifier):
self.assertTrue(item.must_attr("media-type"))
def test_guide_points_at_the_cover_page(self):
reference = self.opf.must_find("guide").must_find("reference")
self.assertEqual(reference.must_attr("type"), "cover")
self.assertEqual(reference.must_attr("href"), "text/cover.xhtml")
def test_ncx_has_matching_uid_and_navpoints(self):
ncx = parse_xml(self.archive.read(epub.NCX_PATH))
self.assertEqual(ncx.must_attr("version"), "2005-1")
uid = [
m for m in ncx.must_find("head").must_find_all("meta") if m.attr("name") == "dtb:uid"
]
self.assertEqual(uid[0].must_attr("content"), make_book().identifier)
self.assertEqual(len(ncx.must_find("navMap").must_find_all("navPoint")), 2)
class EpubContentTests(unittest.TestCase):
def build(self, **overrides) -> tuple[zipfile.ZipFile, bytes]:
data = epub.build_epub(make_book(**overrides))
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
return archive, data
def test_article_document_contains_the_body_and_title(self):
archive, _ = self.build()
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("Body text.", text)
self.assertIn("A Test Article &amp; Its Friends", text)
def test_article_document_carries_a_byline_and_date(self):
archive, _ = self.build()
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("A. Writer", text)
self.assertIn("2026-11-14", text)
def test_source_link_can_be_suppressed(self):
archive, _ = self.build(include_source_link=False)
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertNotIn("Originally published", text)
def test_empty_body_still_produces_a_readable_document(self):
archive, _ = self.build(content_xhtml="")
parse_xml(archive.read(epub.ARTICLE_PATH))
self.assertIn("no content", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
def test_title_with_markup_is_escaped(self):
archive, _ = self.build(title='<script>alert("x")</script>')
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertNotIn("<script>", text)
parse_xml(archive.read(epub.ARTICLE_PATH))
def test_stylesheet_is_embedded(self):
archive, _ = self.build(stylesheet="body { color: red; }")
self.assertEqual(archive.read(epub.CSS_PATH).decode("utf-8"), "body { color: red; }")
def test_default_stylesheet_is_not_empty(self):
archive, _ = self.build()
self.assertIn("line-height", archive.read(epub.CSS_PATH).decode("utf-8"))
class EpubImageTests(unittest.TestCase):
def test_images_are_embedded_and_declared(self):
cover = epub.BookImage(
href="images/img-1.png", media_type="image/png", data=png_bytes(800, 400)
)
data = epub.build_epub(make_book(images=[cover]))
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
self.assertIn("OEBPS/images/img-1.png", archive.namelist())
opf = parse_xml(archive.read(epub.OPF_PATH))
item = opf.must_find("manifest").must_find("item")
self.assertIsNotNone(item)
def test_cover_image_is_marked_and_referenced(self):
cover = epub.BookImage(
href="images/img-1.png",
media_type="image/png",
data=png_bytes(800, 400),
is_cover=True,
)
data = epub.build_epub(make_book(images=[cover]))
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
opf = parse_xml(archive.read(epub.OPF_PATH))
covers = [
item
for item in opf.must_find("manifest").must_find_all("item")
if item.attr("properties") == "cover-image"
]
self.assertEqual(len(covers), 1)
self.assertEqual(covers[0].must_attr("id"), "cover-image")
# The EPUB 2 spelling for older readers.
metas = [
m for m in opf.must_find("metadata").must_find_all("meta") if m.attr("name") == "cover"
]
self.assertEqual(metas[0].must_attr("content"), "cover-image")
# And the cover page shows it, with a path relative to text/.
self.assertIn("../images/img-1.png", archive.read(epub.COVER_PATH).decode("utf-8"))
def test_cover_image_is_not_promoted_without_a_real_cover(self):
small = epub.BookImage(
href="images/img-1.png", media_type="image/png", data=png_bytes(16, 16)
)
book = make_book(images=[small])
_choose = getattr(epub, "_choose_cover")
_choose(book)
self.assertIsNone(book.cover_image)
def test_deterministic_output_for_identical_input(self):
left = epub.build_epub(make_book())
right = epub.build_epub(make_book())
self.assertEqual(left, right)
class BuildArticleEpubTests(unittest.TestCase):
"""The end-to-end assembly step, with and without an image fetcher."""
class Item:
def __init__(self, content: str, title: str = "An Article") -> None:
self.id = "tag:google.com,2005:reader/item/0000000000000001"
self.title = title
self.content_html = content
self.author = "A. Writer"
self.url = "https://example.com/a/1"
self.feed_title = "Example Tech"
self.published = 1_700_000_000
self.updated = 1_700_000_000
self.published_at = datetime.fromtimestamp(1_700_000_000, tz=UTC)
self.updated_at = datetime.fromtimestamp(1_700_000_000, tz=UTC)
class Config:
include_images = True
max_images_per_article = 20
cover_image = True
include_source_link = True
stylesheet = None
max_epub_bytes = 33_554_432
class Catalog:
language = "en"
author = "FreshRSS"
publisher = "Example Press"
class FakeFetcher:
"""Returns one PNG for every URL, recording what it was asked for."""
def __init__(self, fail: bool = False) -> None:
self.asked: list[str] = []
self.fail = fail
def fetch_all(self, urls):
self.asked.extend(urls)
if self.fail:
return {url: None for url in urls}
return {
url: FetchedImage(url=url, media_type="image/png", data=png_bytes(800, 400))
for url in urls
}
def build(self, **overrides) -> tuple[zipfile.ZipFile, bytes]:
item = overrides.pop("item", self.Item("<p>Hello</p>"))
config = overrides.pop("config", self.Config())
fetcher = overrides.pop("fetcher", None)
data = epub.build_article_epub(
item, epub_config=config, catalog_config=self.Catalog(), fetcher=fetcher
)
archive = zipfile.ZipFile(BytesIO(data))
self.addCleanup(archive.close)
return archive, data
def test_book_is_valid_without_a_fetcher(self):
archive, _ = self.build()
parse_xml(archive.read(epub.OPF_PATH))
self.assertIn("Hello", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
def test_article_identifier_is_stable(self):
_, first = self.build()
_, second = self.build()
self.assertEqual(first, second)
def test_remote_images_are_embedded_when_a_fetcher_is_supplied(self):
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
fetcher = self.FakeFetcher()
archive, _ = self.build(item=item, fetcher=fetcher)
self.assertEqual(fetcher.asked, ["https://cdn.example.com/a.png"])
self.assertIn("OEBPS/images/img-1.png", archive.namelist())
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("../images/img-1.png", article)
self.assertNotIn("https://cdn.example.com", article)
def test_unfetchable_images_are_dropped_from_the_body(self):
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
archive, _ = self.build(item=item, fetcher=self.FakeFetcher(fail=True))
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertNotIn("<img", article)
self.assertNotIn("cdn.example.com", article)
def test_lead_image_becomes_the_cover(self):
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
archive, _ = self.build(item=item, fetcher=self.FakeFetcher())
opf = parse_xml(archive.read(epub.OPF_PATH))
covers = [
item
for item in opf.must_find("manifest").must_find_all("item")
if item.attr("properties") == "cover-image"
]
self.assertEqual(len(covers), 1)
def test_images_are_left_remote_when_inlining_is_off(self):
class NoImages(self.Config):
include_images = False
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
archive, _ = self.build(item=item, config=NoImages(), fetcher=self.FakeFetcher())
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
self.assertIn("https://cdn.example.com/a.png", article)
def test_image_count_is_capped(self):
class OneImage(self.Config):
max_images_per_article = 1
item = self.Item(
'<img src="https://cdn.example.com/a.png"><img src="https://cdn.example.com/b.png">'
)
fetcher = self.FakeFetcher()
archive, _ = self.build(item=item, config=OneImage(), fetcher=fetcher)
self.assertEqual(len(fetcher.asked), 1)
self.assertEqual(len(archive.namelist()), 9)
def test_oversized_book_is_refused(self):
class TinyLimit(self.Config):
max_epub_bytes = 1024
with self.assertRaises(epub.EpubTooLargeError):
self.build(config=TinyLimit(), fetcher=self.FakeFetcher())
def test_stylesheet_path_falls_back_when_unreadable(self):
class BadStylesheet(self.Config):
stylesheet = "/nonexistent/stylesheet.css"
archive, _ = self.build(config=BadStylesheet())
self.assertIn("line-height", archive.read(epub.CSS_PATH).decode("utf-8"))
def test_sanitised_body_has_no_scripts(self):
item = self.Item("<p>ok</p><script>alert(1)</script>")
archive, _ = self.build(item=item)
self.assertNotIn("alert", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
def test_parsed_content_respects_the_byte_cap(self):
item = self.Item("<p>" + "word " * 20_000 + "</p>")
data = epub.build_article_epub(
item,
epub_config=self.Config(),
catalog_config=self.Catalog(),
max_content_bytes=4096,
)
self.assertLess(len(data), 100_000)
if __name__ == "__main__":
unittest.main()

414
tests/test_freshrss.py Normal file
View File

@@ -0,0 +1,414 @@
"""Tests for the FreshRSS client, against an in-process FreshRSS stand-in.
These assert on the wire format as much as on the parsed result: the query
parameters the client sends (``r=d``, ``n=``, ``xt=``, ``ck=``) and the form body
it posts for ``edit-tag`` are the parts that silently break against a real server,
so they are checked explicitly.
"""
from __future__ import annotations
import unittest
from datetime import UTC, datetime
from app.config import FreshRSSConfig
from app.freshrss import (
READ_TAG,
READING_LIST,
STARRED,
AuthenticationError,
FreshRSSClient,
FreshRSSError,
NotFoundError,
TransientError,
label_stream,
)
from tests.stub_freshrss import build_demo_catalog
TECH_FEED = "feed/https://example.com/tech/atom.xml"
NEWS_FEED = "feed/https://news.example.org/rss?source=daily"
SOLO_FEED = "feed/https://solo.example.net/feed.xml"
def make_config(stub, **overrides) -> FreshRSSConfig:
"""A client config aimed at the stub, with fast, non-retrying defaults."""
values = {
"api_base": stub.api_url,
"username": stub.username,
"api_password": stub.password,
"timeout_seconds": 5.0,
"retries": 0,
}
values.update(overrides)
return FreshRSSConfig(**values)
class ClientTestCase(unittest.TestCase):
"""Starts a fresh stub per test."""
def setUp(self):
self.stub = build_demo_catalog().start()
self.addCleanup(self.stub.stop)
def client(self, **overrides) -> FreshRSSClient:
client = FreshRSSClient(
make_config(self.stub, **overrides),
# Retry timing is not what these tests are about, and sleeping would
# only make the suite slow.
sleep=lambda _seconds: None,
)
self.addCleanup(client.close)
return client
class AuthenticationTests(ClientTestCase):
def test_login_posts_credentials_and_reads_the_auth_line(self):
self.client().list_subscriptions()
logins = self.stub.find_requests("/accounts/ClientLogin")
self.assertEqual(len(logins), 1)
self.assertEqual(logins[0]["method"], "POST")
self.assertIn("Email=tyler", logins[0]["body"])
self.assertIn("Passwd=api-password", logins[0]["body"])
def test_subsequent_calls_use_the_googlelogin_header(self):
client = self.client()
client.list_subscriptions()
client.list_subscriptions()
requests = self.stub.find_requests("/subscription/list")
self.assertEqual(len(requests), 2)
for entry in requests:
self.assertEqual(entry["authorization"], "GoogleLogin auth=stub-auth-token")
# The token is cached, so only one login happened.
self.assertEqual(len(self.stub.find_requests("/accounts/ClientLogin")), 1)
def test_bad_password_raises_authentication_error(self):
client = self.client(api_password="wrong")
with self.assertRaises(AuthenticationError):
client.list_subscriptions()
def test_credentials_are_form_encoded(self):
# A password with & or = must not corrupt the form body.
self.stub.password = "p&ss=word+plus"
client = self.client(api_password="p&ss=word+plus")
client.list_subscriptions()
body = self.stub.find_requests("/accounts/ClientLogin")[0]["body"]
self.assertIn("Passwd=p%26ss%3Dword%2Bplus", body)
class SubscriptionTests(ClientTestCase):
def test_subscriptions_are_parsed_with_categories(self):
feeds = {feed.id: feed for feed in self.client().list_subscriptions()}
self.assertEqual(len(feeds), 3)
self.assertEqual(feeds[TECH_FEED].title, "Example Tech")
self.assertEqual(feeds[TECH_FEED].categories, ["Tech"])
self.assertEqual(feeds[NEWS_FEED].categories, ["News"])
def test_feed_without_a_category_is_recognised(self):
feeds = {feed.id: feed for feed in self.client().list_subscriptions()}
self.assertEqual(feeds[SOLO_FEED].categories, [])
self.assertEqual(feeds[SOLO_FEED].primary_category, "Uncategorised")
def test_subscriptions_are_sorted_by_title(self):
titles = [feed.title for feed in self.client().list_subscriptions()]
self.assertEqual(titles, sorted(titles, key=str.casefold))
def test_query_asks_for_json(self):
self.client().list_subscriptions()
query = self.stub.find_requests("/subscription/list")[0]["query"]
self.assertEqual(query["output"], ["json"])
def test_non_json_response_is_reported_clearly(self):
self.stub.corrupt_json = True
with self.assertRaises(FreshRSSError) as caught:
self.client().list_subscriptions()
self.assertIn("not JSON", str(caught.exception))
def test_probe_reports_success_and_feed_count(self):
ok, detail = self.client().probe()
self.assertTrue(ok)
self.assertEqual(detail, "3 feeds")
def test_probe_reports_failure_without_raising(self):
ok, detail = self.client(api_password="wrong").probe()
self.assertFalse(ok)
self.assertTrue(detail)
class StreamTests(ClientTestCase):
def test_stream_query_parameters(self):
self.client().stream_items(TECH_FEED, count=7, exclude_read=True)
request = self.stub.find_requests("/stream/contents/")[0]
self.assertEqual(request["query"]["output"], ["json"])
self.assertEqual(request["query"]["r"], ["d"])
self.assertEqual(request["query"]["n"], ["7"])
self.assertEqual(request["query"]["xt"], [READ_TAG])
self.assertTrue(request["query"]["ck"][0].isdigit())
def test_stream_id_is_path_encoded(self):
self.client().stream_items(NEWS_FEED, count=1)
raw_path = self.stub.find_requests("/stream/contents/")[0]["raw_path"]
# The feed URL contains "?" and "/" which must not leak into the path.
self.assertNotIn("?", raw_path.split("?")[0].split("/api/greader.php")[1])
self.assertIn("%3F", raw_path)
def test_unread_filter_is_applied_upstream(self):
client = self.client()
everything = client.stream_items(TECH_FEED, count=50)
unread = client.stream_items(TECH_FEED, count=50, exclude_read=True)
self.assertEqual(len(everything), 3)
self.assertEqual(len(unread), 2)
self.assertNotIn("Already Read Post", [item.title for item in unread])
def test_count_bounds_the_result(self):
items = self.client().stream_items(TECH_FEED, count=2)
self.assertEqual(len(items), 2)
def test_newest_first(self):
items = self.client().stream_items(TECH_FEED, count=10)
published = [item.published for item in items]
self.assertEqual(published, sorted(published, reverse=True))
def test_older_than_is_passed_through(self):
self.client().stream_items(TECH_FEED, count=5, older_than=1_699_500_000)
self.assertEqual(
self.stub.find_requests("/stream/contents/")[0]["query"]["ot"],
["1699500000"],
)
def test_label_stream_selects_label_articles(self):
items = self.client().stream_items(label_stream("News"), count=10)
self.assertEqual({item.feed_id for item in items}, {NEWS_FEED})
def test_reading_list_covers_every_feed(self):
items = self.client().stream_items(READING_LIST, count=50)
self.assertEqual({item.feed_id for item in items}, {TECH_FEED, NEWS_FEED, SOLO_FEED})
def test_starred_stream_returns_only_starred(self):
items = self.client().stream_items(STARRED, count=10)
self.assertEqual([item.title for item in items], ["Starred Investigation"])
self.assertTrue(items[0].is_starred)
def test_missing_stream_is_treated_as_empty(self):
# A feed unsubscribed upstream must not break the catalog.
self.stub.fail_next(404, path="/stream/contents/")
self.assertEqual(self.client().stream_items(TECH_FEED, count=5), [])
self.stub.fail_next(400, path="/stream/contents/")
self.assertEqual(self.client().stream_items(TECH_FEED, count=5), [])
class ItemParsingTests(ClientTestCase):
def items_by_title(self, stream=TECH_FEED) -> dict:
return {item.title: item for item in self.client().stream_items(stream, count=50)}
def test_full_content_is_preferred_over_the_summary(self):
self.stub.add_item(
TECH_FEED,
99,
"Full versus summary",
content="<p>FULL BODY</p>",
summary="<p>teaser</p>",
)
item = self.items_by_title()["Full versus summary"]
self.assertIn("FULL BODY", item.content_html)
def test_summary_is_used_when_there_is_no_content(self):
self.stub.add_item(
TECH_FEED,
98,
"Summary only",
summary="<p>only teaser</p>",
)
self.stub.items[TECH_FEED][-1].pop("content")
item = self.items_by_title()["Summary only"]
self.assertIn("only teaser", item.content_html)
def test_item_fields_are_parsed(self):
item = self.items_by_title()["A Tech Post & Its Friends"]
self.assertEqual(item.feed_id, TECH_FEED)
self.assertEqual(item.feed_title, "Example Tech")
self.assertEqual(item.author, "A. Writer")
self.assertEqual(item.url, "https://example.com/articles/1")
self.assertEqual(item.published, 1_700_000_100)
self.assertTrue(item.title.startswith("A Tech Post"))
def test_missing_author_is_allowed(self):
item = self.items_by_title(NEWS_FEED)["Markets Dip Slightly"]
self.assertIsNone(item.author)
def test_read_state_is_parsed(self):
items = self.items_by_title()
self.assertTrue(items["Already Read Post"].is_read)
self.assertFalse(items["A Tech Post & Its Friends"].is_read)
def test_kept_unread_overrides_the_read_tag(self):
# FreshRSS keeps both tags when a read item is explicitly marked unread.
item = self.items_by_title()["Already Read Post"]
item.categories.append("user/-/state/com.google/kept-unread")
self.assertFalse(item.is_read)
def test_malformed_entries_are_skipped(self):
self.stub.inject_raw_item(TECH_FEED, {"title": "no id"})
self.stub.inject_raw_item(TECH_FEED, "not even an object")
titles = [item.title for item in self.client().stream_items(TECH_FEED, count=50)]
self.assertNotIn("no id", titles)
self.assertEqual(len(titles), 3)
def test_untitled_item_gets_a_placeholder(self):
self.stub.add_item(TECH_FEED, 97, "", content="<p>x</p>")
titles = [item.title for item in self.client().stream_items(TECH_FEED, count=50)]
self.assertIn("(untitled)", titles)
def test_timestamps_are_utc_datetimes(self):
item = self.items_by_title()["A Tech Post & Its Friends"]
self.assertEqual(item.published_at, datetime(2023, 11, 14, 22, 15, 0, tzinfo=UTC))
self.assertEqual(item.published_at.tzinfo, UTC)
def test_broken_timestamp_falls_back_to_the_epoch(self):
# Every date field has to be unusable for the epoch fallback to apply;
# while `updated` survives, using it is the correct behaviour.
self.stub.add_item(TECH_FEED, 96, "Broken date", content="<p>x</p>")
entry = self.stub.items[TECH_FEED][-1]
for field in ("published", "updated", "timestampUsec"):
entry.pop(field, None)
item = self.items_by_title()["Broken date"]
self.assertEqual(item.published, 0)
self.assertEqual(item.published_at, datetime(1970, 1, 1, tzinfo=UTC))
def test_unusable_publication_date_falls_back_to_updated(self):
self.stub.add_item(TECH_FEED, 94, "Only updated", content="<p>x</p>")
entry = self.stub.items[TECH_FEED][-1]
entry["published"] = "not-a-number"
entry.pop("timestampUsec", None)
entry["updated"] = 1_600_000_000
item = self.items_by_title()["Only updated"]
self.assertEqual(item.published_at, datetime(2020, 9, 13, 12, 26, 40, tzinfo=UTC))
def test_timestamp_usec_is_used_when_published_is_absent(self):
self.stub.add_item(TECH_FEED, 95, "Usec only", content="<p>x</p>")
entry = self.stub.items[TECH_FEED][-1]
entry.pop("published")
entry["timestampUsec"] = "1699999999000000"
item = self.items_by_title()["Usec only"]
self.assertEqual(item.published, 1_699_999_999)
class UnreadCountTests(ClientTestCase):
def test_unread_counts_are_parsed(self):
counts = self.client().unread_counts()
self.assertEqual(counts[TECH_FEED], 2)
self.assertEqual(counts[NEWS_FEED], 2)
def test_unsupported_counts_raise_not_found(self):
# The caller latches this and carries on without counts.
self.stub.supports_unread_counts = False
with self.assertRaises(NotFoundError):
self.client().unread_counts()
class MarkReadTests(ClientTestCase):
def test_mark_read_fetches_the_write_token_and_posts_ids(self):
client = self.client()
identifier = self.stub.items[TECH_FEED][0]["id"]
self.assertEqual(client.mark_read([identifier]), 1)
token_requests = self.stub.find_requests("/reader/api/0/token")
self.assertEqual(len(token_requests), 1)
edits = self.stub.find_requests("/reader/api/0/edit-tag")
self.assertEqual(len(edits), 1)
self.assertEqual(edits[0]["method"], "POST")
body = edits[0]["body"]
self.assertIn("T=stub-write-token", body)
self.assertIn("a=user%2F-%2Fstate%2Fcom.google%2Fread", body)
self.assertIn("i=tag%3Agoogle.com%2C2005%3Areader%2Fitem", body)
self.assertIn(identifier, self.stub.read)
def test_mark_read_sends_one_i_parameter_per_id(self):
ids = [entry["id"] for entry in self.stub.items[TECH_FEED]]
self.assertEqual(self.client().mark_read(ids), 3)
body = self.stub.find_requests("/reader/api/0/edit-tag")[0]["body"]
self.assertEqual(body.count("&i="), 3)
def test_item_id_survives_form_encoding(self):
identifier = "tag:google.com,2005:reader/item/00000000000000ff"
self.client().mark_read([identifier])
from urllib.parse import parse_qs
body = self.stub.find_requests("/reader/api/0/edit-tag")[0]["body"]
self.assertEqual(parse_qs(body)["i"], [identifier])
def test_mark_read_is_idempotent(self):
client = self.client()
identifier = self.stub.items[TECH_FEED][0]["id"]
client.mark_read([identifier])
client.mark_read([identifier])
self.assertIn(identifier, self.stub.read)
self.assertEqual(len(self.stub.find_requests("/reader/api/0/edit-tag")), 2)
def test_mark_read_with_no_ids_makes_no_requests(self):
self.assertEqual(self.client().mark_read([]), 0)
self.assertEqual(self.stub.requests, [])
def test_stale_write_token_is_refreshed_and_retried(self):
client = self.client()
identifier = self.stub.items[TECH_FEED][0]["id"]
# Warm the token cache, then rotate the token upstream so the cached one
# is rejected. The client must re-authenticate and retry rather than lose
# the mark-read.
client.mark_read([identifier])
self.stub.write_token = "rotated-token"
self.stub.clear_read()
client.mark_read([identifier])
self.assertIn(identifier, self.stub.read)
self.assertEqual(len(self.stub.find_requests("/reader/api/0/edit-tag")), 3)
def test_marking_read_removes_the_item_from_the_unread_stream(self):
client = self.client()
identifier = self.stub.items[TECH_FEED][0]["id"]
client.mark_read([identifier])
unread = client.stream_items(TECH_FEED, count=50, exclude_read=True)
self.assertNotIn(identifier, [item.id for item in unread])
class FailureHandlingTests(ClientTestCase):
def test_transient_status_is_retried(self):
client = self.client(retries=2)
self.stub.fail_next(503, path="/subscription/list")
feeds = client.list_subscriptions()
self.assertEqual(len(feeds), 3)
self.assertEqual(len(self.stub.find_requests("/subscription/list")), 2)
def test_retries_are_bounded(self):
client = self.client(retries=1)
self.stub.fail_next(503, count=5, path="/subscription/list")
with self.assertRaises(TransientError):
client.list_subscriptions()
self.assertEqual(len(self.stub.find_requests("/subscription/list")), 2)
def test_server_error_is_not_swallowed(self):
self.stub.fail_next(500, path="/subscription/list")
with self.assertRaises(FreshRSSError) as caught:
self.client(retries=0).list_subscriptions()
self.assertEqual(caught.exception.status, 500)
def test_unreachable_upstream_raises_transient_error(self):
# Bind a port, then close it, so nothing is listening.
import socket
probe = socket.socket()
probe.bind(("127.0.0.1", 0))
port = probe.getsockname()[1]
probe.close()
config = make_config(self.stub, api_base=f"http://127.0.0.1:{port}/api/greader.php")
with self.assertRaises(TransientError):
FreshRSSClient(config).list_subscriptions()
def test_error_body_is_included_in_the_message(self):
self.stub.fail_next(500, path="/subscription/list")
with self.assertRaises(FreshRSSError) as caught:
self.client(retries=0).list_subscriptions()
self.assertIn("injected failure", str(caught.exception))
if __name__ == "__main__":
unittest.main()

543
tests/test_server.py Normal file
View File

@@ -0,0 +1,543 @@
"""End-to-end tests: a real server, talking to a real (stand-in) FreshRSS.
These run the whole stack rather than units of it: the configuration is loaded
through its validation path, the HTTP server serves on a socket, the FreshRSS
client speaks HTTP to the stub, and the EPUB that comes back is opened and
checked. The claim that matters most -- that downloading a book marks the
matching article read upstream, and only a completed download does -- can only be
verified here, by looking at what the stub received.
Image embedding is turned off in this configuration so the suite stays offline;
image handling has its own tests with a stand-in fetcher.
"""
from __future__ import annotations
import base64
import json
import tempfile
import threading
import time
import unittest
import urllib.error
import urllib.parse
import urllib.request
import zipfile
from io import BytesIO
from pathlib import Path
from app import server as app_server
from app.config import load_config, password_digest
from app.epub import ARTICLE_PATH, OPF_PATH
from tests.stub_freshrss import READING_LIST, build_demo_catalog, item_id
from tests.support import parse_xml
class _NoRedirects(urllib.request.HTTPRedirectHandler):
"""Surface a redirect as its own status instead of following it.
urlopen follows redirects transparently, so a test that asserts on a 302 would
otherwise be looking at the status of whatever the redirect pointed at.
"""
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
USERNAME = "reader"
PASSWORD = "reader-password"
TECH_FEED = "feed/https://example.com/tech/atom.xml"
class ServerTestCase(unittest.TestCase):
"""Starts a stub FreshRSS and the real server for each test."""
include_images = False
def setUp(self):
self.stub = build_demo_catalog().start()
self.addCleanup(self.stub.stop)
# The demo account ships one already-read article, so assertions about
# marking read are written against the change, not the absolute state.
self.baseline_read = set(self.stub.read)
self._dir = tempfile.TemporaryDirectory()
self.addCleanup(self._dir.cleanup)
self._no_redirects = urllib.request.build_opener(_NoRedirects)
self.config = self._write_config()
self.httpd, self.application = app_server.build_server(self.config)
self.port = self.httpd.server_address[1]
self.thread = threading.Thread(
target=self.httpd.serve_forever, kwargs={"poll_interval": 0.02}, daemon=True
)
self.thread.start()
self.addCleanup(self._stop_server)
def _stop_server(self):
self.httpd.shutdown()
self.httpd.server_close()
self.thread.join(timeout=5)
@property
def base(self) -> str:
return f"http://127.0.0.1:{self.port}"
def _write_config(self):
document = {
"server": {"host": "127.0.0.1", "port": 0, "log_level": "critical"},
"freshrss": {
"url": self.stub.api_url,
"username": self.stub.username,
"api_password": self.stub.password,
"items_per_page": 2,
"timeout_seconds": 5,
"retries": 0,
},
"catalog": {"title": "Test Catalog", "author": "Tester"},
"epub": {"include_images": self.include_images},
"auth": {
"users": [{"username": USERNAME, "password_sha256": password_digest(PASSWORD)}]
},
"cache": {"subscriptions_ttl_seconds": 1, "stream_ttl_seconds": 1},
}
path = Path(self._dir.name) / "config.json"
path.write_text(json.dumps(document), encoding="utf-8")
return load_config(str(path))
# -- HTTP helpers ----------------------------------------------------
def request(self, path, *, auth=True, method="GET", headers=None, follow_redirects=True):
request = urllib.request.Request(self.base + path, method=method)
if auth:
token = base64.b64encode(f"{USERNAME}:{PASSWORD}".encode()).decode()
request.add_header("Authorization", f"Basic {token}")
for key, value in (headers or {}).items():
request.add_header(key, value)
opener = urllib.request.urlopen if follow_redirects else self._no_redirects.open
try:
with opener(request, timeout=10) as response:
return response.status, dict(response.headers), response.read()
except urllib.error.HTTPError as error:
# With redirects suppressed, a 3xx arrives here, which is the point.
# The error object owns the response socket, so it is closed rather
# than left for the garbage collector.
try:
body = error.read()
finally:
error.close()
return error.code, dict(error.headers), body
def wait_for_mark_read(self, expected: set[str], timeout: float = 5.0) -> set[str]:
"""Wait for a mark-read to land upstream (returns what landed).
The server marks an article read only after the body has been delivered,
so from the client's side that upstream effect is asynchronous: a test
asserting immediately would be racing the handler thread. It is
deliberately not the other way round -- marking before delivery would
consume articles whose transfer failed.
"""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
landed = self.stub.read - self.baseline_read
if landed >= expected:
return landed
time.sleep(0.02)
return self.stub.read - self.baseline_read
def get_xml(self, path, *, auth=True):
status, headers, body = self.request(path, auth=auth)
self.assertEqual(status, 200, body[:400])
node = parse_xml(body)
return node, headers
def entries(self, node):
return {entry.must_find("title").all_text(): entry for entry in node.must_find_all("entry")}
def follow(self, entry, rel: str) -> str:
for link in entry.must_find_all("link"):
if link.attr("rel") == rel:
return link.must_attr("href")
raise AssertionError(f"no {rel!r} link in entry")
def acquisition_path(self, section="unread", label="Tech", feed_title="Example Tech") -> str:
"""Walk the catalog down to one feed's acquisition feed.
Shared by the suite rather than owned by one test class: the download,
pagination, search and serialisation tests all start from a feed.
"""
node, _ = self.get_xml(f"/opds/{section}")
category = self.follow(self.entries(node)[label], "subsection")
node, _ = self.get_xml(urllib.parse.urlsplit(category).path)
return urllib.parse.urlsplit(self.follow(self.entries(node)[feed_title], "subsection")).path
def download_path(self, title="A Tech Post & Its Friends", section="unread") -> str:
"""The EPUB acquisition URL for one article."""
node, _ = self.get_xml(self.acquisition_path(section=section))
entry = self.entries(node)[title]
return urllib.parse.urlsplit(
self.follow(entry, "http://opds-spec.org/acquisition/open-access")
).path
class AuthenticationTests(ServerTestCase):
def test_catalog_requires_credentials(self):
status, headers, _body = self.request("/opds", auth=False)
self.assertEqual(status, 401)
self.assertIn("Basic", headers.get("WWW-Authenticate", ""))
self.assertIn("Test" if False else "OPDS", headers.get("WWW-Authenticate", ""))
def test_wrong_password_is_rejected(self):
token = base64.b64encode(f"{USERNAME}:wrong".encode()).decode()
status, _headers, _body = self.request(
"/opds", auth=False, headers={"Authorization": f"Basic {token}"}
)
self.assertEqual(status, 401)
def test_health_needs_no_credentials_and_touches_nothing(self):
status, _headers, body = self.request("/healthz", auth=False)
self.assertEqual(status, 200)
self.assertEqual(json.loads(body), {"status": "ok"})
self.assertEqual(self.stub.requests, [])
def test_malformed_authorization_header_is_rejected(self):
for value in ("Basic", "Bearer xyz", "Basic !!!not-base64", "Basic " + "aGk="):
with self.subTest(value=value):
status, _headers, _body = self.request(
"/opds", auth=False, headers={"Authorization": value}
)
self.assertEqual(status, 401)
def test_diagnostics_requires_credentials(self):
self.assertEqual(self.request("/opds/healthz", auth=False)[0], 401)
def test_diagnostics_reports_upstream_state(self):
status, _headers, body = self.request("/opds/healthz")
self.assertEqual(status, 200)
payload = json.loads(body)
self.assertTrue(payload["upstream"]["reachable"])
self.assertEqual(payload["catalog"]["feeds"], 3)
self.assertTrue(payload["mark_read"])
class NavigationTests(ServerTestCase):
def test_bare_root_redirects_to_the_catalog(self):
# Redirects must be suppressed, or this would see the 401 that /opds
# answers an unauthenticated follower with.
status, headers, _body = self.request("/", auth=False, follow_redirects=False)
self.assertEqual(status, 302)
self.assertTrue(headers["Location"].endswith("/opds"))
def test_root_lists_the_sections(self):
node, headers = self.get_xml("/opds")
self.assertEqual(node.must_find("title").all_text(), "Test Catalog")
titles = set(self.entries(node))
self.assertIn("Unread", titles)
self.assertIn("Read", titles)
self.assertIn("Starred", titles)
def test_navigation_feed_declares_its_profile(self):
_node, headers = self.get_xml("/opds")
self.assertEqual(
headers["Content-Type"],
"application/atom+xml;profile=opds-catalog;kind=navigation",
)
def test_root_offers_a_search_link(self):
node, _headers = self.get_xml("/opds")
rels = {link.attr("rel") for link in node.must_find_all("link")}
self.assertIn("search", rels)
self.assertIn("self", rels)
def test_section_lists_categories_with_unread_counts(self):
node, _headers = self.get_xml("/opds/unread")
entries = self.entries(node)
self.assertIn("Tech", entries)
self.assertIn("News", entries)
self.assertIn("Uncategorised", entries)
self.assertIn("unread", entries["Tech"].must_find("content").all_text())
def test_category_lists_its_feeds(self):
category = self.follow(self.entries(self.get_xml("/opds/unread")[0])["Tech"], "subsection")
node, _headers = self.get_xml(urllib.parse.urlsplit(category).path)
titles = set(self.entries(node))
self.assertIn("Example Tech", titles)
self.assertIn("All in this category", titles)
def test_unknown_path_is_404(self):
self.assertEqual(self.request("/opds/nope")[0], 404)
def test_undecodable_identifier_is_404(self):
self.assertEqual(self.request("/opds/unread/c/!!!not-base64")[0], 404)
def test_post_is_not_allowed(self):
self.assertEqual(self.request("/opds", method="POST")[0], 405)
def test_opensearch_descriptor_is_served(self):
node, headers = self.get_xml("/opds/opensearch.xml")
self.assertEqual(headers["Content-Type"], "application/opensearchdescription+xml")
url = node.must_find("Url")
self.assertIn("searchTerms", url.must_attr("template"))
class AcquisitionTests(ServerTestCase):
def test_acquisition_feed_lists_articles(self):
node, headers = self.get_xml(self.acquisition_path())
self.assertEqual(
headers["Content-Type"],
"application/atom+xml;profile=opds-catalog;kind=acquisition",
)
titles = set(self.entries(node))
self.assertIn("A Tech Post & Its Friends", titles)
def test_read_articles_are_absent_from_the_unread_feed(self):
titles = set(self.entries(self.get_xml(self.acquisition_path())[0]))
self.assertNotIn("Already Read Post", titles)
def test_read_section_contains_read_articles(self):
path = self.acquisition_path(section="read")
titles = set(self.entries(self.get_xml(path)[0]))
self.assertIn("Already Read Post", titles)
def test_entry_offers_an_epub_acquisition_link(self):
node, _headers = self.get_xml(self.acquisition_path())
entry = self.entries(node)["A Tech Post & Its Friends"]
href = self.follow(entry, "http://opds-spec.org/acquisition/open-access")
self.assertTrue(href.endswith(".epub"))
types = {link.attr("type") for link in entry.must_find_all("link")}
self.assertIn("application/epub+zip", types)
# The generic relation is offered too, because readers disagree.
self.assertIn(
"http://opds-spec.org/acquisition",
{link.attr("rel") for link in entry.must_find_all("link")},
)
def test_entry_links_back_to_the_original_article(self):
node, _headers = self.get_xml(self.acquisition_path())
entry = self.entries(node)["A Tech Post & Its Friends"]
self.assertTrue(self.follow(entry, "alternate").startswith("https://"))
def test_entry_carries_its_feed_as_a_category(self):
node, _headers = self.get_xml(self.acquisition_path())
entry = self.entries(node)["A Tech Post & Its Friends"]
labels = {category.attr("label") for category in entry.must_find_all("category")}
self.assertIn("Example Tech", labels)
def test_articles_are_newest_first(self):
node, _headers = self.get_xml(self.acquisition_path())
stamps = [entry.must_find("updated").all_text() for entry in node.must_find_all("entry")]
self.assertEqual(stamps, sorted(stamps, reverse=True))
def test_pagination_link_appears_when_there_is_more(self):
# The test configuration serves two items per page, and the reading list
# holds five unread articles.
node, _headers = self.get_xml("/opds/unread/all")
rels = {link.attr("rel") for link in node.must_find_all("link")}
self.assertIn("next", rels)
self.assertIn("first", rels)
self.assertNotIn("previous", rels)
def test_second_page_is_reachable_and_different(self):
path = "/opds/unread/all"
first, _ = self.get_xml(path)
second, _ = self.get_xml(path + "?page=2")
first_ids = {entry.must_find("id").all_text() for entry in first.must_find_all("entry")}
second_ids = {entry.must_find("id").all_text() for entry in second.must_find_all("entry")}
self.assertTrue(second_ids)
self.assertFalse(first_ids & second_ids)
def test_starred_section_lists_starred_articles(self):
titles = set(self.entries(self.get_xml("/opds/starred")[0]))
self.assertIn("Starred Investigation", titles)
def test_search_finds_articles_in_the_window(self):
node, _headers = self.get_xml("/opds/search?q=Tech+Post")
titles = set(self.entries(node))
self.assertIn("A Tech Post & Its Friends", titles)
self.assertNotIn("Markets Dip Slightly", titles)
def test_empty_search_returns_an_empty_feed(self):
# A search with no term is a valid, empty acquisition feed rather than an
# error, so a reader that sends one sees an empty list.
node, _headers = self.get_xml("/opds/search?q=")
self.assertEqual(node.find_all("entry"), [])
def test_bad_page_number_falls_back_to_the_first_page(self):
node, _headers = self.get_xml(self.acquisition_path() + "?page=nonsense")
self.assertTrue(node.must_find_all("entry"))
class DownloadTests(ServerTestCase):
"""The claim under test: a completed download is what marks an article read."""
def test_download_returns_an_epub(self):
status, headers, body = self.request(self.download_path())
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "application/epub+zip")
disposition = headers["Content-Disposition"]
self.assertTrue(disposition.startswith("attachment;"))
# Both spellings are sent: a plain name every reader understands, and an
# RFC 5987 one so a non-ASCII title survives.
self.assertIn('filename="a-tech-post-its-friends.epub"', disposition)
self.assertIn("filename*=UTF-8''", disposition)
def test_downloaded_file_is_a_valid_epub(self):
_status, _headers, body = self.request(self.download_path())
archive = zipfile.ZipFile(BytesIO(body))
self.addCleanup(archive.close)
self.assertEqual(archive.namelist()[0], "mimetype")
self.assertEqual(archive.read("mimetype"), b"application/epub+zip")
self.assertEqual(archive.getinfo("mimetype").compress_type, zipfile.ZIP_STORED)
opf = parse_xml(archive.read(OPF_PATH))
self.assertEqual(opf.must_find("title").all_text(), "A Tech Post & Its Friends")
# The body went through the sanitiser on the way in.
article = archive.read(ARTICLE_PATH).decode("utf-8")
self.assertNotIn("alert(", article)
self.assertIn("Entities: &amp;", article)
def test_download_marks_the_article_read_upstream(self):
self.request(self.download_path())
self.assertEqual(self.wait_for_mark_read({item_id(1)}), {item_id(1)})
edits = self.stub.find_requests("/reader/api/0/edit-tag")
self.assertEqual(len(edits), 1)
self.assertIn("i=tag%3Agoogle.com", edits[0]["body"])
def test_downloads_mark_only_their_own_article(self):
self.request(self.download_path())
self.assertEqual(self.wait_for_mark_read({item_id(1)}), {item_id(1)})
def test_head_does_not_mark_read(self):
# A reader probing the URL, or a CDN, must not consume the article.
status, headers, body = self.request(self.download_path(), method="HEAD")
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "application/epub+zip")
self.assertEqual(body, b"")
# Deterministic rather than a race: a HEAD never reaches the mark-read.
self.assertEqual(self.stub.read - self.baseline_read, set())
self.assertEqual(self.stub.find_requests("/reader/api/0/edit-tag"), [])
def test_missing_article_is_404(self):
stream = base64.urlsafe_b64encode(TECH_FEED.encode()).decode().rstrip("=")
absent = (
base64.urlsafe_b64encode(b"tag:google.com,2005:reader/item/nope").decode().rstrip("=")
)
status, _headers, _body = self.request(f"/opds/d/{stream}/{absent}/nope.epub")
self.assertEqual(status, 404)
def test_article_id_is_encoded_in_the_url_and_survives(self):
# FreshRSS ids contain ':' and '/', so the URL must carry them encoded.
path = self.download_path()
self.assertNotIn("tag:google.com", path)
self.assertEqual(path.count("/"), 5)
def test_marked_read_article_leaves_the_unread_queue(self):
path = self.download_path()
self.request(path)
self.wait_for_mark_read({item_id(1)})
# The stream cache is invalidated by the mark-read, so the next browse
# reflects it without waiting for a TTL.
titles = set(self.entries(self.get_xml(self.acquisition_path())[0]))
self.assertNotIn("A Tech Post & Its Friends", titles)
def test_marked_read_article_appears_in_the_read_section(self):
self.request(self.download_path())
self.wait_for_mark_read({item_id(1)})
path = self.acquisition_path(section="read")
titles = set(self.entries(self.get_xml(path)[0]))
self.assertIn("A Tech Post & Its Friends", titles)
def test_mark_read_can_be_disabled(self):
self.application.config.freshrss.mark_read = False
self.request(self.download_path())
self.assertEqual(self.stub.read - self.baseline_read, set())
self.assertEqual(self.stub.find_requests("/reader/api/0/edit-tag"), [])
def test_advertised_unread_count_refreshes_after_a_download(self):
"""The count must not lag the queue it describes.
Unread counts come from the cached account snapshot, so a mark-read has to
drop that cache too -- otherwise a reader downloads an article and is still
told the old total, with one fewer entry underneath it.
"""
def advertised_unread() -> int:
node, _ = self.get_xml("/opds/unread")
for entry in node.must_find_all("entry"):
if entry.must_find("title").all_text() == "All feeds":
return int(entry.must_find("content").all_text().split()[0])
raise AssertionError("no 'All feeds' entry in the unread catalog")
before = advertised_unread()
self.request(self.download_path())
self.wait_for_mark_read({item_id(1)})
self.assertEqual(advertised_unread(), before - 1)
def test_second_download_is_still_served_after_the_queue_moved_on(self):
path = self.download_path()
self.request(path)
status, _headers, body = self.request(path)
self.assertEqual(status, 200)
self.assertTrue(body.startswith(b"PK"))
class UpstreamFailureTests(ServerTestCase):
def test_upstream_outage_is_a_bad_gateway(self):
self.stub.fail_next(503, count=10, path="/subscription/list")
status, _headers, _body = self.request("/opds")
self.assertEqual(status, 502)
def test_catalog_recovers_once_upstream_returns(self):
# Exactly one failure, so the next attempt succeeds.
self.stub.fail_next(503, path="/subscription/list")
self.assertEqual(self.request("/opds")[0], 502)
self.application.invalidate()
self.assertEqual(self.request("/opds")[0], 200)
def test_unread_counts_missing_is_not_fatal(self):
# The feature is optional; the catalog must still browse without it.
self.stub.supports_unread_counts = False
self.application.invalidate()
status, _headers, body = self.request("/opds/unread")
self.assertEqual(status, 200)
node = parse_xml(body)
titles = {entry.must_find("title").all_text() for entry in node.must_find_all("entry")}
self.assertIn("Tech", titles)
def test_unknown_host_is_refused_when_an_allowlist_is_set(self):
self.application.config.server.allowed_hosts = ["catalog.example.com"]
status, _headers, _body = self.request("/opds", auth=False)
self.assertEqual(status, 500)
def test_public_base_url_controls_the_links(self):
self.application.config.server.public_base_url = "https://catalog.example.com/books"
node, _headers = self.get_xml("/opds")
hrefs = {link.must_attr("href") for link in node.must_find_all("link")}
self.assertTrue(
any(href.startswith("https://catalog.example.com/books/opds") for href in hrefs)
)
class SerializationTests(ServerTestCase):
"""Responses must be correct XML even when titles contain markup."""
def test_titles_with_markup_are_escaped(self):
self.stub.subscriptions[0]["title"] = 'Feed & <Friends> "quoted"'
self.application.invalidate()
category = urllib.parse.urlsplit(
self.follow(self.entries(self.get_xml("/opds/unread")[0])["Tech"], "subsection")
).path
node, _headers = self.get_xml(category)
titles = {entry.must_find("title").all_text() for entry in node.must_find_all("entry")}
self.assertIn('Feed & <Friends> "quoted"', titles)
def test_feed_ids_are_stable_across_requests(self):
first, _ = self.get_xml("/opds/unread")
second, _ = self.get_xml("/opds/unread")
self.assertEqual(first.must_find("id").all_text(), second.must_find("id").all_text())
def test_entry_ids_are_unique_within_a_feed(self):
node, _headers = self.get_xml(self.acquisition_path())
ids = [entry.must_find("id").all_text() for entry in node.must_find_all("entry")]
self.assertEqual(len(ids), len(set(ids)))
if __name__ == "__main__":
unittest.main()

399
tests/test_xhtml.py Normal file
View File

@@ -0,0 +1,399 @@
"""Tests for the HTML to XHTML normaliser.
The bar every case is held to: the output must parse with a strict XML parser,
because that is what an EPUB reader does with it.
"""
from __future__ import annotations
import unittest
from app.xhtml import parse_article
from tests.support import assert_well_formed, body, collapse, parse_xml
BASE = "https://example.com/posts/one?utm=x"
def render(raw: str, image_map=None, base: str = BASE, max_input_bytes: int = 0) -> str:
"""Parse and serialise, asserting well-formedness on the way out."""
article = parse_article(raw, base, max_input_bytes=max_input_bytes)
fragment = article.to_xhtml(image_map)
assert_well_formed(unittest.TestCase(), fragment)
return fragment
class MalformedInputTests(unittest.TestCase):
"""Feed HTML is broken in predictable ways; none of it may reach the output."""
def test_unclosed_paragraphs_are_balanced(self):
out = render("<p>one<p>two<p>three")
self.assertEqual(body(out).count("<p>"), 3)
self.assertEqual(body(out).count("</p>"), 3)
self.assertIn("three", out)
def test_block_element_closes_open_paragraph(self):
# A naive stack would nest the div inside the paragraph.
out = render("<p>lead<div>block</div>trailing")
self.assertIn("<p>lead</p>", out)
self.assertIn("<div>block</div>", out)
self.assertNotIn("<p>lead<div>", out)
def test_unclosed_list_items(self):
out = render("<ul><li>a<li>b<li>c</ul>")
self.assertEqual(body(out).count("<li>"), 3)
self.assertEqual(body(out).count("</li>"), 3)
def test_stray_end_tag_is_ignored(self):
out = render("<p>text</span></div></p>")
self.assertIn("<p>text</p>", out)
def test_unclosed_table_row_and_cells(self):
out = render("<table><tr><td>a<td>b<tr><td>c</table>")
self.assertEqual(body(out).count("<tr>"), 2)
self.assertEqual(body(out).count("<td>"), 3)
self.assertIn("<td>c</td>", out)
def test_self_closing_non_void_element_swallows_nothing(self):
# `<div/>` must not open a subtree that eats the rest of the document, and
# the empty element it leaves behind is noise worth dropping.
out = render("<div/>after")
self.assertIn("after", out)
self.assertEqual(body(out), "after")
def test_void_elements_are_self_closed(self):
out = render("a<br>b<hr>c")
self.assertIn("<br/>", out)
self.assertIn("<hr/>", out)
self.assertNotIn("<br>", out)
def test_control_characters_are_removed(self):
# NUL and vertical tab cannot be represented in XML even as entities.
out = render("<p>be\x00fore\x0baft\x0cft</p>")
self.assertNotIn("\x00", out)
self.assertNotIn("\x0b", out)
self.assertNotIn("\x0c", out)
self.assertIn("before", out)
def test_unpaired_surrogate_is_removed(self):
out = render("<p>ok\ud800tail</p>")
self.assertNotIn("\ud800", out)
self.assertIn("oktail", out)
def test_zero_width_characters_are_removed(self):
out = render("<p>a\u200bb\ufeffc</p>")
self.assertEqual(collapse(parse_xml(out).all_text()), "abc")
def test_comments_are_dropped(self):
out = render("<p>keep<!-- drop me --> this</p>")
self.assertNotIn("drop me", out)
def test_processing_instruction_and_doctype_are_dropped(self):
out = render("<?xml version='1.0'?><!DOCTYPE html><p>body</p>")
self.assertNotIn("DOCTYPE", out)
self.assertIn("body", out)
def test_deeply_nested_wrappers_keep_their_text(self):
out = render("<section><article><aside>deep</aside></article></section>")
self.assertIn("deep", out)
self.assertNotIn("<section>", out)
self.assertNotIn("<aside>", out)
class EscapingTests(unittest.TestCase):
"""Nothing in the output may be interpretable as markup or as an entity."""
def test_bare_ampersand_is_escaped(self):
out = render("<p>Tom & Jerry</p>")
self.assertIn("Tom &amp; Jerry", out)
def test_named_and_numeric_entities_are_decoded_then_reescaped(self):
out = render("<p>&nbsp;&#233;&amp;</p>")
# &nbsp; must not survive as an entity, and must not break the parse.
self.assertNotIn("&nbsp;", out)
self.assertIn("\u00a0", out)
self.assertIn("\u00e9", out)
def test_text_that_looks_like_xml_is_escaped(self):
out = render("<p>&lt;script&gt;alert(1)&lt;/script&gt;</p>")
self.assertIn("&lt;script&gt;", out)
self.assertEqual(body(out).count("<script>"), 0)
def test_quotes_in_attributes_are_escaped(self):
out = render('<p><a href="https://e.com/?q=&quot;x&quot;" title="a&quot;b">t</a></p>')
self.assertIn("&quot;", out)
def test_cdata_section_is_unwrapped_to_text(self):
out = render("<p><![CDATA[raw text]]></p>")
self.assertIn("raw text", out)
def test_gets_escaped_inside_attributes_only_once(self):
out = render('<p><a href="https://e.com/a?b=1&amp;c=2">t</a></p>')
self.assertIn('href="https://e.com/a?b=1&amp;c=2"', out)
class DangerousContentTests(unittest.TestCase):
"""Scriptable and resource-fetching constructs must not survive."""
def test_script_element_and_body_are_removed(self):
out = render("<p>before</p><script>alert('xss')</script><p>after</p>")
self.assertNotIn("script", out.lower())
self.assertIn("before", out)
self.assertIn("after", out)
def test_unclosed_script_does_not_swallow_forever(self):
out = render("<script>alert(1)</script><p>visible</p>")
self.assertIn("visible", out)
def test_style_element_is_removed(self):
out = render("<style>p{color:red}</style><p>text</p>")
self.assertNotIn("color:red", out)
def test_iframe_and_object_are_removed_with_contents(self):
out = render('<iframe src="https://e.com/x"></iframe><p>t</p>')
self.assertNotIn("iframe", out)
def test_style_attribute_is_dropped(self):
out = render('<p style="background:url(javascript:1)">t</p>')
self.assertNotIn("style", out)
def test_event_handler_attributes_are_dropped(self):
out = render('<p onclick="evil()" onmouseover="evil()">t</p>')
self.assertNotIn("onclick", out)
self.assertNotIn("onmouseover", out)
def test_class_attribute_is_dropped(self):
# class fights the book's own stylesheet and means nothing offline.
out = render('<p class="x">t</p>')
self.assertNotIn("class", body(out))
def test_id_is_kept_for_anchor_targets(self):
out = render('<p id="section-2">t</p>')
self.assertIn('id="section-2"', out)
def test_invalid_id_is_dropped(self):
# An id has to be a usable XML name; anything else is not an anchor. The
# quote case needs single-quoted markup, or the parser would read a
# shorter, valid value and the case would prove nothing.
cases = {
"leading digit": '<p id="9lives">t</p>',
"space": '<p id="has space">t</p>',
"slash": '<p id="sla/sh">t</p>',
"quote": "<p id='quote\"d'>t</p>",
"empty": '<p id="">t</p>',
"too long": f'<p id="{"a" * 100}">t</p>',
}
for label, markup in cases.items():
self.assertNotIn("id=", body(render(markup)), label)
def test_javascript_url_is_dropped(self):
out = render('<a href="javascript:alert(1)">click</a>')
self.assertNotIn("javascript", out)
self.assertIn("click", out)
def test_data_url_link_is_dropped(self):
out = render('<a href="data:text/html,<script>x</script>">t</a>')
self.assertNotIn("data:", out)
def test_data_url_image_is_dropped(self):
out = render('<img src="data:image/png;base64,AAAA"><p>t</p>')
self.assertNotIn("data:", out)
self.assertNotIn("<img", out)
def test_img_without_source_is_dropped(self):
out = render('<p><img alt="no source"></p>')
self.assertNotIn("<img", out)
def test_only_href_survives_on_links(self):
out = render('<a href="https://e.com/a" target="_blank" rel="noopener" download>x</a>')
self.assertIn('href="https://e.com/a"', out)
self.assertNotIn("target", out)
self.assertNotIn("rel=", out)
class UrlResolutionTests(unittest.TestCase):
"""Bodies are re-based into the book, so relative URLs must be resolved."""
def test_root_relative_link_uses_article_origin(self):
out = render('<a href="/about">about</a>')
self.assertIn('href="https://example.com/about"', out)
def test_document_relative_link_uses_article_directory(self):
out = render('<a href="sibling.html">s</a>')
self.assertIn('href="https://example.com/posts/sibling.html"', out)
def test_protocol_relative_link_is_resolved(self):
out = render('<a href="//cdn.example.com/x">x</a>')
self.assertIn('href="https://cdn.example.com/x"', out)
def test_mailto_link_is_kept(self):
out = render('<a href="mailto:someone@example.com">mail</a>')
self.assertIn('href="mailto:someone@example.com"', out)
def test_anchor_only_link_stays_in_document(self):
# A same-document link must not be rewritten to the source web page: the
# matching id is kept, so it jumps within the book instead.
out = render('<p id="section">target</p><a href="#section">jump</a>')
self.assertIn('href="#section"', out)
self.assertIn('id="section"', out)
def test_relative_image_is_absolutised(self):
article = parse_article('<img src="images/a.png">', BASE)
self.assertEqual(article.image_urls, ["https://example.com/posts/images/a.png"])
def test_blockquote_cite_is_validated(self):
out = render('<blockquote cite="javascript:x">q</blockquote>')
self.assertNotIn("cite", out)
out = render('<blockquote cite="https://e.com/src">q</blockquote>')
self.assertIn('cite="https://e.com/src"', out)
class AnchorIdTests(unittest.TestCase):
"""Ids survive so an article's own table of contents works offline."""
def test_duplicate_ids_are_renamed_not_dropped(self):
out = render('<p id="a">1</p><p id="a">2</p><p id="a">3</p>')
self.assertIn('id="a"', out)
self.assertIn('id="a-2"', out)
self.assertIn('id="a-3"', out)
def test_id_on_void_element_is_dropped(self):
out = render('<img id="x" src="a.png">')
self.assertNotIn('id="x"', out)
def test_renamed_ids_do_not_collide_with_existing_ones(self):
out = render('<p id="a">1</p><p id="a-2">2</p><p id="a">3</p>')
self.assertEqual(body(out).count('id="a-2"'), 1)
self.assertIn('id="a-3"', out)
def test_unique_ids_are_left_alone(self):
out = render('<p id="one">1</p><p id="two">2</p>')
self.assertIn('id="one"', out)
self.assertIn('id="two"', out)
def test_an_element_keeping_only_an_id_is_not_pruned(self):
out = render('<div id="anchor"></div><p>text</p>')
self.assertIn('id="anchor"', out)
class ImageHandlingTests(unittest.TestCase):
"""Images are collected for the fetcher and rewritten by the caller."""
def test_image_urls_are_collected_in_order_and_deduplicated(self):
article = parse_article('<p><img src="a.png"><img src="a.png"><img src="b.png"></p>', BASE)
self.assertEqual(
article.image_urls,
["https://example.com/posts/a.png", "https://example.com/posts/b.png"],
)
def test_remote_urls_are_kept_when_no_map_is_given(self):
out = render('<img src="a.png" alt="alt text">')
self.assertIn('src="https://example.com/posts/a.png"', out)
self.assertIn('alt="alt text"', out)
def test_map_rewrites_src_to_local_path(self):
url = "https://example.com/posts/a.png"
out = render('<img src="a.png">', image_map={url: "images/img-1.png"})
self.assertIn('src="images/img-1.png"', out)
def test_none_in_map_drops_image_and_empty_wrapper(self):
url = "https://example.com/posts/a.png"
out = render('<div><img src="a.png"></div><p>text</p>', image_map={url: None})
self.assertNotIn("<img", out)
self.assertNotIn("<div>", out)
self.assertIn("<p>text</p>", out)
def test_alt_and_title_are_preserved_for_localised_images(self):
url = "https://example.com/posts/a.png"
out = render('<img src="a.png" alt="A" title="T">', image_map={url: "images/i.png"})
self.assertIn('alt="A"', out)
self.assertIn('title="T"', out)
class WhitespaceAndStructureTests(unittest.TestCase):
def test_whitespace_is_collapsed_outside_pre(self):
out = render("<p>one\n\n two\t\tthree</p>")
self.assertIn("<p>one two three</p>", out)
def test_whitespace_is_preserved_inside_pre(self):
out = render("<pre>def f():\n return 1</pre>")
self.assertIn("def f():\n return 1", out)
def test_empty_paragraphs_are_pruned(self):
out = render("<p> </p><p>&nbsp;</p><p>real</p>")
self.assertEqual(body(out).count("<p>"), 1)
self.assertIn("real", out)
def test_empty_table_cells_are_kept(self):
# An empty cell is layout, not noise, so pruning must leave it alone.
out = render("<table><tr><td></td><td>b</td></tr></table>")
self.assertEqual(body(out).count("<td></td>"), 1)
def test_plain_text_body_is_wrapped_in_a_paragraph(self):
out = render("just words with no markup")
self.assertEqual(body(out), "<p>just words with no markup</p>")
def test_whitespace_only_body_renders_empty(self):
article = parse_article(" \n ", BASE)
self.assertEqual(article.to_xhtml(), "")
def test_empty_input_renders_empty(self):
self.assertEqual(parse_article("", BASE).to_xhtml(), "")
def test_heading_levels_survive(self):
out = render("<h1>a</h1><h2>b</h2><h3>c</h3>")
for tag in ("h1", "h2", "h3"):
self.assertIn(f"<{tag}>", out)
def test_colspan_is_clamped(self):
out = render('<td colspan="99999">x</td>')
self.assertNotIn("colspan", out)
out = render('<td colspan="2">x</td>')
self.assertIn('colspan="2"', out)
def test_element_structure_is_preserved(self):
out = render("<ul><li>a<ul><li>b</li></ul></li></ul>")
tree = parse_xml(out)
# The fragment's own root is the article wrapper.
self.assertEqual(tree.must_attr("class"), "article-body")
outer = tree.must_find("ul")
items = outer.must_find_all("li")
self.assertEqual(len(items), 2)
# The nested list belongs to the first item, not to the outer list.
self.assertTrue(items[0].find_all("ul"))
class ArticleApiTests(unittest.TestCase):
def test_text_extraction_flattens_and_collapses(self):
article = parse_article("<p>One</p> <p>Two &amp; three</p>", BASE)
self.assertEqual(article.text(), "One Two & three")
def test_text_extraction_is_limited_at_a_word_boundary(self):
article = parse_article("<p>alpha beta gamma delta</p>", BASE)
text = article.text(limit=12)
self.assertTrue(text.endswith("\u2026"))
self.assertLessEqual(len(text), 13)
self.assertIn("alpha beta", text)
def test_truncation_flag_and_output_validity(self):
article = parse_article("<p>" + "word " * 5000 + "</p>", BASE, max_input_bytes=2000)
self.assertTrue(article.truncated)
assert_well_formed(self, article.to_xhtml())
def test_no_truncation_when_under_limit(self):
article = parse_article("<p>short</p>", BASE, max_input_bytes=10_000)
self.assertFalse(article.truncated)
def test_output_is_deterministic(self):
raw = '<p>a &amp; b</p><div><img src="i.png"><ul><li>x</ul></div>'
self.assertEqual(render(raw), render(raw))
def test_image_urls_are_cached_between_calls(self):
article = parse_article('<img src="a.png">', BASE)
self.assertIs(article.image_urls, article.image_urls)
def test_output_wrapper_has_the_expected_class(self):
self.assertTrue(render("<p>x</p>").startswith('<div class="article-body">'))
if __name__ == "__main__":
unittest.main()

145
tools/fake-freshrss.py Executable file
View File

@@ -0,0 +1,145 @@
#!/usr/bin/env python3
"""Run a fake FreshRSS, so the whole server can be tried without an account.
Serves the same in-process stand-in the test suite uses: three feeds across two
categories, a starred article, an already-read article, a feed with no category,
and article bodies that exercise the sanitiser. It also writes a ready-to-run
configuration pointing at itself, with a known demo password.
Run locally::
python3 tools/fake-freshrss.py
python3 -m app --config config.demo.json # in another shell
# then browse http://127.0.0.1:8080/opds as demo / demo
Or as the other half of the demo compose stack, where it binds the whole
container interface and writes a configuration addressing itself by service name
(see ``docker-compose.demo.yaml``).
Nothing here is a real credential. With the default loopback bind it is
reachable only from this machine; the demo compose stack puts both containers on
an internal network with no published ports other than the catalogue.
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT))
from app.config import password_digest # noqa: E402
from tests.stub_freshrss import build_demo_catalog # noqa: E402
DEMO_USER = "demo"
DEMO_PASSWORD = "demo"
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--write-config",
metavar="PATH",
default="config.demo.json",
help="where to write a configuration that points at this fake server",
)
app = parser.add_argument_group("the server being configured")
app.add_argument("--app-port", type=int, default=8080, help="port for the server")
app.add_argument(
"--app-host",
default="127.0.0.1",
help="address the server should bind (use 0.0.0.0 inside a container)",
)
api = parser.add_argument_group("this fake FreshRSS")
api.add_argument(
"--api-port",
type=int,
default=0,
help="port to listen on; 0 asks the operating system for a free one",
)
api.add_argument(
"--api-bind",
default="127.0.0.1",
help="address to listen on (use 0.0.0.0 so another container can reach it)",
)
api.add_argument(
"--api-public-host",
default=None,
help="host name to write into the configuration (a compose service name, "
"for instance); defaults to the bind address",
)
return parser
def demo_configuration(stub, arguments: argparse.Namespace) -> dict:
"""The configuration that points a server at this fake account."""
return {
"_comment": "Written by tools/fake-freshrss.py; not a real configuration.",
"server": {
"host": arguments.app_host,
"port": arguments.app_port,
"log_level": "info",
},
"freshrss": {
"url": stub.api_url,
"username": stub.username,
"api_password": stub.password,
"items_per_page": 5,
# A short window so the demo reflects a download immediately rather
# than showing a stale unread count for minutes.
"retries": 1,
},
"catalog": {"title": "Demo Catalog", "author": "Demo"},
"epub": {"include_images": False},
"auth": {
"users": [{"username": DEMO_USER, "password_sha256": password_digest(DEMO_PASSWORD)}]
},
"cache": {"subscriptions_ttl_seconds": 30, "stream_ttl_seconds": 10},
}
def main(argv: list[str] | None = None) -> int:
arguments = build_parser().parse_args(argv)
stub = build_demo_catalog(
bind_host=arguments.api_bind,
bind_port=arguments.api_port,
public_host=arguments.api_public_host,
).start()
path = Path(arguments.write_config)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(demo_configuration(stub, arguments), indent=2) + "\n",
encoding="utf-8",
)
browse_host = "127.0.0.1" if arguments.app_host == "0.0.0.0" else arguments.app_host
print(f"fake FreshRSS listening on {stub.api_url}", flush=True)
print(f" api user: {stub.username}", flush=True)
print(f" api password: {stub.password}", flush=True)
print(f"wrote {path}", flush=True)
print(f"catalogue will be at http://{browse_host}:{arguments.app_port}/opds", flush=True)
print(f" sign in as {DEMO_USER} / {DEMO_PASSWORD}", flush=True)
if arguments.app_host == "127.0.0.1":
print(f"\nnow run, in another shell:\n python3 -m app --config {path}", flush=True)
print("\npress ctrl-c to stop", flush=True)
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
print("\nstopping", flush=True)
finally:
stub.stop()
return 0
if __name__ == "__main__":
raise SystemExit(main())