Initial commit
This commit is contained in:
12
.dockerignore
Normal file
12
.dockerignore
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
.git
|
||||||
|
.gitignore
|
||||||
|
.pi
|
||||||
|
.env
|
||||||
|
config.json
|
||||||
|
# Both Dockerfiles share this context. The main image copies only app/, so the
|
||||||
|
# test and tool sources stay out of it; the demo image needs them.
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*.md
|
||||||
|
!README.md
|
||||||
|
Makefile
|
||||||
8
.env.example
Normal file
8
.env.example
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
# Copy to .env and fill in, then `docker compose up -d --build`.
|
||||||
|
|
||||||
|
# FreshRSS API password: Settings -> Account -> API in FreshRSS.
|
||||||
|
# This is NOT your login password.
|
||||||
|
FRESHRSS_API_PASSWORD=replace-me
|
||||||
|
|
||||||
|
# sha256 of the password your e-reader will use, from: make hash-password
|
||||||
|
OPDS_PASSWORD_SHA256=replace-me-with-64-hex-characters
|
||||||
26
.gitignore
vendored
Normal file
26
.gitignore
vendored
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
# Local configuration and secrets -- never commit these.
|
||||||
|
config.json
|
||||||
|
.env
|
||||||
|
*.local.json
|
||||||
|
|
||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*.egg-info/
|
||||||
|
.venv/
|
||||||
|
venv/
|
||||||
|
|
||||||
|
# Test and tool output
|
||||||
|
# Tool and linter caches
|
||||||
|
.pytest_cache/
|
||||||
|
.ruff_cache/
|
||||||
|
.mypy_cache/
|
||||||
|
|
||||||
|
.coverage
|
||||||
|
htmlcov/
|
||||||
|
*.epub
|
||||||
|
|
||||||
|
# Editors
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
*.swp
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"id": "b880b783d",
|
||||||
|
"name": "Docker image build",
|
||||||
|
"command": "cd /media/git/odps-from-rss && docker build -t opds-from-rss:latest . 2>&1",
|
||||||
|
"description": "Build the container image to verify the zero-dependency Dockerfile works end to end.",
|
||||||
|
"status": "completed",
|
||||||
|
"outputPath": ".pi/tasks/01a1e156-4fa0-722f-83e8-c971d55b7a0e-2566715/b880b783d.output",
|
||||||
|
"cwd": "/media/git/odps-from-rss",
|
||||||
|
"startTime": 1794783944805,
|
||||||
|
"endTime": 1794783954328,
|
||||||
|
"exitCode": 0,
|
||||||
|
"signal": null,
|
||||||
|
"pid": 2607703,
|
||||||
|
"bytesWritten": 3628,
|
||||||
|
"isAgent": false,
|
||||||
|
"surviveReload": false,
|
||||||
|
"notified": true,
|
||||||
|
"notifyOnCompletion": true,
|
||||||
|
"triggerOnCompletion": true,
|
||||||
|
"timeoutSeconds": 900,
|
||||||
|
"shellPolicy": {
|
||||||
|
"policy": "inherit",
|
||||||
|
"executable": "/bin/bash",
|
||||||
|
"argvPrefix": [
|
||||||
|
"-c"
|
||||||
|
],
|
||||||
|
"dialect": "bash"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
#0 building with "default" instance using docker driver
|
||||||
|
|
||||||
|
#1 [internal] load build definition from Dockerfile
|
||||||
|
#1 transferring dockerfile:
|
||||||
|
#1 transferring dockerfile: 1.07kB done
|
||||||
|
#1 DONE 0.2s
|
||||||
|
|
||||||
|
#2 [internal] load metadata for docker.io/library/python:3.13-alpine
|
||||||
|
#2 DONE 1.2s
|
||||||
|
|
||||||
|
#3 [internal] load .dockerignore
|
||||||
|
#3 transferring context:
|
||||||
|
#3 transferring context: 139B done
|
||||||
|
#3 DONE 0.2s
|
||||||
|
|
||||||
|
#4 [internal] load build context
|
||||||
|
#4 ...
|
||||||
|
|
||||||
|
#5 [1/5] FROM docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e
|
||||||
|
#5 resolve docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e 0.4s done
|
||||||
|
#5 DONE 0.4s
|
||||||
|
|
||||||
|
#4 [internal] load build context
|
||||||
|
#4 transferring context: 491.01kB done
|
||||||
|
#4 DONE 0.6s
|
||||||
|
|
||||||
|
#5 [1/5] FROM docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e
|
||||||
|
#5 DONE 1.1s
|
||||||
|
|
||||||
|
#5 [1/5] FROM docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e
|
||||||
|
#5 extracting sha256:881fc932939efef04ca120ad44bbd2c0917851eb247aa4995428afb1e80ede17
|
||||||
|
#5 sha256:17af2db0edc9c7ff966b9f7e8424caed1203bc5cec5e3c061f3516ef5b8d9ece 248B / 248B 0.1s done
|
||||||
|
#5 extracting sha256:881fc932939efef04ca120ad44bbd2c0917851eb247aa4995428afb1e80ede17 0.3s done
|
||||||
|
#5 sha256:f9abc654947e09e316e7703544bd7e68123d383aeb70c5a9cf15be03e71721ce 0B / 12.67MB 0.2s
|
||||||
|
#5 sha256:ce6949f6b5329553c2f3fc626421da4f70d633cfddb6ad2227f3ac0b7205bd8f 0B / 412.00kB 0.2s
|
||||||
|
#5 sha256:f9abc654947e09e316e7703544bd7e68123d383aeb70c5a9cf15be03e71721ce 4.19MB / 12.67MB 0.3s
|
||||||
|
#5 sha256:ce6949f6b5329553c2f3fc626421da4f70d633cfddb6ad2227f3ac0b7205bd8f 412.00kB / 412.00kB 0.2s done
|
||||||
|
#5 extracting sha256:ce6949f6b5329553c2f3fc626421da4f70d633cfddb6ad2227f3ac0b7205bd8f
|
||||||
|
#5 sha256:f9abc654947e09e316e7703544bd7e68123d383aeb70c5a9cf15be03e71721ce 12.67MB / 12.67MB 0.5s done
|
||||||
|
#5 extracting sha256:ce6949f6b5329553c2f3fc626421da4f70d633cfddb6ad2227f3ac0b7205bd8f 0.2s done
|
||||||
|
#5 DONE 1.7s
|
||||||
|
|
||||||
|
#5 [1/5] FROM docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e
|
||||||
|
#5 extracting sha256:f9abc654947e09e316e7703544bd7e68123d383aeb70c5a9cf15be03e71721ce
|
||||||
|
#5 extracting sha256:f9abc654947e09e316e7703544bd7e68123d383aeb70c5a9cf15be03e71721ce 0.3s done
|
||||||
|
#5 DONE 2.0s
|
||||||
|
|
||||||
|
#5 [1/5] FROM docker.io/library/python:3.13-alpine@sha256:9ef1a599f863231c3d91e941389bfdc4a2d6070557243d54df1a4e57015b960e
|
||||||
|
#5 extracting sha256:17af2db0edc9c7ff966b9f7e8424caed1203bc5cec5e3c061f3516ef5b8d9ece 0.1s done
|
||||||
|
#5 DONE 2.1s
|
||||||
|
|
||||||
|
#6 [2/5] RUN apk add --no-cache ca-certificates && adduser -D -u 10001 -h /srv app
|
||||||
|
#6 0.739 OK: 11.4 MiB in 29 packages
|
||||||
|
#6 DONE 1.0s
|
||||||
|
|
||||||
|
#7 [3/5] WORKDIR /srv
|
||||||
|
#7 DONE 0.3s
|
||||||
|
|
||||||
|
#8 [4/5] COPY app/ ./app/
|
||||||
|
#8 DONE 0.3s
|
||||||
|
|
||||||
|
#9 [5/5] COPY config.example.json ./
|
||||||
|
#9 DONE 0.3s
|
||||||
|
|
||||||
|
#10 exporting to image
|
||||||
|
#10 exporting layers
|
||||||
|
#10 exporting layers 1.6s done
|
||||||
|
#10 exporting manifest sha256:701ef954b346b1f95b3ce1975be05310709337bda6897169db926e7ce7e93d18 0.1s done
|
||||||
|
#10 exporting config sha256:b098ef38aca29a44c9e971d5f7e128f7bb3dc43e4711f916fa24ad042ec443a1 0.1s done
|
||||||
|
#10 exporting attestation manifest sha256:0bdcacb0668771798a65cf1b8fa04c1460748786ea2b9d166b1436a416750f34 0.1s done
|
||||||
|
#10 exporting manifest list sha256:28c19d6010bbc1d8452a93d32e1ad8aa1e8f96b9a5e2d147bf93e99f4ac29f24
|
||||||
|
#10 exporting manifest list sha256:28c19d6010bbc1d8452a93d32e1ad8aa1e8f96b9a5e2d147bf93e99f4ac29f24 0.1s done
|
||||||
|
#10 naming to docker.io/library/opds-from-rss:latest 0.0s done
|
||||||
|
#10 unpacking to docker.io/library/opds-from-rss:latest
|
||||||
|
#10 unpacking to docker.io/library/opds-from-rss:latest 0.8s done
|
||||||
|
#10 DONE 2.9s
|
||||||
27
Dockerfile
Normal file
27
Dockerfile
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# The whole server is the Python standard library, so the image needs no build
|
||||||
|
# toolchain, no wheels and no virtual environment -- just an interpreter, a trust
|
||||||
|
# store for upstream TLS, and the code.
|
||||||
|
FROM python:3.13-alpine
|
||||||
|
|
||||||
|
RUN apk add --no-cache ca-certificates \
|
||||||
|
&& adduser -D -u 10001 -h /srv app
|
||||||
|
|
||||||
|
ENV PYTHONUNBUFFERED=1 \
|
||||||
|
PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
CONFIG_PATH=/config/config.json
|
||||||
|
|
||||||
|
WORKDIR /srv
|
||||||
|
COPY app/ ./app/
|
||||||
|
COPY config.example.json ./
|
||||||
|
|
||||||
|
# Runs as an unprivileged user with no writable path: the server keeps nothing on
|
||||||
|
# disk, so the container can be run read-only (see docker-compose.yaml).
|
||||||
|
USER app
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
# The health endpoint answers without credentials and never calls upstream, so
|
||||||
|
# this checks the process, not the network.
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
|
||||||
|
CMD ["python", "-c", "import sys, urllib.request; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/healthz', timeout=4).status == 200 else 1)"]
|
||||||
|
|
||||||
|
ENTRYPOINT ["python", "-m", "app"]
|
||||||
31
Dockerfile.demo
Normal file
31
Dockerfile.demo
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
# The fake FreshRSS from the demo stack.
|
||||||
|
#
|
||||||
|
# It serves the same in-process stand-in the test suite uses, which is why this
|
||||||
|
# image needs tests/ and tools/ -- the production image copies only app/ and stays
|
||||||
|
# minimal. Nothing here talks to a real account: the account is three feeds of
|
||||||
|
# sample articles held in memory.
|
||||||
|
FROM python:3.13-alpine
|
||||||
|
|
||||||
|
RUN adduser -D -u 10001 -h /srv app
|
||||||
|
|
||||||
|
ENV PYTHONUNBUFFERED=1 \
|
||||||
|
PYTHONDONTWRITEBYTECODE=1
|
||||||
|
|
||||||
|
WORKDIR /srv
|
||||||
|
COPY app/ ./app/
|
||||||
|
COPY tests/ ./tests/
|
||||||
|
COPY tools/ ./tools/
|
||||||
|
|
||||||
|
USER app
|
||||||
|
EXPOSE 8099
|
||||||
|
|
||||||
|
# Only says the fake server is answering; it is what the catalogue waits for.
|
||||||
|
HEALTHCHECK --interval=2s --timeout=3s --start-period=2s --retries=15 \
|
||||||
|
CMD ["python", "-c", "import sys, urllib.request; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8099/healthz', timeout=2).status == 200 else 1)"]
|
||||||
|
|
||||||
|
ENTRYPOINT ["python", "tools/fake-freshrss.py"]
|
||||||
|
# Binds the whole container interface so the catalogue can reach it by service
|
||||||
|
# name, and writes its generated configuration to /tmp purely so the demo
|
||||||
|
# credentials show up in the log.
|
||||||
|
CMD ["--api-bind=0.0.0.0", "--api-port=8099", "--api-public-host=fresh", \
|
||||||
|
"--app-host=0.0.0.0", "--app-port=8080", "--write-config=/tmp/config.json"]
|
||||||
71
Makefile
Normal file
71
Makefile
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
# Deployment and development helpers. Nothing here needs any dependency beyond
|
||||||
|
# Python 3.11+ and, for the container targets, Docker.
|
||||||
|
|
||||||
|
CONFIG ?= config.json
|
||||||
|
IMAGE ?= opds-from-rss:latest
|
||||||
|
PYTHON ?= python3
|
||||||
|
|
||||||
|
.PHONY: help test run check demo demo-up demo-down build up down logs shell hash-password lint clean
|
||||||
|
|
||||||
|
help:
|
||||||
|
@echo "make test run the test suite (no dependencies needed)"
|
||||||
|
@echo "make run run the server locally with $(CONFIG)"
|
||||||
|
@echo "make check validate $(CONFIG) and print the effective settings"
|
||||||
|
@echo "make hash-password print the sha256 digest for auth.users[].password_sha256"
|
||||||
|
@echo "make demo serve a built-in fake FreshRSS, for a smoke test"
|
||||||
|
@echo "make demo-up run the self-contained demo stack (catalogue + fake FreshRSS)"
|
||||||
|
@echo "make demo-down stop the demo stack"
|
||||||
|
@echo "make build build the container image"
|
||||||
|
@echo "make up / down start or stop the container via docker compose"
|
||||||
|
@echo "make logs follow the container logs"
|
||||||
|
@echo "make shell open a shell in the image"
|
||||||
|
@echo "make lint byte-compile and check formatting of the sources"
|
||||||
|
@echo "make clean remove caches and local configuration"
|
||||||
|
|
||||||
|
test:
|
||||||
|
$(PYTHON) -m unittest discover -s tests -t . -v
|
||||||
|
|
||||||
|
lint:
|
||||||
|
$(PYTHON) -m compileall -q app tests tools
|
||||||
|
@echo "sources compile cleanly"
|
||||||
|
|
||||||
|
run:
|
||||||
|
$(PYTHON) -m app --config $(CONFIG)
|
||||||
|
|
||||||
|
check:
|
||||||
|
$(PYTHON) -m app --config $(CONFIG) --check-config
|
||||||
|
|
||||||
|
hash-password:
|
||||||
|
@$(PYTHON) -c "import getpass, hashlib; print(hashlib.sha256(getpass.getpass('Password: ').encode()).hexdigest())"
|
||||||
|
|
||||||
|
demo:
|
||||||
|
$(PYTHON) tools/fake-freshrss.py
|
||||||
|
|
||||||
|
# The self-contained stack: builds, starts, and prints where to point a reader.
|
||||||
|
demo-up:
|
||||||
|
docker compose -f docker-compose.demo.yaml up -d --build
|
||||||
|
@echo ""
|
||||||
|
@echo "catalogue: http://127.0.0.1:8088/opds (sign in as demo / demo)"
|
||||||
|
@echo "logs: make demo-down | docker compose -f docker-compose.demo.yaml logs -f"
|
||||||
|
|
||||||
|
demo-down:
|
||||||
|
docker compose -f docker-compose.demo.yaml down
|
||||||
|
|
||||||
|
build:
|
||||||
|
docker build -t $(IMAGE) .
|
||||||
|
|
||||||
|
up:
|
||||||
|
@test -f config.json || { echo "config.json is missing: cp config.example.json config.json, then edit it"; exit 1; }
|
||||||
|
docker compose up -d --build
|
||||||
|
|
||||||
|
down:
|
||||||
|
docker compose down
|
||||||
|
|
||||||
|
logs:
|
||||||
|
docker compose logs -f
|
||||||
|
|
||||||
|
shell:
|
||||||
|
docker run --rm -it --entrypoint /bin/sh $(IMAGE)
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -rf app/__pycache__ tests/__pycache__ tools/__pycache__ .env config.json
|
||||||
247
README.md
Normal file
247
README.md
Normal file
@@ -0,0 +1,247 @@
|
|||||||
|
# opds-from-rss
|
||||||
|
|
||||||
|
Serve a **FreshRSS** account as an **OPDS 1.2** catalog, where every article is
|
||||||
|
downloadable as an **EPUB generated on the fly**. Downloading an article marks it
|
||||||
|
read in FreshRSS.
|
||||||
|
|
||||||
|
It is a single small container with **no runtime dependencies** — the whole server
|
||||||
|
is the Python standard library — and it keeps **nothing on disk**. The catalog is
|
||||||
|
rebuilt from the FreshRSS API as you browse it, and each book is assembled in
|
||||||
|
memory at the moment it is requested.
|
||||||
|
|
||||||
|
```text
|
||||||
|
e-reader ──OPDS──▶ categories ──▶ feeds ──▶ articles ──▶ EPUB (built in memory)
|
||||||
|
│ │
|
||||||
|
└──────────────── browse the unread queue ◀──────────────┘
|
||||||
|
mark read upstream
|
||||||
|
```
|
||||||
|
|
||||||
|
## How it works
|
||||||
|
|
||||||
|
* **Your account's shape becomes the catalog.** FreshRSS category labels become
|
||||||
|
navigation feeds, subscriptions become feed entries, and unread articles become
|
||||||
|
the acquisition entries you download.
|
||||||
|
* **One article, one book.** Each article is converted to an EPUB 3 (with an EPUB 2
|
||||||
|
NCX fallback) whose body is the content FreshRSS already stores, normalised into
|
||||||
|
strict XHTML. Images are fetched and embedded so the book reads offline.
|
||||||
|
* **Downloading is what marks it read.** After the EPUB has been delivered, the
|
||||||
|
matching item is tagged read upstream. A `HEAD` probe or an interrupted transfer
|
||||||
|
does **not** consume the article, and the call is idempotent.
|
||||||
|
* **Nothing is persisted.** No database, no cache directory, no temp files. The
|
||||||
|
container can run with a read-only filesystem, which the shipped Compose file
|
||||||
|
does.
|
||||||
|
|
||||||
|
## Try it out first
|
||||||
|
|
||||||
|
There is a self-contained demo stack that needs no configuration and no FreshRSS
|
||||||
|
account: it runs the catalogue against a fake FreshRSS holding three sample feeds.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make demo-up # or: docker compose -f docker-compose.demo.yaml up --build
|
||||||
|
```
|
||||||
|
|
||||||
|
Then open **http://127.0.0.1:8088/opds** and sign in as **demo / demo**. The fake
|
||||||
|
account holds three feeds across two categories, a starred article and an
|
||||||
|
already-read one. Downloading a book marks that article read in the fake server, so
|
||||||
|
the unread queue visibly drains as you read.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make demo-down # stop it; all demo state is in memory, so nothing persists
|
||||||
|
```
|
||||||
|
|
||||||
|
It publishes an unencrypted port on loopback with a known password, so keep it for
|
||||||
|
trying things out — for real use, follow the quick start below.
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cp config.example.json config.json # edit: your FreshRSS URL, user, categories
|
||||||
|
cp .env.example .env # edit: the two secrets
|
||||||
|
make hash-password # paste the digest into config.json (auth.users)
|
||||||
|
docker compose up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
Then point your e-reader's OPDS browser at `http://<host>:8080/opds` and sign in
|
||||||
|
with the user name and password from `auth.users`.
|
||||||
|
|
||||||
|
### FreshRSS prerequisites
|
||||||
|
|
||||||
|
You need an **API password**, which is separate from your login password:
|
||||||
|
FreshRSS → **Settings → Account → API**. Put that in `FRESHRSS_API_PASSWORD`.
|
||||||
|
The URL in `config.json` may be written with or without the `/api/greader.php`
|
||||||
|
suffix; it is added for you, and any path prefix (`https://host/freshrss`) is kept.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
`config.json` is the only input. `config.example.json` documents every setting
|
||||||
|
in place, and a key beginning with `_` is treated as a comment — ignored, never
|
||||||
|
interpolated, and never reported as unknown.
|
||||||
|
|
||||||
|
Any string value may reference a secret instead of containing one:
|
||||||
|
|
||||||
|
| Syntax | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `${VAR}` | The environment variable; the server refuses to start if it is unset |
|
||||||
|
| `${VAR:-default}` | The environment variable, or a default when unset |
|
||||||
|
| `${file:/run/secrets/x}` | The contents of a file (Docker secrets); relative paths resolve against `config.json`. The trailing newline is trimmed. |
|
||||||
|
| `$${` | A literal `${` |
|
||||||
|
|
||||||
|
The settings most worth knowing about:
|
||||||
|
|
||||||
|
| Setting | Default | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `server.host` | `127.0.0.1` | **A container needs `0.0.0.0`**, which `config.example.json` sets. Exposure is controlled by the published port. |
|
||||||
|
| `server.port` | `8080` | `0` asks the OS for a free port. |
|
||||||
|
| `server.public_base_url` | unset | Pins the catalog's links. Set it in production so a spoofed `Host` header cannot shape them. |
|
||||||
|
| `server.trust_proxy_headers` | `false` | Enables `X-Forwarded-Proto/Host/Prefix`. Only behind a proxy you control. |
|
||||||
|
| `server.allowed_hosts` | `[]` | Optional allowlist; other `Host` values are refused. |
|
||||||
|
| `server.tls.cert_file` / `key_file` | unset | In-process TLS, if you would rather not run a reverse proxy. |
|
||||||
|
| `freshrss.items_per_page` | `30` | Articles per catalog page. |
|
||||||
|
| `freshrss.mark_read` | `true` | Set `false` for a read-only mirror. |
|
||||||
|
| `freshrss.read_window_items` | `200` | How far back the **Read** section looks (see limitations). |
|
||||||
|
| `catalog.show_read_section` | `true` | Adds a Read tree beside the Unread queue. |
|
||||||
|
| `catalog.show_starred` | `true` | Adds a Starred shelf. |
|
||||||
|
| `catalog.exclude_categories` / `exclude_feeds` | `[]` | Leave things out of the catalog entirely. |
|
||||||
|
| `epub.include_images` | `true` | Fetch and embed article images so books work offline. |
|
||||||
|
| `epub.cover_image` | `true` | Use the article's lead image as the book cover. |
|
||||||
|
| `epub.allowed_image_hosts` | `[]` | When set, only these hosts' images are embedded. |
|
||||||
|
| `auth.users[].password_sha256` | — | Preferred over `password`. Generate with `make hash-password`. |
|
||||||
|
| `cache.*` | 300s / 60s / 900s | In-memory TTLs. Nothing is written to disk. |
|
||||||
|
|
||||||
|
Validate before deploying — every problem is reported at once, with its JSON path:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make check # or: python3 -m app --config config.json --check-config
|
||||||
|
```
|
||||||
|
|
||||||
|
## What the catalog looks like
|
||||||
|
|
||||||
|
| Path | Kind | Contents |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `/opds` | navigation | Unread, Read, Starred, All unread |
|
||||||
|
| `/opds/unread` · `/opds/read` | navigation | Categories, with unread counts |
|
||||||
|
| `/opds/unread/c/<category>` | navigation | The feeds in that category |
|
||||||
|
| `/opds/unread/c/<category>/f/<feed>` | acquisition | Unread articles, newest first |
|
||||||
|
| `/opds/unread/all` · `/opds/read/all` | acquisition | The whole reading list, paginated |
|
||||||
|
| `/opds/starred` | acquisition | Starred articles |
|
||||||
|
| `/opds/search?q=…` | acquisition | Search over the recent window |
|
||||||
|
| `/opds/d/<feed>/<item>/<slug>.epub` | — | The EPUB, and the mark-read |
|
||||||
|
| `/opds/opensearch.xml` | — | OpenSearch description, so readers show a search box |
|
||||||
|
| `/healthz` | — | Liveness. No credentials, no upstream call |
|
||||||
|
| `/opds/healthz` | — | Diagnostics: upstream reachability, feed count, cache, mark-read state |
|
||||||
|
|
||||||
|
Opaque identifiers in paths are base64url, because FreshRSS ids contain `/`, `?`
|
||||||
|
and `:`. The `<slug>.epub` segment is decoration — it names the saved file and the
|
||||||
|
server ignores it when looking the article up.
|
||||||
|
|
||||||
|
Links are absolute, and every feed carries `rel="self"`, `rel="start"` and
|
||||||
|
`rel="search"`, with `rel="first"`/`previous`/`next` for pagination. Acquisition
|
||||||
|
entries link the EPUB twice (`open-access` and the generic acquisition relation),
|
||||||
|
because readers disagree about which to look for, plus `alternate` to the original
|
||||||
|
article.
|
||||||
|
|
||||||
|
### Reader setup
|
||||||
|
|
||||||
|
* **KOReader** — Tools → OPDS catalog → add `http://<host>:8080/opds` with your
|
||||||
|
credentials. Use `https://` if you put a proxy in front.
|
||||||
|
* **Calibre** — *Get books* → *Add a feed* → paste the same URL.
|
||||||
|
* **Moon+ Reader / Librera / FBReader** — add a *custom OPDS* catalog with that
|
||||||
|
URL and the same credentials.
|
||||||
|
|
||||||
|
Only OPDS **1.2 (Atom)** is served, deliberately: that is what shipping readers
|
||||||
|
implement. OPDS 2.0 JSON would be a second, unused surface.
|
||||||
|
|
||||||
|
## Deploying
|
||||||
|
|
||||||
|
The Compose file runs the container read-only, with all capabilities dropped, no
|
||||||
|
new privileges, published on **loopback only**, and with secrets coming from
|
||||||
|
`.env`. Put a TLS-terminating reverse proxy (or a Cloudflare tunnel, or similar)
|
||||||
|
in front of it — see below.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make up # build and start
|
||||||
|
make logs # follow
|
||||||
|
make down # stop
|
||||||
|
```
|
||||||
|
|
||||||
|
To reach it from the LAN instead of through a proxy, change the published port to
|
||||||
|
`"8080:8080"` in `docker-compose.yaml` and set `server.public_base_url`. Only do
|
||||||
|
that on a network you trust.
|
||||||
|
|
||||||
|
### Security notes
|
||||||
|
|
||||||
|
* **HTTP Basic requires TLS.** Basic credentials are base64-encoded, not
|
||||||
|
encrypted. Terminate TLS at a proxy, or set `server.tls.*` so the container does
|
||||||
|
it itself. The server logs a warning at startup when it is serving plain HTTP.
|
||||||
|
* **Authentication is mandatory and deliberate.** There is no "auth off" switch:
|
||||||
|
the catalog is your entire reading queue, and an unauthenticated OPDS endpoint on
|
||||||
|
a routable address is an open book. Passwords are compared as sha256 digests in
|
||||||
|
constant time, an unknown user takes the same time as a wrong password, and
|
||||||
|
repeated failures from one address are throttled (`auth.max_failures_per_minute`,
|
||||||
|
`auth.throttle_seconds`).
|
||||||
|
* **Image fetching is SSRF-guarded.** Image URLs come from feed content, so they are
|
||||||
|
attacker-controlled if you follow a hostile feed. Only `http`/`https` is
|
||||||
|
followed; every address a hostname resolves to is checked against the loopback,
|
||||||
|
link-local, private, multicast and reserved ranges (including v4-mapped v6
|
||||||
|
forms); the connection is dialled to the *validated address* rather than
|
||||||
|
re-resolving the name, which closes the DNS-rebinding window; TLS still verifies
|
||||||
|
the real hostname; redirects are validated hop by hop; and bodies are capped per
|
||||||
|
image and per article. A refused image is simply dropped from the book.
|
||||||
|
* **Set `server.public_base_url`** in production, and `server.allowed_hosts` if the
|
||||||
|
service is directly reachable, so a spoofed `Host` header cannot shape the links
|
||||||
|
in the catalog.
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make test # 275 tests, no dependencies beyond Python 3.11+
|
||||||
|
make run # run locally against config.json
|
||||||
|
make demo # serve a built-in fake FreshRSS and try the whole thing offline
|
||||||
|
```
|
||||||
|
|
||||||
|
The suite needs no network: an in-process FreshRSS stand-in
|
||||||
|
(`tests/stub_freshrss.py`) speaks the real wire format, and the end-to-end tests
|
||||||
|
drive the actual HTTP server against it — browsing the catalog, downloading an
|
||||||
|
EPUB, unzipping it, and asserting on the `edit-tag` request that FreshRSS received.
|
||||||
|
`tools/fake-freshrss.py` exposes the same stub for a manual smoke test.
|
||||||
|
|
||||||
|
Layout: `config` validation, `urls` identifiers, `cache` TTL cache, `xhtml`
|
||||||
|
HTML→XHTML, `images` sniffing/sizing, `fetch` guarded fetching, `epub` the EPUB
|
||||||
|
writer, `freshrss` the API client, `auth`, `opds` the Atom builders, `server`
|
||||||
|
routing and catalog logic.
|
||||||
|
|
||||||
|
## Limitations, honestly
|
||||||
|
|
||||||
|
* **The Read section is bounded.** Unread filtering happens upstream (exact), but
|
||||||
|
read items are filtered locally because only the *unread exclusion* is a
|
||||||
|
documented upstream parameter — so Read shows at most `read_window_items` recent
|
||||||
|
articles. The same window bounds `/opds/search`, which is a local filter rather
|
||||||
|
than an upstream search call.
|
||||||
|
* **Article bodies come from FreshRSS**, not from the original pages. Fast,
|
||||||
|
offline-friendly, and it works for paywalled or JavaScript-heavy sites — but a
|
||||||
|
feed that only publishes teasers yields teaser-length books. Pair it with
|
||||||
|
FreshRSS's own full-text (Readability) configuration if you need more.
|
||||||
|
* **No cover thumbnails in the catalog.** Cover images are used inside the EPUB
|
||||||
|
(`cover-image`), but navigation feeds do not advertise thumbnails, to avoid
|
||||||
|
making every reader fetch one image per entry.
|
||||||
|
* **`javascript:`/`data:` URLs, scripts, styles, iframes and SVG do not survive**
|
||||||
|
into a book, by design. Neither do inline styles or `class` attributes: the book
|
||||||
|
gets one plain stylesheet instead (override with `epub.stylesheet`).
|
||||||
|
* **Basic auth only.** No OAuth, no per-user catalogs: every user sees the whole
|
||||||
|
account.
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
| Symptom | Cause |
|
||||||
|
| --- | --- |
|
||||||
|
| Container exits immediately, `configuration file not found` | `config.json` is not at `/config/config.json`; check the volume mount, or set `CONFIG_PATH`. |
|
||||||
|
| `environment variable FRESHRSS_API_PASSWORD is not set` | `.env` is missing or unrenamed; `cp .env.example .env`. |
|
||||||
|
| Reader shows an empty catalog | `freshrss.url` or the API password is wrong. `curl -u user:pass http://host:8080/opds/healthz` says which. |
|
||||||
|
| Reader prompts for a password repeatedly | Wrong `auth.users` entry; note that passwords are compared as digests, so a `password_sha256` must match exactly. |
|
||||||
|
| Catalog loads, downloads fail with 502 | The container cannot reach FreshRSS, or the image host refused; check `docker compose logs`. |
|
||||||
|
| Books are missing images | The images were refused by the SSRF guard or exceeded `epub.max_image_bytes`; the log says which, per URL. |
|
||||||
|
| A downloaded article is still listed as unread | The reader refetched the catalog before the mark-read landed, or `freshrss.mark_read` is `false`. |
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT.
|
||||||
42
app/__init__.py
Normal file
42
app/__init__.py
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
"""OPDS server backed by FreshRSS.
|
||||||
|
|
||||||
|
An OPDS 1.2 catalog whose shelves mirror FreshRSS categories and feeds, serving
|
||||||
|
one EPUB per article, generated in memory at download time. Nothing is written to
|
||||||
|
disk: the catalog is rebuilt from the FreshRSS API as it is browsed, and a
|
||||||
|
completed EPUB download marks the corresponding article read upstream.
|
||||||
|
|
||||||
|
Layout, in dependency order:
|
||||||
|
|
||||||
|
``config``
|
||||||
|
The JSON document, its interpolation (``${ENV}``, ``${file:...}``) and its
|
||||||
|
validation. Everything else is configured from here.
|
||||||
|
``urls``
|
||||||
|
Identifier and filename helpers: base64url for path identifiers, stable
|
||||||
|
``urn:uuid`` values for Atom ids, and safe download filenames.
|
||||||
|
``cache``
|
||||||
|
A bounded in-memory TTL cache with per-key single-flight, so a reader
|
||||||
|
browsing quickly does not turn into a burst of upstream requests.
|
||||||
|
``xhtml``
|
||||||
|
Feed HTML into well-formed XHTML. Everything the reader's XML parser sees
|
||||||
|
has been rebuilt by this module.
|
||||||
|
``images``
|
||||||
|
Media-type sniffing and dimension parsing, so a book's manifest is honest
|
||||||
|
about its bytes and a cover can be chosen without an imaging library.
|
||||||
|
``fetch``
|
||||||
|
The one place that fetches a host the operator did not choose. Article
|
||||||
|
images are fetched through an SSRF guard that dials a validated address.
|
||||||
|
``epub``
|
||||||
|
The EPUB 3 writer, assembled entirely in memory.
|
||||||
|
``freshrss``
|
||||||
|
The FreshRSS client, over its Google Reader compatible API.
|
||||||
|
``auth``
|
||||||
|
HTTP Basic authentication, with a throttle on failed attempts.
|
||||||
|
``opds``
|
||||||
|
OPDS 1.2 Atom document builders.
|
||||||
|
``server``
|
||||||
|
Routing, the catalog logic, and EPUB delivery.
|
||||||
|
|
||||||
|
Run it with ``python -m app --config config.json``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__version__ = "1.0.0"
|
||||||
127
app/__main__.py
Normal file
127
app/__main__.py
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
"""Entry point: load the configuration, then serve.
|
||||||
|
|
||||||
|
Startup is deliberately noisy about anything that will be hard to diagnose later:
|
||||||
|
warnings from configuration validation, the effective settings (with secrets
|
||||||
|
redacted), and any degraded optional feature. An OPDS server that is reachable
|
||||||
|
but shows an empty catalog should say why in its own log, not require a
|
||||||
|
debugging session.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import signal
|
||||||
|
import ssl
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
|
||||||
|
import app.server as app_server
|
||||||
|
from app import __version__
|
||||||
|
from app import config as app_config
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger("app")
|
||||||
|
|
||||||
|
_LOG_FORMAT = "%(asctime)s %(levelname)-7s %(name)s: %(message)s"
|
||||||
|
_LOG_DATEFORMAT = "%Y-%m-%dT%H:%M:%S%z"
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog="opds-from-rss",
|
||||||
|
description="Serve a FreshRSS account as an OPDS 1.2 catalog of EPUBs.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--config",
|
||||||
|
metavar="PATH",
|
||||||
|
help="configuration file (default: $CONFIG_PATH, /config/config.json, config.json)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--check-config",
|
||||||
|
action="store_true",
|
||||||
|
help="validate the configuration and print it, then exit",
|
||||||
|
)
|
||||||
|
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def configure_logging(level: str) -> None:
|
||||||
|
logging.basicConfig(level=level.upper(), format=_LOG_FORMAT, datefmt=_LOG_DATEFORMAT)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
arguments = build_parser().parse_args(argv)
|
||||||
|
# Logging is configured twice: once coarsely so a configuration failure is
|
||||||
|
# reported properly, then again at the level the configuration asks for.
|
||||||
|
configure_logging("info")
|
||||||
|
try:
|
||||||
|
settings = app_config.load_config(arguments.config)
|
||||||
|
except app_config.ConfigError as exc:
|
||||||
|
# The exception already lists every problem with its path.
|
||||||
|
print(str(exc), file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
configure_logging(settings.server.log_level)
|
||||||
|
for warning in settings.warnings:
|
||||||
|
LOGGER.warning("configuration: %s", warning)
|
||||||
|
for line in app_config.describe_config(settings).splitlines():
|
||||||
|
LOGGER.info("%s", line)
|
||||||
|
|
||||||
|
if arguments.check_config:
|
||||||
|
LOGGER.info("configuration is valid; not starting the server")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
httpd, application = app_server.build_server(settings)
|
||||||
|
if settings.server.tls_cert_file and settings.server.tls_key_file:
|
||||||
|
_enable_tls(httpd, settings.server.tls_cert_file, settings.server.tls_key_file)
|
||||||
|
LOGGER.info("serving HTTPS directly (no reverse proxy required)")
|
||||||
|
else:
|
||||||
|
LOGGER.warning(
|
||||||
|
"serving plain HTTP: put a TLS-terminating proxy in front of this, "
|
||||||
|
"because HTTP Basic credentials are only base64-encoded"
|
||||||
|
)
|
||||||
|
_install_signal_handlers(httpd)
|
||||||
|
|
||||||
|
LOGGER.info(
|
||||||
|
"listening on %s:%s; catalog at %s/opds",
|
||||||
|
settings.server.host,
|
||||||
|
settings.server.port,
|
||||||
|
settings.server.public_base_url or f"http://{settings.server.host}:{settings.server.port}",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
httpd.serve_forever(poll_interval=0.25)
|
||||||
|
finally:
|
||||||
|
LOGGER.info("shutting down")
|
||||||
|
httpd.server_close()
|
||||||
|
application.client.close()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _enable_tls(httpd, cert_file: str, key_file: str) -> None:
|
||||||
|
"""Wrap the listening socket, so the service can be secure on its own.
|
||||||
|
|
||||||
|
The paths are passed in rather than read off the configuration so that the
|
||||||
|
"both or neither" guarantee made during validation is visible here too.
|
||||||
|
"""
|
||||||
|
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||||
|
context.load_cert_chain(cert_file, key_file)
|
||||||
|
httpd.socket = context.wrap_socket(httpd.socket, server_side=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _install_signal_handlers(httpd) -> None:
|
||||||
|
"""Stop cleanly on SIGINT/SIGTERM, which is how a container is stopped.
|
||||||
|
|
||||||
|
``shutdown`` blocks until the serve loop exits and must not be called from the
|
||||||
|
thread running it, so the handler hands the work to a short-lived thread.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def stop(signum: int, _frame: object) -> None:
|
||||||
|
LOGGER.info("received %s", signal.Signals(signum).name)
|
||||||
|
threading.Thread(target=httpd.shutdown, daemon=True).start()
|
||||||
|
|
||||||
|
for handled in (signal.SIGINT, signal.SIGTERM):
|
||||||
|
signal.signal(handled, stop)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
186
app/auth.py
Normal file
186
app/auth.py
Normal file
@@ -0,0 +1,186 @@
|
|||||||
|
"""HTTP Basic authentication, with a throttle on failed attempts.
|
||||||
|
|
||||||
|
Basic is the only scheme every OPDS reader UI can drive: the reader prompts for a
|
||||||
|
user name and password, and the server answers an unauthenticated request with
|
||||||
|
``401`` and a ``WWW-Authenticate`` header. That header is also the whole reason
|
||||||
|
this service must sit behind TLS -- the credentials travel base64-encoded, which
|
||||||
|
is encoding, not encryption.
|
||||||
|
|
||||||
|
Two things this module is careful about:
|
||||||
|
|
||||||
|
*The comparison is constant time.* Passwords are stored as sha256 digests
|
||||||
|
(``make hash-password``) and compared with :func:`hmac.compare_digest`, and an
|
||||||
|
unknown user name is compared against a dummy digest so that a missing account and
|
||||||
|
a wrong password take the same time. Otherwise the response time tells an
|
||||||
|
attacker which half of the guess was right.
|
||||||
|
|
||||||
|
*Failures are throttled.* Basic auth can be guessed at line speed, so an address
|
||||||
|
that fails repeatedly is refused for a cooling-off period before its credentials
|
||||||
|
are even examined.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import binascii
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
from app import config as app_config
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Compared against when the user name is unknown, so that the work done -- and
|
||||||
|
# therefore the time taken -- does not depend on whether the account exists.
|
||||||
|
_DUMMY_DIGEST = hashlib.sha256(b"no such user").hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
class UnauthorizedError(Exception):
|
||||||
|
"""Credentials were missing, malformed, or wrong. Answered with 401."""
|
||||||
|
|
||||||
|
def __init__(self, message: str = "authentication required") -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
|
||||||
|
|
||||||
|
class RateLimitedError(Exception):
|
||||||
|
"""Too many failed attempts from this address. Answered with 429."""
|
||||||
|
|
||||||
|
def __init__(self, retry_after: int) -> None:
|
||||||
|
super().__init__("too many failed authentication attempts")
|
||||||
|
self.retry_after = max(1, int(retry_after))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _Attempts:
|
||||||
|
"""Failed-attempt bookkeeping for one client address."""
|
||||||
|
|
||||||
|
timestamps: list[float] = field(default_factory=list)
|
||||||
|
|
||||||
|
def within_window(self, now: float, window: float) -> int:
|
||||||
|
return sum(1 for moment in self.timestamps if now - moment < window)
|
||||||
|
|
||||||
|
|
||||||
|
class Authenticator:
|
||||||
|
"""Checks Basic credentials and throttles repeated failures."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
config: app_config.AuthConfig,
|
||||||
|
*,
|
||||||
|
clock=time.monotonic,
|
||||||
|
) -> None:
|
||||||
|
self._config = config
|
||||||
|
self._clock = clock
|
||||||
|
# Digest lookups avoid a linear scan that would leak user names by timing.
|
||||||
|
self._users = {user.username: user.password_sha256 for user in config.users}
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._attempts: dict[str, _Attempts] = {}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def challenge(self) -> str:
|
||||||
|
"""The ``WWW-Authenticate`` value that makes a reader prompt."""
|
||||||
|
realm = self._config.realm.replace('"', "")
|
||||||
|
return f'Basic realm="{realm}", charset="UTF-8"'
|
||||||
|
|
||||||
|
def authenticate(self, authorization: str | None, client: str) -> str:
|
||||||
|
"""Return the authenticated user name, or raise.
|
||||||
|
|
||||||
|
``client`` identifies the address for throttling; only failed attempts are
|
||||||
|
counted, and a success clears the record.
|
||||||
|
"""
|
||||||
|
self._check_throttle(client)
|
||||||
|
username, password = _parse_basic(authorization)
|
||||||
|
if username is None or not self._matches(username, password):
|
||||||
|
self._record_failure(client)
|
||||||
|
# The source address is what matters for spotting a brute-force run, so
|
||||||
|
# that is the warning. The attempted name is attacker-supplied and
|
||||||
|
# close to an identifier, so it is kept for debugging only, and the
|
||||||
|
# password is never logged at any level.
|
||||||
|
LOGGER.warning("rejected authentication attempt from %s", client)
|
||||||
|
LOGGER.debug("rejected credentials for user %r from %s", username, client)
|
||||||
|
raise UnauthorizedError
|
||||||
|
self._clear_failures(client)
|
||||||
|
return username
|
||||||
|
|
||||||
|
def _matches(self, username: str, password: str | None) -> bool:
|
||||||
|
if password is None:
|
||||||
|
return False
|
||||||
|
expected = self._users.get(username, _DUMMY_DIGEST)
|
||||||
|
offered = hashlib.sha256(password.encode("utf-8")).hexdigest()
|
||||||
|
return hmac.compare_digest(expected, offered)
|
||||||
|
|
||||||
|
# -- throttling ------------------------------------------------------
|
||||||
|
|
||||||
|
def _check_throttle(self, client: str) -> None:
|
||||||
|
window = self._config.throttle_seconds
|
||||||
|
with self._lock:
|
||||||
|
record = self._attempts.get(client)
|
||||||
|
if record is None:
|
||||||
|
return
|
||||||
|
now = self._clock()
|
||||||
|
recent = record.within_window(now, window)
|
||||||
|
if recent < self._config.max_failures_per_minute:
|
||||||
|
return
|
||||||
|
oldest = min(record.timestamps)
|
||||||
|
retry_after = max(1, int(window - (now - oldest)) + 1)
|
||||||
|
raise RateLimitedError(retry_after)
|
||||||
|
|
||||||
|
def _record_failure(self, client: str) -> None:
|
||||||
|
now = self._clock()
|
||||||
|
window = self._config.throttle_seconds
|
||||||
|
with self._lock:
|
||||||
|
record = self._attempts.setdefault(client, _Attempts())
|
||||||
|
record.timestamps.append(now)
|
||||||
|
# Trim anything that has aged out, and drop the record entirely when
|
||||||
|
# it is empty, so a long uptime cannot accumulate addresses.
|
||||||
|
record.timestamps = [moment for moment in record.timestamps if now - moment < window]
|
||||||
|
if not record.timestamps:
|
||||||
|
del self._attempts[client]
|
||||||
|
elif len(self._attempts) > 4096:
|
||||||
|
self._evict_oldest(now, window)
|
||||||
|
|
||||||
|
def _clear_failures(self, client: str) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._attempts.pop(client, None)
|
||||||
|
|
||||||
|
def _evict_oldest(self, now: float, window: float) -> None:
|
||||||
|
"""Bound the table: a flood of addresses must not become a memory leak."""
|
||||||
|
stale = [
|
||||||
|
address
|
||||||
|
for address, record in self._attempts.items()
|
||||||
|
if not record.within_window(now, window)
|
||||||
|
]
|
||||||
|
for address in stale:
|
||||||
|
del self._attempts[address]
|
||||||
|
while len(self._attempts) > 4096:
|
||||||
|
oldest = min(
|
||||||
|
self._attempts,
|
||||||
|
key=lambda address: min(self._attempts[address].timestamps),
|
||||||
|
)
|
||||||
|
del self._attempts[oldest]
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_basic(authorization: str | None) -> tuple[str | None, str | None]:
|
||||||
|
"""Split a Basic header into its credentials.
|
||||||
|
|
||||||
|
Returns ``(None, None)`` for anything that is not a well-formed Basic header,
|
||||||
|
which the caller treats exactly like a wrong password.
|
||||||
|
"""
|
||||||
|
if not authorization:
|
||||||
|
return None, None
|
||||||
|
scheme, _, payload = authorization.partition(" ")
|
||||||
|
if scheme.lower() != "basic" or not payload:
|
||||||
|
return None, None
|
||||||
|
try:
|
||||||
|
decoded = base64.b64decode(payload.strip(), validate=True).decode("utf-8")
|
||||||
|
except (binascii.Error, UnicodeDecodeError):
|
||||||
|
return None, None
|
||||||
|
# Only the first colon separates: a password may contain colons.
|
||||||
|
username, separator, password = decoded.partition(":")
|
||||||
|
if not separator or not username:
|
||||||
|
return None, None
|
||||||
|
return username, password
|
||||||
157
app/cache.py
Normal file
157
app/cache.py
Normal file
@@ -0,0 +1,157 @@
|
|||||||
|
"""Bounded in-memory TTL cache with per-key single-flight.
|
||||||
|
|
||||||
|
The server keeps nothing on disk, which means every navigation click and every
|
||||||
|
download is backed by a live FreshRSS API call. An OPDS reader browses a tree
|
||||||
|
and re-requests feeds freely, so without a cache a single reader produces a small
|
||||||
|
burst of API traffic per tap. This cache absorbs that burst and nothing more:
|
||||||
|
entries expire on a short TTL and are evicted when the cache is full.
|
||||||
|
|
||||||
|
Single-flight matters as much as caching here. Readers often open several
|
||||||
|
connections at once (browsing navigation feeds in parallel, or prefetching cover
|
||||||
|
requests), and without it a cold cache would issue the same FreshRSS request
|
||||||
|
several times over. ``get_or_compute`` guarantees one in-flight computation per
|
||||||
|
key while still letting unrelated keys run concurrently.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections.abc import Callable
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class TTLCache:
|
||||||
|
"""A thread-safe, size-bounded cache of values with a per-entry TTL."""
|
||||||
|
|
||||||
|
def __init__(self, max_entries: int = 512) -> None:
|
||||||
|
# Config validation guarantees an int here, so no defensive coercion.
|
||||||
|
self._max_entries = max(1, max_entries)
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._values: dict[str, tuple[float, Any]] = {}
|
||||||
|
self._key_locks: dict[str, threading.Lock] = {}
|
||||||
|
self.hits = 0
|
||||||
|
self.misses = 0
|
||||||
|
self.evictions = 0
|
||||||
|
|
||||||
|
def get(self, key: str, *, now: float | None = None) -> Any | None:
|
||||||
|
"""Return a live value, or None when missing or expired."""
|
||||||
|
moment = time.monotonic() if now is None else now
|
||||||
|
with self._lock:
|
||||||
|
entry = self._values.get(key)
|
||||||
|
if entry is None:
|
||||||
|
self.misses += 1
|
||||||
|
return None
|
||||||
|
expires_at, value = entry
|
||||||
|
if expires_at <= moment:
|
||||||
|
del self._values[key]
|
||||||
|
self.misses += 1
|
||||||
|
return None
|
||||||
|
# Re-insert so the newest entries sit at the end of the ordering and
|
||||||
|
# the eviction below drops genuinely cold keys first.
|
||||||
|
del self._values[key]
|
||||||
|
self._values[key] = (expires_at, value)
|
||||||
|
self.hits += 1
|
||||||
|
return value
|
||||||
|
|
||||||
|
def set(self, key: str, value: Any, ttl_seconds: float) -> None:
|
||||||
|
"""Store a value, evicting the coldest entry if the cache is full."""
|
||||||
|
if ttl_seconds <= 0:
|
||||||
|
return
|
||||||
|
expires_at = time.monotonic() + ttl_seconds
|
||||||
|
with self._lock:
|
||||||
|
self._values.pop(key, None)
|
||||||
|
self._values[key] = (expires_at, value)
|
||||||
|
while len(self._values) > self._max_entries:
|
||||||
|
oldest, _ = next(iter(self._values.items()))
|
||||||
|
del self._values[oldest]
|
||||||
|
self.evictions += 1
|
||||||
|
|
||||||
|
def get_or_compute(self, key: str, ttl_seconds: float, compute: Callable[[], Any]) -> Any:
|
||||||
|
"""Return a cached value, computing it at most once per key at a time.
|
||||||
|
|
||||||
|
Exceptions from ``compute`` are never cached, so a transient FreshRSS
|
||||||
|
failure is retried on the next request instead of being remembered.
|
||||||
|
"""
|
||||||
|
cached = self.get(key)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
key_lock = self._key_locks.get(key)
|
||||||
|
if key_lock is None:
|
||||||
|
key_lock = threading.Lock()
|
||||||
|
self._key_locks[key] = key_lock
|
||||||
|
|
||||||
|
with key_lock:
|
||||||
|
# Another thread may have filled the key while this one waited.
|
||||||
|
cached = self.get(key)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
value = compute()
|
||||||
|
self.set(key, value, ttl_seconds)
|
||||||
|
return value
|
||||||
|
|
||||||
|
def invalidate(self, prefix: str = "") -> int:
|
||||||
|
"""Drop every key starting with ``prefix`` (all keys when empty)."""
|
||||||
|
with self._lock:
|
||||||
|
if not prefix:
|
||||||
|
removed = len(self._values)
|
||||||
|
self._values.clear()
|
||||||
|
else:
|
||||||
|
doomed = [key for key in self._values if key.startswith(prefix)]
|
||||||
|
for key in doomed:
|
||||||
|
del self._values[key]
|
||||||
|
removed = len(doomed)
|
||||||
|
# Collapse the lock table too: keys are unbounded over a long uptime
|
||||||
|
# and a stale lock object is pure dead weight.
|
||||||
|
if not prefix or removed:
|
||||||
|
self._key_locks.clear()
|
||||||
|
return removed
|
||||||
|
|
||||||
|
def stats(self) -> dict[str, int]:
|
||||||
|
"""Return a snapshot for the diagnostics endpoint."""
|
||||||
|
with self._lock:
|
||||||
|
return {
|
||||||
|
"entries": len(self._values),
|
||||||
|
"max_entries": self._max_entries,
|
||||||
|
"hits": self.hits,
|
||||||
|
"misses": self.misses,
|
||||||
|
"evictions": self.evictions,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class FailureLatch:
|
||||||
|
"""Suppress a repeated failing optional call for a bounded period.
|
||||||
|
|
||||||
|
Unread counts and upstream search are enhancements: when one of them is
|
||||||
|
unsupported by the FreshRSS build in front of us, retrying on every request
|
||||||
|
would add latency to every page load for no benefit. The latch remembers the
|
||||||
|
failure for a TTL, reports it once on the way in, and then stays quiet.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, ttl_seconds: float = 300.0) -> None:
|
||||||
|
self._ttl = ttl_seconds
|
||||||
|
self._locked_until: dict[str, float] = {}
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
|
def is_open(self, key: str) -> bool:
|
||||||
|
"""True when calls for ``key`` should be attempted again."""
|
||||||
|
with self._lock:
|
||||||
|
until = self._locked_until.get(key, 0.0)
|
||||||
|
return time.monotonic() >= until
|
||||||
|
|
||||||
|
def trip(self, key: str) -> bool:
|
||||||
|
"""Mark ``key`` failed; return True the first time within the window."""
|
||||||
|
with self._lock:
|
||||||
|
now = time.monotonic()
|
||||||
|
first = now >= self._locked_until.get(key, 0.0)
|
||||||
|
self._locked_until[key] = now + self._ttl
|
||||||
|
return first
|
||||||
|
|
||||||
|
def clear(self, key: str) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._locked_until.pop(key, None)
|
||||||
841
app/config.py
Normal file
841
app/config.py
Normal file
@@ -0,0 +1,841 @@
|
|||||||
|
"""JSON configuration: loading, interpolation, validation.
|
||||||
|
|
||||||
|
Configuration is a single JSON document, which keeps deployment to "mount a file
|
||||||
|
and run the container". Two things make that practical for a service holding
|
||||||
|
credentials:
|
||||||
|
|
||||||
|
*Interpolation.* Any string value may contain ``${VAR}`` (an environment
|
||||||
|
variable, optionally with a ``:-default``), ``${file:/run/secrets/x}`` (a file,
|
||||||
|
which is how Docker secrets arrive), or ``$${`` for a literal ``${``. Secrets
|
||||||
|
therefore never have to be written into the JSON document that gets committed.
|
||||||
|
|
||||||
|
*Aggregated validation.* Every problem found is reported in one pass with the
|
||||||
|
JSON path that caused it, because a container that exits on the first of five
|
||||||
|
typos is a slow way to configure a service. Unknown keys are reported too, as
|
||||||
|
warnings: a misspelled ``mark_read`` that silently defaulted to something else is
|
||||||
|
exactly the kind of thing that costs an hour.
|
||||||
|
|
||||||
|
*Comments.* JSON has no comment syntax, so any key beginning with an underscore
|
||||||
|
is treated as one: it is ignored, never interpolated, and never reported as
|
||||||
|
unknown. That lets ``config.example.json`` explain itself.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
CONFIG_PATH_ENV = "CONFIG_PATH"
|
||||||
|
|
||||||
|
# Default listen address. Loopback is the safe default for a service that holds
|
||||||
|
# a whole reading queue: running it directly on a host binds only to that host.
|
||||||
|
# A container is the one case that needs the wildcard, because the published port
|
||||||
|
# only reaches the process through the container's own interface -- so the
|
||||||
|
# shipped config.example.json and docker-compose.yaml set it explicitly, and there
|
||||||
|
# is exactly one place (the mounted configuration) that decides exposure.
|
||||||
|
DEFAULT_BIND_HOST = "127.0.0.1"
|
||||||
|
_DEFAULT_PATHS = ("/config/config.json", "config.json")
|
||||||
|
|
||||||
|
_INTERPOLATION_RE = re.compile(r"\$\{([^{}]*)\}")
|
||||||
|
_SHA256_RE = re.compile(r"\A[0-9a-fA-F]{64}\Z")
|
||||||
|
_ESCAPED_DOLLAR = "\x00escaped-dollar\x00"
|
||||||
|
|
||||||
|
GREADER_SUFFIX = "/api/greader.php"
|
||||||
|
|
||||||
|
_LOG_LEVELS = frozenset({"debug", "info", "warning", "error", "critical"})
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigError(Exception):
|
||||||
|
"""Raised when the configuration cannot be used.
|
||||||
|
|
||||||
|
Carries every problem found, so a human sees the whole list at once.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, problems: list[str], source: str = "") -> None:
|
||||||
|
self.problems = problems
|
||||||
|
self.source = source
|
||||||
|
where = f" in {source}" if source else ""
|
||||||
|
detail = "\n".join(f" - {problem}" for problem in problems)
|
||||||
|
super().__init__(f"invalid configuration{where}:\n{detail}")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ServerConfig:
|
||||||
|
"""HTTP listener settings."""
|
||||||
|
|
||||||
|
host: str = DEFAULT_BIND_HOST
|
||||||
|
port: int = 8080
|
||||||
|
log_level: str = "info"
|
||||||
|
# Absolute base URL for links. When unset, links are built from the request
|
||||||
|
# Host header, which is fine behind a proxy but lets a spoofed Host shape the
|
||||||
|
# catalog URLs -- so a public deployment should set this.
|
||||||
|
public_base_url: str | None = None
|
||||||
|
# Honour X-Forwarded-Proto/Host/Prefix. Only enable behind a proxy you trust,
|
||||||
|
# because the headers are client-controlled otherwise.
|
||||||
|
trust_proxy_headers: bool = False
|
||||||
|
# Optional Host allowlist; a request with any other Host is refused.
|
||||||
|
allowed_hosts: list[str] = field(default_factory=list)
|
||||||
|
tls_cert_file: str | None = None
|
||||||
|
tls_key_file: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FreshRSSConfig:
|
||||||
|
"""Upstream FreshRSS connection."""
|
||||||
|
|
||||||
|
api_base: str
|
||||||
|
username: str
|
||||||
|
api_password: str
|
||||||
|
verify_tls: bool = True
|
||||||
|
ca_file: str | None = None
|
||||||
|
timeout_seconds: float = 15.0
|
||||||
|
retries: int = 2
|
||||||
|
# Mark the article read upstream once its EPUB has been delivered.
|
||||||
|
mark_read: bool = True
|
||||||
|
items_per_page: int = 30
|
||||||
|
max_items_per_request: int = 200
|
||||||
|
# How far back the Read section looks. Read items are filtered client-side
|
||||||
|
# (only the unread filter is a documented upstream parameter), so this bounds
|
||||||
|
# the history that section can show.
|
||||||
|
read_window_items: int = 200
|
||||||
|
# Article body cap before parsing; prevents one enormous feed item from
|
||||||
|
# dominating memory.
|
||||||
|
max_content_bytes: int = 1_048_576
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class CatalogConfig:
|
||||||
|
"""OPDS catalog presentation."""
|
||||||
|
|
||||||
|
title: str = "FreshRSS OPDS"
|
||||||
|
author: str = "FreshRSS"
|
||||||
|
language: str = "en"
|
||||||
|
publisher: str | None = None
|
||||||
|
show_unread_counts: bool = True
|
||||||
|
show_read_section: bool = True
|
||||||
|
show_starred: bool = True
|
||||||
|
hide_empty_categories: bool = False
|
||||||
|
exclude_categories: list[str] = field(default_factory=list)
|
||||||
|
exclude_feeds: list[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class EpubConfig:
|
||||||
|
"""EPUB generation."""
|
||||||
|
|
||||||
|
include_images: bool = True
|
||||||
|
allowed_image_hosts: list[str] = field(default_factory=list)
|
||||||
|
blocked_image_hosts: list[str] = field(default_factory=list)
|
||||||
|
max_images_per_article: int = 20
|
||||||
|
max_image_bytes: int = 2_097_152
|
||||||
|
max_total_image_bytes: int = 16_777_216
|
||||||
|
image_timeout_seconds: float = 10.0
|
||||||
|
cover_image: bool = True
|
||||||
|
include_source_link: bool = True
|
||||||
|
stylesheet: str | None = None
|
||||||
|
max_epub_bytes: int = 33_554_432
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AuthUser:
|
||||||
|
"""One Basic-auth principal. Passwords are always compared as digests."""
|
||||||
|
|
||||||
|
username: str
|
||||||
|
password_sha256: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AuthConfig:
|
||||||
|
"""HTTP Basic credentials.
|
||||||
|
|
||||||
|
Authentication is mandatory by design: this catalog exposes the whole of a
|
||||||
|
reading queue, and an unauthenticated OPDS endpoint on a routable address is
|
||||||
|
an open book. Basic auth must therefore sit behind TLS (a reverse proxy, or
|
||||||
|
the TLS options under ``server``), because the header is not encrypted.
|
||||||
|
"""
|
||||||
|
|
||||||
|
realm: str = "OPDS"
|
||||||
|
users: list[AuthUser] = field(default_factory=list)
|
||||||
|
max_failures_per_minute: int = 10
|
||||||
|
throttle_seconds: float = 60.0
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class CacheConfig:
|
||||||
|
"""In-memory cache lifetimes. Nothing is ever written to disk."""
|
||||||
|
|
||||||
|
subscriptions_ttl_seconds: float = 300.0
|
||||||
|
stream_ttl_seconds: float = 60.0
|
||||||
|
item_ttl_seconds: float = 900.0
|
||||||
|
max_entries: int = 512
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Config:
|
||||||
|
"""The fully resolved configuration."""
|
||||||
|
|
||||||
|
server: ServerConfig
|
||||||
|
freshrss: FreshRSSConfig
|
||||||
|
catalog: CatalogConfig
|
||||||
|
epub: EpubConfig
|
||||||
|
auth: AuthConfig
|
||||||
|
cache: CacheConfig
|
||||||
|
source_path: str = ""
|
||||||
|
warnings: list[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_config_path(explicit: str | None = None) -> str:
|
||||||
|
"""Find the configuration file: flag, then env, then the usual locations."""
|
||||||
|
if explicit:
|
||||||
|
return explicit
|
||||||
|
from_env = os.environ.get(CONFIG_PATH_ENV)
|
||||||
|
if from_env:
|
||||||
|
return from_env
|
||||||
|
for candidate in _DEFAULT_PATHS:
|
||||||
|
if Path(candidate).is_file():
|
||||||
|
return candidate
|
||||||
|
return _DEFAULT_PATHS[0]
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(path: str | None = None) -> Config:
|
||||||
|
"""Load, interpolate and validate the configuration file."""
|
||||||
|
source = resolve_config_path(path)
|
||||||
|
raw_text = _read_config_text(source)
|
||||||
|
try:
|
||||||
|
document = json.loads(raw_text)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
detail = f"not valid JSON: {exc.msg} (line {exc.lineno}, column {exc.colno})"
|
||||||
|
raise ConfigError([detail], source) from exc
|
||||||
|
if not isinstance(document, dict):
|
||||||
|
raise ConfigError(["the top level of the configuration must be a JSON object"], source)
|
||||||
|
|
||||||
|
base_dir = Path(source).resolve().parent
|
||||||
|
problems: list[str] = []
|
||||||
|
interpolated = _interpolate(document, "", base_dir, problems)
|
||||||
|
if problems:
|
||||||
|
raise ConfigError(problems, source)
|
||||||
|
if not isinstance(interpolated, dict): # pragma: no cover - shape is preserved
|
||||||
|
raise ConfigError(["internal error: configuration root is not an object"], source)
|
||||||
|
|
||||||
|
warnings: list[str] = []
|
||||||
|
validator = _Validator()
|
||||||
|
config = Config(
|
||||||
|
server=_build_server(validator, interpolated),
|
||||||
|
freshrss=_build_freshrss(validator, interpolated),
|
||||||
|
catalog=_build_catalog(validator, interpolated),
|
||||||
|
epub=_build_epub(validator, interpolated),
|
||||||
|
auth=_build_auth(validator, interpolated),
|
||||||
|
cache=_build_cache(validator, interpolated),
|
||||||
|
source_path=str(Path(source).resolve()),
|
||||||
|
warnings=warnings,
|
||||||
|
)
|
||||||
|
warnings.extend(_unknown_keys(interpolated, _KNOWN_KEYS))
|
||||||
|
if validator.problems:
|
||||||
|
raise ConfigError(validator.problems, source)
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def password_digest(password: str) -> str:
|
||||||
|
"""Digest a password for storage in ``auth.users[].password_sha256``."""
|
||||||
|
return hashlib.sha256(password.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def normalise_api_base(url: str) -> str:
|
||||||
|
"""Normalise a FreshRSS URL to its Google Reader API base.
|
||||||
|
|
||||||
|
Mirrors the firmware client's rule so the same address works in both places:
|
||||||
|
the ``/api/greader.php`` suffix is appended when absent and left alone when
|
||||||
|
present, and any path prefix in between (``/freshrss``) is preserved.
|
||||||
|
"""
|
||||||
|
base = url.strip().rstrip("/")
|
||||||
|
if base.lower().endswith(GREADER_SUFFIX):
|
||||||
|
return base
|
||||||
|
return base + GREADER_SUFFIX
|
||||||
|
|
||||||
|
|
||||||
|
def describe_config(config: Config) -> str:
|
||||||
|
"""Render the effective configuration for logs, with secrets redacted."""
|
||||||
|
users = ", ".join(user.username for user in config.auth.users) or "(none)"
|
||||||
|
lines = [
|
||||||
|
f"config file: {config.source_path}",
|
||||||
|
f"listener: {config.server.host}:{config.server.port} "
|
||||||
|
f"(log level {config.server.log_level}"
|
||||||
|
+ (", TLS on" if config.server.tls_cert_file else "")
|
||||||
|
+ ")",
|
||||||
|
f"freshrss: {config.freshrss.api_base} as {config.freshrss.username} "
|
||||||
|
f"(password: {_mask(config.freshrss.api_password)}, mark read: "
|
||||||
|
f"{'yes' if config.freshrss.mark_read else 'no'})",
|
||||||
|
f"catalog: {config.catalog.title!r}, {config.freshrss.items_per_page} items/page, "
|
||||||
|
f"read section: {'yes' if config.catalog.show_read_section else 'no'}",
|
||||||
|
f"epub images: {'yes' if config.epub.include_images else 'no'}",
|
||||||
|
f"auth users: {users}",
|
||||||
|
]
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def _mask(secret: str) -> str:
|
||||||
|
if not secret:
|
||||||
|
return "(empty)"
|
||||||
|
return "***" if len(secret) <= 8 else f"***({len(secret)} chars)"
|
||||||
|
|
||||||
|
|
||||||
|
def _read_config_text(source: str) -> str:
|
||||||
|
try:
|
||||||
|
return Path(source).read_text(encoding="utf-8")
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
raise ConfigError(
|
||||||
|
[
|
||||||
|
f"configuration file not found: {source}",
|
||||||
|
f"set {CONFIG_PATH_ENV} or pass --config; "
|
||||||
|
"see config.example.json for a starting point",
|
||||||
|
],
|
||||||
|
source,
|
||||||
|
) from exc
|
||||||
|
except OSError as exc:
|
||||||
|
raise ConfigError([f"cannot read configuration file: {exc.strerror}"], source) from exc
|
||||||
|
|
||||||
|
|
||||||
|
# -- interpolation -------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _interpolate(value: Any, path: str, base_dir: Path, problems: list[str]) -> Any:
|
||||||
|
"""Recursively expand ``${VAR}`` and ``${file:...}`` in every string."""
|
||||||
|
if isinstance(value, str):
|
||||||
|
return _interpolate_string(value, path, base_dir, problems)
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {
|
||||||
|
key: (
|
||||||
|
# A `_`-prefixed key is a comment: left exactly as written, so a
|
||||||
|
# comment that mentions ${SOMETHING} stays inert.
|
||||||
|
item
|
||||||
|
if key.startswith("_")
|
||||||
|
else _interpolate(item, f"{path}.{key}" if path else key, base_dir, problems)
|
||||||
|
)
|
||||||
|
for key, item in value.items()
|
||||||
|
}
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [
|
||||||
|
_interpolate(item, f"{path}[{index}]", base_dir, problems)
|
||||||
|
for index, item in enumerate(value)
|
||||||
|
]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _interpolate_string(value: str, path: str, base_dir: Path, problems: list[str]) -> str:
|
||||||
|
if "${" not in value:
|
||||||
|
return value
|
||||||
|
# Protect an escaped literal `${` before scanning for real references.
|
||||||
|
text = value.replace("$${", _ESCAPED_DOLLAR + "{")
|
||||||
|
|
||||||
|
def replace(match: re.Match[str]) -> str:
|
||||||
|
expression = match.group(1).strip()
|
||||||
|
if expression.startswith("file:"):
|
||||||
|
return _read_secret_file(expression[len("file:") :].strip(), path, base_dir, problems)
|
||||||
|
name, separator, default = expression.partition(":-")
|
||||||
|
name = name.strip()
|
||||||
|
if not name:
|
||||||
|
problems.append(f"{path}: empty ${{}} reference")
|
||||||
|
return ""
|
||||||
|
found = os.environ.get(name)
|
||||||
|
if found is not None:
|
||||||
|
return found
|
||||||
|
if separator:
|
||||||
|
return default
|
||||||
|
problems.append(
|
||||||
|
f"{path}: environment variable {name} is not set (use ${{{name}:-default}} "
|
||||||
|
"to make it optional, or ${file:path} to read a secret from a file)"
|
||||||
|
)
|
||||||
|
return ""
|
||||||
|
|
||||||
|
result = _INTERPOLATION_RE.sub(replace, text)
|
||||||
|
return result.replace(_ESCAPED_DOLLAR + "{", "${")
|
||||||
|
|
||||||
|
|
||||||
|
def _read_secret_file(filename: str, path: str, base_dir: Path, problems: list[str]) -> str:
|
||||||
|
if not filename:
|
||||||
|
problems.append(f"{path}: ${{file:}} needs a path")
|
||||||
|
return ""
|
||||||
|
candidate = Path(filename)
|
||||||
|
if not candidate.is_absolute():
|
||||||
|
candidate = base_dir / candidate
|
||||||
|
try:
|
||||||
|
# Strip the trailing newline every secret-management tool adds.
|
||||||
|
return candidate.read_text(encoding="utf-8").strip()
|
||||||
|
except OSError as exc:
|
||||||
|
problems.append(f"{path}: cannot read secret file {candidate}: {exc.strerror}")
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
# -- validation ----------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class _Validator:
|
||||||
|
"""Collects type and range problems while building the config dataclasses.
|
||||||
|
|
||||||
|
Problems are qualified by the section being validated, so a message reads
|
||||||
|
``freshrss.url: required, but missing`` rather than the ambiguous
|
||||||
|
``url: required, but missing``. That matters as soon as two sections share a
|
||||||
|
key name, which they do (``timeout_seconds`` lives in three of them).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.problems: list[str] = []
|
||||||
|
self._prefix = ""
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def within(self, path: str) -> Iterator[None]:
|
||||||
|
"""Qualify every problem reported inside this block with ``path``."""
|
||||||
|
previous = self._prefix
|
||||||
|
self._prefix = f"{previous}.{path}" if previous else path
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
self._prefix = previous
|
||||||
|
|
||||||
|
def qualify(self, key: str | None = None) -> str:
|
||||||
|
"""Full path for a key, in the section currently being validated."""
|
||||||
|
if not key:
|
||||||
|
return self._prefix
|
||||||
|
return f"{self._prefix}.{key}" if self._prefix else key
|
||||||
|
|
||||||
|
def problem(self, message: str, key: str | None = None) -> None:
|
||||||
|
"""Report a problem against the current section, or against one key."""
|
||||||
|
location = self.qualify(key)
|
||||||
|
self.problems.append(f"{location}: {message}" if location else message)
|
||||||
|
|
||||||
|
def section(self, raw: dict[str, Any], name: str) -> dict[str, Any]:
|
||||||
|
value = raw.get(name)
|
||||||
|
if value is None:
|
||||||
|
return {}
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
self.problem("must be a JSON object", key=name)
|
||||||
|
return {}
|
||||||
|
return value
|
||||||
|
|
||||||
|
def string(
|
||||||
|
self,
|
||||||
|
raw: dict[str, Any],
|
||||||
|
key: str,
|
||||||
|
default: str | None = None,
|
||||||
|
*,
|
||||||
|
required: bool = False,
|
||||||
|
) -> Any:
|
||||||
|
if key not in raw or raw[key] is None:
|
||||||
|
if required:
|
||||||
|
self.problem("required, but missing", key=key)
|
||||||
|
return default
|
||||||
|
value = raw[key]
|
||||||
|
if not isinstance(value, str):
|
||||||
|
self.problem("must be a string", key=key)
|
||||||
|
return default
|
||||||
|
text = value.strip()
|
||||||
|
if not text:
|
||||||
|
if required:
|
||||||
|
self.problem("required, but empty", key=key)
|
||||||
|
return default
|
||||||
|
return text
|
||||||
|
|
||||||
|
def one_of(self, raw: dict[str, Any], key: str, allowed: frozenset[str], default: str) -> str:
|
||||||
|
value = self.string(raw, key, default)
|
||||||
|
if value is None or value not in allowed:
|
||||||
|
self.problem(f"must be one of {', '.join(sorted(allowed))} (got {value!r})", key=key)
|
||||||
|
return default
|
||||||
|
return value
|
||||||
|
|
||||||
|
def boolean(self, raw: dict[str, Any], key: str, default: bool) -> bool:
|
||||||
|
if key not in raw or raw[key] is None:
|
||||||
|
return default
|
||||||
|
value = raw[key]
|
||||||
|
if not isinstance(value, bool):
|
||||||
|
self.problem("must be true or false", key=key)
|
||||||
|
return default
|
||||||
|
return value
|
||||||
|
|
||||||
|
def integer(
|
||||||
|
self,
|
||||||
|
raw: dict[str, Any],
|
||||||
|
key: str,
|
||||||
|
default: int,
|
||||||
|
*,
|
||||||
|
minimum: int | None = None,
|
||||||
|
maximum: int | None = None,
|
||||||
|
) -> int:
|
||||||
|
if key not in raw or raw[key] is None:
|
||||||
|
return default
|
||||||
|
value = raw[key]
|
||||||
|
# A bool is an int in Python; accepting it here would turn `true` into 1.
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int):
|
||||||
|
self.problem("must be a whole number", key=key)
|
||||||
|
return default
|
||||||
|
if minimum is not None and value < minimum:
|
||||||
|
self.problem(f"must be at least {minimum} (got {value})", key=key)
|
||||||
|
return default
|
||||||
|
if maximum is not None and value > maximum:
|
||||||
|
self.problem(f"must be at most {maximum} (got {value})", key=key)
|
||||||
|
return default
|
||||||
|
return value
|
||||||
|
|
||||||
|
def number(self, raw: dict[str, Any], key: str, default: float, *, above: float = 0) -> float:
|
||||||
|
if key not in raw or raw[key] is None:
|
||||||
|
return default
|
||||||
|
value = raw[key]
|
||||||
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||||
|
self.problem("must be a number", key=key)
|
||||||
|
return default
|
||||||
|
if value <= above:
|
||||||
|
self.problem(f"must be greater than {above:g} (got {value:g})", key=key)
|
||||||
|
return default
|
||||||
|
return float(value)
|
||||||
|
|
||||||
|
def string_list(self, raw: dict[str, Any], key: str) -> list[str]:
|
||||||
|
if key not in raw or raw[key] is None:
|
||||||
|
return []
|
||||||
|
value = raw[key]
|
||||||
|
if not isinstance(value, list):
|
||||||
|
self.problem("must be an array of strings", key=key)
|
||||||
|
return []
|
||||||
|
items: list[str] = []
|
||||||
|
for index, item in enumerate(value):
|
||||||
|
if not isinstance(item, str):
|
||||||
|
self.problem("must be a string", key=f"{key}[{index}]")
|
||||||
|
continue
|
||||||
|
if item.strip():
|
||||||
|
items.append(item.strip())
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def _build_server(validator: _Validator, raw: dict[str, Any]) -> ServerConfig:
|
||||||
|
section = validator.section(raw, "server")
|
||||||
|
with validator.within("server"):
|
||||||
|
with validator.within("tls"):
|
||||||
|
tls = validator.section(section, "tls")
|
||||||
|
cert = validator.string(tls, "cert_file")
|
||||||
|
key = validator.string(tls, "key_file")
|
||||||
|
if bool(cert) != bool(key):
|
||||||
|
validator.problem("cert_file and key_file must be set together")
|
||||||
|
elif cert:
|
||||||
|
_require_readable_file(validator, cert, "cert_file")
|
||||||
|
_require_readable_file(validator, key, "key_file")
|
||||||
|
return ServerConfig(
|
||||||
|
host=validator.string(section, "host", DEFAULT_BIND_HOST) or DEFAULT_BIND_HOST,
|
||||||
|
# Port 0 asks the operating system for a free port, which is what the test
|
||||||
|
# suite uses and what a socket-activated or ephemeral deployment wants.
|
||||||
|
port=validator.integer(section, "port", 8080, minimum=0, maximum=65535),
|
||||||
|
log_level=validator.one_of(section, "log_level", _LOG_LEVELS, "info"),
|
||||||
|
public_base_url=validator.string(section, "public_base_url"),
|
||||||
|
trust_proxy_headers=validator.boolean(section, "trust_proxy_headers", False),
|
||||||
|
allowed_hosts=[
|
||||||
|
host.lower() for host in validator.string_list(section, "allowed_hosts")
|
||||||
|
],
|
||||||
|
tls_cert_file=cert,
|
||||||
|
tls_key_file=key,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_freshrss(validator: _Validator, raw: dict[str, Any]) -> FreshRSSConfig:
|
||||||
|
section = validator.section(raw, "freshrss")
|
||||||
|
with validator.within("freshrss"):
|
||||||
|
url = validator.string(section, "url", required=True)
|
||||||
|
if url and not url.lower().startswith(("http://", "https://")):
|
||||||
|
validator.problem(f"must start with http:// or https:// (got {url!r})", key="url")
|
||||||
|
username = validator.string(section, "username", required=True)
|
||||||
|
password = validator.string(section, "api_password", required=True)
|
||||||
|
ca_file = validator.string(section, "ca_file")
|
||||||
|
if ca_file:
|
||||||
|
_require_readable_file(validator, ca_file, "ca_file")
|
||||||
|
items_per_page = validator.integer(section, "items_per_page", 30, minimum=1, maximum=500)
|
||||||
|
max_items = validator.integer(
|
||||||
|
section, "max_items_per_request", 200, minimum=1, maximum=5000
|
||||||
|
)
|
||||||
|
if max_items < items_per_page:
|
||||||
|
validator.problem(
|
||||||
|
f"max_items_per_request ({max_items}) must be at least "
|
||||||
|
f"items_per_page ({items_per_page})"
|
||||||
|
)
|
||||||
|
return FreshRSSConfig(
|
||||||
|
api_base=normalise_api_base(url) if url else "",
|
||||||
|
username=username or "",
|
||||||
|
api_password=password or "",
|
||||||
|
verify_tls=validator.boolean(section, "verify_tls", True),
|
||||||
|
ca_file=ca_file,
|
||||||
|
timeout_seconds=validator.number(section, "timeout_seconds", 15.0),
|
||||||
|
retries=validator.integer(section, "retries", 2, minimum=0, maximum=10),
|
||||||
|
mark_read=validator.boolean(section, "mark_read", True),
|
||||||
|
items_per_page=items_per_page,
|
||||||
|
max_items_per_request=max_items,
|
||||||
|
read_window_items=validator.integer(
|
||||||
|
section, "read_window_items", 200, minimum=1, maximum=5000
|
||||||
|
),
|
||||||
|
max_content_bytes=validator.integer(
|
||||||
|
section, "max_content_bytes", 1_048_576, minimum=4096, maximum=67_108_864
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_catalog(validator: _Validator, raw: dict[str, Any]) -> CatalogConfig:
|
||||||
|
section = validator.section(raw, "catalog")
|
||||||
|
with validator.within("catalog"):
|
||||||
|
return CatalogConfig(
|
||||||
|
title=validator.string(section, "title", "FreshRSS OPDS") or "FreshRSS OPDS",
|
||||||
|
author=validator.string(section, "author", "FreshRSS") or "FreshRSS",
|
||||||
|
language=validator.string(section, "language", "en") or "en",
|
||||||
|
publisher=validator.string(section, "publisher"),
|
||||||
|
show_unread_counts=validator.boolean(section, "show_unread_counts", True),
|
||||||
|
show_read_section=validator.boolean(section, "show_read_section", True),
|
||||||
|
show_starred=validator.boolean(section, "show_starred", True),
|
||||||
|
hide_empty_categories=validator.boolean(section, "hide_empty_categories", False),
|
||||||
|
exclude_categories=validator.string_list(section, "exclude_categories"),
|
||||||
|
exclude_feeds=validator.string_list(section, "exclude_feeds"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_epub(validator: _Validator, raw: dict[str, Any]) -> EpubConfig:
|
||||||
|
section = validator.section(raw, "epub")
|
||||||
|
with validator.within("epub"):
|
||||||
|
stylesheet = validator.string(section, "stylesheet")
|
||||||
|
if stylesheet:
|
||||||
|
_require_readable_file(validator, stylesheet, "stylesheet")
|
||||||
|
return EpubConfig(
|
||||||
|
include_images=validator.boolean(section, "include_images", True),
|
||||||
|
allowed_image_hosts=[
|
||||||
|
host.lower() for host in validator.string_list(section, "allowed_image_hosts")
|
||||||
|
],
|
||||||
|
blocked_image_hosts=[
|
||||||
|
host.lower() for host in validator.string_list(section, "blocked_image_hosts")
|
||||||
|
],
|
||||||
|
max_images_per_article=validator.integer(
|
||||||
|
section, "max_images_per_article", 20, minimum=0, maximum=200
|
||||||
|
),
|
||||||
|
max_image_bytes=validator.integer(
|
||||||
|
section, "max_image_bytes", 2_097_152, minimum=1024, maximum=52_428_800
|
||||||
|
),
|
||||||
|
max_total_image_bytes=validator.integer(
|
||||||
|
section,
|
||||||
|
"max_total_image_bytes",
|
||||||
|
16_777_216,
|
||||||
|
minimum=1024,
|
||||||
|
maximum=268_435_456,
|
||||||
|
),
|
||||||
|
image_timeout_seconds=validator.number(section, "image_timeout_seconds", 10.0),
|
||||||
|
cover_image=validator.boolean(section, "cover_image", True),
|
||||||
|
include_source_link=validator.boolean(section, "include_source_link", True),
|
||||||
|
stylesheet=stylesheet,
|
||||||
|
max_epub_bytes=validator.integer(
|
||||||
|
section, "max_epub_bytes", 33_554_432, minimum=4096, maximum=268_435_456
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_auth(validator: _Validator, raw: dict[str, Any]) -> AuthConfig:
|
||||||
|
section = validator.section(raw, "auth")
|
||||||
|
users: list[AuthUser] = []
|
||||||
|
with validator.within("auth"):
|
||||||
|
raw_users = section.get("users")
|
||||||
|
if raw_users is None:
|
||||||
|
validator.problem(
|
||||||
|
"required -- this catalog serves your entire reading queue, so it "
|
||||||
|
"will not start without credentials "
|
||||||
|
"(generate a digest with: make hash-password)",
|
||||||
|
key="users",
|
||||||
|
)
|
||||||
|
elif not isinstance(raw_users, list):
|
||||||
|
validator.problem("must be an array of user objects", key="users")
|
||||||
|
elif not raw_users:
|
||||||
|
validator.problem("must contain at least one user", key="users")
|
||||||
|
else:
|
||||||
|
seen: set[str] = set()
|
||||||
|
for index, entry in enumerate(raw_users):
|
||||||
|
with validator.within(f"users[{index}]"):
|
||||||
|
users.append(_build_auth_user(validator, entry, seen))
|
||||||
|
return AuthConfig(
|
||||||
|
realm=validator.string(section, "realm", "OPDS") or "OPDS",
|
||||||
|
users=users,
|
||||||
|
max_failures_per_minute=validator.integer(
|
||||||
|
section, "max_failures_per_minute", 10, minimum=1, maximum=10_000
|
||||||
|
),
|
||||||
|
throttle_seconds=validator.number(section, "throttle_seconds", 60.0),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_auth_user(validator: _Validator, entry: Any, seen: set[str]) -> AuthUser:
|
||||||
|
"""Build one user; the caller has already entered its ``users[i]`` context."""
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
validator.problem("must be a JSON object")
|
||||||
|
return AuthUser("", "")
|
||||||
|
username = validator.string(entry, "username", required=True) or ""
|
||||||
|
if username and username in seen:
|
||||||
|
validator.problem(f"duplicate username {username!r}", key="username")
|
||||||
|
seen.add(username)
|
||||||
|
|
||||||
|
plain = _validated_plain_password(validator, entry)
|
||||||
|
digest = _validated_digest(validator, entry)
|
||||||
|
if plain is not None and digest is not None:
|
||||||
|
validator.problem("set either password or password_sha256, not both")
|
||||||
|
return AuthUser(username, "")
|
||||||
|
if digest is not None:
|
||||||
|
return AuthUser(username, digest)
|
||||||
|
if plain is not None:
|
||||||
|
LOGGER.warning(
|
||||||
|
"%s.password is in clear text in the configuration file; "
|
||||||
|
"prefer password_sha256 (make hash-password)",
|
||||||
|
validator.qualify(),
|
||||||
|
)
|
||||||
|
return AuthUser(username, password_digest(plain))
|
||||||
|
validator.problem("needs password or password_sha256")
|
||||||
|
return AuthUser(username, "")
|
||||||
|
|
||||||
|
|
||||||
|
def _validated_plain_password(validator: _Validator, entry: dict[str, Any]) -> str | None:
|
||||||
|
"""Return a clear-text password from config, or None when absent/invalid."""
|
||||||
|
value = entry.get("password")
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if not isinstance(value, str) or not value:
|
||||||
|
validator.problem("must be a non-empty string", key="password")
|
||||||
|
return None
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _validated_digest(validator: _Validator, entry: dict[str, Any]) -> str | None:
|
||||||
|
"""Return a validated sha256 digest from config, or None when absent/invalid."""
|
||||||
|
value = entry.get("password_sha256")
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if not isinstance(value, str) or not _SHA256_RE.match(value):
|
||||||
|
validator.problem("must be 64 hex characters (make hash-password)", key="password_sha256")
|
||||||
|
return None
|
||||||
|
return value.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _build_cache(validator: _Validator, raw: dict[str, Any]) -> CacheConfig:
|
||||||
|
section = validator.section(raw, "cache")
|
||||||
|
with validator.within("cache"):
|
||||||
|
return CacheConfig(
|
||||||
|
subscriptions_ttl_seconds=validator.number(section, "subscriptions_ttl_seconds", 300.0),
|
||||||
|
stream_ttl_seconds=validator.number(section, "stream_ttl_seconds", 60.0),
|
||||||
|
item_ttl_seconds=validator.number(section, "item_ttl_seconds", 900.0),
|
||||||
|
max_entries=validator.integer(section, "max_entries", 512, minimum=16, maximum=100_000),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_readable_file(validator: _Validator, filename: str, key: str) -> None:
|
||||||
|
if not Path(filename).is_file():
|
||||||
|
validator.problem(f"no such file: {filename}", key=key)
|
||||||
|
|
||||||
|
|
||||||
|
_KNOWN_KEYS: dict[str, set[str]] = {
|
||||||
|
"server": {
|
||||||
|
"host",
|
||||||
|
"port",
|
||||||
|
"log_level",
|
||||||
|
"public_base_url",
|
||||||
|
"trust_proxy_headers",
|
||||||
|
"allowed_hosts",
|
||||||
|
"tls",
|
||||||
|
},
|
||||||
|
"server.tls": {"cert_file", "key_file"},
|
||||||
|
"freshrss": {
|
||||||
|
"url",
|
||||||
|
"username",
|
||||||
|
"api_password",
|
||||||
|
"verify_tls",
|
||||||
|
"ca_file",
|
||||||
|
"timeout_seconds",
|
||||||
|
"retries",
|
||||||
|
"mark_read",
|
||||||
|
"items_per_page",
|
||||||
|
"max_items_per_request",
|
||||||
|
"read_window_items",
|
||||||
|
"max_content_bytes",
|
||||||
|
},
|
||||||
|
"catalog": {
|
||||||
|
"title",
|
||||||
|
"author",
|
||||||
|
"language",
|
||||||
|
"publisher",
|
||||||
|
"show_unread_counts",
|
||||||
|
"show_read_section",
|
||||||
|
"show_starred",
|
||||||
|
"hide_empty_categories",
|
||||||
|
"exclude_categories",
|
||||||
|
"exclude_feeds",
|
||||||
|
},
|
||||||
|
"epub": {
|
||||||
|
"include_images",
|
||||||
|
"allowed_image_hosts",
|
||||||
|
"blocked_image_hosts",
|
||||||
|
"max_images_per_article",
|
||||||
|
"max_image_bytes",
|
||||||
|
"max_total_image_bytes",
|
||||||
|
"image_timeout_seconds",
|
||||||
|
"cover_image",
|
||||||
|
"include_source_link",
|
||||||
|
"stylesheet",
|
||||||
|
"max_epub_bytes",
|
||||||
|
},
|
||||||
|
"auth": {"realm", "users", "max_failures_per_minute", "throttle_seconds"},
|
||||||
|
"auth.users[]": {"username", "password", "password_sha256"},
|
||||||
|
"cache": {
|
||||||
|
"subscriptions_ttl_seconds",
|
||||||
|
"stream_ttl_seconds",
|
||||||
|
"item_ttl_seconds",
|
||||||
|
"max_entries",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _unknown_keys(document: dict[str, Any], known: dict[str, set[str]]) -> list[str]:
|
||||||
|
"""Report keys the server would otherwise silently ignore."""
|
||||||
|
warnings: list[str] = []
|
||||||
|
sections = {name for name in known if "." not in name}
|
||||||
|
for key in document:
|
||||||
|
if key in sections or key.startswith("_"):
|
||||||
|
continue
|
||||||
|
suggestions = _closest(key, sections)
|
||||||
|
hint = f" (did you mean {suggestions}?)" if suggestions else ""
|
||||||
|
warnings.append(f"{key}: unknown top-level setting, ignored{hint}")
|
||||||
|
for section, allowed in known.items():
|
||||||
|
if "." in section:
|
||||||
|
continue
|
||||||
|
value = document.get(section)
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
continue
|
||||||
|
for key in value:
|
||||||
|
if key not in allowed and not key.startswith("_"):
|
||||||
|
suggestions = _closest(key, allowed)
|
||||||
|
hint = f" (did you mean {suggestions}?)" if suggestions else ""
|
||||||
|
warnings.append(f"{section}.{key}: unknown setting, ignored{hint}")
|
||||||
|
if section == "server" and isinstance(value.get("tls"), dict):
|
||||||
|
for key in value["tls"]:
|
||||||
|
if key not in known["server.tls"] and not key.startswith("_"):
|
||||||
|
warnings.append(f"server.tls.{key}: unknown setting, ignored")
|
||||||
|
if section == "auth" and isinstance(value.get("users"), list):
|
||||||
|
for index, entry in enumerate(value["users"]):
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
for key in entry:
|
||||||
|
if key not in known["auth.users[]"] and not key.startswith("_"):
|
||||||
|
warnings.append(f"auth.users[{index}].{key}: unknown setting, ignored")
|
||||||
|
return warnings
|
||||||
|
|
||||||
|
|
||||||
|
def _closest(key: str, candidates: set[str]) -> str:
|
||||||
|
"""Suggest a near-miss key name, cheaply."""
|
||||||
|
lowered = key.lower().replace("-", "_")
|
||||||
|
for candidate in sorted(candidates):
|
||||||
|
if candidate.lower().replace("-", "_") == lowered:
|
||||||
|
return repr(candidate)
|
||||||
|
for candidate in sorted(candidates):
|
||||||
|
if candidate.lower().startswith(lowered[:4]) and len(lowered) >= 4:
|
||||||
|
return repr(candidate)
|
||||||
|
return ""
|
||||||
492
app/epub.py
Normal file
492
app/epub.py
Normal file
@@ -0,0 +1,492 @@
|
|||||||
|
"""EPUB 3 generation, entirely in memory.
|
||||||
|
|
||||||
|
An EPUB is a ZIP with a strict shape, and a reader that rejects a book usually
|
||||||
|
does so because of a shape violation rather than because of its content. The
|
||||||
|
rules that matter here, and that the tests pin down:
|
||||||
|
|
||||||
|
* ``mimetype`` is the first entry, stored uncompressed, with no extra field. A
|
||||||
|
ZIP where it is deflated or not first is not an EPUB.
|
||||||
|
* ``META-INF/container.xml`` names the OPF, which is the package document.
|
||||||
|
* XML everywhere: the OPF, the navigation document, the NCX fallback and every
|
||||||
|
content document have to parse with a strict XML parser. Article bodies are
|
||||||
|
normalised by :mod:`app.xhtml` before they come anywhere near this module.
|
||||||
|
* A publication needs ``dc:identifier``, ``dc:title``, ``dc:language`` and a
|
||||||
|
``dcterms:modified`` timestamp; a navigation document marked ``properties="nav"``
|
||||||
|
is mandatory, and the NCX is kept as the EPUB 2 fallback because older readers
|
||||||
|
still look for it.
|
||||||
|
* Every manifest href resolves to a real entry, and every spine idref resolves to
|
||||||
|
a manifest item. Both are checked in the tests.
|
||||||
|
|
||||||
|
Nothing touches the filesystem: the ZIP is assembled in a ``BytesIO`` and handed
|
||||||
|
back as ``bytes``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import logging
|
||||||
|
import zipfile
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app import images, urls, xhtml
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
MIMETYPE = "application/epub+zip"
|
||||||
|
|
||||||
|
# Where each kind of part lives inside the package.
|
||||||
|
OPF_PATH = "OEBPS/content.opf"
|
||||||
|
NAV_PATH = "OEBPS/nav.xhtml"
|
||||||
|
NCX_PATH = "OEBPS/toc.ncx"
|
||||||
|
CSS_PATH = "OEBPS/styles/main.css"
|
||||||
|
COVER_PATH = "OEBPS/text/cover.xhtml"
|
||||||
|
ARTICLE_PATH = "OEBPS/text/article.xhtml"
|
||||||
|
IMAGE_DIR = "OEBPS/images"
|
||||||
|
|
||||||
|
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
|
||||||
|
OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
|
||||||
|
DC_NAMESPACE = "http://purl.org/dc/elements/1.1/"
|
||||||
|
NCX_NAMESPACE = "http://www.daisy.org/z3986/2005/ncx/"
|
||||||
|
OPS_NAMESPACE = "http://www.idpf.org/2007/ops"
|
||||||
|
|
||||||
|
# A deliberately plain stylesheet. It sets a readable measure and gives code,
|
||||||
|
# quotes and images somewhere sane to sit, and it never fights the reader's own
|
||||||
|
# typography: no colours, no font families, no absolute sizes.
|
||||||
|
DEFAULT_STYLESHEET = """\
|
||||||
|
body { margin: 5%; line-height: 1.5; }
|
||||||
|
h1, h2, h3 { line-height: 1.25; page-break-after: avoid; }
|
||||||
|
.article-title { margin-bottom: 0.2em; }
|
||||||
|
.byline { margin-top: 0; font-size: 0.9em; opacity: 0.8; }
|
||||||
|
img { max-width: 100%; height: auto; }
|
||||||
|
figure { margin: 1em 0; }
|
||||||
|
figcaption { font-size: 0.9em; font-style: italic; }
|
||||||
|
blockquote { margin: 1em 1.5em; }
|
||||||
|
pre { white-space: pre-wrap; word-wrap: break-word; }
|
||||||
|
code, pre { font-family: monospace; }
|
||||||
|
table { border-collapse: collapse; max-width: 100%; }
|
||||||
|
td, th { border: 1px solid currentColor; padding: 0.3em 0.5em; }
|
||||||
|
hr { border: none; border-top: 1px solid currentColor; }
|
||||||
|
.source { margin-top: 2em; font-size: 0.85em; opacity: 0.8; }
|
||||||
|
.cover { text-align: center; }
|
||||||
|
.cover img { max-height: 90%; }
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Content documents sit one directory below the images.
|
||||||
|
_TEXT_PREFIX = "../"
|
||||||
|
|
||||||
|
# A literal middle dot rather than the HTML entity ·: XML predefines only
|
||||||
|
# amp, lt, gt, quot and apos, and this package ships no DOCTYPE that could define
|
||||||
|
# any others, so an HTML entity here would make the document unparseable.
|
||||||
|
BYLINE_SEPARATOR = " \u00b7 "
|
||||||
|
|
||||||
|
|
||||||
|
class EpubTooLargeError(Exception):
|
||||||
|
"""The assembled book exceeded the configured size cap."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class BookImage:
|
||||||
|
"""An image embedded in the book."""
|
||||||
|
|
||||||
|
href: str # relative to the OPF, e.g. "images/img-1.png"
|
||||||
|
media_type: str
|
||||||
|
data: bytes
|
||||||
|
is_cover: bool = False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def document_href(self) -> str:
|
||||||
|
"""The same image, as referenced from a document in ``text/``."""
|
||||||
|
return _TEXT_PREFIX + self.href
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Book:
|
||||||
|
"""Everything needed to write one EPUB."""
|
||||||
|
|
||||||
|
identifier: str
|
||||||
|
title: str
|
||||||
|
language: str
|
||||||
|
modified: datetime
|
||||||
|
content_xhtml: str
|
||||||
|
published: datetime | None = None
|
||||||
|
author: str | None = None
|
||||||
|
publisher: str | None = None
|
||||||
|
source_url: str | None = None
|
||||||
|
feed_title: str | None = None
|
||||||
|
images: list[BookImage] = field(default_factory=list)
|
||||||
|
stylesheet: str = DEFAULT_STYLESHEET
|
||||||
|
include_source_link: bool = True
|
||||||
|
|
||||||
|
@property
|
||||||
|
def cover_image(self) -> BookImage | None:
|
||||||
|
for image in self.images:
|
||||||
|
if image.is_cover:
|
||||||
|
return image
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def build_epub(book: Book) -> bytes:
|
||||||
|
"""Assemble a book into EPUB bytes."""
|
||||||
|
buffer = io.BytesIO()
|
||||||
|
# ZIP entry timestamps come from the book so that identical input produces
|
||||||
|
# identical bytes; a book that changes only because it was rebuilt would
|
||||||
|
# otherwise defeat every cache between here and the reader.
|
||||||
|
stamp = _zip_timestamp(book.modified)
|
||||||
|
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||||
|
# The mimetype entry must come first and must not be compressed, which is
|
||||||
|
# why it is written through an explicit ZipInfo rather than writestr.
|
||||||
|
info = zipfile.ZipInfo("mimetype", date_time=stamp)
|
||||||
|
info.compress_type = zipfile.ZIP_STORED
|
||||||
|
info.external_attr = 0o644 << 16
|
||||||
|
archive.writestr(info, MIMETYPE)
|
||||||
|
|
||||||
|
_write(archive, "META-INF/container.xml", _container_xml(), stamp)
|
||||||
|
_write(archive, OPF_PATH, _opf_document(book), stamp)
|
||||||
|
_write(archive, NAV_PATH, _nav_document(book), stamp)
|
||||||
|
_write(archive, NCX_PATH, _ncx_document(book), stamp)
|
||||||
|
_write(archive, CSS_PATH, book.stylesheet, stamp)
|
||||||
|
_write(archive, COVER_PATH, _cover_document(book), stamp)
|
||||||
|
_write(archive, ARTICLE_PATH, _article_document(book), stamp)
|
||||||
|
for image in book.images:
|
||||||
|
package_dir = OPF_PATH.rsplit("/", 1)[0]
|
||||||
|
_write(archive, f"{package_dir}/{image.href}", image.data, stamp)
|
||||||
|
return buffer.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def build_article_epub(
|
||||||
|
item,
|
||||||
|
*,
|
||||||
|
epub_config,
|
||||||
|
catalog_config,
|
||||||
|
fetcher=None,
|
||||||
|
max_content_bytes: int = 1_048_576,
|
||||||
|
) -> bytes:
|
||||||
|
"""Turn one FreshRSS item into an EPUB.
|
||||||
|
|
||||||
|
The article body comes from FreshRSS, normalised by :mod:`app.xhtml`. Images
|
||||||
|
are embedded when a fetcher is supplied, and any that cannot be fetched are
|
||||||
|
dropped from the body rather than left pointing at the network -- a book that
|
||||||
|
claims to work offline should.
|
||||||
|
"""
|
||||||
|
article = xhtml.parse_article(
|
||||||
|
item.content_html, base_url=item.url or "", max_input_bytes=max_content_bytes
|
||||||
|
)
|
||||||
|
book_images: list[BookImage] = []
|
||||||
|
image_map: dict[str, str | None] | None = None
|
||||||
|
|
||||||
|
if epub_config.include_images and fetcher is not None:
|
||||||
|
wanted = article.image_urls[: epub_config.max_images_per_article]
|
||||||
|
fetched = fetcher.fetch_all(wanted)
|
||||||
|
image_map = {}
|
||||||
|
for url in wanted:
|
||||||
|
image = fetched.get(url)
|
||||||
|
if image is None:
|
||||||
|
image_map[url] = None
|
||||||
|
continue
|
||||||
|
index = len(book_images) + 1
|
||||||
|
href = f"images/img-{index}{images.MEDIA_TYPES[image.media_type]}"
|
||||||
|
book_images.append(BookImage(href=href, media_type=image.media_type, data=image.data))
|
||||||
|
image_map[url] = _TEXT_PREFIX + href
|
||||||
|
elif article.image_urls:
|
||||||
|
# Keeping remote URLs means the OPF has to declare remote resources, which
|
||||||
|
# is handled by the caller not setting include_images.
|
||||||
|
image_map = None
|
||||||
|
|
||||||
|
book = Book(
|
||||||
|
identifier=urls.stable_urn("book", item.id),
|
||||||
|
title=item.title,
|
||||||
|
language=catalog_config.language,
|
||||||
|
modified=item.updated_at,
|
||||||
|
published=item.published_at,
|
||||||
|
content_xhtml=article.to_xhtml(image_map),
|
||||||
|
author=item.author or catalog_config.author,
|
||||||
|
publisher=catalog_config.publisher,
|
||||||
|
source_url=item.url,
|
||||||
|
feed_title=item.feed_title,
|
||||||
|
images=book_images,
|
||||||
|
include_source_link=epub_config.include_source_link,
|
||||||
|
stylesheet=_load_stylesheet(epub_config.stylesheet),
|
||||||
|
)
|
||||||
|
if epub_config.cover_image:
|
||||||
|
_choose_cover(book)
|
||||||
|
data = build_epub(book)
|
||||||
|
if len(data) > epub_config.max_epub_bytes:
|
||||||
|
raise EpubTooLargeError(
|
||||||
|
f"book is {len(data)} bytes, over the {epub_config.max_epub_bytes} byte cap"
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def _load_stylesheet(path: str | None) -> str:
|
||||||
|
"""Use the configured stylesheet, falling back to the built-in one."""
|
||||||
|
if not path:
|
||||||
|
return DEFAULT_STYLESHEET
|
||||||
|
try:
|
||||||
|
return Path(path).read_text(encoding="utf-8")
|
||||||
|
except OSError as exc:
|
||||||
|
LOGGER.warning("cannot read stylesheet %s (%s); using the built-in one", path, exc)
|
||||||
|
return DEFAULT_STYLESHEET
|
||||||
|
|
||||||
|
|
||||||
|
def _choose_cover(book: Book) -> None:
|
||||||
|
"""Promote the first suitably large image to be the book's cover.
|
||||||
|
|
||||||
|
A reader's library view shows a cover or a blank tile, and a feed's lead image
|
||||||
|
is almost always the right one. Images whose size cannot be read are skipped:
|
||||||
|
promoting an unknown image risks a cover that is a tracking pixel.
|
||||||
|
"""
|
||||||
|
for index, image in enumerate(book.images):
|
||||||
|
size = images.dimensions(image.data)
|
||||||
|
if size is None:
|
||||||
|
continue
|
||||||
|
width, height = size
|
||||||
|
if images.looks_like_a_cover(width, height):
|
||||||
|
book.images[index] = BookImage(
|
||||||
|
href=image.href,
|
||||||
|
media_type=image.media_type,
|
||||||
|
data=image.data,
|
||||||
|
is_cover=True,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def _write(
|
||||||
|
archive: zipfile.ZipFile,
|
||||||
|
path: str,
|
||||||
|
content: str | bytes,
|
||||||
|
stamp: tuple[int, int, int, int, int, int],
|
||||||
|
) -> None:
|
||||||
|
"""Write one entry, encoding text parts and passing binary parts through."""
|
||||||
|
info = zipfile.ZipInfo(path, date_time=stamp)
|
||||||
|
info.compress_type = zipfile.ZIP_DEFLATED
|
||||||
|
info.external_attr = 0o644 << 16
|
||||||
|
data = content if isinstance(content, bytes) else content.encode("utf-8")
|
||||||
|
archive.writestr(info, data)
|
||||||
|
|
||||||
|
|
||||||
|
def _zip_timestamp(moment: datetime) -> tuple[int, int, int, int, int, int]:
|
||||||
|
"""A ZIP timestamp, clamped to the range the format can express (from 1980)."""
|
||||||
|
year = max(1980, min(2107, moment.year))
|
||||||
|
return (year, moment.month, moment.day, moment.hour, moment.minute, moment.second)
|
||||||
|
|
||||||
|
|
||||||
|
# -- documents -----------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _container_xml() -> str:
|
||||||
|
return (
|
||||||
|
'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||||
|
'<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">\n'
|
||||||
|
" <rootfiles>\n"
|
||||||
|
f' <rootfile full-path="{OPF_PATH}" media-type="application/oebps-package+xml"/>\n'
|
||||||
|
" </rootfiles>\n"
|
||||||
|
"</container>\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _opf_document(book: Book) -> str:
|
||||||
|
metadata = [
|
||||||
|
f' <dc:identifier id="pub-id">{_escape(book.identifier)}</dc:identifier>',
|
||||||
|
f" <dc:title>{_escape(book.title)}</dc:title>",
|
||||||
|
f" <dc:language>{_escape(book.language)}</dc:language>",
|
||||||
|
]
|
||||||
|
if book.author:
|
||||||
|
metadata.append(f" <dc:creator>{_escape(book.author)}</dc:creator>")
|
||||||
|
if book.publisher:
|
||||||
|
metadata.append(f" <dc:publisher>{_escape(book.publisher)}</dc:publisher>")
|
||||||
|
if book.published:
|
||||||
|
metadata.append(f" <dc:date>{_timestamp(book.published)}</dc:date>")
|
||||||
|
if book.source_url:
|
||||||
|
metadata.append(f" <dc:source>{_escape(book.source_url)}</dc:source>")
|
||||||
|
metadata.append(f' <meta property="dcterms:modified">{_timestamp(book.modified)}</meta>')
|
||||||
|
if book.cover_image is not None:
|
||||||
|
# The EPUB 2 spelling of the same statement, for readers that predate the
|
||||||
|
# cover-image property below.
|
||||||
|
metadata.append(' <meta name="cover" content="cover-image"/>')
|
||||||
|
|
||||||
|
manifest = [
|
||||||
|
' <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>',
|
||||||
|
' <item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>',
|
||||||
|
' <item id="css" href="styles/main.css" media-type="text/css"/>',
|
||||||
|
' <item id="cover" href="text/cover.xhtml" media-type="application/xhtml+xml"/>',
|
||||||
|
' <item id="article" href="text/article.xhtml" media-type="application/xhtml+xml"/>',
|
||||||
|
]
|
||||||
|
cover = book.cover_image
|
||||||
|
for index, image in enumerate(book.images, start=1):
|
||||||
|
properties = ' properties="cover-image"' if image.is_cover else ""
|
||||||
|
identifier = "cover-image" if image.is_cover else f"image-{index}"
|
||||||
|
manifest.append(
|
||||||
|
f' <item id="{identifier}" href="{_escape(image.href)}" '
|
||||||
|
f'media-type="{image.media_type}"{properties}/>'
|
||||||
|
)
|
||||||
|
if cover is None and book.images:
|
||||||
|
# Images are embedded rather than linked, so nothing to declare here; the
|
||||||
|
# branch exists to keep the intent obvious if remote images are ever kept.
|
||||||
|
pass
|
||||||
|
|
||||||
|
spine = [
|
||||||
|
' <itemref idref="cover"/>',
|
||||||
|
' <itemref idref="article"/>',
|
||||||
|
]
|
||||||
|
return "\n".join(
|
||||||
|
[
|
||||||
|
'<?xml version="1.0" encoding="utf-8"?>',
|
||||||
|
f'<package xmlns="{OPF_NAMESPACE}" version="3.0" '
|
||||||
|
f'unique-identifier="pub-id" xml:lang="{_escape(book.language)}">',
|
||||||
|
f' <metadata xmlns:dc="{DC_NAMESPACE}">',
|
||||||
|
*metadata,
|
||||||
|
" </metadata>",
|
||||||
|
" <manifest>",
|
||||||
|
*manifest,
|
||||||
|
" </manifest>",
|
||||||
|
' <spine toc="ncx">',
|
||||||
|
*spine,
|
||||||
|
" </spine>",
|
||||||
|
" <guide>",
|
||||||
|
' <reference type="cover" title="Cover" href="text/cover.xhtml"/>',
|
||||||
|
" </guide>",
|
||||||
|
"</package>",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _nav_document(book: Book) -> str:
|
||||||
|
return "\n".join(
|
||||||
|
[
|
||||||
|
'<?xml version="1.0" encoding="utf-8"?>',
|
||||||
|
f'<html xmlns="{XHTML_NAMESPACE}" xmlns:epub="{OPS_NAMESPACE}" '
|
||||||
|
f'xml:lang="{_escape(book.language)}" lang="{_escape(book.language)}">',
|
||||||
|
" <head>",
|
||||||
|
" <title>Contents</title>",
|
||||||
|
' <meta charset="utf-8"/>',
|
||||||
|
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
|
||||||
|
" </head>",
|
||||||
|
" <body>",
|
||||||
|
' <nav epub:type="toc" id="toc">',
|
||||||
|
" <h1>Contents</h1>",
|
||||||
|
" <ol>",
|
||||||
|
f' <li><a href="text/cover.xhtml">{_escape(book.title)}</a></li>',
|
||||||
|
' <li><a href="text/article.xhtml">Article</a></li>',
|
||||||
|
" </ol>",
|
||||||
|
" </nav>",
|
||||||
|
" </body>",
|
||||||
|
"</html>",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ncx_document(book: Book) -> str:
|
||||||
|
points = [
|
||||||
|
("navpoint-1", 1, "Cover", "text/cover.xhtml"),
|
||||||
|
("navpoint-2", 2, "Article", "text/article.xhtml"),
|
||||||
|
]
|
||||||
|
body = []
|
||||||
|
for identifier, order, label, href in points:
|
||||||
|
body.extend(
|
||||||
|
[
|
||||||
|
f' <navPoint id="{identifier}" playOrder="{order}">',
|
||||||
|
f" <navLabel><text>{_escape(label)}</text></navLabel>",
|
||||||
|
f' <content src="{href}"/>',
|
||||||
|
" </navPoint>",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return "\n".join(
|
||||||
|
[
|
||||||
|
'<?xml version="1.0" encoding="utf-8"?>',
|
||||||
|
f'<ncx xmlns="{NCX_NAMESPACE}" version="2005-1">',
|
||||||
|
" <head>",
|
||||||
|
f' <meta name="dtb:uid" content="{_escape(book.identifier)}"/>',
|
||||||
|
' <meta name="dtb:depth" content="1"/>',
|
||||||
|
' <meta name="dtb:totalPageCount" content="0"/>',
|
||||||
|
' <meta name="dtb:maxPageNumber" content="0"/>',
|
||||||
|
" </head>",
|
||||||
|
f" <docTitle><text>{_escape(book.title)}</text></docTitle>",
|
||||||
|
" <navMap>",
|
||||||
|
*body,
|
||||||
|
" </navMap>",
|
||||||
|
"</ncx>",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _cover_document(book: Book) -> str:
|
||||||
|
lines = [
|
||||||
|
'<?xml version="1.0" encoding="utf-8"?>',
|
||||||
|
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
|
||||||
|
f'lang="{_escape(book.language)}">',
|
||||||
|
" <head>",
|
||||||
|
f" <title>{_escape(book.title)}</title>",
|
||||||
|
' <meta charset="utf-8"/>',
|
||||||
|
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
|
||||||
|
" </head>",
|
||||||
|
' <body class="cover">',
|
||||||
|
]
|
||||||
|
cover = book.cover_image
|
||||||
|
if cover is not None:
|
||||||
|
lines.append(f' <img src="{_escape(cover.document_href)}" alt="{_escape(book.title)}"/>')
|
||||||
|
lines.append(f' <h1 class="article-title">{_escape(book.title)}</h1>')
|
||||||
|
if book.author:
|
||||||
|
lines.append(f' <p class="byline">{_escape(book.author)}</p>')
|
||||||
|
if book.feed_title:
|
||||||
|
lines.append(f' <p class="byline">{_escape(book.feed_title)}</p>')
|
||||||
|
if book.published:
|
||||||
|
lines.append(f' <p class="byline">{_date_only(book.published)}</p>')
|
||||||
|
lines.extend([" </body>", "</html>", ""])
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def _article_document(book: Book) -> str:
|
||||||
|
byline_bits = []
|
||||||
|
if book.author:
|
||||||
|
byline_bits.append(_escape(book.author))
|
||||||
|
if book.published:
|
||||||
|
byline_bits.append(_date_only(book.published))
|
||||||
|
lines = [
|
||||||
|
'<?xml version="1.0" encoding="utf-8"?>',
|
||||||
|
f'<html xmlns="{XHTML_NAMESPACE}" xml:lang="{_escape(book.language)}" '
|
||||||
|
f'lang="{_escape(book.language)}">',
|
||||||
|
" <head>",
|
||||||
|
f" <title>{_escape(book.title)}</title>",
|
||||||
|
' <meta charset="utf-8"/>',
|
||||||
|
' <link rel="stylesheet" type="text/css" href="styles/main.css"/>',
|
||||||
|
" </head>",
|
||||||
|
" <body>",
|
||||||
|
f' <h1 class="article-title">{_escape(book.title)}</h1>',
|
||||||
|
]
|
||||||
|
if byline_bits:
|
||||||
|
lines.append(f' <p class="byline">{BYLINE_SEPARATOR.join(byline_bits)}</p>')
|
||||||
|
if book.content_xhtml:
|
||||||
|
lines.append(book.content_xhtml)
|
||||||
|
else:
|
||||||
|
# Never ship an empty content document: a reader that opens a blank page
|
||||||
|
# looks broken, and the original is the most useful thing to show instead.
|
||||||
|
lines.append(" <p>This article had no content to download.</p>")
|
||||||
|
if book.include_source_link and book.source_url:
|
||||||
|
lines.append(
|
||||||
|
' <p class="source">Originally published at '
|
||||||
|
f'<a href="{_escape(book.source_url)}">{_escape(book.source_url)}</a>.</p>'
|
||||||
|
)
|
||||||
|
lines.extend([" </body>", "</html>", ""])
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
# -- formatting ----------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _escape(text: str) -> str:
|
||||||
|
"""XML-escape text for a document body or an attribute value."""
|
||||||
|
return xhtml.escape_text(text).replace('"', """)
|
||||||
|
|
||||||
|
|
||||||
|
def _timestamp(moment: datetime) -> str:
|
||||||
|
"""An EPUB date: UTC, seconds precision, ``Z`` suffix."""
|
||||||
|
return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
|
||||||
|
|
||||||
|
def _date_only(moment: datetime) -> str:
|
||||||
|
return moment.astimezone(UTC).strftime("%Y-%m-%d")
|
||||||
339
app/fetch.py
Normal file
339
app/fetch.py
Normal file
@@ -0,0 +1,339 @@
|
|||||||
|
"""Fetching article images for embedding, with an SSRF guard.
|
||||||
|
|
||||||
|
Image URLs come from feed content, which means they are attacker-controlled by
|
||||||
|
anyone who can publish to a feed this account follows. That makes this the one
|
||||||
|
place in the server that makes an outbound request to a host the operator did not
|
||||||
|
choose, so the guard is the point of the module rather than an extra.
|
||||||
|
|
||||||
|
What is enforced, and why:
|
||||||
|
|
||||||
|
*Only http and https*, so ``file:``, ``gopher:`` and friends are unreachable.
|
||||||
|
*Every address a name resolves to is checked* against the private, loopback,
|
||||||
|
link-local, multicast and reserved ranges. If a name resolves to a mixture, the
|
||||||
|
whole fetch is refused rather than picking the acceptable address -- a mixed
|
||||||
|
answer is itself the attack.
|
||||||
|
*The connection is dialled to the validated address, not to the name.* This is
|
||||||
|
what closes the DNS-rebinding window: a name that answered with a public address
|
||||||
|
during validation cannot answer with ``169.254.169.254`` a millisecond later,
|
||||||
|
because the name is never resolved a second time. TLS still verifies the name
|
||||||
|
the URL used, so a valid certificate for the real host is required.
|
||||||
|
*Redirects are followed manually*, so each hop is validated again; ``urllib``'s
|
||||||
|
automatic redirect handling would happily follow the first hop to
|
||||||
|
``http://127.0.0.1:8080/``.
|
||||||
|
*Bodies are capped*, per image and across the whole article, so one hostile feed
|
||||||
|
cannot exhaust memory.
|
||||||
|
|
||||||
|
A refusal is a normal outcome: the image is dropped from the book and the article
|
||||||
|
is still built.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import http.client
|
||||||
|
import ipaddress
|
||||||
|
import logging
|
||||||
|
import socket
|
||||||
|
import ssl
|
||||||
|
import threading
|
||||||
|
import urllib.parse
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from app import images
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
USER_AGENT = "odps-from-rss/1.0 (+image fetch for EPUB embedding)"
|
||||||
|
|
||||||
|
_ALLOWED_SCHEMES = ("http", "https")
|
||||||
|
_MAX_REDIRECTS = 3
|
||||||
|
_READ_CHUNK = 64 * 1024
|
||||||
|
# Refuse to even start reading a body larger than the cap, when the server tells
|
||||||
|
# us the size up front. A little slack covers a mislabelled Content-Length.
|
||||||
|
_CONTENT_LENGTH_SLACK = 4096
|
||||||
|
|
||||||
|
|
||||||
|
class ImageFetchError(Exception):
|
||||||
|
"""The URL was refused by the guard, or the fetch failed."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FetchedImage:
|
||||||
|
"""An image that was fetched, validated and identified."""
|
||||||
|
|
||||||
|
url: str
|
||||||
|
media_type: str
|
||||||
|
data: bytes
|
||||||
|
|
||||||
|
@property
|
||||||
|
def filename(self) -> str:
|
||||||
|
"""A safe file name for the EPUB, with the extension from the media type."""
|
||||||
|
return "image" + images.MEDIA_TYPES.get(self.media_type, ".bin")
|
||||||
|
|
||||||
|
|
||||||
|
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
|
||||||
|
"""HTTPS to a validated address, verifying the hostname from the URL.
|
||||||
|
|
||||||
|
``HTTPSConnection`` would use its own host for SNI and certificate
|
||||||
|
verification, which here is a bare IP address and would always fail. Dialling
|
||||||
|
the address but verifying the name keeps both properties: no second DNS
|
||||||
|
lookup, and a real certificate check.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
address: str,
|
||||||
|
port: int,
|
||||||
|
*,
|
||||||
|
server_hostname: str,
|
||||||
|
context: ssl.SSLContext,
|
||||||
|
timeout: float,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(address, port, timeout=timeout, context=context)
|
||||||
|
self._verify_hostname = server_hostname
|
||||||
|
|
||||||
|
def connect(self) -> None:
|
||||||
|
http.client.HTTPConnection.connect(self)
|
||||||
|
self.sock = self._context.wrap_socket( # type: ignore[union-attr]
|
||||||
|
self.sock, server_hostname=self._verify_hostname
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ImageFetcher:
|
||||||
|
"""Fetches article images concurrently, refusing anything unsafe."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
allowed_hosts: list[str] | None = None,
|
||||||
|
blocked_hosts: list[str] | None = None,
|
||||||
|
max_image_bytes: int = 2_097_152,
|
||||||
|
max_total_bytes: int = 16_777_216,
|
||||||
|
timeout: float = 10.0,
|
||||||
|
max_workers: int = 4,
|
||||||
|
allow_private_networks: bool = False,
|
||||||
|
) -> None:
|
||||||
|
self._allowed_hosts = [host.lower() for host in (allowed_hosts or [])]
|
||||||
|
self._blocked_hosts = [host.lower() for host in (blocked_hosts or [])]
|
||||||
|
self._max_image_bytes = max_image_bytes
|
||||||
|
self._max_total_bytes = max_total_bytes
|
||||||
|
self._timeout = timeout
|
||||||
|
self._max_workers = max(1, max_workers)
|
||||||
|
# The upstream FreshRSS may be on a LAN, but article images never have a
|
||||||
|
# legitimate reason to point into one. Kept as a parameter so a test can
|
||||||
|
# exercise the guard, not as a configuration option.
|
||||||
|
self._allow_private_networks = allow_private_networks
|
||||||
|
self._tls_context = ssl.create_default_context()
|
||||||
|
self._budget_lock = threading.Lock()
|
||||||
|
self._budget_used = 0
|
||||||
|
|
||||||
|
def fetch_all(self, urls: list[str]) -> dict[str, FetchedImage | None]:
|
||||||
|
"""Fetch each URL, mapping it to an image or to None when refused.
|
||||||
|
|
||||||
|
Order is preserved by the caller, which walks the document's images; this
|
||||||
|
method only has to answer "did it arrive?". Concurrent, because a article
|
||||||
|
with a dozen images should not cost a dozen sequential round trips.
|
||||||
|
"""
|
||||||
|
unique: list[str] = []
|
||||||
|
for url in urls:
|
||||||
|
if url not in unique:
|
||||||
|
unique.append(url)
|
||||||
|
if not unique:
|
||||||
|
return {}
|
||||||
|
with self._budget_lock:
|
||||||
|
self._budget_used = 0
|
||||||
|
workers = min(self._max_workers, len(unique))
|
||||||
|
results: dict[str, FetchedImage | None] = {}
|
||||||
|
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="image") as pool:
|
||||||
|
for url, result in zip(unique, pool.map(self._fetch_safely, unique), strict=True):
|
||||||
|
results[url] = result
|
||||||
|
return results
|
||||||
|
|
||||||
|
# -- internals -------------------------------------------------------
|
||||||
|
|
||||||
|
def _fetch_safely(self, url: str) -> FetchedImage | None:
|
||||||
|
"""Fetch one image, turning every failure into a dropped image."""
|
||||||
|
try:
|
||||||
|
return self._fetch(url)
|
||||||
|
except ImageFetchError as exc:
|
||||||
|
LOGGER.info("not embedding image %s: %s", url, exc)
|
||||||
|
except (OSError, http.client.HTTPException, ssl.SSLError) as exc:
|
||||||
|
LOGGER.info("could not fetch image %s: %s", url, exc)
|
||||||
|
except Exception:
|
||||||
|
# One bad image must never break the book it was going into.
|
||||||
|
LOGGER.exception("unexpected failure fetching image %s", url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _fetch(self, url: str) -> FetchedImage:
|
||||||
|
target = url
|
||||||
|
for _ in range(_MAX_REDIRECTS + 1):
|
||||||
|
parsed = urllib.parse.urlsplit(target)
|
||||||
|
scheme = parsed.scheme.lower()
|
||||||
|
host = parsed.hostname
|
||||||
|
if scheme not in _ALLOWED_SCHEMES:
|
||||||
|
raise ImageFetchError(f"scheme {scheme or '(none)'!r} is not allowed")
|
||||||
|
if not host:
|
||||||
|
raise ImageFetchError("no host in URL")
|
||||||
|
if parsed.username or parsed.password:
|
||||||
|
raise ImageFetchError("credentials in an image URL are not followed")
|
||||||
|
self._check_host_allowed(host)
|
||||||
|
address = self._resolve(host)
|
||||||
|
port = parsed.port or (443 if scheme == "https" else 80)
|
||||||
|
status, headers, body = self._request_once(scheme, address, host, port, parsed)
|
||||||
|
if status in (301, 302, 303, 307, 308):
|
||||||
|
location = headers.get("location")
|
||||||
|
if not location:
|
||||||
|
raise ImageFetchError("redirect without a location")
|
||||||
|
# Resolve relative redirects against the URL that produced them.
|
||||||
|
target = urllib.parse.urljoin(target, location)
|
||||||
|
LOGGER.debug("following image redirect to %s", target)
|
||||||
|
continue
|
||||||
|
if status != 200:
|
||||||
|
raise ImageFetchError(f"server answered {status}")
|
||||||
|
declared = headers.get("content-type")
|
||||||
|
media_type = images.identify(body, declared)
|
||||||
|
if media_type is None:
|
||||||
|
raise ImageFetchError(f"not an embeddable image ({declared or 'no type'})")
|
||||||
|
return FetchedImage(url=url, media_type=media_type, data=body)
|
||||||
|
raise ImageFetchError("too many redirects")
|
||||||
|
|
||||||
|
def _check_host_allowed(self, host: str) -> None:
|
||||||
|
lowered = host.lower()
|
||||||
|
if any(_host_matches(lowered, blocked) for blocked in self._blocked_hosts):
|
||||||
|
raise ImageFetchError("host is on the blocked list")
|
||||||
|
if self._allowed_hosts and not any(
|
||||||
|
_host_matches(lowered, allowed) for allowed in self._allowed_hosts
|
||||||
|
):
|
||||||
|
raise ImageFetchError("host is not on the allowed list")
|
||||||
|
|
||||||
|
def _resolve(self, host: str) -> str:
|
||||||
|
"""Validate every address for a host and return the one to dial."""
|
||||||
|
try:
|
||||||
|
infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
|
||||||
|
except socket.gaierror as exc:
|
||||||
|
raise ImageFetchError(f"cannot resolve host ({exc.strerror or exc})") from exc
|
||||||
|
addresses: list[str] = []
|
||||||
|
for info in infos:
|
||||||
|
address = str(info[4][0])
|
||||||
|
if address not in addresses:
|
||||||
|
addresses.append(address)
|
||||||
|
if not addresses:
|
||||||
|
raise ImageFetchError("host resolved to no addresses")
|
||||||
|
for address in addresses:
|
||||||
|
reason = self._address_refusal(address)
|
||||||
|
if reason is not None:
|
||||||
|
# Refuse the whole host rather than picking a good address: a
|
||||||
|
# mixed answer is the signature of a rebinding attempt.
|
||||||
|
raise ImageFetchError(f"{address} is refused ({reason})")
|
||||||
|
return addresses[0]
|
||||||
|
|
||||||
|
def _address_refusal(self, address: str) -> str | None:
|
||||||
|
try:
|
||||||
|
parsed = ipaddress.ip_address(address)
|
||||||
|
except ValueError:
|
||||||
|
return "unparseable address"
|
||||||
|
if self._allow_private_networks:
|
||||||
|
return None
|
||||||
|
# A v4-mapped v6 address is judged as the v4 address it is, and this has to
|
||||||
|
# come first: ::ffff:127.0.0.1 is neither loopback nor private when it is
|
||||||
|
# tested as a v6 address, so the checks below would let it through.
|
||||||
|
if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None:
|
||||||
|
return self._address_refusal(str(parsed.ipv4_mapped))
|
||||||
|
if parsed.is_loopback:
|
||||||
|
return "loopback"
|
||||||
|
if parsed.is_link_local:
|
||||||
|
return "link-local"
|
||||||
|
if parsed.is_private:
|
||||||
|
return "private network"
|
||||||
|
if parsed.is_multicast:
|
||||||
|
return "multicast"
|
||||||
|
if parsed.is_reserved or parsed.is_unspecified:
|
||||||
|
return "reserved"
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _request_once(
|
||||||
|
self,
|
||||||
|
scheme: str,
|
||||||
|
address: str,
|
||||||
|
host: str,
|
||||||
|
port: int,
|
||||||
|
parsed: urllib.parse.SplitResult,
|
||||||
|
) -> tuple[int, dict[str, str], bytes]:
|
||||||
|
"""One request, to a pinned address, reading at most the byte cap."""
|
||||||
|
if scheme == "https":
|
||||||
|
connection: http.client.HTTPConnection = _PinnedHTTPSConnection(
|
||||||
|
address,
|
||||||
|
port,
|
||||||
|
server_hostname=host,
|
||||||
|
context=self._tls_context,
|
||||||
|
timeout=self._timeout,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
connection = http.client.HTTPConnection(address, port, timeout=self._timeout)
|
||||||
|
path = urllib.parse.urlunsplit(("", "", parsed.path or "/", parsed.query, ""))
|
||||||
|
headers = {
|
||||||
|
"Host": _host_header(host, port, scheme),
|
||||||
|
"User-Agent": USER_AGENT,
|
||||||
|
"Accept": "image/*",
|
||||||
|
"Connection": "close",
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
connection.request("GET", path, headers=headers)
|
||||||
|
response = connection.getresponse()
|
||||||
|
status = response.status
|
||||||
|
response_headers = {key.lower(): value for key, value in response.getheaders()}
|
||||||
|
if status != 200:
|
||||||
|
response.read(_READ_CHUNK)
|
||||||
|
return status, response_headers, b""
|
||||||
|
return status, response_headers, self._read_capped(response, response_headers)
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
def _read_capped(self, response: http.client.HTTPResponse, headers: dict[str, str]) -> bytes:
|
||||||
|
"""Read a response body, refusing one that exceeds the budget."""
|
||||||
|
limit = self._image_limit(headers)
|
||||||
|
chunks: list[bytes] = []
|
||||||
|
total = 0
|
||||||
|
while True:
|
||||||
|
chunk = response.read(_READ_CHUNK)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
total += len(chunk)
|
||||||
|
if total > limit:
|
||||||
|
raise ImageFetchError(f"image is larger than {limit} bytes")
|
||||||
|
chunks.append(chunk)
|
||||||
|
self._spend(total)
|
||||||
|
return b"".join(chunks)
|
||||||
|
|
||||||
|
def _image_limit(self, headers: dict[str, str]) -> int:
|
||||||
|
"""The smaller of the per-image cap and what is left of the total budget."""
|
||||||
|
declared = headers.get("content-length")
|
||||||
|
limit = self._max_image_bytes
|
||||||
|
if declared and declared.isdigit():
|
||||||
|
size = int(declared)
|
||||||
|
if size > self._max_image_bytes:
|
||||||
|
raise ImageFetchError(f"image declares {size} bytes")
|
||||||
|
with self._budget_lock:
|
||||||
|
remaining = max(0, self._max_total_bytes - self._budget_used)
|
||||||
|
if remaining == 0:
|
||||||
|
# The article's total allowance is spent; the rest of its images are
|
||||||
|
# dropped rather than growing the book further.
|
||||||
|
raise ImageFetchError("the article's image budget is exhausted")
|
||||||
|
return min(limit, remaining)
|
||||||
|
|
||||||
|
def _spend(self, size: int) -> None:
|
||||||
|
with self._budget_lock:
|
||||||
|
self._budget_used += size
|
||||||
|
|
||||||
|
|
||||||
|
def _host_matches(host: str, pattern: str) -> bool:
|
||||||
|
"""Exact host or subdomain match, so ``example.com`` covers ``cdn.example.com``."""
|
||||||
|
return host == pattern or host.endswith("." + pattern)
|
||||||
|
|
||||||
|
|
||||||
|
def _host_header(host: str, port: int, scheme: str) -> str:
|
||||||
|
"""A Host header, including the port only when it is not the default."""
|
||||||
|
default = 443 if scheme == "https" else 80
|
||||||
|
if port == default:
|
||||||
|
return host
|
||||||
|
return f"{host}:{port}"
|
||||||
598
app/freshrss.py
Normal file
598
app/freshrss.py
Normal file
@@ -0,0 +1,598 @@
|
|||||||
|
"""FreshRSS client over its Google Reader compatible API.
|
||||||
|
|
||||||
|
Only the endpoints the firmware client already proved out against a real server
|
||||||
|
are used (see ``lib/FreshRss/src/FreshRssSync.cpp`` in the reader tree):
|
||||||
|
|
||||||
|
* ``POST accounts/ClientLogin`` returns the ``Auth`` token that goes into an
|
||||||
|
``Authorization: GoogleLogin auth=...`` header.
|
||||||
|
* ``GET reader/api/0/subscription/list?output=json`` lists feeds, each with its
|
||||||
|
category labels.
|
||||||
|
* ``GET reader/api/0/stream/contents/<stream>?output=json&r=d&n=&xt=`` fetches
|
||||||
|
items. ``r=d`` is newest first, ``n`` bounds the count, and ``xt`` excludes
|
||||||
|
items carrying a tag -- which is how the unread queue is filtered server-side.
|
||||||
|
* ``GET reader/api/0/token`` returns the write token, and
|
||||||
|
``POST reader/api/0/edit-tag`` with ``T=`` and repeated ``i=`` marks items read.
|
||||||
|
|
||||||
|
Deliberately absent: anything that would need an undocumented parameter. Where a
|
||||||
|
filter is not expressible with these calls (read items), the caller filters
|
||||||
|
locally rather than assuming an endpoint behaves a certain way.
|
||||||
|
|
||||||
|
Note the two endpoints this module never touches: ``unread-count`` is optional
|
||||||
|
and its failure is tolerated, and nothing here fetches article bodies from the
|
||||||
|
open internet -- the body FreshRSS already stores is the source of truth.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import ssl
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app import config as app_config
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Google Reader stream and tag identifiers.
|
||||||
|
READING_LIST = "user/-/state/com.google/reading-list"
|
||||||
|
STARRED = "user/-/state/com.google/starred"
|
||||||
|
READ_TAG = "user/-/state/com.google/read"
|
||||||
|
STARRED_TAG = "user/-/state/com.google/starred"
|
||||||
|
KEPT_UNREAD_TAG = "user/-/state/com.google/kept-unread"
|
||||||
|
|
||||||
|
USER_AGENT = "odps-from-rss/1.0 (+https://github.com/; OPDS 1.2 catalog)"
|
||||||
|
|
||||||
|
# Statuses worth retrying: the upstream is briefly unhappy, not wrong.
|
||||||
|
_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504, 507, 509})
|
||||||
|
_AUTH_STATUSES = frozenset({401, 403})
|
||||||
|
_MISSING_STATUSES = frozenset({400, 404})
|
||||||
|
|
||||||
|
|
||||||
|
class FreshRSSError(Exception):
|
||||||
|
"""An upstream call failed."""
|
||||||
|
|
||||||
|
def __init__(self, message: str, *, status: int | None = None) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
class AuthenticationError(FreshRSSError):
|
||||||
|
"""The API password or username was rejected."""
|
||||||
|
|
||||||
|
|
||||||
|
class TransientError(FreshRSSError):
|
||||||
|
"""The upstream could not be reached, or answered with a retryable status."""
|
||||||
|
|
||||||
|
|
||||||
|
class NotFoundError(FreshRSSError):
|
||||||
|
"""The requested stream or item does not exist upstream."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Subscription:
|
||||||
|
"""One feed in the FreshRSS account."""
|
||||||
|
|
||||||
|
id: str
|
||||||
|
title: str
|
||||||
|
categories: list[str] = field(default_factory=list)
|
||||||
|
html_url: str | None = None
|
||||||
|
unread: int | None = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def primary_category(self) -> str:
|
||||||
|
"""The label this feed is shelved under, or a stable placeholder."""
|
||||||
|
return self.categories[0] if self.categories else UNCATEGORISED
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Item:
|
||||||
|
"""One article."""
|
||||||
|
|
||||||
|
id: str
|
||||||
|
title: str
|
||||||
|
published: int
|
||||||
|
updated: int
|
||||||
|
feed_id: str
|
||||||
|
feed_title: str
|
||||||
|
content_html: str = ""
|
||||||
|
author: str | None = None
|
||||||
|
url: str | None = None
|
||||||
|
feed_url: str | None = None
|
||||||
|
categories: list[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_read(self) -> bool:
|
||||||
|
"""True when the item is read upstream.
|
||||||
|
|
||||||
|
FreshRSS keeps both the read and the kept-unread tag on an item that was
|
||||||
|
read and then explicitly marked unread again, and there the explicit
|
||||||
|
signal is the one that reflects what the reader wants.
|
||||||
|
"""
|
||||||
|
if KEPT_UNREAD_TAG in self.categories:
|
||||||
|
return False
|
||||||
|
return READ_TAG in self.categories
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_starred(self) -> bool:
|
||||||
|
return STARRED_TAG in self.categories
|
||||||
|
|
||||||
|
@property
|
||||||
|
def published_at(self) -> datetime:
|
||||||
|
"""Publication time as an aware UTC datetime.
|
||||||
|
|
||||||
|
Falls back to the epoch rather than raising: a feed with a broken date is
|
||||||
|
not a reason to fail the whole catalog.
|
||||||
|
"""
|
||||||
|
stamp = self.published or self.updated or 0
|
||||||
|
try:
|
||||||
|
return datetime.fromtimestamp(stamp, tz=UTC)
|
||||||
|
except (OverflowError, OSError, ValueError):
|
||||||
|
return datetime(1970, 1, 1, tzinfo=UTC)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def updated_at(self) -> datetime:
|
||||||
|
stamp = self.updated or self.published or 0
|
||||||
|
try:
|
||||||
|
return datetime.fromtimestamp(stamp, tz=UTC)
|
||||||
|
except (OverflowError, OSError, ValueError):
|
||||||
|
return self.published_at
|
||||||
|
|
||||||
|
|
||||||
|
UNCATEGORISED = "Uncategorised"
|
||||||
|
|
||||||
|
|
||||||
|
def label_stream(label: str) -> str:
|
||||||
|
"""Stream id for a category label."""
|
||||||
|
return f"user/-/label/{label}"
|
||||||
|
|
||||||
|
|
||||||
|
class FreshRSSClient:
|
||||||
|
"""A small client for one FreshRSS account.
|
||||||
|
|
||||||
|
Instances are safe to share between threads: the only mutable state is the
|
||||||
|
cached session token, which is guarded by a lock.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
config: app_config.FreshRSSConfig,
|
||||||
|
*,
|
||||||
|
clock: Any = time.time,
|
||||||
|
sleep: Any = time.sleep,
|
||||||
|
) -> None:
|
||||||
|
self._config = config
|
||||||
|
self._clock = clock
|
||||||
|
# Injectable so tests can exercise retry logic without waiting it out.
|
||||||
|
self._sleep = sleep
|
||||||
|
self._auth_token: str | None = None
|
||||||
|
self._write_token: str | None = None
|
||||||
|
self._write_token_fetched_at = 0.0
|
||||||
|
self._token_lock = threading.Lock()
|
||||||
|
self._tls_context = _build_tls_context(config)
|
||||||
|
self._opener = urllib.request.build_opener(
|
||||||
|
urllib.request.HTTPSHandler(context=self._tls_context)
|
||||||
|
)
|
||||||
|
|
||||||
|
# -- public API ------------------------------------------------------
|
||||||
|
|
||||||
|
def list_subscriptions(self) -> list[Subscription]:
|
||||||
|
"""Every feed in the account, with its category labels."""
|
||||||
|
payload = self._get_json("/reader/api/0/subscription/list")
|
||||||
|
raw_subscriptions = payload.get("subscriptions")
|
||||||
|
if not isinstance(raw_subscriptions, list):
|
||||||
|
raise FreshRSSError("subscription list had no subscriptions array")
|
||||||
|
subscriptions: list[Subscription] = []
|
||||||
|
for entry in raw_subscriptions:
|
||||||
|
parsed = _parse_subscription(entry)
|
||||||
|
if parsed is not None:
|
||||||
|
subscriptions.append(parsed)
|
||||||
|
subscriptions.sort(key=lambda feed: feed.title.casefold())
|
||||||
|
return subscriptions
|
||||||
|
|
||||||
|
def unread_counts(self) -> dict[str, int]:
|
||||||
|
"""Unread counts keyed by stream id. Optional upstream feature.
|
||||||
|
|
||||||
|
Raises whatever the client raises, so the caller can latch the failure and
|
||||||
|
carry on without counts -- the catalog is perfectly usable without them.
|
||||||
|
"""
|
||||||
|
payload = self._get_json("/reader/api/0/unread-count", {"all": "true"})
|
||||||
|
counts: dict[str, int] = {}
|
||||||
|
for entry in payload.get("unreadcounts") or []:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
stream_id = entry.get("id")
|
||||||
|
count = entry.get("count")
|
||||||
|
if isinstance(stream_id, str) and isinstance(count, int):
|
||||||
|
counts[stream_id] = count
|
||||||
|
return counts
|
||||||
|
|
||||||
|
def stream_items(
|
||||||
|
self,
|
||||||
|
stream_id: str,
|
||||||
|
*,
|
||||||
|
count: int,
|
||||||
|
exclude_read: bool = False,
|
||||||
|
older_than: int | None = None,
|
||||||
|
) -> list[Item]:
|
||||||
|
"""Fetch items from one stream, newest first.
|
||||||
|
|
||||||
|
A stream that has disappeared upstream (``400``/``404``) is reported as an
|
||||||
|
empty result rather than an error: a feed you unsubscribed from should not
|
||||||
|
break the catalog that still lists it.
|
||||||
|
"""
|
||||||
|
params = {
|
||||||
|
"output": "json",
|
||||||
|
"r": "d",
|
||||||
|
"n": str(max(1, count)),
|
||||||
|
# Cache killer: asks FreshRSS not to serve a cached response.
|
||||||
|
"ck": str(int(self._clock())),
|
||||||
|
}
|
||||||
|
if exclude_read:
|
||||||
|
params["xt"] = READ_TAG
|
||||||
|
if older_than:
|
||||||
|
params["ot"] = str(int(older_than))
|
||||||
|
path = "/reader/api/0/stream/contents/" + urllib.parse.quote(stream_id, safe="")
|
||||||
|
try:
|
||||||
|
payload = self._get_json(path, params)
|
||||||
|
except NotFoundError:
|
||||||
|
LOGGER.info("stream %s is gone upstream; treating it as empty", stream_id)
|
||||||
|
return []
|
||||||
|
raw_items = payload.get("items")
|
||||||
|
if not isinstance(raw_items, list):
|
||||||
|
return []
|
||||||
|
items = [item for item in (_parse_item(entry) for entry in raw_items) if item]
|
||||||
|
return items
|
||||||
|
|
||||||
|
def mark_read(self, item_ids: list[str]) -> int:
|
||||||
|
"""Mark items read upstream. Returns the number of ids sent.
|
||||||
|
|
||||||
|
Idempotent: adding the read tag to an already-read item is a no-op
|
||||||
|
upstream, so a repeated download of the same book is harmless.
|
||||||
|
"""
|
||||||
|
wanted = [item_id for item_id in item_ids if item_id]
|
||||||
|
if not wanted:
|
||||||
|
return 0
|
||||||
|
token = self._write_token_value()
|
||||||
|
form: list[tuple[str, str]] = [
|
||||||
|
("T", token),
|
||||||
|
("a", READ_TAG),
|
||||||
|
]
|
||||||
|
form.extend(("i", item_id) for item_id in wanted)
|
||||||
|
# edit-tag needs both the session header and the write token: the header
|
||||||
|
# authenticates the caller, the token authorises the write.
|
||||||
|
try:
|
||||||
|
self._request(
|
||||||
|
"POST",
|
||||||
|
"/reader/api/0/edit-tag",
|
||||||
|
body=urllib.parse.urlencode(form),
|
||||||
|
authenticated=True,
|
||||||
|
)
|
||||||
|
except AuthenticationError:
|
||||||
|
# Either the session or the write token outlived its validity; drop
|
||||||
|
# both, fetch fresh ones, and retry the write once.
|
||||||
|
self._forget_tokens()
|
||||||
|
form[0] = ("T", self._write_token_value())
|
||||||
|
self._request(
|
||||||
|
"POST",
|
||||||
|
"/reader/api/0/edit-tag",
|
||||||
|
body=urllib.parse.urlencode(form),
|
||||||
|
authenticated=True,
|
||||||
|
)
|
||||||
|
return len(wanted)
|
||||||
|
|
||||||
|
def probe(self) -> tuple[bool, str]:
|
||||||
|
"""Check credentials and reachability, for the diagnostics endpoint."""
|
||||||
|
try:
|
||||||
|
subscriptions = self.list_subscriptions()
|
||||||
|
except FreshRSSError as exc:
|
||||||
|
return False, str(exc)
|
||||||
|
return True, f"{len(subscriptions)} feeds"
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
"""Release upstream resources (connections are per-request)."""
|
||||||
|
self._forget_tokens()
|
||||||
|
|
||||||
|
# -- transport -------------------------------------------------------
|
||||||
|
|
||||||
|
def _forget_tokens(self) -> None:
|
||||||
|
with self._token_lock:
|
||||||
|
self._auth_token = None
|
||||||
|
self._write_token = None
|
||||||
|
self._write_token_fetched_at = 0.0
|
||||||
|
|
||||||
|
def _auth_header(self) -> dict[str, str]:
|
||||||
|
with self._token_lock:
|
||||||
|
token = self._auth_token
|
||||||
|
if token is None:
|
||||||
|
token = self._login()
|
||||||
|
with self._token_lock:
|
||||||
|
self._auth_token = token
|
||||||
|
return {"Authorization": f"GoogleLogin auth={token}"}
|
||||||
|
|
||||||
|
def _write_token_value(self) -> str:
|
||||||
|
with self._token_lock:
|
||||||
|
token = self._write_token
|
||||||
|
fetched_at = self._write_token_fetched_at
|
||||||
|
# Refresh hourly: the token is cheap to fetch and this bounds how long a
|
||||||
|
# stale one can survive a password change upstream.
|
||||||
|
if token and self._clock() - fetched_at < 3600:
|
||||||
|
return token
|
||||||
|
response = self._request("GET", "/reader/api/0/token", authenticated=True)
|
||||||
|
token = response.decode("utf-8", "replace").strip()
|
||||||
|
if not token:
|
||||||
|
raise FreshRSSError("upstream returned an empty write token")
|
||||||
|
with self._token_lock:
|
||||||
|
self._write_token = token
|
||||||
|
self._write_token_fetched_at = self._clock()
|
||||||
|
return token
|
||||||
|
|
||||||
|
def _login(self) -> str:
|
||||||
|
"""Exchange the API password for an Auth token."""
|
||||||
|
form = urllib.parse.urlencode(
|
||||||
|
{"Email": self._config.username, "Passwd": self._config.api_password}
|
||||||
|
)
|
||||||
|
response = self._request("POST", "/accounts/ClientLogin", body=form)
|
||||||
|
text = response.decode("utf-8", "replace")
|
||||||
|
for line in text.splitlines():
|
||||||
|
if line.startswith("Auth="):
|
||||||
|
value = line[len("Auth=") :].strip()
|
||||||
|
if value:
|
||||||
|
LOGGER.debug("authenticated with FreshRSS as %s", self._config.username)
|
||||||
|
return value
|
||||||
|
raise AuthenticationError("FreshRSS rejected the API password")
|
||||||
|
|
||||||
|
def _get_json(self, path: str, params: dict[str, str] | None = None) -> dict[str, Any]:
|
||||||
|
query = dict(params or {})
|
||||||
|
query.setdefault("output", "json")
|
||||||
|
response = self._request("GET", path, query=query, authenticated=True)
|
||||||
|
try:
|
||||||
|
payload = json.loads(response.decode("utf-8", "replace"))
|
||||||
|
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||||
|
raise FreshRSSError(f"upstream sent a response that is not JSON ({exc})") from exc
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise FreshRSSError("upstream sent a JSON value that is not an object")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
def _request(
|
||||||
|
self,
|
||||||
|
method: str,
|
||||||
|
path: str,
|
||||||
|
*,
|
||||||
|
query: dict[str, str] | None = None,
|
||||||
|
body: str | None = None,
|
||||||
|
authenticated: bool = False,
|
||||||
|
) -> bytes:
|
||||||
|
"""Perform one upstream request, retrying transient failures."""
|
||||||
|
url = self._config.api_base + path
|
||||||
|
if query:
|
||||||
|
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||||
|
data = body.encode("utf-8") if body is not None else None
|
||||||
|
attempts = max(1, self._config.retries + 1)
|
||||||
|
last_error: Exception | None = None
|
||||||
|
|
||||||
|
for attempt in range(attempts):
|
||||||
|
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
||||||
|
if body is not None:
|
||||||
|
headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||||
|
if authenticated:
|
||||||
|
headers.update(self._auth_header())
|
||||||
|
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||||
|
try:
|
||||||
|
with self._opener.open(request, timeout=self._config.timeout_seconds) as reply:
|
||||||
|
return reply.read()
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
status = exc.code
|
||||||
|
try:
|
||||||
|
detail = _read_error_body(exc)
|
||||||
|
finally:
|
||||||
|
# The error object owns the response socket. Leaving it open
|
||||||
|
# leaks one connection for every failed upstream request.
|
||||||
|
exc.close()
|
||||||
|
if status in _AUTH_STATUSES:
|
||||||
|
raise AuthenticationError(
|
||||||
|
f"FreshRSS refused the request ({status}); check the API password",
|
||||||
|
status=status,
|
||||||
|
) from exc
|
||||||
|
if status in _MISSING_STATUSES:
|
||||||
|
raise NotFoundError(
|
||||||
|
f"upstream has no such resource ({status})", status=status
|
||||||
|
) from exc
|
||||||
|
if status in _RETRY_STATUSES:
|
||||||
|
# A retryable status stays transient even once the attempts run
|
||||||
|
# out, so the caller can tell "upstream is unwell" from
|
||||||
|
# "upstream says this request is wrong".
|
||||||
|
error = TransientError(f"upstream returned {status}{detail}", status=status)
|
||||||
|
if attempt + 1 < attempts:
|
||||||
|
last_error = error
|
||||||
|
self._backoff(attempt, status)
|
||||||
|
continue
|
||||||
|
raise error from exc
|
||||||
|
raise FreshRSSError(f"upstream returned {status}{detail}", status=status) from exc
|
||||||
|
except urllib.error.URLError as exc:
|
||||||
|
last_error = TransientError(f"cannot reach FreshRSS: {exc.reason}")
|
||||||
|
if attempt + 1 < attempts:
|
||||||
|
self._backoff(attempt, None)
|
||||||
|
continue
|
||||||
|
raise last_error from exc
|
||||||
|
except TimeoutError as exc:
|
||||||
|
last_error = TransientError("the upstream request timed out")
|
||||||
|
if attempt + 1 < attempts:
|
||||||
|
self._backoff(attempt, None)
|
||||||
|
continue
|
||||||
|
raise last_error from exc
|
||||||
|
except ssl.SSLError as exc:
|
||||||
|
raise FreshRSSError(f"TLS failure talking to FreshRSS: {exc}") from exc
|
||||||
|
|
||||||
|
# Only reachable if the loop exhausted its attempts without raising.
|
||||||
|
raise last_error or TransientError("the upstream request failed")
|
||||||
|
|
||||||
|
def _backoff(self, attempt: int, status: int | None) -> None:
|
||||||
|
delay = min(2.0, 0.2 * (2**attempt))
|
||||||
|
LOGGER.warning(
|
||||||
|
"upstream %s, retrying in %.1fs%s",
|
||||||
|
f"returned {status}" if status else "unreachable",
|
||||||
|
delay,
|
||||||
|
f" (attempt {attempt + 1})" if attempt else "",
|
||||||
|
)
|
||||||
|
self._sleep(delay)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_tls_context(config: app_config.FreshRSSConfig) -> ssl.SSLContext:
|
||||||
|
"""Build the TLS context for upstream calls.
|
||||||
|
|
||||||
|
An operator pointing at a LAN instance with a private CA sets ``ca_file``;
|
||||||
|
disabling verification is possible but logged loudly, because it silently
|
||||||
|
removes the protection against a machine-in-the-middle reading the API
|
||||||
|
password.
|
||||||
|
"""
|
||||||
|
if not config.verify_tls:
|
||||||
|
LOGGER.warning(
|
||||||
|
"freshrss.verify_tls is false: TLS certificates are not verified. "
|
||||||
|
"The API password and article contents are exposed to anyone able to "
|
||||||
|
"intercept the connection."
|
||||||
|
)
|
||||||
|
context = ssl.create_default_context()
|
||||||
|
context.check_hostname = False
|
||||||
|
context.verify_mode = ssl.CERT_NONE
|
||||||
|
return context
|
||||||
|
context = ssl.create_default_context()
|
||||||
|
if config.ca_file:
|
||||||
|
context.load_verify_locations(cafile=config.ca_file)
|
||||||
|
return context
|
||||||
|
|
||||||
|
|
||||||
|
def _read_error_body(error: urllib.error.HTTPError) -> str:
|
||||||
|
"""Include a short, sanitised excerpt of an error body in the message."""
|
||||||
|
try:
|
||||||
|
raw = error.read(512)
|
||||||
|
except Exception:
|
||||||
|
# A diagnostic excerpt must never mask the error it is describing.
|
||||||
|
return ""
|
||||||
|
text = raw.decode("utf-8", "replace").strip().replace("\n", " ")
|
||||||
|
if not text:
|
||||||
|
return ""
|
||||||
|
return f": {text[:200]}"
|
||||||
|
|
||||||
|
|
||||||
|
def _text(value: Any) -> str:
|
||||||
|
"""A trimmed string from upstream JSON, or an empty string.
|
||||||
|
|
||||||
|
Upstream is an arbitrary FreshRSS build, so every field is treated as
|
||||||
|
untrusted input of unknown type rather than as the shape the docs suggest.
|
||||||
|
"""
|
||||||
|
return value.strip() if isinstance(value, str) else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _as_int(value: Any) -> int:
|
||||||
|
"""Best-effort integer from upstream JSON, tolerating strings."""
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return 0
|
||||||
|
if isinstance(value, int):
|
||||||
|
return value
|
||||||
|
if isinstance(value, float):
|
||||||
|
return int(value)
|
||||||
|
if isinstance(value, str):
|
||||||
|
try:
|
||||||
|
return int(value)
|
||||||
|
except ValueError:
|
||||||
|
return 0
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_subscription(entry: Any) -> Subscription | None:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
return None
|
||||||
|
stream_id = entry.get("id")
|
||||||
|
if not isinstance(stream_id, str) or not stream_id:
|
||||||
|
return None
|
||||||
|
categories: list[str] = []
|
||||||
|
for category in entry.get("categories") or []:
|
||||||
|
if not isinstance(category, dict):
|
||||||
|
continue
|
||||||
|
label = _text(category.get("label"))
|
||||||
|
if label and label not in categories:
|
||||||
|
categories.append(label)
|
||||||
|
return Subscription(
|
||||||
|
id=stream_id,
|
||||||
|
title=_text(entry.get("title")) or stream_id,
|
||||||
|
categories=categories,
|
||||||
|
html_url=_text(entry.get("htmlUrl")) or None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_item(entry: Any) -> Item | None:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
return None
|
||||||
|
item_id = entry.get("id")
|
||||||
|
if not isinstance(item_id, str) or not item_id:
|
||||||
|
return None
|
||||||
|
|
||||||
|
raw_origin = entry.get("origin")
|
||||||
|
origin: dict[str, Any] = raw_origin if isinstance(raw_origin, dict) else {}
|
||||||
|
feed_id = _text(origin.get("streamId"))
|
||||||
|
return Item(
|
||||||
|
id=item_id,
|
||||||
|
title=_text(entry.get("title")) or "(untitled)",
|
||||||
|
published=_as_int(entry.get("published")) or _timestamp_usec(entry),
|
||||||
|
updated=_as_int(entry.get("updated")) or _timestamp_usec(entry),
|
||||||
|
feed_id=feed_id,
|
||||||
|
feed_title=_text(origin.get("title")) or feed_id,
|
||||||
|
content_html=_extract_content(entry),
|
||||||
|
author=_text(entry.get("author")) or None,
|
||||||
|
url=_extract_url(entry, origin),
|
||||||
|
feed_url=_text(origin.get("htmlUrl")) or None,
|
||||||
|
categories=[tag for tag in (entry.get("categories") or []) if isinstance(tag, str)],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _timestamp_usec(entry: dict[str, Any]) -> int:
|
||||||
|
"""Derive seconds from Google Reader's microsecond timestamp, when present."""
|
||||||
|
raw = entry.get("timestampUsec")
|
||||||
|
if isinstance(raw, str) and raw.isdigit():
|
||||||
|
return int(raw) // 1_000_000
|
||||||
|
if isinstance(raw, int):
|
||||||
|
return raw // 1_000_000
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_content(entry: dict[str, Any]) -> str:
|
||||||
|
"""Prefer the full body over the summary.
|
||||||
|
|
||||||
|
FreshRSS stores whichever it has; ``content`` is the article body when the
|
||||||
|
feed provides one, and ``summary`` is the fallback.
|
||||||
|
"""
|
||||||
|
for key in ("content", "summary"):
|
||||||
|
block = entry.get(key)
|
||||||
|
if isinstance(block, str) and block.strip():
|
||||||
|
return block
|
||||||
|
if isinstance(block, dict):
|
||||||
|
inner = block.get("content")
|
||||||
|
if isinstance(inner, str) and inner.strip():
|
||||||
|
return inner
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_url(entry: dict[str, Any], origin: dict[str, Any]) -> str | None:
|
||||||
|
alternates = entry.get("alternate")
|
||||||
|
if isinstance(alternates, list):
|
||||||
|
for candidate in alternates:
|
||||||
|
if not isinstance(candidate, dict):
|
||||||
|
continue
|
||||||
|
href = candidate.get("href")
|
||||||
|
if not isinstance(href, str) or not href:
|
||||||
|
continue
|
||||||
|
kind = candidate.get("type")
|
||||||
|
if kind in (None, "text/html"):
|
||||||
|
return href
|
||||||
|
for candidate in alternates:
|
||||||
|
if isinstance(candidate, dict) and isinstance(candidate.get("href"), str):
|
||||||
|
return candidate["href"]
|
||||||
|
html_url = origin.get("htmlUrl")
|
||||||
|
return html_url if isinstance(html_url, str) and html_url else None
|
||||||
193
app/images.py
Normal file
193
app/images.py
Normal file
@@ -0,0 +1,193 @@
|
|||||||
|
"""Image metadata: media type sniffing and dimension parsing.
|
||||||
|
|
||||||
|
Two small jobs, both in service of the EPUB writer.
|
||||||
|
|
||||||
|
*Sniffing.* A feed image is as likely to arrive as ``application/octet-stream``
|
||||||
|
or ``text/plain`` as with a correct ``Content-Type``. An EPUB manifest entry
|
||||||
|
whose media type does not match the bytes is invalid, so the type is decided from
|
||||||
|
the actual magic bytes and the response header is only a fallback.
|
||||||
|
|
||||||
|
*Dimensions.* Picking a cover means knowing which image is big enough to be one.
|
||||||
|
Reading a PNG header or a JPEG start-of-frame costs nothing and avoids pulling in
|
||||||
|
an imaging library (and its C dependencies) just to answer "is this wide?".
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import struct
|
||||||
|
|
||||||
|
# Types the EPUB writer will embed. SVG is deliberately absent: it can carry
|
||||||
|
# script and external references, and an image in an article body has no reason
|
||||||
|
# to need either.
|
||||||
|
MEDIA_TYPES = {
|
||||||
|
"image/jpeg": ".jpg",
|
||||||
|
"image/png": ".png",
|
||||||
|
"image/gif": ".gif",
|
||||||
|
"image/webp": ".webp",
|
||||||
|
}
|
||||||
|
|
||||||
|
_MEDIA_TYPE_ALIASES = {
|
||||||
|
"image/jpg": "image/jpeg",
|
||||||
|
"image/pjpeg": "image/jpeg",
|
||||||
|
"image/x-png": "image/png",
|
||||||
|
"image/x-gif": "image/gif",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def normalise_media_type(raw: str | None) -> str | None:
|
||||||
|
"""Map a Content-Type header onto a media type this server will embed."""
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
# Strip parameters: "image/jpeg; charset=binary" is still a JPEG.
|
||||||
|
base = raw.split(";", 1)[0].strip().lower()
|
||||||
|
base = _MEDIA_TYPE_ALIASES.get(base, base)
|
||||||
|
return base if base in MEDIA_TYPES else None
|
||||||
|
|
||||||
|
|
||||||
|
def sniff_media_type(data: bytes) -> str | None:
|
||||||
|
"""Identify an image from its leading bytes, or None if unrecognised."""
|
||||||
|
if len(data) < 12:
|
||||||
|
return None
|
||||||
|
if data.startswith(b"\xff\xd8\xff"):
|
||||||
|
return "image/jpeg"
|
||||||
|
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||||
|
return "image/png"
|
||||||
|
if data.startswith((b"GIF87a", b"GIF89a")):
|
||||||
|
return "image/gif"
|
||||||
|
if data.startswith(b"RIFF") and data[8:12] == b"WEBP":
|
||||||
|
return "image/webp"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def identify(data: bytes, declared: str | None) -> str | None:
|
||||||
|
"""Decide the media type for a fetched image.
|
||||||
|
|
||||||
|
The bytes win. A declared type is used only when the bytes are recognisable
|
||||||
|
as nothing else, and never to *override* a successful sniff -- otherwise a
|
||||||
|
mislabelled payload would produce an invalid EPUB manifest.
|
||||||
|
"""
|
||||||
|
sniffed = sniff_media_type(data)
|
||||||
|
if sniffed is not None:
|
||||||
|
return sniffed
|
||||||
|
return normalise_media_type(declared)
|
||||||
|
|
||||||
|
|
||||||
|
def dimensions(data: bytes) -> tuple[int, int] | None:
|
||||||
|
"""Return ``(width, height)`` in pixels, or None when it cannot be read.
|
||||||
|
|
||||||
|
Only the container headers are parsed: the value is used to decide whether an
|
||||||
|
image is a plausible cover, so "unknown" is a perfectly good answer.
|
||||||
|
"""
|
||||||
|
sniffed = sniff_media_type(data)
|
||||||
|
try:
|
||||||
|
if sniffed == "image/png":
|
||||||
|
return _png_dimensions(data)
|
||||||
|
if sniffed == "image/gif":
|
||||||
|
return _gif_dimensions(data)
|
||||||
|
if sniffed == "image/jpeg":
|
||||||
|
return _jpeg_dimensions(data)
|
||||||
|
if sniffed == "image/webp":
|
||||||
|
return _webp_dimensions(data)
|
||||||
|
except (struct.error, IndexError, ValueError):
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _png_dimensions(data: bytes) -> tuple[int, int] | None:
|
||||||
|
# IHDR is required to be the first chunk: 8 signature bytes, then length and
|
||||||
|
# type, so width and height sit at a fixed offset.
|
||||||
|
if len(data) < 24:
|
||||||
|
return None
|
||||||
|
width, height = struct.unpack(">II", data[16:24])
|
||||||
|
return (width, height) if width and height else None
|
||||||
|
|
||||||
|
|
||||||
|
def _gif_dimensions(data: bytes) -> tuple[int, int] | None:
|
||||||
|
if len(data) < 10:
|
||||||
|
return None
|
||||||
|
width, height = struct.unpack("<HH", data[6:10])
|
||||||
|
return (width, height) if width and height else None
|
||||||
|
|
||||||
|
|
||||||
|
def _webp_dimensions(data: bytes) -> tuple[int, int] | None:
|
||||||
|
if len(data) < 30:
|
||||||
|
return None
|
||||||
|
chunk = data[12:16]
|
||||||
|
if chunk == b"VP8X":
|
||||||
|
width = int.from_bytes(data[24:27], "little") + 1
|
||||||
|
height = int.from_bytes(data[27:30], "little") + 1
|
||||||
|
return (width, height)
|
||||||
|
if chunk == b"VP8 ":
|
||||||
|
# Lossy: dimensions are 14 bytes into the frame header, after the
|
||||||
|
# start code, with 14 bits of width and 14 of height.
|
||||||
|
if len(data) < 30:
|
||||||
|
return None
|
||||||
|
start = 26
|
||||||
|
width = int.from_bytes(data[start : start + 2], "little") & 0x3FFF
|
||||||
|
height = int.from_bytes(data[start + 2 : start + 4], "little") & 0x3FFF
|
||||||
|
return (width, height) if width and height else None
|
||||||
|
if chunk == b"VP8L":
|
||||||
|
bits = int.from_bytes(data[21:25], "little")
|
||||||
|
width = (bits & 0x3FFF) + 1
|
||||||
|
height = ((bits >> 14) & 0x3FFF) + 1
|
||||||
|
return (width, height)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# JPEG start-of-frame markers, which carry the dimensions.
|
||||||
|
_SOF_MARKERS = frozenset(
|
||||||
|
{
|
||||||
|
0xC0,
|
||||||
|
0xC1,
|
||||||
|
0xC2,
|
||||||
|
0xC3,
|
||||||
|
0xC5,
|
||||||
|
0xC6,
|
||||||
|
0xC7,
|
||||||
|
0xC9,
|
||||||
|
0xCA,
|
||||||
|
0xCB,
|
||||||
|
0xCD,
|
||||||
|
0xCE,
|
||||||
|
0xCF,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _jpeg_dimensions(data: bytes) -> tuple[int, int] | None:
|
||||||
|
"""Walk the JPEG segment chain to the start-of-frame marker.
|
||||||
|
|
||||||
|
A JPEG has no fixed header offset, so the segments have to be stepped over
|
||||||
|
until the frame header appears. This is bounded work: it stops at the first
|
||||||
|
frame marker, which always precedes the image data.
|
||||||
|
"""
|
||||||
|
index = 2 # skip the SOI marker (FF D8)
|
||||||
|
limit = len(data)
|
||||||
|
while index + 9 < limit:
|
||||||
|
if data[index] != 0xFF:
|
||||||
|
index += 1
|
||||||
|
continue
|
||||||
|
marker = data[index + 1]
|
||||||
|
if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7:
|
||||||
|
index += 2
|
||||||
|
continue
|
||||||
|
if marker == 0xDA: # start of scan: no frame header found
|
||||||
|
return None
|
||||||
|
length = struct.unpack(">H", data[index + 2 : index + 4])[0]
|
||||||
|
if length < 2:
|
||||||
|
return None
|
||||||
|
if marker in _SOF_MARKERS:
|
||||||
|
height, width = struct.unpack(">HH", data[index + 5 : index + 9])
|
||||||
|
return (width, height) if width and height else None
|
||||||
|
index += 2 + length
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def looks_like_a_cover(width: int, height: int) -> bool:
|
||||||
|
"""Whether an image is a plausible book cover.
|
||||||
|
|
||||||
|
A lead image is usually landscape and wide; a thumbnail, icon or tracking
|
||||||
|
pixel is not. The bar is deliberately low -- it only has to beat a blank
|
||||||
|
tile in a reader's library view.
|
||||||
|
"""
|
||||||
|
return width >= 400 and height >= 200
|
||||||
308
app/opds.py
Normal file
308
app/opds.py
Normal file
@@ -0,0 +1,308 @@
|
|||||||
|
"""OPDS 1.2 catalog documents.
|
||||||
|
|
||||||
|
The dialect is Atom with the OPDS profile, because that is what ebook readers
|
||||||
|
actually implement: KOReader, Calibre, Moon+ Reader, Librera, FBReader and Aldiko
|
||||||
|
all browse OPDS 1.2, and its navigation/acquisition split maps exactly onto the
|
||||||
|
category/feed/article tree this server mirrors out of FreshRSS.
|
||||||
|
|
||||||
|
The two document kinds, and the rules that matter:
|
||||||
|
|
||||||
|
*Navigation feeds* (``kind=navigation``) are directories. Each entry is a
|
||||||
|
``rel="subsection"`` link to another feed, carrying a text description such as an
|
||||||
|
unread count.
|
||||||
|
|
||||||
|
*Acquisition feeds* (``kind=acquisition``) list articles. Each entry carries an
|
||||||
|
``rel="http://opds-spec.org/acquisition/open-access"`` link with an EPUB media
|
||||||
|
type, which is what a reader downloads. A plain ``http://opds-spec.org/acquisition``
|
||||||
|
link to the same URL is also emitted: readers disagree about which relation to
|
||||||
|
look for, and both are valid ways to say the same thing. An ``alternate`` link
|
||||||
|
points at the original article, which readers show as "open in browser".
|
||||||
|
|
||||||
|
Every feed carries ``rel="self"`` with the profile in its type, plus ``rel="start"``
|
||||||
|
back to the root, because readers navigate by these rather than by URL. Pagination
|
||||||
|
uses ``rel="next"``/``rel="previous"``/``rel="first"``.
|
||||||
|
|
||||||
|
Documents are built as strings rather than through a DOM library: the output is
|
||||||
|
small, fully determined, and easy to read in a test failure. Every interpolated
|
||||||
|
value goes through an escaper, and the tests parse the result with a strict XML
|
||||||
|
parser.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from app import xhtml
|
||||||
|
|
||||||
|
ATOM_NAMESPACE = "http://www.w3.org/2005/Atom"
|
||||||
|
OPDS_NAMESPACE = "http://opds-spec.org/2010/catalog"
|
||||||
|
OPENSEARCH_NAMESPACE = "http://a9.com/-/spec/opensearch/1.1/"
|
||||||
|
DC_NAMESPACE = "http://purl.org/dc/elements/1.1/"
|
||||||
|
|
||||||
|
NAVIGATION_TYPE = "application/atom+xml;profile=opds-catalog;kind=navigation"
|
||||||
|
ACQUISITION_TYPE = "application/atom+xml;profile=opds-catalog;kind=acquisition"
|
||||||
|
OPENSEARCH_TYPE = "application/opensearchdescription+xml"
|
||||||
|
|
||||||
|
EPUB_TYPE = "application/epub+zip"
|
||||||
|
HTML_TYPE = "text/html"
|
||||||
|
|
||||||
|
ACQUISITION_REL = "http://opds-spec.org/acquisition"
|
||||||
|
OPEN_ACCESS_REL = "http://opds-spec.org/acquisition/open-access"
|
||||||
|
SUBSECTION_REL = "subsection"
|
||||||
|
ALTERNATE_REL = "alternate"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class NavEntry:
|
||||||
|
"""One row of a navigation feed: a link deeper into the catalog."""
|
||||||
|
|
||||||
|
title: str
|
||||||
|
entry_id: str
|
||||||
|
href: str
|
||||||
|
updated: datetime
|
||||||
|
description: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AcquisitionEntry:
|
||||||
|
"""One row of an acquisition feed: an article, as a downloadable book."""
|
||||||
|
|
||||||
|
title: str
|
||||||
|
entry_id: str
|
||||||
|
updated: datetime
|
||||||
|
download_href: str
|
||||||
|
author: str | None = None
|
||||||
|
summary: str | None = None
|
||||||
|
language: str | None = None
|
||||||
|
alternate_href: str | None = None
|
||||||
|
categories: tuple[tuple[str, str], ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PageLinks:
|
||||||
|
"""Pagination targets, already absolute. Omitting one omits its link."""
|
||||||
|
|
||||||
|
first: str | None = None
|
||||||
|
previous: str | None = None
|
||||||
|
next: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FeedLinks:
|
||||||
|
"""Everything a reader needs to place a feed in the catalog.
|
||||||
|
|
||||||
|
Pagination lives only in :attr:`pages`: an earlier version also carried
|
||||||
|
``next_url``/``previous_url``, and the renderer read those while callers set
|
||||||
|
``pages``, so the links silently never appeared.
|
||||||
|
"""
|
||||||
|
|
||||||
|
self_url: str
|
||||||
|
start_url: str | None = None
|
||||||
|
search_url: str | None = None
|
||||||
|
pages: PageLinks = field(default_factory=PageLinks)
|
||||||
|
|
||||||
|
|
||||||
|
def navigation_feed(
|
||||||
|
*,
|
||||||
|
feed_id: str,
|
||||||
|
title: str,
|
||||||
|
updated: datetime,
|
||||||
|
links: FeedLinks,
|
||||||
|
entries: list[NavEntry],
|
||||||
|
author: str = "OPDS",
|
||||||
|
subtitle: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""Render a navigation feed."""
|
||||||
|
return _feed(
|
||||||
|
feed_id=feed_id,
|
||||||
|
title=title,
|
||||||
|
updated=updated,
|
||||||
|
links=links,
|
||||||
|
entries=[_nav_entry_xml(entry) for entry in entries],
|
||||||
|
kind_type=NAVIGATION_TYPE,
|
||||||
|
author=author,
|
||||||
|
subtitle=subtitle,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def acquisition_feed(
|
||||||
|
*,
|
||||||
|
feed_id: str,
|
||||||
|
title: str,
|
||||||
|
updated: datetime,
|
||||||
|
links: FeedLinks,
|
||||||
|
entries: list[AcquisitionEntry],
|
||||||
|
author: str = "OPDS",
|
||||||
|
subtitle: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""Render an acquisition feed."""
|
||||||
|
return _feed(
|
||||||
|
feed_id=feed_id,
|
||||||
|
title=title,
|
||||||
|
updated=updated,
|
||||||
|
links=links,
|
||||||
|
entries=[_acquisition_entry_xml(entry) for entry in entries],
|
||||||
|
kind_type=ACQUISITION_TYPE,
|
||||||
|
author=author,
|
||||||
|
subtitle=subtitle,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def opensearch_description(
|
||||||
|
*,
|
||||||
|
search_url: str,
|
||||||
|
short_name: str,
|
||||||
|
description: str,
|
||||||
|
output_type: str = ACQUISITION_TYPE,
|
||||||
|
) -> str:
|
||||||
|
"""Render an OpenSearch description document.
|
||||||
|
|
||||||
|
Readers that support search advertise it by fetching this document, which is
|
||||||
|
why the root navigation feed links to it with ``rel="search"``.
|
||||||
|
"""
|
||||||
|
return "\n".join(
|
||||||
|
[
|
||||||
|
'<?xml version="1.0" encoding="utf-8"?>',
|
||||||
|
f'<OpenSearchDescription xmlns="{OPENSEARCH_NAMESPACE}">',
|
||||||
|
f" <ShortName>{_escape(short_name)}</ShortName>",
|
||||||
|
f" <Description>{_escape(description)}</Description>",
|
||||||
|
" <InputEncoding>UTF-8</InputEncoding>",
|
||||||
|
" <OutputEncoding>UTF-8</OutputEncoding>",
|
||||||
|
f' <Url type="{_attribute(output_type)}" '
|
||||||
|
f'template="{_attribute(search_url)}?q={{searchTerms}}&page={{startPage?}}"/>',
|
||||||
|
"</OpenSearchDescription>",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# -- feed assembly -------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _feed(
|
||||||
|
*,
|
||||||
|
feed_id: str,
|
||||||
|
title: str,
|
||||||
|
updated: datetime,
|
||||||
|
links: FeedLinks,
|
||||||
|
entries: list[str],
|
||||||
|
kind_type: str,
|
||||||
|
author: str,
|
||||||
|
subtitle: str | None,
|
||||||
|
) -> str:
|
||||||
|
lines = [
|
||||||
|
'<?xml version="1.0" encoding="utf-8"?>',
|
||||||
|
f'<feed xmlns="{ATOM_NAMESPACE}" xmlns:opds="{OPDS_NAMESPACE}" xmlns:dc="{DC_NAMESPACE}">',
|
||||||
|
f" <id>{_escape(feed_id)}</id>",
|
||||||
|
f" <title>{_escape(title)}</title>",
|
||||||
|
f" <updated>{_timestamp(updated)}</updated>",
|
||||||
|
]
|
||||||
|
if subtitle:
|
||||||
|
lines.append(f" <subtitle>{_escape(subtitle)}</subtitle>")
|
||||||
|
lines.append(f" <author><name>{_escape(author)}</name></author>")
|
||||||
|
lines.extend(_feed_links_xml(links, kind_type))
|
||||||
|
lines.extend(entries)
|
||||||
|
lines.extend(["</feed>", ""])
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def _feed_links_xml(links: FeedLinks, kind_type: str) -> list[str]:
|
||||||
|
"""Self, start, search and pagination links, in that order."""
|
||||||
|
rendered = [
|
||||||
|
f' <link rel="self" href="{_attribute(links.self_url)}" type="{_attribute(kind_type)}"/>'
|
||||||
|
]
|
||||||
|
if links.start_url:
|
||||||
|
rendered.append(
|
||||||
|
f' <link rel="start" href="{_attribute(links.start_url)}" '
|
||||||
|
f'type="{_attribute(NAVIGATION_TYPE)}"/>'
|
||||||
|
)
|
||||||
|
if links.search_url:
|
||||||
|
rendered.append(
|
||||||
|
f' <link rel="search" href="{_attribute(links.search_url)}" '
|
||||||
|
f'type="{_attribute(OPENSEARCH_TYPE)}"/>'
|
||||||
|
)
|
||||||
|
first = links.pages.first
|
||||||
|
if first:
|
||||||
|
rendered.append(
|
||||||
|
f' <link rel="first" href="{_attribute(first)}" type="{_attribute(kind_type)}"/>'
|
||||||
|
)
|
||||||
|
if links.pages.previous:
|
||||||
|
rendered.append(
|
||||||
|
f' <link rel="previous" href="{_attribute(links.pages.previous)}" '
|
||||||
|
f'type="{_attribute(kind_type)}"/>'
|
||||||
|
)
|
||||||
|
if links.pages.next:
|
||||||
|
rendered.append(
|
||||||
|
f' <link rel="next" href="{_attribute(links.pages.next)}" '
|
||||||
|
f'type="{_attribute(kind_type)}"/>'
|
||||||
|
)
|
||||||
|
return rendered
|
||||||
|
|
||||||
|
|
||||||
|
def _nav_entry_xml(entry: NavEntry) -> str:
|
||||||
|
lines = [
|
||||||
|
" <entry>",
|
||||||
|
f" <title>{_escape(entry.title)}</title>",
|
||||||
|
f" <id>{_escape(entry.entry_id)}</id>",
|
||||||
|
f" <updated>{_timestamp(entry.updated)}</updated>",
|
||||||
|
]
|
||||||
|
if entry.description:
|
||||||
|
lines.append(f' <content type="text">{_escape(entry.description)}</content>')
|
||||||
|
lines.extend(
|
||||||
|
[
|
||||||
|
f' <link rel="{SUBSECTION_REL}" href="{_attribute(entry.href)}" '
|
||||||
|
f'type="{_attribute(NAVIGATION_TYPE)}"/>',
|
||||||
|
" </entry>",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def _acquisition_entry_xml(entry: AcquisitionEntry) -> str:
|
||||||
|
lines = [
|
||||||
|
" <entry>",
|
||||||
|
f" <title>{_escape(entry.title)}</title>",
|
||||||
|
f" <id>{_escape(entry.entry_id)}</id>",
|
||||||
|
f" <updated>{_timestamp(entry.updated)}</updated>",
|
||||||
|
]
|
||||||
|
if entry.author:
|
||||||
|
lines.append(f" <author><name>{_escape(entry.author)}</name></author>")
|
||||||
|
if entry.summary:
|
||||||
|
lines.append(f' <summary type="text">{_escape(entry.summary)}</summary>')
|
||||||
|
if entry.language:
|
||||||
|
lines.append(f" <dc:language>{_escape(entry.language)}</dc:language>")
|
||||||
|
for term, label in entry.categories:
|
||||||
|
lines.append(f' <category term="{_attribute(term)}" label="{_attribute(label)}"/>')
|
||||||
|
# Two acquisition relations to the same resource, because readers disagree
|
||||||
|
# about which one to look for; both are valid.
|
||||||
|
for rel in (OPEN_ACCESS_REL, ACQUISITION_REL):
|
||||||
|
lines.append(
|
||||||
|
f' <link rel="{rel}" href="{_attribute(entry.download_href)}" '
|
||||||
|
f'type="{_attribute(EPUB_TYPE)}"/>'
|
||||||
|
)
|
||||||
|
if entry.alternate_href:
|
||||||
|
lines.append(
|
||||||
|
f' <link rel="{ALTERNATE_REL}" href="{_attribute(entry.alternate_href)}" '
|
||||||
|
f'type="{_attribute(HTML_TYPE)}"/>'
|
||||||
|
)
|
||||||
|
lines.append(" </entry>")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
# -- formatting ----------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _escape(text: str) -> str:
|
||||||
|
"""Escape text for a document body."""
|
||||||
|
return xhtml.escape_text(text)
|
||||||
|
|
||||||
|
|
||||||
|
def _attribute(value: str) -> str:
|
||||||
|
"""Escape a value for a double-quoted attribute."""
|
||||||
|
return xhtml.escape_attribute(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _timestamp(moment: datetime) -> str:
|
||||||
|
"""Atom dates are RFC 3339 in UTC."""
|
||||||
|
return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
1075
app/server.py
Normal file
1075
app/server.py
Normal file
File diff suppressed because it is too large
Load Diff
120
app/urls.py
Normal file
120
app/urls.py
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
"""Identifier, slug and URL helpers.
|
||||||
|
|
||||||
|
FreshRSS ids do not survive a URL path or a Content-Disposition filename
|
||||||
|
unharmed. Item ids look like ``tag:google.com,2005:reader/item/0000000000000abc``
|
||||||
|
and stream ids look like ``feed/https://example.com/atom.xml?x=1``, so both
|
||||||
|
carry characters that are structural in a path (``/``, ``?``, ``#``) and are
|
||||||
|
long enough to blow past filename limits.
|
||||||
|
|
||||||
|
The rule this module enforces: every opaque id that appears in a path is
|
||||||
|
base64url-encoded, and the human-readable part of a download URL is a slug that
|
||||||
|
the server ignores when it looks the item up. That keeps routes unambiguous
|
||||||
|
without any escaping tricks, and lets a reader name the saved file sensibly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import binascii
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
# Fixed namespace for every derived urn:uuid in this server. Hardcoded so that
|
||||||
|
# feed and entry ids stay stable across restarts and across hostnames -- a client
|
||||||
|
# that cached an entry id must still recognise it tomorrow.
|
||||||
|
_UUID_NAMESPACE = uuid.UUID("b7437da1-d469-46b1-a582-0af386fadb9d")
|
||||||
|
|
||||||
|
_B64URL_RE = re.compile(r"\A[A-Za-z0-9_-]+\Z")
|
||||||
|
# Everything that is not an ASCII lowercase letter or digit becomes a dash, so
|
||||||
|
# the result is safe in a path segment, a filename, and a shell.
|
||||||
|
_SLUG_JUNK_RE = re.compile(r"[^a-z0-9]+")
|
||||||
|
|
||||||
|
|
||||||
|
def b64url_encode(value: str) -> str:
|
||||||
|
"""Encode a UTF-8 string as unpadded base64url (path-segment safe)."""
|
||||||
|
raw = value.encode("utf-8")
|
||||||
|
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
def b64url_decode(value: str) -> str:
|
||||||
|
"""Decode an unpadded base64url string, raising ValueError on junk.
|
||||||
|
|
||||||
|
Strict on purpose: this parses untrusted path input, so anything that is not
|
||||||
|
a well-formed unpadded base64url token -- or that decodes to something other
|
||||||
|
than UTF-8 -- is rejected instead of being coerced.
|
||||||
|
"""
|
||||||
|
if not value or not _B64URL_RE.match(value):
|
||||||
|
raise ValueError("not a base64url token")
|
||||||
|
padding = "=" * (-len(value) % 4)
|
||||||
|
try:
|
||||||
|
raw = base64.urlsafe_b64decode(value + padding)
|
||||||
|
except (binascii.Error, ValueError) as exc:
|
||||||
|
raise ValueError("not a base64url token") from exc
|
||||||
|
try:
|
||||||
|
return raw.decode("utf-8")
|
||||||
|
except UnicodeDecodeError as exc:
|
||||||
|
raise ValueError("base64url token is not UTF-8") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def slugify(text: str, max_length: int = 60) -> str:
|
||||||
|
"""Reduce a title to a conservative filename slug.
|
||||||
|
|
||||||
|
Accented characters are folded to ASCII rather than dropped, non-ASCII
|
||||||
|
scripts fall back to ``article``, and the result is length-capped because a
|
||||||
|
filesystem name is the eventual destination.
|
||||||
|
"""
|
||||||
|
folded = unicodedata.normalize("NFKD", text or "")
|
||||||
|
ascii_text = folded.encode("ascii", "ignore").decode("ascii")
|
||||||
|
slug = _SLUG_JUNK_RE.sub("-", ascii_text.lower()).strip("-")
|
||||||
|
if len(slug) > max_length:
|
||||||
|
slug = slug[:max_length].rstrip("-")
|
||||||
|
return slug or "article"
|
||||||
|
|
||||||
|
|
||||||
|
def stable_urn(kind: str, value: str) -> str:
|
||||||
|
"""Return a deterministic ``urn:uuid`` for a feed, entry or item.
|
||||||
|
|
||||||
|
Atom requires every feed and entry to carry a permanent id. Deriving it from
|
||||||
|
the FreshRSS id rather than from the request URL means the same article keeps
|
||||||
|
one identity no matter which category path the reader reached it through.
|
||||||
|
"""
|
||||||
|
derived = uuid.uuid5(_UUID_NAMESPACE, f"{kind}:{value}")
|
||||||
|
return f"urn:uuid:{derived}"
|
||||||
|
|
||||||
|
|
||||||
|
def content_disposition(ascii_name: str, unicode_name: str | None = None) -> str:
|
||||||
|
"""Build an attachment Content-Disposition with an RFC 5987 fallback.
|
||||||
|
|
||||||
|
Every OPDS reader understands ``filename``; readers that save a title with
|
||||||
|
non-ASCII characters need ``filename*``. Sending both is the only way to get
|
||||||
|
a correct name everywhere, so both are always emitted.
|
||||||
|
"""
|
||||||
|
ascii_name = ascii_name.replace('"', "").replace("\\", "") or "article.epub"
|
||||||
|
header = f'attachment; filename="{ascii_name}"'
|
||||||
|
if unicode_name and unicode_name != ascii_name:
|
||||||
|
encoded = _quote_rfc5987(unicode_name.replace('"', "").replace("\\", ""))
|
||||||
|
if encoded:
|
||||||
|
header += f"; filename*=UTF-8''{encoded}"
|
||||||
|
return header
|
||||||
|
|
||||||
|
|
||||||
|
def _quote_rfc5987(value: str) -> str:
|
||||||
|
out = []
|
||||||
|
for byte in value.encode("utf-8"):
|
||||||
|
char = chr(byte)
|
||||||
|
if char.isalnum() and byte < 0x80 or char in "-._~":
|
||||||
|
out.append(char)
|
||||||
|
else:
|
||||||
|
out.append(f"%{byte:02X}")
|
||||||
|
return "".join(out)
|
||||||
|
|
||||||
|
|
||||||
|
def join_url(base: str, *parts: str) -> str:
|
||||||
|
"""Join URL path parts onto a base, collapsing duplicate slashes."""
|
||||||
|
segments = [base.rstrip("/")]
|
||||||
|
for part in parts:
|
||||||
|
cleaned = part.strip("/")
|
||||||
|
if cleaned:
|
||||||
|
segments.append(cleaned)
|
||||||
|
return "/".join(segments)
|
||||||
747
app/xhtml.py
Normal file
747
app/xhtml.py
Normal file
@@ -0,0 +1,747 @@
|
|||||||
|
"""Turn arbitrary feed HTML into well-formed XHTML for the EPUB pipeline.
|
||||||
|
|
||||||
|
EPUB is XML end to end: the OPF, the NCX, the navigation document and every
|
||||||
|
content document are parsed by a strict XML parser (the reader firmware this
|
||||||
|
server was written alongside uses expat). Feed article bodies are the opposite
|
||||||
|
of strict -- unclosed ``<li>``, stray ``&``, ``<br>`` instead of ``<br/>``,
|
||||||
|
control characters, embedded scripts and tracking pixels. Handing that to the
|
||||||
|
EPUB writer would produce a book that some readers refuse to open, so everything
|
||||||
|
goes through this normaliser first.
|
||||||
|
|
||||||
|
How it stays safe, in order of importance:
|
||||||
|
|
||||||
|
* The input is parsed into an in-memory tree and re-serialised from scratch.
|
||||||
|
Any malformedness in the source therefore cannot reach the output: whatever the
|
||||||
|
parser tolerated is rebuilt with balanced tags, quoted attributes and escaped
|
||||||
|
text. There is no pass-through of raw markup.
|
||||||
|
* Characters that XML 1.0 forbids outright (most C0 controls, unpaired
|
||||||
|
surrogates) are removed, because they are not expressible even when escaped.
|
||||||
|
* Entities are decoded to real Unicode by the parser and re-escaped on output, so
|
||||||
|
a stray ``&`` in the source cannot produce a document that will not parse.
|
||||||
|
* Scriptable and metadata elements are dropped with their contents; every other
|
||||||
|
unknown element is unwrapped, keeping its text.
|
||||||
|
* Only URLs that survive scheme validation reach an attribute, which keeps
|
||||||
|
``javascript:`` and ``data:`` payloads out of the book.
|
||||||
|
|
||||||
|
The tree is kept around after parsing so a caller can collect image URLs, fetch
|
||||||
|
them, and then serialise with the fetched images rewritten to local paths.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
from urllib.parse import urljoin, urlsplit
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Elements kept in the output. Mirrors the firmware reader's sanitiser
|
||||||
|
# (lib/FreshRss/src/FreshRssArticle.cpp) with the additions that make sense on a
|
||||||
|
# device with a real layout engine: images, and the small semantic set EPUB
|
||||||
|
# readers style (q, cite, time, abbr, del, ins, small, mark).
|
||||||
|
KEPT_TAGS = frozenset(
|
||||||
|
{
|
||||||
|
"p",
|
||||||
|
"div",
|
||||||
|
"span",
|
||||||
|
"br",
|
||||||
|
"hr",
|
||||||
|
"h1",
|
||||||
|
"h2",
|
||||||
|
"h3",
|
||||||
|
"h4",
|
||||||
|
"h5",
|
||||||
|
"h6",
|
||||||
|
"ul",
|
||||||
|
"ol",
|
||||||
|
"li",
|
||||||
|
"dl",
|
||||||
|
"dt",
|
||||||
|
"dd",
|
||||||
|
"blockquote",
|
||||||
|
"pre",
|
||||||
|
"code",
|
||||||
|
"em",
|
||||||
|
"i",
|
||||||
|
"strong",
|
||||||
|
"b",
|
||||||
|
"u",
|
||||||
|
"s",
|
||||||
|
"sub",
|
||||||
|
"sup",
|
||||||
|
"a",
|
||||||
|
"img",
|
||||||
|
"figure",
|
||||||
|
"figcaption",
|
||||||
|
"table",
|
||||||
|
"thead",
|
||||||
|
"tbody",
|
||||||
|
"tfoot",
|
||||||
|
"tr",
|
||||||
|
"td",
|
||||||
|
"th",
|
||||||
|
"caption",
|
||||||
|
"small",
|
||||||
|
"mark",
|
||||||
|
"cite",
|
||||||
|
"time",
|
||||||
|
"abbr",
|
||||||
|
"q",
|
||||||
|
"ins",
|
||||||
|
"del",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Dropped along with everything inside them. These are either scriptable,
|
||||||
|
# fetch extra resources, or are document-level metadata that has no place inside
|
||||||
|
# an EPUB content document.
|
||||||
|
DROP_WITH_CONTENT = frozenset(
|
||||||
|
{
|
||||||
|
"script",
|
||||||
|
"style",
|
||||||
|
"iframe",
|
||||||
|
"frame",
|
||||||
|
"frameset",
|
||||||
|
"object",
|
||||||
|
"embed",
|
||||||
|
"applet",
|
||||||
|
"svg",
|
||||||
|
"math",
|
||||||
|
"form",
|
||||||
|
"input",
|
||||||
|
"button",
|
||||||
|
"select",
|
||||||
|
"option",
|
||||||
|
"optgroup",
|
||||||
|
"textarea",
|
||||||
|
"label",
|
||||||
|
"fieldset",
|
||||||
|
"legend",
|
||||||
|
"noscript",
|
||||||
|
"template",
|
||||||
|
"canvas",
|
||||||
|
"video",
|
||||||
|
"audio",
|
||||||
|
"source",
|
||||||
|
"track",
|
||||||
|
"map",
|
||||||
|
"area",
|
||||||
|
"head",
|
||||||
|
"title",
|
||||||
|
"meta",
|
||||||
|
"link",
|
||||||
|
"base",
|
||||||
|
"dialog",
|
||||||
|
"slot",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# HTML void elements: no end tag exists, so the parser must not expect one.
|
||||||
|
VOID_TAGS = frozenset({"br", "hr", "img"})
|
||||||
|
_HTML_VOID = frozenset(
|
||||||
|
{
|
||||||
|
"area",
|
||||||
|
"base",
|
||||||
|
"br",
|
||||||
|
"col",
|
||||||
|
"embed",
|
||||||
|
"hr",
|
||||||
|
"img",
|
||||||
|
"input",
|
||||||
|
"link",
|
||||||
|
"meta",
|
||||||
|
"param",
|
||||||
|
"source",
|
||||||
|
"track",
|
||||||
|
"wbr",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Elements that silently end an open element in HTML. Feeds are full of
|
||||||
|
# unclosed <p> and <li>, and the browser-correct fix is to close the open
|
||||||
|
# element rather than to nest one block inside another (which is what a naive
|
||||||
|
# stack would do, producing <p>text<div>...</div></p>).
|
||||||
|
_IMPLICIT_CLOSE: dict[str, frozenset[str]] = {
|
||||||
|
"p": frozenset(
|
||||||
|
{
|
||||||
|
"address",
|
||||||
|
"article",
|
||||||
|
"aside",
|
||||||
|
"blockquote",
|
||||||
|
"details",
|
||||||
|
"div",
|
||||||
|
"dl",
|
||||||
|
"fieldset",
|
||||||
|
"figcaption",
|
||||||
|
"figure",
|
||||||
|
"footer",
|
||||||
|
"form",
|
||||||
|
"h1",
|
||||||
|
"h2",
|
||||||
|
"h3",
|
||||||
|
"h4",
|
||||||
|
"h5",
|
||||||
|
"h6",
|
||||||
|
"header",
|
||||||
|
"hr",
|
||||||
|
"main",
|
||||||
|
"nav",
|
||||||
|
"ol",
|
||||||
|
"p",
|
||||||
|
"pre",
|
||||||
|
"section",
|
||||||
|
"table",
|
||||||
|
"ul",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"li": frozenset({"li"}),
|
||||||
|
"dt": frozenset({"dt", "dd"}),
|
||||||
|
"dd": frozenset({"dt", "dd"}),
|
||||||
|
"td": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
|
||||||
|
"th": frozenset({"td", "th", "tr", "tbody", "tfoot", "thead"}),
|
||||||
|
"tr": frozenset({"tr", "tbody", "tfoot", "thead"}),
|
||||||
|
"option": frozenset({"option", "optgroup"}),
|
||||||
|
"thead": frozenset({"tbody", "tfoot"}),
|
||||||
|
"tbody": frozenset({"tbody", "tfoot"}),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Attributes kept, per element. Everything else is discarded: inline styles and
|
||||||
|
# layout attributes fight the EPUB stylesheet, and event handlers, tracking
|
||||||
|
# attributes and class names have no meaning in a book.
|
||||||
|
_KEPT_ATTRS: dict[str, frozenset[str]] = {
|
||||||
|
"a": frozenset({"href", "title"}),
|
||||||
|
"img": frozenset({"src", "alt", "title"}),
|
||||||
|
"blockquote": frozenset({"cite"}),
|
||||||
|
"q": frozenset({"cite"}),
|
||||||
|
"time": frozenset({"datetime"}),
|
||||||
|
"ins": frozenset({"datetime"}),
|
||||||
|
"del": frozenset({"datetime"}),
|
||||||
|
"abbr": frozenset({"title"}),
|
||||||
|
"td": frozenset({"colspan", "rowspan"}),
|
||||||
|
"th": frozenset({"colspan", "rowspan"}),
|
||||||
|
"ol": frozenset({"start"}),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Empty structural elements worth removing once their contents are gone (for
|
||||||
|
# example a wrapper whose only child was a rejected image). Table cells are
|
||||||
|
# deliberately absent: an empty cell is layout, not noise.
|
||||||
|
_DROPPABLE_WHEN_EMPTY = frozenset(
|
||||||
|
{
|
||||||
|
"p",
|
||||||
|
"div",
|
||||||
|
"span",
|
||||||
|
"li",
|
||||||
|
"ul",
|
||||||
|
"ol",
|
||||||
|
"dl",
|
||||||
|
"blockquote",
|
||||||
|
"figure",
|
||||||
|
"figcaption",
|
||||||
|
"section",
|
||||||
|
"article",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
_LINK_SCHEMES = ("http", "https", "mailto")
|
||||||
|
_IMAGE_SCHEMES = ("http", "https")
|
||||||
|
|
||||||
|
# XML 1.0 permits #x9, #xA, #xD, #x20-#xD7FF, #xE000-#xFFFD, #x10000-#x10FFFF.
|
||||||
|
# Everything else -- NUL bytes, vertical tabs from mangled feeds, form feeds,
|
||||||
|
# unpaired surrogates -- cannot be represented even as a character reference and
|
||||||
|
# must be deleted before serialisation.
|
||||||
|
_ILLEGAL_XML_RE = re.compile(
|
||||||
|
"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]"
|
||||||
|
)
|
||||||
|
# Collapses runs of layout whitespace. A non-breaking space is deliberately not
|
||||||
|
# matched: feeds use it for intentional spacing, and reflowable text keeps it.
|
||||||
|
_WHITESPACE_RE = re.compile(r"[^\S\xa0]+")
|
||||||
|
# Trims layout whitespace from the edges of a block. Python's str.strip() treats
|
||||||
|
# a non-breaking space as whitespace and would eat one the author typed on
|
||||||
|
# purpose, so the pattern excludes it here too.
|
||||||
|
_BLOCK_TRIM_RE = re.compile(r"\A[^\S\xa0]+|[^\S\xa0]+\Z")
|
||||||
|
_ZERO_WIDTH_RE = re.compile("[\u200b\u200c\u200d\u2060\ufeff]")
|
||||||
|
# XML name rules for an id: starts with a letter or underscore, then a bounded run
|
||||||
|
# of name characters. Anything else is not a usable anchor target.
|
||||||
|
_ID_RE = re.compile(r"\A[A-Za-z_][A-Za-z0-9_.-]{0,63}\Z")
|
||||||
|
_ID_ATTRS = frozenset({"id"})
|
||||||
|
|
||||||
|
# Wrappers that need the attribute filter relaxed for integer attributes.
|
||||||
|
_INT_ATTR_LIMITS = {"colspan": (1, 100), "rowspan": (1, 100), "start": (-10000, 10000)}
|
||||||
|
|
||||||
|
|
||||||
|
class _Element:
|
||||||
|
"""One element node in the parsed tree."""
|
||||||
|
|
||||||
|
__slots__ = ("attrs", "children", "tag")
|
||||||
|
|
||||||
|
def __init__(self, tag: str) -> None:
|
||||||
|
self.tag = tag
|
||||||
|
self.attrs: dict[str, str] = {}
|
||||||
|
self.children: list[_Element | _Text] = []
|
||||||
|
|
||||||
|
|
||||||
|
class _Text:
|
||||||
|
"""One text node in the parsed tree."""
|
||||||
|
|
||||||
|
__slots__ = ("text",)
|
||||||
|
|
||||||
|
def __init__(self, text: str) -> None:
|
||||||
|
self.text = text
|
||||||
|
|
||||||
|
|
||||||
|
class _TreeBuilder(HTMLParser):
|
||||||
|
"""Build a normalised element tree from feed HTML.
|
||||||
|
|
||||||
|
The parser is tolerant by construction -- it never raises on malformed input,
|
||||||
|
it only decides what to keep -- so this class is the single place where all
|
||||||
|
of the "this feed is broken" handling lives.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
# convert_charrefs decodes &/é/ to Unicode for us, so the
|
||||||
|
# serialiser only ever has to escape what it writes itself.
|
||||||
|
super().__init__(convert_charrefs=True)
|
||||||
|
self.root = _Element("#root")
|
||||||
|
self._stack: list[_Element] = [self.root]
|
||||||
|
# Names of open drop-with-content elements; while non-empty, all input is
|
||||||
|
# discarded except the end tag that closes the innermost one.
|
||||||
|
self._suppressed: list[str] = []
|
||||||
|
|
||||||
|
# -- parser callbacks -------------------------------------------------
|
||||||
|
|
||||||
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||||
|
if self._suppressed:
|
||||||
|
if tag in DROP_WITH_CONTENT and tag not in _HTML_VOID:
|
||||||
|
self._suppressed.append(tag)
|
||||||
|
return
|
||||||
|
if tag in DROP_WITH_CONTENT:
|
||||||
|
if tag not in _HTML_VOID:
|
||||||
|
self._suppressed.append(tag)
|
||||||
|
return
|
||||||
|
self._close_implied(tag)
|
||||||
|
if tag in _HTML_VOID and tag not in VOID_TAGS:
|
||||||
|
# Void but not kept (<col>, <wbr>, <param>): it cannot have content,
|
||||||
|
# so it must never open a subtree that would swallow the rest.
|
||||||
|
# br/hr/img are void *and* kept, so they fall through to be emitted.
|
||||||
|
return
|
||||||
|
element = _Element(tag)
|
||||||
|
element.attrs = self._filter_attrs(tag, attrs)
|
||||||
|
self._stack[-1].children.append(element)
|
||||||
|
if tag not in VOID_TAGS:
|
||||||
|
self._stack.append(element)
|
||||||
|
|
||||||
|
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||||
|
# <br/> lands here. A self-closing element is complete: it is appended
|
||||||
|
# and never pushed, so there is nothing left open. A self-closing
|
||||||
|
# drop-with-content element has no content to discard and must not open a
|
||||||
|
# suppression region, or the rest of the document would be swallowed.
|
||||||
|
if self._suppressed or tag in DROP_WITH_CONTENT or tag in _HTML_VOID:
|
||||||
|
return
|
||||||
|
self._close_implied(tag)
|
||||||
|
element = _Element(tag)
|
||||||
|
element.attrs = self._filter_attrs(tag, attrs)
|
||||||
|
self._stack[-1].children.append(element)
|
||||||
|
|
||||||
|
def handle_endtag(self, tag: str) -> None:
|
||||||
|
if self._suppressed:
|
||||||
|
if tag == self._suppressed[-1]:
|
||||||
|
self._suppressed.pop()
|
||||||
|
return
|
||||||
|
if tag in _HTML_VOID:
|
||||||
|
return
|
||||||
|
# Close down to the matching open element, tolerating unclosed tags in
|
||||||
|
# between (this is exactly what browsers do).
|
||||||
|
for index in range(len(self._stack) - 1, 0, -1):
|
||||||
|
if self._stack[index].tag == tag:
|
||||||
|
del self._stack[index:]
|
||||||
|
return
|
||||||
|
# Stray end tag with no matching open element: ignore it.
|
||||||
|
|
||||||
|
def handle_data(self, data: str) -> None:
|
||||||
|
if self._suppressed or not data:
|
||||||
|
return
|
||||||
|
self._stack[-1].children.append(_Text(data))
|
||||||
|
|
||||||
|
def handle_comment(self, data: str) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def handle_decl(self, decl: str) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def handle_pi(self, data: str) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def unknown_decl(self, data: str) -> None:
|
||||||
|
if self._suppressed:
|
||||||
|
return
|
||||||
|
# An HTML parser treats <![CDATA[...]]> as a bogus comment and discards
|
||||||
|
# it, but a feed body that passed through an XML parser upstream can
|
||||||
|
# legitimately contain one, and its payload is real content. The parser
|
||||||
|
# hands over the section body with the closing ]] already stripped.
|
||||||
|
marker = "CDATA["
|
||||||
|
if data.startswith(marker):
|
||||||
|
inner = data[len(marker) :]
|
||||||
|
if inner:
|
||||||
|
self._stack[-1].children.append(_Text(inner))
|
||||||
|
|
||||||
|
# -- helpers ---------------------------------------------------------
|
||||||
|
|
||||||
|
def _close_implied(self, tag: str) -> None:
|
||||||
|
"""Close open elements that HTML would close implicitly."""
|
||||||
|
while len(self._stack) > 1:
|
||||||
|
conditions = _IMPLICIT_CLOSE.get(self._stack[-1].tag)
|
||||||
|
if conditions is None or tag not in conditions:
|
||||||
|
return
|
||||||
|
self._stack.pop()
|
||||||
|
|
||||||
|
def _filter_attrs(self, tag: str, attrs: list[tuple[str, str | None]]) -> dict[str, str]:
|
||||||
|
if tag not in KEPT_TAGS:
|
||||||
|
# The element is unwrapped later, so its attributes are never used.
|
||||||
|
return {}
|
||||||
|
allowed = _KEPT_ATTRS.get(tag, frozenset())
|
||||||
|
if tag not in VOID_TAGS:
|
||||||
|
# Anchor targets survive so an in-book table of contents still jumps
|
||||||
|
# to its section when the article is read offline.
|
||||||
|
allowed = allowed | _ID_ATTRS
|
||||||
|
kept: dict[str, str] = {}
|
||||||
|
for raw_name, raw_value in attrs:
|
||||||
|
name = raw_name.lower()
|
||||||
|
if name not in allowed:
|
||||||
|
continue
|
||||||
|
value = html.unescape(raw_value or "") if raw_value is not None else ""
|
||||||
|
value = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", value)).strip()
|
||||||
|
if name == "id":
|
||||||
|
if not _ID_RE.match(value):
|
||||||
|
continue
|
||||||
|
kept[name] = value
|
||||||
|
continue
|
||||||
|
if name in _INT_ATTR_LIMITS:
|
||||||
|
bounds = _INT_ATTR_LIMITS[name]
|
||||||
|
try:
|
||||||
|
number = int(value)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
if not bounds[0] <= number <= bounds[1]:
|
||||||
|
continue
|
||||||
|
value = str(number)
|
||||||
|
if not value:
|
||||||
|
continue
|
||||||
|
kept[name] = value
|
||||||
|
return kept
|
||||||
|
|
||||||
|
|
||||||
|
class Article:
|
||||||
|
"""A parsed, normalised article body ready for EPUB serialisation."""
|
||||||
|
|
||||||
|
def __init__(self, tree: _Element, base_url: str, truncated: bool) -> None:
|
||||||
|
self._tree = tree
|
||||||
|
self._base_url = base_url
|
||||||
|
self.truncated = truncated
|
||||||
|
self._image_urls: list[str] | None = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def image_urls(self) -> list[str]:
|
||||||
|
"""Absolute http(s) image URLs, in document order, de-duplicated."""
|
||||||
|
if self._image_urls is None:
|
||||||
|
seen: list[str] = []
|
||||||
|
known: set[str] = set()
|
||||||
|
for element in _iter_elements(self._tree):
|
||||||
|
if element.tag != "img":
|
||||||
|
continue
|
||||||
|
src = element.attrs.get("src", "")
|
||||||
|
if src and src not in known:
|
||||||
|
known.add(src)
|
||||||
|
seen.append(src)
|
||||||
|
self._image_urls = seen
|
||||||
|
return self._image_urls
|
||||||
|
|
||||||
|
def text(self, limit: int = 600) -> str:
|
||||||
|
"""Flattened plain text, for summaries and the search index."""
|
||||||
|
parts: list[str] = []
|
||||||
|
for element in _iter_text(self._tree):
|
||||||
|
parts.append(element.text)
|
||||||
|
collapsed = _WHITESPACE_RE.sub(" ", _ILLEGAL_XML_RE.sub("", "".join(parts))).strip()
|
||||||
|
if len(collapsed) <= limit:
|
||||||
|
return collapsed
|
||||||
|
# Cut at a word boundary so a summary never ends mid-word.
|
||||||
|
cut = collapsed[:limit].rsplit(" ", 1)[0]
|
||||||
|
return (cut or collapsed[:limit]).rstrip() + "\u2026"
|
||||||
|
|
||||||
|
def to_xhtml(self, image_map: dict[str, str | None] | None = None) -> str:
|
||||||
|
"""Serialise the body as an XHTML fragment.
|
||||||
|
|
||||||
|
``image_map`` maps an original absolute image URL to the local href to
|
||||||
|
use in its place; a value of None drops that image (and its wrapper, if it
|
||||||
|
becomes empty). Pass None to leave remote image URLs untouched -- the
|
||||||
|
caller must then declare remote resources in the OPF manifest.
|
||||||
|
|
||||||
|
Applying a map rewrites the tree in place, so an Article is rendered
|
||||||
|
once. The server builds a book and serialises it exactly once.
|
||||||
|
"""
|
||||||
|
if image_map is not None:
|
||||||
|
_apply_image_map(self._tree, image_map)
|
||||||
|
# A dropped image can leave an empty wrapper behind.
|
||||||
|
_prune_empty(self._tree)
|
||||||
|
body = _serialise_children(self._tree, inside_pre=False)
|
||||||
|
if not body.strip():
|
||||||
|
return ""
|
||||||
|
return f'<div class="article-body">\n{body}\n</div>'
|
||||||
|
|
||||||
|
|
||||||
|
class _Resolve:
|
||||||
|
"""Decides which URLs survive normalisation, and where they point."""
|
||||||
|
|
||||||
|
def __init__(self, base_url: str) -> None:
|
||||||
|
self.base_url = base_url
|
||||||
|
|
||||||
|
def link(self, raw: str) -> str | None:
|
||||||
|
# A same-document reference is left alone so it resolves against the book
|
||||||
|
# rather than against the web page the article came from. The matching id
|
||||||
|
# was kept by the attribute filter.
|
||||||
|
if raw.startswith("#"):
|
||||||
|
return raw
|
||||||
|
resolved = _absolutise(raw, self.base_url)
|
||||||
|
if resolved is None:
|
||||||
|
return None
|
||||||
|
if urlsplit(resolved).scheme.lower() not in _LINK_SCHEMES:
|
||||||
|
return None
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
def image(self, raw: str) -> str | None:
|
||||||
|
resolved = _absolutise(raw, self.base_url)
|
||||||
|
if resolved is None:
|
||||||
|
return None
|
||||||
|
if urlsplit(resolved).scheme.lower() not in _IMAGE_SCHEMES:
|
||||||
|
return None
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def parse_article(raw_html: str, base_url: str = "", max_input_bytes: int = 0) -> Article:
|
||||||
|
"""Parse a feed article body into a normalised tree.
|
||||||
|
|
||||||
|
``base_url`` is the article's own URL; relative links and images inside the
|
||||||
|
body are resolved against it, which is the only way they keep working once
|
||||||
|
the body is inside a book with a different base.
|
||||||
|
"""
|
||||||
|
text = _ILLEGAL_XML_RE.sub("", _ZERO_WIDTH_RE.sub("", raw_html or ""))
|
||||||
|
truncated = False
|
||||||
|
if max_input_bytes > 0 and len(text.encode("utf-8", "ignore")) > max_input_bytes:
|
||||||
|
# Cut on a character boundary, not a byte boundary, and do not try to be
|
||||||
|
# clever about tags: the parser tolerates a truncated tail.
|
||||||
|
while len(text.encode("utf-8", "ignore")) > max_input_bytes:
|
||||||
|
text = text[: int(len(text) * 0.98) or 1]
|
||||||
|
truncated = True
|
||||||
|
LOGGER.warning("article body truncated to %d bytes before parsing", max_input_bytes)
|
||||||
|
|
||||||
|
builder = _TreeBuilder()
|
||||||
|
builder.feed(text)
|
||||||
|
builder.close()
|
||||||
|
tree = builder.root
|
||||||
|
|
||||||
|
if not _has_elements(tree):
|
||||||
|
# A body with no markup at all (plain text feed) still needs block
|
||||||
|
# structure, or the EPUB would contain a bare text node.
|
||||||
|
paragraph = _Element("p")
|
||||||
|
paragraph.children = list(tree.children)
|
||||||
|
tree.children = [paragraph]
|
||||||
|
|
||||||
|
_resolve_urls(tree, _Resolve(base_url), set())
|
||||||
|
_prune_empty(tree)
|
||||||
|
return Article(tree, base_url, truncated)
|
||||||
|
|
||||||
|
|
||||||
|
def _absolutise(raw: str, base_url: str) -> str | None:
|
||||||
|
candidate = (raw or "").strip()
|
||||||
|
if not candidate:
|
||||||
|
return None
|
||||||
|
# Control characters and raw whitespace inside a URL are always corruption.
|
||||||
|
candidate = _ILLEGAL_XML_RE.sub("", candidate)
|
||||||
|
if any(char.isspace() for char in candidate):
|
||||||
|
candidate = candidate.split()[0]
|
||||||
|
try:
|
||||||
|
if base_url:
|
||||||
|
return urljoin(base_url, candidate)
|
||||||
|
return candidate
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_urls(element: _Element, resolve: _Resolve, seen_ids: set[str]) -> None:
|
||||||
|
"""Validate URLs and de-duplicate anchor ids, in document order.
|
||||||
|
|
||||||
|
Runs once, at parse time, so everything downstream (the image collector, the
|
||||||
|
fetcher, the OPF manifest) sees final absolute URLs and there is no second
|
||||||
|
chance for a bad scheme to slip through.
|
||||||
|
"""
|
||||||
|
for child in element.children:
|
||||||
|
if isinstance(child, _Text):
|
||||||
|
continue
|
||||||
|
_dedupe_id(child, seen_ids)
|
||||||
|
if child.tag == "a":
|
||||||
|
href = child.attrs.get("href")
|
||||||
|
if href is not None:
|
||||||
|
target = resolve.link(href)
|
||||||
|
if target is None:
|
||||||
|
child.attrs.pop("href", None)
|
||||||
|
else:
|
||||||
|
child.attrs["href"] = target
|
||||||
|
elif child.tag == "img":
|
||||||
|
src = child.attrs.get("src")
|
||||||
|
target = resolve.image(src) if src else None
|
||||||
|
if target is None:
|
||||||
|
# Mark it for pruning: an image with no usable source is noise.
|
||||||
|
child.tag = "#dropped-image"
|
||||||
|
else:
|
||||||
|
child.attrs["src"] = target
|
||||||
|
elif child.tag in ("blockquote", "q"):
|
||||||
|
cite = child.attrs.get("cite")
|
||||||
|
if cite is not None:
|
||||||
|
target = resolve.link(cite)
|
||||||
|
if target is None:
|
||||||
|
child.attrs.pop("cite", None)
|
||||||
|
else:
|
||||||
|
child.attrs["cite"] = target
|
||||||
|
_resolve_urls(child, resolve, seen_ids)
|
||||||
|
|
||||||
|
|
||||||
|
def _dedupe_id(element: _Element, seen_ids: set[str]) -> None:
|
||||||
|
"""Make an element's id unique within the document.
|
||||||
|
|
||||||
|
Duplicate ids are invalid and would make the first anchor win, so a repeat is
|
||||||
|
renamed rather than dropped -- dropping it would break the link pointing at
|
||||||
|
it. One book holds one article, so a document-wide set is enough.
|
||||||
|
"""
|
||||||
|
anchor = element.attrs.get("id")
|
||||||
|
if anchor is None:
|
||||||
|
return
|
||||||
|
if anchor not in seen_ids:
|
||||||
|
seen_ids.add(anchor)
|
||||||
|
return
|
||||||
|
index = 2
|
||||||
|
while f"{anchor}-{index}" in seen_ids:
|
||||||
|
index += 1
|
||||||
|
renamed = f"{anchor}-{index}"
|
||||||
|
seen_ids.add(renamed)
|
||||||
|
element.attrs["id"] = renamed
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_image_map(element: _Element, image_map: dict[str, str | None]) -> None:
|
||||||
|
"""Point images at their local copies inside the book.
|
||||||
|
|
||||||
|
A URL mapped to None means the fetch failed or the image was refused, so the
|
||||||
|
element is marked for pruning instead of being left pointing at the network.
|
||||||
|
"""
|
||||||
|
for child in element.children:
|
||||||
|
if isinstance(child, _Text):
|
||||||
|
continue
|
||||||
|
if child.tag == "img":
|
||||||
|
src = child.attrs.get("src")
|
||||||
|
if src is not None and src in image_map:
|
||||||
|
target = image_map[src]
|
||||||
|
if target is None:
|
||||||
|
child.tag = "#dropped-image"
|
||||||
|
else:
|
||||||
|
child.attrs["src"] = target
|
||||||
|
_apply_image_map(child, image_map)
|
||||||
|
|
||||||
|
|
||||||
|
def _prune_empty(node: _Element) -> None:
|
||||||
|
"""Remove rejected images and the empty wrappers they leave behind."""
|
||||||
|
kept: list[_Element | _Text] = []
|
||||||
|
for child in node.children:
|
||||||
|
if isinstance(child, _Element):
|
||||||
|
if child.tag == "#dropped-image":
|
||||||
|
continue
|
||||||
|
_prune_empty(child)
|
||||||
|
if _is_noise(child):
|
||||||
|
continue
|
||||||
|
kept.append(child)
|
||||||
|
node.children = kept
|
||||||
|
|
||||||
|
|
||||||
|
def _is_noise(element: _Element) -> bool:
|
||||||
|
"""True for a droppable wrapper with no content left inside it.
|
||||||
|
|
||||||
|
Whitespace-only text does not count as content (``<p> </p>`` is noise),
|
||||||
|
but a retained ``id`` does: it is the target of a link.
|
||||||
|
"""
|
||||||
|
if element.tag not in _DROPPABLE_WHEN_EMPTY or element.attrs:
|
||||||
|
return False
|
||||||
|
for child in element.children:
|
||||||
|
if isinstance(child, _Element):
|
||||||
|
return False
|
||||||
|
if child.text.strip():
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _has_elements(element: _Element) -> bool:
|
||||||
|
"""True when the tree contains at least one element node."""
|
||||||
|
return any(True for _ in _iter_elements(element))
|
||||||
|
|
||||||
|
|
||||||
|
def _serialise_children(node: _Element, inside_pre: bool) -> str:
|
||||||
|
return "".join(_serialise(child, inside_pre) for child in node.children)
|
||||||
|
|
||||||
|
|
||||||
|
def _serialise(node: _Element | _Text, inside_pre: bool) -> str:
|
||||||
|
if isinstance(node, _Text):
|
||||||
|
if inside_pre:
|
||||||
|
return escape_text(node.text)
|
||||||
|
# Collapse whitespace outside <pre>: feed HTML arrives indented, and that
|
||||||
|
# indentation is pure bytes in the book.
|
||||||
|
return escape_text(_WHITESPACE_RE.sub(" ", node.text))
|
||||||
|
if node.tag not in KEPT_TAGS:
|
||||||
|
# Unwrapped elements keep their children but lose their own tag, so no
|
||||||
|
# unknown element can appear in the output.
|
||||||
|
return _serialise_children(node, inside_pre)
|
||||||
|
|
||||||
|
child_pre = inside_pre or node.tag == "pre"
|
||||||
|
inner = _serialise_children(node, child_pre)
|
||||||
|
if node.tag in ("p", "div", "li", "td", "th", "blockquote", "figcaption", "caption"):
|
||||||
|
inner = _BLOCK_TRIM_RE.sub("", inner)
|
||||||
|
attributes = "".join(
|
||||||
|
f' {name}="{escape_attribute(value)}"' for name, value in _ordered_attrs(node)
|
||||||
|
)
|
||||||
|
if node.tag in VOID_TAGS:
|
||||||
|
return f"<{node.tag}{attributes}/>"
|
||||||
|
return f"<{node.tag}{attributes}>{inner}</{node.tag}>"
|
||||||
|
|
||||||
|
|
||||||
|
def _ordered_attrs(element: _Element) -> list[tuple[str, str]]:
|
||||||
|
"""Stable attribute order, with the most meaningful attribute first."""
|
||||||
|
priority = {"id": 0, "href": 1, "src": 2, "alt": 3, "title": 4}
|
||||||
|
return sorted(element.attrs.items(), key=lambda item: (priority.get(item[0], 9), item[0]))
|
||||||
|
|
||||||
|
|
||||||
|
def escape_text(text: str) -> str:
|
||||||
|
return text.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||||
|
|
||||||
|
|
||||||
|
def escape_attribute(value: str) -> str:
|
||||||
|
# Attribute values are normalised by XML parsers anyway (tab/newline become
|
||||||
|
# spaces), so do it explicitly rather than leaving it implicit.
|
||||||
|
text = value.replace("\t", " ").replace("\n", " ").replace("\r", " ")
|
||||||
|
return (
|
||||||
|
text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_elements(node: _Element) -> Iterator[_Element]:
|
||||||
|
"""Depth-first traversal over element nodes only."""
|
||||||
|
for child in node.children:
|
||||||
|
if isinstance(child, _Element):
|
||||||
|
yield child
|
||||||
|
yield from _iter_elements(child)
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_text(node: _Element) -> Iterator[_Text]:
|
||||||
|
"""Depth-first traversal over text nodes only."""
|
||||||
|
for child in node.children:
|
||||||
|
if isinstance(child, _Text):
|
||||||
|
yield child
|
||||||
|
else:
|
||||||
|
yield from _iter_text(child)
|
||||||
36
config.demo.json
Normal file
36
config.demo.json
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
{
|
||||||
|
"_comment": "Demo configuration for `docker compose -f docker-compose.demo.yaml up`. It points at the fake FreshRSS in that stack and logs in as demo/demo. Not for real use; see config.example.json instead.",
|
||||||
|
"server": {
|
||||||
|
"host": "0.0.0.0",
|
||||||
|
"port": 8080,
|
||||||
|
"log_level": "info"
|
||||||
|
},
|
||||||
|
"freshrss": {
|
||||||
|
"_url": "The demo stack's fake FreshRSS, addressed by its compose service name.",
|
||||||
|
"url": "http://fresh:8099/api/greader.php",
|
||||||
|
"username": "tyler",
|
||||||
|
"api_password": "${DEMO_API_PASSWORD:-api-password}",
|
||||||
|
"items_per_page": 5,
|
||||||
|
"retries": 1
|
||||||
|
},
|
||||||
|
"catalog": {
|
||||||
|
"title": "Demo Catalog",
|
||||||
|
"author": "Demo"
|
||||||
|
},
|
||||||
|
"epub": {
|
||||||
|
"include_images": false
|
||||||
|
},
|
||||||
|
"auth": {
|
||||||
|
"_users": "demo/demo -- a throwaway login for the demo stack.",
|
||||||
|
"users": [
|
||||||
|
{
|
||||||
|
"username": "demo",
|
||||||
|
"password_sha256": "2a97516c354b68848cdbd8f54a226a0a55b21ed138e207ad6c5cbb9c00aa5aea"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"cache": {
|
||||||
|
"subscriptions_ttl_seconds": 30,
|
||||||
|
"stream_ttl_seconds": 10
|
||||||
|
}
|
||||||
|
}
|
||||||
134
config.example.json
Normal file
134
config.example.json
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
{
|
||||||
|
"_about": "OPDS server backed by FreshRSS. Copy to config.json and edit. Any key starting with an underscore is a comment: it is ignored, never interpolated, and never reported as unknown. Strings may use ${ENV_VAR}, ${ENV_VAR:-default} or ${file:/run/secrets/name}, so secrets do not have to live in this file.",
|
||||||
|
|
||||||
|
"server": {
|
||||||
|
"_host": "0.0.0.0 is required *inside a container*, because the published port only reaches the process through the container's own interface. It is not the 'exposed on every host interface' case: exposure is decided by the port mapping in docker-compose.yaml, which publishes on 127.0.0.1 by default. Running this directly on a host should keep the 127.0.0.1 default instead.",
|
||||||
|
"host": "0.0.0.0",
|
||||||
|
"_port": "0 asks the operating system for a free port, which is handy for a local run or a test; leave it at 8080 for a normal deployment.",
|
||||||
|
"port": 8080,
|
||||||
|
"log_level": "info",
|
||||||
|
|
||||||
|
"_public_base_url": "Absolute base URL for catalog links, e.g. https://opds.example.com. Leave unset to build links from the request Host header. Set it when the service is reachable at more than one name, or when you want the catalog URLs pinned against a spoofed Host header.",
|
||||||
|
"public_base_url": null,
|
||||||
|
|
||||||
|
"_trust_proxy_headers": "Honour X-Forwarded-Proto/Host/Prefix. Enable only behind a reverse proxy you control: without one, a client can set those headers itself.",
|
||||||
|
"trust_proxy_headers": false,
|
||||||
|
|
||||||
|
"_allowed_hosts": "Optional Host allowlist. When set, a request with any other Host is refused with 400. Useful when the service is directly reachable.",
|
||||||
|
"allowed_hosts": [],
|
||||||
|
|
||||||
|
"_tls": "Optional in-process TLS. Usually the reverse proxy terminates TLS instead, but this lets the container be secure on its own. Basic auth is unencrypted, so one of the two is required.",
|
||||||
|
"tls": {
|
||||||
|
"cert_file": null,
|
||||||
|
"key_file": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
"freshrss": {
|
||||||
|
"_url": "Your FreshRSS address. The /api/greader.php suffix is optional here; it is appended when missing, and any path prefix is kept.",
|
||||||
|
"url": "https://rss.example.com",
|
||||||
|
"username": "tyler",
|
||||||
|
|
||||||
|
"_api_password": "Settings -> Account -> API in FreshRSS. This is the API password, which is separate from your login password. Prefer ${file:...} or ${ENV_VAR} over writing it here.",
|
||||||
|
"api_password": "${FRESHRSS_API_PASSWORD}",
|
||||||
|
|
||||||
|
"verify_tls": true,
|
||||||
|
"_ca_file": "Optional CA bundle, for a FreshRSS instance behind a private certificate authority.",
|
||||||
|
"ca_file": null,
|
||||||
|
|
||||||
|
"timeout_seconds": 15,
|
||||||
|
"retries": 2,
|
||||||
|
|
||||||
|
"_mark_read": "Mark the article read in FreshRSS once its EPUB has been fully delivered to the reader.",
|
||||||
|
"mark_read": true,
|
||||||
|
|
||||||
|
"items_per_page": 30,
|
||||||
|
"max_items_per_request": 200,
|
||||||
|
|
||||||
|
"_read_window_items": "How far back the Read section looks. Unread filtering happens upstream, but read items are filtered here, so this bounds the history that section can show.",
|
||||||
|
"read_window_items": 200,
|
||||||
|
|
||||||
|
"_max_content_bytes": "Cap on one article body before it is parsed, so a single enormous feed item cannot dominate memory.",
|
||||||
|
"max_content_bytes": 1048576
|
||||||
|
},
|
||||||
|
|
||||||
|
"catalog": {
|
||||||
|
"title": "FreshRSS OPDS",
|
||||||
|
"author": "Tyler",
|
||||||
|
"language": "en",
|
||||||
|
"_publisher": "Optional dc:publisher for every generated book.",
|
||||||
|
"publisher": null,
|
||||||
|
|
||||||
|
"_show_unread_counts": "Append unread counts to category and feed rows. Costs one extra upstream call per catalog refresh, and degrades quietly if unsupported.",
|
||||||
|
"show_unread_counts": true,
|
||||||
|
|
||||||
|
"_show_read_section": "Expose a Read tree next to the Unread queue, for browsing history without mixing it into the queue.",
|
||||||
|
"show_read_section": true,
|
||||||
|
|
||||||
|
"_show_starred": "Expose starred items as their own shelf.",
|
||||||
|
"show_starred": true,
|
||||||
|
|
||||||
|
"_hide_empty_categories": "Hide categories that contain no feeds (or no unread items, when unread counts are on).",
|
||||||
|
"hide_empty_categories": false,
|
||||||
|
|
||||||
|
"_exclude_categories": "Category labels to leave out of the catalog entirely.",
|
||||||
|
"exclude_categories": [],
|
||||||
|
|
||||||
|
"_exclude_feeds": "Feed titles to leave out of the catalog entirely.",
|
||||||
|
"exclude_feeds": []
|
||||||
|
},
|
||||||
|
|
||||||
|
"epub": {
|
||||||
|
"_include_images": "Download article images and embed them in the book, so it reads correctly offline. Image URLs come from feed content, so they are fetched through an SSRF guard: private, loopback and link-local addresses are refused.",
|
||||||
|
"include_images": true,
|
||||||
|
|
||||||
|
"_allowed_image_hosts": "When non-empty, only images from these hosts may be embedded (no scheme, exact host or subdomain suffix). Use for a locked-down deployment.",
|
||||||
|
"allowed_image_hosts": [],
|
||||||
|
|
||||||
|
"_blocked_image_hosts": "Hosts to refuse even when inlining is on, e.g. a tracking pixel domain.",
|
||||||
|
"blocked_image_hosts": [],
|
||||||
|
|
||||||
|
"max_images_per_article": 20,
|
||||||
|
"max_image_bytes": 2097152,
|
||||||
|
"max_total_image_bytes": 16777216,
|
||||||
|
"image_timeout_seconds": 10,
|
||||||
|
|
||||||
|
"_cover_image": "Use the first suitably large image as the book cover, so a reader's library view shows something other than a blank tile.",
|
||||||
|
"cover_image": true,
|
||||||
|
|
||||||
|
"_include_source_link": "Add a link back to the original article, plus a dc:source entry.",
|
||||||
|
"include_source_link": true,
|
||||||
|
|
||||||
|
"_stylesheet": "Optional path to a CSS file to use instead of the built-in stylesheet.",
|
||||||
|
"stylesheet": null,
|
||||||
|
|
||||||
|
"_max_epub_bytes": "Refuse to build a book larger than this, so one pathological article cannot exhaust memory.",
|
||||||
|
"max_epub_bytes": 33554432
|
||||||
|
},
|
||||||
|
|
||||||
|
"auth": {
|
||||||
|
"_realm": "The realm shown in the reader's password prompt.",
|
||||||
|
"realm": "OPDS",
|
||||||
|
|
||||||
|
"_users": "HTTP Basic credentials. This catalog serves your entire reading queue, so it will not start without at least one user. Use password_sha256 (make hash-password) rather than password: the served catalogue is public once TLS is in front of it, but the config file is not.",
|
||||||
|
"users": [
|
||||||
|
{
|
||||||
|
"username": "tyler",
|
||||||
|
"_password_sha256": "Generate with: make hash-password",
|
||||||
|
"password_sha256": "${OPDS_PASSWORD_SHA256}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
|
||||||
|
"_max_failures_per_minute": "Failed attempts from one address before it is throttled. Basic auth is brute-forceable, so this is on by default.",
|
||||||
|
"max_failures_per_minute": 10,
|
||||||
|
"throttle_seconds": 60
|
||||||
|
},
|
||||||
|
|
||||||
|
"cache": {
|
||||||
|
"_about": "Lifetimes for the in-memory cache. Nothing is written to disk. Lower values make the catalog more responsive to upstream changes; higher values reduce FreshRSS traffic while a reader browses.",
|
||||||
|
"subscriptions_ttl_seconds": 300,
|
||||||
|
"stream_ttl_seconds": 60,
|
||||||
|
"item_ttl_seconds": 900,
|
||||||
|
"max_entries": 512
|
||||||
|
}
|
||||||
|
}
|
||||||
58
docker-compose.demo.yaml
Normal file
58
docker-compose.demo.yaml
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
# A self-contained demo: the catalogue plus a fake FreshRSS, so the whole
|
||||||
|
# thing can be tried with one command, without handing a real account over.
|
||||||
|
#
|
||||||
|
# docker compose -f docker-compose.demo.yaml up --build
|
||||||
|
# open http://127.0.0.1:8088/opds and sign in as demo / demo
|
||||||
|
#
|
||||||
|
# The fake account holds three feeds across two categories, a starred
|
||||||
|
# article and an already-read one. Downloading a book marks that article
|
||||||
|
# read in the fake server, so the unread queue drains as you use it.
|
||||||
|
#
|
||||||
|
# This is a demonstration, not a deployment: it publishes an unencrypted
|
||||||
|
# port on loopback with a known password. For real use, follow README.md
|
||||||
|
# with config.example.json and docker-compose.yaml.
|
||||||
|
name: opds-from-rss-demo
|
||||||
|
|
||||||
|
services:
|
||||||
|
fresh:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile.demo
|
||||||
|
image: opds-from-rss-demo-freshrss:latest
|
||||||
|
container_name: opds-demo-fresh
|
||||||
|
restart: "no"
|
||||||
|
# No ports: the catalogue reaches it over the compose network, and nothing
|
||||||
|
# else needs to.
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
cap_drop:
|
||||||
|
- ALL
|
||||||
|
|
||||||
|
opds:
|
||||||
|
build: .
|
||||||
|
image: opds-from-rss:latest
|
||||||
|
container_name: opds-demo-server
|
||||||
|
restart: "no"
|
||||||
|
depends_on:
|
||||||
|
fresh:
|
||||||
|
# Wait for the fake server to answer, so a reader's first tap does
|
||||||
|
# not land on an upstream that is still starting up.
|
||||||
|
condition: service_healthy
|
||||||
|
environment:
|
||||||
|
CONFIG_PATH: /config/config.json
|
||||||
|
volumes:
|
||||||
|
- ./config.demo.json:/config/config.json:ro
|
||||||
|
ports:
|
||||||
|
# 8088 on the host, because 8080 is a popular neighbour; the
|
||||||
|
# container itself always listens on 8080.
|
||||||
|
- "127.0.0.1:8088:8080"
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
cap_drop:
|
||||||
|
- ALL
|
||||||
36
docker-compose.yaml
Normal file
36
docker-compose.yaml
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
services:
|
||||||
|
opds-from-rss:
|
||||||
|
build: .
|
||||||
|
image: opds-from-rss:latest
|
||||||
|
container_name: opds-from-rss
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
# Secrets (the FreshRSS API password and the catalogue password digest) come
|
||||||
|
# from .env rather than from config.json, so the mounted file stays safe to
|
||||||
|
# keep in version control. Optional, so that `docker compose config` and a
|
||||||
|
# first `up` work before the file exists -- configuration validation then
|
||||||
|
# reports the missing variable by name.
|
||||||
|
env_file:
|
||||||
|
- path: .env
|
||||||
|
required: false
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
- ./config.json:/config/config.json:ro
|
||||||
|
|
||||||
|
# Published on loopback only. Put a TLS-terminating reverse proxy (or a
|
||||||
|
# tunnel) in front, because HTTP Basic credentials are only base64-encoded.
|
||||||
|
# To reach it from the LAN instead, publish "8080:8080" deliberately.
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:8080:8080"
|
||||||
|
|
||||||
|
# The server never writes to disk, so the container filesystem can be
|
||||||
|
# read-only. /tmp is mounted only because some Python internals expect it
|
||||||
|
# to exist; nothing here writes to it.
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp
|
||||||
|
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
cap_drop:
|
||||||
|
- ALL
|
||||||
35
pyproject.toml
Normal file
35
pyproject.toml
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=68"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "opds-from-rss"
|
||||||
|
version = "1.0.0"
|
||||||
|
description = "OPDS 1.2 catalog backed by FreshRSS, serving one EPUB per article"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.11"
|
||||||
|
license = { text = "MIT" }
|
||||||
|
# No runtime dependencies, deliberately: the whole server is the Python standard
|
||||||
|
# library. That keeps the image small, the supply chain empty, and the EPUB
|
||||||
|
# writer auditable.
|
||||||
|
dependencies = []
|
||||||
|
|
||||||
|
[project.urls]
|
||||||
|
Homepage = "https://github.com/"
|
||||||
|
|
||||||
|
[tool.pyright]
|
||||||
|
# The application package sits at the repository root rather than under src/.
|
||||||
|
include = ["app", "tests", "tools"]
|
||||||
|
extraPaths = ["."]
|
||||||
|
pythonVersion = "3.11"
|
||||||
|
typeCheckingMode = "standard"
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
line-length = 100
|
||||||
|
target-version = "py311"
|
||||||
|
|
||||||
|
[tool.ruff.lint]
|
||||||
|
select = ["E", "F", "W", "I", "N", "UP", "B", "C4", "SIM", "RUF"]
|
||||||
|
|
||||||
|
[tool.setuptools]
|
||||||
|
packages = ["app"]
|
||||||
12
tests/__init__.py
Normal file
12
tests/__init__.py
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
"""Test package.
|
||||||
|
|
||||||
|
Logging is quietened here so that expected warnings from exercised code paths
|
||||||
|
(for example the clear-text password notice) do not drown out test results. A
|
||||||
|
test that asserts on a log message can raise the level itself.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logging.getLogger("app").setLevel(logging.CRITICAL)
|
||||||
605
tests/stub_freshrss.py
Normal file
605
tests/stub_freshrss.py
Normal file
@@ -0,0 +1,605 @@
|
|||||||
|
"""An in-process FreshRSS stand-in.
|
||||||
|
|
||||||
|
This exists so the client can be tested against the wire format a real FreshRSS
|
||||||
|
produces, rather than against a mock that would happily accept anything. The
|
||||||
|
response shapes here mirror the Google Reader API responses the firmware client
|
||||||
|
consumes (``lib/FreshRss/src/FreshRssSync.cpp``), including the parts that are
|
||||||
|
easy to get wrong: ``Auth=`` embedded in a multi-line ClientLogin body, category
|
||||||
|
labels nested inside each subscription, item payloads that put the body under
|
||||||
|
``content``/``summary``, and one ``i=`` parameter per id on ``edit-tag``.
|
||||||
|
|
||||||
|
It also records every request, so tests can assert on the exact query parameters
|
||||||
|
the client sent -- which is how the unread filter and the page size are verified.
|
||||||
|
|
||||||
|
The same class backs ``tools/fake-freshrss.py``, so the server can be smoke
|
||||||
|
tested by hand without pointing it at a real account.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import threading
|
||||||
|
import urllib.parse
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
API_PREFIX = "/api/greader.php"
|
||||||
|
|
||||||
|
READ_TAG = "user/-/state/com.google/read"
|
||||||
|
STARRED_TAG = "user/-/state/com.google/starred"
|
||||||
|
READING_LIST = "user/-/state/com.google/reading-list"
|
||||||
|
|
||||||
|
|
||||||
|
def _tags(entry: Any) -> list[str]:
|
||||||
|
"""Category tags on an entry, tolerating a malformed payload.
|
||||||
|
|
||||||
|
Tests deliberately inject non-object entries to check that the client skips
|
||||||
|
them, so the stub's own filtering must survive them too.
|
||||||
|
"""
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
return []
|
||||||
|
tags = entry.get("categories")
|
||||||
|
if not isinstance(tags, list):
|
||||||
|
return []
|
||||||
|
return [tag for tag in tags if isinstance(tag, str)]
|
||||||
|
|
||||||
|
|
||||||
|
def _published(entry: Any) -> int:
|
||||||
|
"""Sort key that tolerates a malformed entry."""
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
return 0
|
||||||
|
value = entry.get("published")
|
||||||
|
return value if isinstance(value, int) else 0
|
||||||
|
|
||||||
|
|
||||||
|
def _with_read_state(entry: Any, read_ids: set[str]) -> Any:
|
||||||
|
"""The entry as the server would send it, with the read tag applied."""
|
||||||
|
if not isinstance(entry, dict) or entry.get("id") not in read_ids:
|
||||||
|
return entry
|
||||||
|
if READ_TAG in _tags(entry):
|
||||||
|
return entry
|
||||||
|
projected = dict(entry)
|
||||||
|
projected["categories"] = [*_tags(entry), READ_TAG]
|
||||||
|
return projected
|
||||||
|
|
||||||
|
|
||||||
|
def _is_tagged(entry: Any, tag: str, read_ids: set[str]) -> bool:
|
||||||
|
"""Whether an entry carries a tag, including tracked read state.
|
||||||
|
|
||||||
|
Marking an item read adds the read tag on the server, so an ``xt=read`` query
|
||||||
|
stops returning it. Without modelling that, a test could not tell whether
|
||||||
|
mark-read actually took effect.
|
||||||
|
"""
|
||||||
|
if tag in _tags(entry):
|
||||||
|
return True
|
||||||
|
return tag == READ_TAG and isinstance(entry, dict) and entry.get("id") in read_ids
|
||||||
|
|
||||||
|
|
||||||
|
def item_id(index: int) -> str:
|
||||||
|
"""A FreshRSS-shaped item id: the exact form that needs URL-safe encoding."""
|
||||||
|
return f"tag:google.com,2005:reader/item/{index:016x}"
|
||||||
|
|
||||||
|
|
||||||
|
class StubFreshRSS:
|
||||||
|
"""A scriptable FreshRSS with just enough behaviour to trust the client."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
username: str = "tyler",
|
||||||
|
password: str = "api-password",
|
||||||
|
page_limit: int = 1000,
|
||||||
|
bind_host: str = "127.0.0.1",
|
||||||
|
bind_port: int = 0,
|
||||||
|
public_host: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.username = username
|
||||||
|
self.password = password
|
||||||
|
self.page_limit = page_limit
|
||||||
|
self._bind_host = bind_host
|
||||||
|
self._bind_port = bind_port
|
||||||
|
# The name other containers should use to reach this one. It defaults to
|
||||||
|
# the bind address, which is right for a local test and wrong inside a
|
||||||
|
# compose network, where the service name is the address.
|
||||||
|
self._public_host = public_host or bind_host
|
||||||
|
self.auth_token = "stub-auth-token"
|
||||||
|
self.write_token = "stub-write-token"
|
||||||
|
self.subscriptions: list[dict[str, Any]] = []
|
||||||
|
# Items are typed loosely on purpose: this class stands in for an upstream
|
||||||
|
# that can send anything, and tests rely on being able to inject malformed
|
||||||
|
# payloads to check that the client skips them.
|
||||||
|
self.items: dict[str, list[Any]] = {}
|
||||||
|
self.read: set[str] = set()
|
||||||
|
self.requests: list[dict[str, Any]] = []
|
||||||
|
self.supports_unread_counts = True
|
||||||
|
# When set, subscription list replies with 200 and a body that is not JSON,
|
||||||
|
# so the client's response handling can be tested against a proxy or an
|
||||||
|
# error page rather than against well-formed output.
|
||||||
|
self.corrupt_json = False
|
||||||
|
# Injected failures as (status, path fragment or None) pairs.
|
||||||
|
self._failures: list[tuple[int, str | None]] = []
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._server: ThreadingHTTPServer | None = None
|
||||||
|
self._thread: threading.Thread | None = None
|
||||||
|
self.port = 0
|
||||||
|
|
||||||
|
# -- scripting -------------------------------------------------------
|
||||||
|
|
||||||
|
def add_feed(
|
||||||
|
self,
|
||||||
|
stream_id: str,
|
||||||
|
title: str,
|
||||||
|
categories: list[str] | None = None,
|
||||||
|
*,
|
||||||
|
html_url: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""Register a feed under one or more category labels."""
|
||||||
|
self.subscriptions.append(
|
||||||
|
{
|
||||||
|
"id": stream_id,
|
||||||
|
"title": title,
|
||||||
|
"categories": [
|
||||||
|
{"id": f"user/-/label/{label}", "label": label} for label in (categories or [])
|
||||||
|
],
|
||||||
|
"htmlUrl": html_url or stream_id.removeprefix("feed/"),
|
||||||
|
"url": stream_id.removeprefix("feed/"),
|
||||||
|
"sortid": f"{len(self.subscriptions):08x}",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.items.setdefault(stream_id, [])
|
||||||
|
return stream_id
|
||||||
|
|
||||||
|
def add_item(
|
||||||
|
self,
|
||||||
|
stream_id: str,
|
||||||
|
index: int,
|
||||||
|
title: str,
|
||||||
|
*,
|
||||||
|
content: str = "<p>body</p>",
|
||||||
|
summary: str | None = None,
|
||||||
|
published: int = 1_700_000_000,
|
||||||
|
author: str | None = "A. Writer",
|
||||||
|
url: str | None = None,
|
||||||
|
read: bool = False,
|
||||||
|
starred: bool = False,
|
||||||
|
categories: list[str] | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""Add an article to a feed, returning its id."""
|
||||||
|
identifier = item_id(index)
|
||||||
|
tags = [READING_LIST, *(categories or [])]
|
||||||
|
if read:
|
||||||
|
tags.append(READ_TAG)
|
||||||
|
if starred:
|
||||||
|
tags.append(STARRED_TAG)
|
||||||
|
entry: dict[str, Any] = {
|
||||||
|
"id": identifier,
|
||||||
|
"title": title,
|
||||||
|
"published": published,
|
||||||
|
"updated": published,
|
||||||
|
"timestampUsec": str(published * 1_000_000),
|
||||||
|
"crawlTimeMsec": str(published * 1000),
|
||||||
|
"categories": tags,
|
||||||
|
"alternate": [
|
||||||
|
{
|
||||||
|
"href": url or f"https://example.com/articles/{index}",
|
||||||
|
"type": "text/html",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"origin": {
|
||||||
|
"streamId": stream_id,
|
||||||
|
"title": self._feed_title(stream_id),
|
||||||
|
"htmlUrl": stream_id.removeprefix("feed/"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if author is not None:
|
||||||
|
entry["author"] = author
|
||||||
|
entry["content"] = {"content": content, "direction": "ltr"}
|
||||||
|
entry["summary"] = {
|
||||||
|
"content": summary if summary is not None else content,
|
||||||
|
"direction": "ltr",
|
||||||
|
}
|
||||||
|
self.items.setdefault(stream_id, []).append(entry)
|
||||||
|
if read:
|
||||||
|
self.read.add(identifier)
|
||||||
|
return identifier
|
||||||
|
|
||||||
|
def inject_raw_item(self, stream_id: str, payload: Any) -> None:
|
||||||
|
"""Push an arbitrary payload into a stream.
|
||||||
|
|
||||||
|
Upstream is not guaranteed to answer with well-formed objects, so tests
|
||||||
|
need a way to send junk through the real client path rather than trusting
|
||||||
|
that it would be handled. Guarded because handler threads read this.
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
self.items.setdefault(stream_id, []).append(payload)
|
||||||
|
|
||||||
|
def fail_next(self, status: int, count: int = 1, *, path: str | None = None) -> None:
|
||||||
|
"""Make the next ``count`` requests fail with ``status``.
|
||||||
|
|
||||||
|
``path`` narrows it to requests whose path contains that fragment, which
|
||||||
|
matters because the client logs in before anything else: an untargeted
|
||||||
|
injected failure would be spent on the login and never reach the call the
|
||||||
|
test is about. Guarded because it is called between requests while
|
||||||
|
handler threads are running.
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
self._failures.extend([(status, path)] * count)
|
||||||
|
|
||||||
|
def _feed_title(self, stream_id: str) -> str:
|
||||||
|
for feed in self.subscriptions:
|
||||||
|
if feed["id"] == stream_id:
|
||||||
|
return feed["title"]
|
||||||
|
return stream_id
|
||||||
|
|
||||||
|
# -- lifecycle -------------------------------------------------------
|
||||||
|
|
||||||
|
@property
|
||||||
|
def base_url(self) -> str:
|
||||||
|
return f"http://{self._public_host}:{self.port}"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def api_url(self) -> str:
|
||||||
|
return self.base_url + API_PREFIX
|
||||||
|
|
||||||
|
def start(self) -> StubFreshRSS:
|
||||||
|
handler = _make_handler(self)
|
||||||
|
server = ThreadingHTTPServer((self._bind_host, self._bind_port), handler)
|
||||||
|
self._server = server
|
||||||
|
# Read the port back, because a request for port 0 means "any free port"
|
||||||
|
# and only the operating system knows which one that turned out to be.
|
||||||
|
self.port = server.server_address[1]
|
||||||
|
# A short poll interval keeps teardown cheap: shutdown() only returns once
|
||||||
|
# the serve loop notices the flag, and the default half second would be
|
||||||
|
# paid by every test that starts a stub.
|
||||||
|
self._thread = threading.Thread(
|
||||||
|
target=server.serve_forever,
|
||||||
|
kwargs={"poll_interval": 0.02},
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
self._thread.start()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
if self._server is not None:
|
||||||
|
self._server.shutdown()
|
||||||
|
self._server.server_close()
|
||||||
|
self._server = None
|
||||||
|
if self._thread is not None:
|
||||||
|
self._thread.join(timeout=5)
|
||||||
|
self._thread = None
|
||||||
|
|
||||||
|
def __enter__(self) -> StubFreshRSS:
|
||||||
|
return self.start()
|
||||||
|
|
||||||
|
def __exit__(
|
||||||
|
self,
|
||||||
|
exc_type: type[BaseException] | None,
|
||||||
|
exc_value: BaseException | None,
|
||||||
|
traceback: object,
|
||||||
|
) -> None:
|
||||||
|
self.stop()
|
||||||
|
|
||||||
|
# -- request handling ------------------------------------------------
|
||||||
|
|
||||||
|
def record(self, entry: dict[str, Any]) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self.requests.append(entry)
|
||||||
|
|
||||||
|
def next_failure(self, path: str) -> int | None:
|
||||||
|
"""The next injected failure that applies to ``path``, if any."""
|
||||||
|
with self._lock:
|
||||||
|
for index, (status, target) in enumerate(self._failures):
|
||||||
|
if target is None or target in path:
|
||||||
|
del self._failures[index]
|
||||||
|
return status
|
||||||
|
return None
|
||||||
|
|
||||||
|
def find_requests(self, fragment: str) -> list[dict[str, Any]]:
|
||||||
|
"""Every recorded request whose path contains ``fragment``.
|
||||||
|
|
||||||
|
A substring match rather than a suffix, because a stream path is followed
|
||||||
|
by the encoded stream id (``/stream/contents/feed%2F...``) and so never
|
||||||
|
ends where the interesting part of the route does.
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
return [entry for entry in self.requests if fragment in entry["path"]]
|
||||||
|
|
||||||
|
def mark_read(self, identifiers: list[str]) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self.read.update(identifiers)
|
||||||
|
|
||||||
|
def clear_read(self) -> None:
|
||||||
|
"""Forget every read mark, for tests that re-run a marking step."""
|
||||||
|
with self._lock:
|
||||||
|
self.read.clear()
|
||||||
|
|
||||||
|
def read_snapshot(self) -> set[str]:
|
||||||
|
"""A copy of the read marks, taken under the lock."""
|
||||||
|
with self._lock:
|
||||||
|
return set(self.read)
|
||||||
|
|
||||||
|
def subscription_payload(self) -> dict[str, Any]:
|
||||||
|
return {"subscriptions": self.subscriptions}
|
||||||
|
|
||||||
|
def unread_count_payload(self) -> dict[str, Any]:
|
||||||
|
counts = []
|
||||||
|
for feed in self.subscriptions:
|
||||||
|
unread = sum(
|
||||||
|
1
|
||||||
|
for entry in self.items.get(feed["id"], [])
|
||||||
|
if isinstance(entry, dict) and entry.get("id") not in self.read
|
||||||
|
)
|
||||||
|
counts.append(
|
||||||
|
{
|
||||||
|
"id": feed["id"],
|
||||||
|
"count": unread,
|
||||||
|
"newestItemTimestampUsec": "1700000000000000",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return {"max": self.page_limit, "unreadcounts": counts}
|
||||||
|
|
||||||
|
def stream_payload(self, stream_id: str, query: dict[str, list[str]]) -> dict[str, Any]:
|
||||||
|
"""Items for a stream, honouring ``xt``, ``ot``, ``n`` and ``r``."""
|
||||||
|
read_ids = self.read_snapshot()
|
||||||
|
if stream_id == READING_LIST:
|
||||||
|
entries = [entry for group in self.items.values() for entry in group]
|
||||||
|
elif stream_id == STARRED_TAG:
|
||||||
|
entries = [
|
||||||
|
entry
|
||||||
|
for group in self.items.values()
|
||||||
|
for entry in group
|
||||||
|
if _is_tagged(entry, STARRED_TAG, read_ids)
|
||||||
|
]
|
||||||
|
elif stream_id.startswith("user/-/label/"):
|
||||||
|
label = stream_id.rsplit("/", 1)[-1]
|
||||||
|
labelled = {
|
||||||
|
feed["id"]
|
||||||
|
for feed in self.subscriptions
|
||||||
|
if any(category["label"] == label for category in feed["categories"])
|
||||||
|
}
|
||||||
|
entries = [
|
||||||
|
entry
|
||||||
|
for feed_id, group in self.items.items()
|
||||||
|
if feed_id in labelled
|
||||||
|
for entry in group
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
entries = list(self.items.get(stream_id, []))
|
||||||
|
|
||||||
|
excluded = set(query.get("xt", []))
|
||||||
|
if excluded:
|
||||||
|
entries = [
|
||||||
|
entry
|
||||||
|
for entry in entries
|
||||||
|
if not any(_is_tagged(entry, tag, read_ids) for tag in excluded)
|
||||||
|
]
|
||||||
|
older_than = query.get("ot", [None])[0]
|
||||||
|
if older_than and older_than.isdigit():
|
||||||
|
entries = [
|
||||||
|
entry
|
||||||
|
for entry in entries
|
||||||
|
if not isinstance(entry, dict) or _published(entry) >= int(older_than)
|
||||||
|
]
|
||||||
|
descending = query.get("r", ["d"])[0] != "o"
|
||||||
|
entries.sort(key=_published, reverse=descending)
|
||||||
|
# Marking an item read adds the tag to the item itself upstream, so the
|
||||||
|
# returned payload has to carry it too -- otherwise a client could never
|
||||||
|
# observe a read mark it just made.
|
||||||
|
entries = [_with_read_state(entry, read_ids) for entry in entries]
|
||||||
|
limit = query.get("n", [None])[0]
|
||||||
|
if limit and limit.isdigit():
|
||||||
|
entries = entries[: int(limit)]
|
||||||
|
return {"id": stream_id, "updated": 1_700_000_000, "items": entries}
|
||||||
|
|
||||||
|
|
||||||
|
def _make_handler(stub: StubFreshRSS):
|
||||||
|
"""Build the request handler class bound to one stub instance."""
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
protocol_version = "HTTP/1.1"
|
||||||
|
server_version = "StubFreshRSS/1.0"
|
||||||
|
|
||||||
|
# The parameter must keep the base class's name for keyword compatibility,
|
||||||
|
# even though it shadows the builtin.
|
||||||
|
def log_message(self, format: str, *args: object) -> None:
|
||||||
|
"""Silence the default access log; tests assert on recorded requests."""
|
||||||
|
|
||||||
|
# -- plumbing ----------------------------------------------------
|
||||||
|
|
||||||
|
def _respond(self, status: int, body: bytes, content_type: str) -> None:
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", content_type)
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
if body and self.command != "HEAD":
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def _respond_json(self, payload: Any, status: int = 200) -> None:
|
||||||
|
self._respond(
|
||||||
|
status,
|
||||||
|
json.dumps(payload).encode("utf-8"),
|
||||||
|
"application/json; charset=utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _record(self, body: str) -> tuple[str, dict[str, list[str]]]:
|
||||||
|
parsed = urllib.parse.urlsplit(self.path)
|
||||||
|
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
|
||||||
|
route = parsed.path
|
||||||
|
if route.startswith(API_PREFIX):
|
||||||
|
route = route[len(API_PREFIX) :]
|
||||||
|
stub.record(
|
||||||
|
{
|
||||||
|
"method": self.command,
|
||||||
|
"path": route,
|
||||||
|
"raw_path": self.path,
|
||||||
|
"query": query,
|
||||||
|
"body": body,
|
||||||
|
"authorization": self.headers.get("Authorization", ""),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return route, query
|
||||||
|
|
||||||
|
def _authorised(self) -> bool:
|
||||||
|
expected = f"GoogleLogin auth={stub.auth_token}"
|
||||||
|
return self.headers.get("Authorization", "") == expected
|
||||||
|
|
||||||
|
def _drain_body(self) -> str:
|
||||||
|
length = int(self.headers.get("Content-Length") or 0)
|
||||||
|
if not length:
|
||||||
|
return ""
|
||||||
|
return self.rfile.read(length).decode("utf-8", "replace")
|
||||||
|
|
||||||
|
# -- routes ------------------------------------------------------
|
||||||
|
|
||||||
|
def do_HEAD(self) -> None:
|
||||||
|
self.do_GET()
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
route, query = self._record("")
|
||||||
|
failure = stub.next_failure(route)
|
||||||
|
if failure is not None:
|
||||||
|
self._respond_json({"error": "injected failure"}, status=failure)
|
||||||
|
return
|
||||||
|
|
||||||
|
if route == "/healthz":
|
||||||
|
# Unauthenticated on purpose: it only says the server answered, and
|
||||||
|
# it is what the demo compose stack waits on before starting the
|
||||||
|
# catalogue. Real FreshRSS has no such route.
|
||||||
|
self._respond_json({"status": "ok"})
|
||||||
|
return
|
||||||
|
if route == "/accounts/ClientLogin":
|
||||||
|
self._respond_json({"error": "ClientLogin is a POST"}, status=405)
|
||||||
|
return
|
||||||
|
if route == "/reader/api/0/subscription/list":
|
||||||
|
if not self._require_auth():
|
||||||
|
return
|
||||||
|
if stub.corrupt_json:
|
||||||
|
self._respond(200, b"<html>login page</html>", "text/html")
|
||||||
|
return
|
||||||
|
self._respond_json(stub.subscription_payload())
|
||||||
|
return
|
||||||
|
if route == "/reader/api/0/unread-count":
|
||||||
|
if not self._require_auth():
|
||||||
|
return
|
||||||
|
if not stub.supports_unread_counts:
|
||||||
|
self._respond_json({"error": "unsupported"}, status=404)
|
||||||
|
return
|
||||||
|
self._respond_json(stub.unread_count_payload())
|
||||||
|
return
|
||||||
|
if route == "/reader/api/0/token":
|
||||||
|
if not self._require_auth():
|
||||||
|
return
|
||||||
|
self._respond(200, stub.write_token.encode("utf-8"), "text/plain")
|
||||||
|
return
|
||||||
|
if route.startswith("/reader/api/0/stream/contents/"):
|
||||||
|
if not self._require_auth():
|
||||||
|
return
|
||||||
|
stream_id = urllib.parse.unquote(route[len("/reader/api/0/stream/contents/") :])
|
||||||
|
self._respond_json(stub.stream_payload(stream_id, query))
|
||||||
|
return
|
||||||
|
self._respond_json({"error": "not found"}, status=404)
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
body = self._drain_body()
|
||||||
|
route, _query = self._record(body)
|
||||||
|
failure = stub.next_failure(route)
|
||||||
|
if failure is not None:
|
||||||
|
self._respond_json({"error": "injected failure"}, status=failure)
|
||||||
|
return
|
||||||
|
|
||||||
|
if route == "/accounts/ClientLogin":
|
||||||
|
self._handle_login(body)
|
||||||
|
return
|
||||||
|
if route == "/reader/api/0/edit-tag":
|
||||||
|
if not self._require_auth():
|
||||||
|
return
|
||||||
|
self._handle_edit_tag(body)
|
||||||
|
return
|
||||||
|
self._respond_json({"error": "not found"}, status=404)
|
||||||
|
|
||||||
|
def _require_auth(self) -> bool:
|
||||||
|
if self._authorised():
|
||||||
|
return True
|
||||||
|
self._respond_json({"error": "unauthorised"}, status=401)
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _handle_login(self, body: str) -> None:
|
||||||
|
form = urllib.parse.parse_qs(body)
|
||||||
|
username = form.get("Email", [""])[0]
|
||||||
|
password = form.get("Passwd", [""])[0]
|
||||||
|
if username != stub.username or password != stub.password:
|
||||||
|
self._respond_json({"error": "BadAuthentication"}, status=403)
|
||||||
|
return
|
||||||
|
# The real endpoint returns three lines; only Auth matters to us, and a
|
||||||
|
# parser that grabs the first "=" would get SID instead.
|
||||||
|
payload = f"SID=stub-sid\nLSID=stub-lsid\nAuth={stub.auth_token}\n"
|
||||||
|
self._respond(200, payload.encode("utf-8"), "text/plain; charset=utf-8")
|
||||||
|
|
||||||
|
def _handle_edit_tag(self, body: str) -> None:
|
||||||
|
form = urllib.parse.parse_qs(body, keep_blank_values=True)
|
||||||
|
token = form.get("T", [""])[0]
|
||||||
|
if token != stub.write_token:
|
||||||
|
self._respond_json({"error": "Invalid token"}, status=401)
|
||||||
|
return
|
||||||
|
if form.get("a", [""])[0] != READ_TAG:
|
||||||
|
self._respond_json({"error": "unexpected tag"}, status=400)
|
||||||
|
return
|
||||||
|
stub.mark_read(form.get("i", []))
|
||||||
|
self._respond(200, b"OK", "text/plain; charset=utf-8")
|
||||||
|
|
||||||
|
return Handler
|
||||||
|
|
||||||
|
|
||||||
|
def build_demo_catalog(**stub_options: Any) -> StubFreshRSS:
|
||||||
|
"""A small but realistic account, used by tests and the manual smoke tool.
|
||||||
|
|
||||||
|
Three feeds across two categories, with a read item, a starred item, an item
|
||||||
|
in a feed that has no category, and content that exercises the sanitiser
|
||||||
|
(relative links, a script tag, an unclosed list, an entity).
|
||||||
|
|
||||||
|
Keyword options are forwarded to :class:`StubFreshRSS`, which is how
|
||||||
|
``tools/fake-freshrss.py`` serves this same account to a container --
|
||||||
|
``bind_host``, ``bind_port`` and ``public_host``, since a compose network needs
|
||||||
|
a wildcard bind, a fixed port and a service name in place of loopback. The
|
||||||
|
defaults are the loopback bind and ephemeral port the tests rely on.
|
||||||
|
"""
|
||||||
|
stub = StubFreshRSS(**stub_options)
|
||||||
|
tech = stub.add_feed("feed/https://example.com/tech/atom.xml", "Example Tech", ["Tech"])
|
||||||
|
news = stub.add_feed(
|
||||||
|
"feed/https://news.example.org/rss?source=daily",
|
||||||
|
"Daily News",
|
||||||
|
["News"],
|
||||||
|
)
|
||||||
|
lonely = stub.add_feed("feed/https://solo.example.net/feed.xml", "Solo Blog")
|
||||||
|
|
||||||
|
stub.add_item(
|
||||||
|
tech,
|
||||||
|
1,
|
||||||
|
"A Tech Post & Its Friends",
|
||||||
|
content=(
|
||||||
|
'<p>Intro with an <a href="/relative-link">in-page link</a>.</p>'
|
||||||
|
"<script>alert('nope')</script>"
|
||||||
|
"<ul><li>first<li>second</ul>"
|
||||||
|
'<p><img src="/images/diagram.png" alt="A diagram"></p>'
|
||||||
|
"<p>Entities: & <tag> done.</p>"
|
||||||
|
),
|
||||||
|
published=1_700_000_100,
|
||||||
|
)
|
||||||
|
stub.add_item(tech, 2, "An Older Tech Post", published=1_699_000_000)
|
||||||
|
stub.add_item(
|
||||||
|
tech,
|
||||||
|
3,
|
||||||
|
"Already Read Post",
|
||||||
|
published=1_698_000_000,
|
||||||
|
read=True,
|
||||||
|
)
|
||||||
|
stub.add_item(news, 4, "Markets Dip Slightly", published=1_700_000_200, author=None)
|
||||||
|
stub.add_item(
|
||||||
|
news,
|
||||||
|
5,
|
||||||
|
"Starred Investigation",
|
||||||
|
published=1_700_000_300,
|
||||||
|
starred=True,
|
||||||
|
)
|
||||||
|
stub.add_item(lonely, 6, "Uncategorised Musings", published=1_700_000_050)
|
||||||
|
return stub
|
||||||
235
tests/support.py
Normal file
235
tests/support.py
Normal file
@@ -0,0 +1,235 @@
|
|||||||
|
"""Shared test helpers.
|
||||||
|
|
||||||
|
The XML helper here is deliberately not ``xml.etree``: that API is documented as
|
||||||
|
unsafe for untrusted documents because it will expand entities and honour a DTD.
|
||||||
|
Even though every document these tests parse is one this project generated, the
|
||||||
|
parser is configured to refuse DTDs, entity declarations and external entity
|
||||||
|
resolution outright. Refusing them is stronger than trusting the parser's
|
||||||
|
amplification limits, and it keeps the test suite free of the pattern that makes
|
||||||
|
reviewers (and linters) rightly nervous.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from xml.parsers import expat
|
||||||
|
|
||||||
|
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
|
||||||
|
|
||||||
|
_WS_RE = re.compile(r"\s+")
|
||||||
|
|
||||||
|
|
||||||
|
def local_name(tag: str) -> str:
|
||||||
|
"""The part of a tag after any namespace prefix.
|
||||||
|
|
||||||
|
Expat is used without namespace processing here, so a prefixed element arrives
|
||||||
|
as ``dc:title``. Structural assertions care about the element, not the
|
||||||
|
prefix, so lookups compare the local name and read naturally.
|
||||||
|
"""
|
||||||
|
return tag.rsplit(":", 1)[-1]
|
||||||
|
|
||||||
|
|
||||||
|
class XmlNode:
|
||||||
|
"""A minimal element tree node, enough for structural assertions."""
|
||||||
|
|
||||||
|
__slots__ = ("attrs", "children", "tag", "text")
|
||||||
|
|
||||||
|
def __init__(self, tag: str, attrs: dict[str, str] | None = None) -> None:
|
||||||
|
self.tag = tag
|
||||||
|
self.attrs: dict[str, str] = attrs or {}
|
||||||
|
self.children: list[XmlNode] = []
|
||||||
|
self.text = ""
|
||||||
|
|
||||||
|
def attr(self, name: str, default: str | None = None) -> str | None:
|
||||||
|
"""Return an attribute value (None when absent)."""
|
||||||
|
return self.attrs.get(name, default)
|
||||||
|
|
||||||
|
def must_find(self, tag: str) -> XmlNode:
|
||||||
|
"""The first descendant with this tag, failing loudly when absent.
|
||||||
|
|
||||||
|
Structural assertions want to say "the OPF has no <manifest>" rather than
|
||||||
|
hand back a None that only explodes on the next attribute access.
|
||||||
|
"""
|
||||||
|
found = self.find(tag)
|
||||||
|
if found is None:
|
||||||
|
raise AssertionError(f"no <{tag}> element in <{self.tag}>")
|
||||||
|
return found
|
||||||
|
|
||||||
|
def must_find_all(self, tag: str) -> list[XmlNode]:
|
||||||
|
"""Every descendant with this tag, failing loudly when there are none."""
|
||||||
|
found = self.find_all(tag)
|
||||||
|
if not found:
|
||||||
|
raise AssertionError(f"no <{tag}> elements in <{self.tag}>")
|
||||||
|
return found
|
||||||
|
|
||||||
|
def must_attr(self, name: str) -> str:
|
||||||
|
"""An attribute value that must be present."""
|
||||||
|
value = self.attrs.get(name)
|
||||||
|
if value is None:
|
||||||
|
raise AssertionError(f"no {name!r} attribute on <{self.tag}>")
|
||||||
|
return value
|
||||||
|
|
||||||
|
def find_all(self, tag: str) -> list[XmlNode]:
|
||||||
|
"""All descendants matching a tag name, in document order.
|
||||||
|
|
||||||
|
Matches the local name, so ``find("title")`` finds ``dc:title`` while
|
||||||
|
``find("dc:title")`` still works.
|
||||||
|
"""
|
||||||
|
wanted = local_name(tag)
|
||||||
|
found: list[XmlNode] = []
|
||||||
|
for child in self.children:
|
||||||
|
if child.tag == tag or local_name(child.tag) == wanted:
|
||||||
|
found.append(child)
|
||||||
|
found.extend(child.find_all(tag))
|
||||||
|
return found
|
||||||
|
|
||||||
|
def find(self, tag: str) -> XmlNode | None:
|
||||||
|
"""First descendant with a matching tag name, or None."""
|
||||||
|
matches = self.find_all(tag)
|
||||||
|
return matches[0] if matches else None
|
||||||
|
|
||||||
|
def all_text(self) -> str:
|
||||||
|
"""Concatenated text of this node and its descendants."""
|
||||||
|
return self.text + "".join(child.all_text() for child in self.children)
|
||||||
|
|
||||||
|
def direct_text(self) -> str:
|
||||||
|
"""Text belonging to this node only, ignoring descendants."""
|
||||||
|
return self.text
|
||||||
|
|
||||||
|
|
||||||
|
def parse_xml(data: str | bytes) -> XmlNode:
|
||||||
|
"""Parse XML into an :class:`XmlNode` tree, refusing any DTD or entity.
|
||||||
|
|
||||||
|
Raises ``expat.ExpatError`` for malformed input and ``ValueError`` for a
|
||||||
|
document that tries to declare a doctype or an entity.
|
||||||
|
"""
|
||||||
|
parser = expat.ParserCreate()
|
||||||
|
parser.buffer_text = True
|
||||||
|
# Parameter entities come from a DTD, which we never accept.
|
||||||
|
parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_NEVER)
|
||||||
|
|
||||||
|
def reject_doctype(doctype_name, system_id, public_id, has_internal_subset):
|
||||||
|
raise ValueError("doctype declarations are not allowed in generated XML")
|
||||||
|
|
||||||
|
def reject_entity(*_args):
|
||||||
|
raise ValueError("entity declarations are not allowed in generated XML")
|
||||||
|
|
||||||
|
def reject_external(*_args):
|
||||||
|
# Returning 0 makes expat fail the parse instead of resolving the entity.
|
||||||
|
return 0
|
||||||
|
|
||||||
|
parser.StartDoctypeDeclHandler = reject_doctype
|
||||||
|
parser.EntityDeclHandler = reject_entity
|
||||||
|
parser.ExternalEntityRefHandler = reject_external
|
||||||
|
|
||||||
|
root = XmlNode("#document")
|
||||||
|
stack = [root]
|
||||||
|
|
||||||
|
def start_element(tag: str, attrs: dict[str, str]) -> None:
|
||||||
|
stack.append(XmlNode(tag, dict(attrs)))
|
||||||
|
|
||||||
|
def end_element(_tag: str) -> None:
|
||||||
|
node = stack.pop()
|
||||||
|
stack[-1].children.append(node)
|
||||||
|
|
||||||
|
def characters(data: str) -> None:
|
||||||
|
stack[-1].text += data
|
||||||
|
|
||||||
|
parser.StartElementHandler = start_element
|
||||||
|
parser.EndElementHandler = end_element
|
||||||
|
parser.CharacterDataHandler = characters
|
||||||
|
parser.Parse(data, True)
|
||||||
|
if not root.children:
|
||||||
|
raise ValueError("XML document has no root element")
|
||||||
|
return root.children[0]
|
||||||
|
|
||||||
|
|
||||||
|
def assert_well_formed(testcase, fragment: str) -> None:
|
||||||
|
"""Assert that ``fragment`` parses as XML inside a single wrapper root."""
|
||||||
|
markup = f'<root xmlns="{XHTML_NAMESPACE}">{fragment}</root>'
|
||||||
|
try:
|
||||||
|
parse_xml(markup)
|
||||||
|
except (expat.ExpatError, ValueError) as exc: # pragma: no cover - on failure
|
||||||
|
testcase.fail(f"output is not well-formed XML: {exc}\n{fragment}")
|
||||||
|
|
||||||
|
|
||||||
|
def collapse(text: str) -> str:
|
||||||
|
"""Collapse all whitespace, for assertions that do not care about layout."""
|
||||||
|
return _WS_RE.sub(" ", text).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def body(fragment: str) -> str:
|
||||||
|
"""Return the inner markup of the article wrapper the sanitiser emits."""
|
||||||
|
prefix = '<div class="article-body">'
|
||||||
|
if fragment.startswith(prefix):
|
||||||
|
fragment = fragment[len(prefix) :]
|
||||||
|
suffix = "</div>"
|
||||||
|
if fragment.endswith(suffix):
|
||||||
|
fragment = fragment[: -len(suffix)]
|
||||||
|
return fragment.strip()
|
||||||
|
|
||||||
|
|
||||||
|
# -- image builders ------------------------------------------------------
|
||||||
|
#
|
||||||
|
# These produce bytes that are structurally real -- correct magic numbers and
|
||||||
|
# container headers -- so sniffing and dimension parsing are exercised against the
|
||||||
|
# formats rather than against hand-written expectations. Pixel data is minimal
|
||||||
|
# because nothing here ever decodes an image.
|
||||||
|
|
||||||
|
|
||||||
|
def png_bytes(width: int = 800, height: int = 400) -> bytes:
|
||||||
|
"""A structurally valid PNG of the given size."""
|
||||||
|
import struct
|
||||||
|
import zlib
|
||||||
|
|
||||||
|
def chunk(tag: bytes, payload: bytes) -> bytes:
|
||||||
|
crc = zlib.crc32(tag + payload) & 0xFFFFFFFF
|
||||||
|
return struct.pack(">I", len(payload)) + tag + payload + struct.pack(">I", crc)
|
||||||
|
|
||||||
|
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||||
|
rows = b"".join(b"\x00" + b"\xff\x00\x00" * width for _ in range(height))
|
||||||
|
return (
|
||||||
|
b"\x89PNG\r\n\x1a\n"
|
||||||
|
+ chunk(b"IHDR", header)
|
||||||
|
+ chunk(b"IDAT", zlib.compress(rows, 9))
|
||||||
|
+ chunk(b"IEND", b"")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def jpeg_bytes(width: int = 800, height: int = 400) -> bytes:
|
||||||
|
"""SOI + APP0 + SOF0 + EOI: a JPEG the dimension walker can read."""
|
||||||
|
import struct
|
||||||
|
|
||||||
|
app0 = b"\xff\xe0" + struct.pack(">H", 16) + b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00"
|
||||||
|
# The frame header carries height and width, in that order.
|
||||||
|
sof0 = (
|
||||||
|
b"\xff\xc0"
|
||||||
|
+ struct.pack(">H", 17)
|
||||||
|
+ b"\x08"
|
||||||
|
+ struct.pack(">HH", height, width)
|
||||||
|
+ b"\x03\x01\x11\x00\x02\x11\x01\x03\x11\x01"
|
||||||
|
)
|
||||||
|
return b"\xff\xd8" + app0 + sof0 + b"\xff\xd9"
|
||||||
|
|
||||||
|
|
||||||
|
def gif_bytes(width: int = 320, height: int = 200) -> bytes:
|
||||||
|
"""A GIF header with the given logical screen size."""
|
||||||
|
import struct
|
||||||
|
|
||||||
|
return b"GIF89a" + struct.pack("<HH", width, height) + b"\xf7\x00\x00" + b"\x00" * 20
|
||||||
|
|
||||||
|
|
||||||
|
def webp_bytes(width: int = 800, height: int = 400) -> bytes:
|
||||||
|
"""A lossless VP8X WebP header with the given canvas size (stored minus one)."""
|
||||||
|
import struct
|
||||||
|
|
||||||
|
return (
|
||||||
|
b"RIFF"
|
||||||
|
+ struct.pack("<I", 22)
|
||||||
|
+ b"WEBP"
|
||||||
|
+ b"VP8X"
|
||||||
|
+ struct.pack("<I", 10)
|
||||||
|
+ b"\x00\x00\x00\x00"
|
||||||
|
+ (width - 1).to_bytes(3, "little")
|
||||||
|
+ (height - 1).to_bytes(3, "little")
|
||||||
|
)
|
||||||
487
tests/test_config.py
Normal file
487
tests/test_config.py
Normal file
@@ -0,0 +1,487 @@
|
|||||||
|
"""Tests for configuration loading, interpolation and validation."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.config import (
|
||||||
|
DEFAULT_BIND_HOST,
|
||||||
|
ConfigError,
|
||||||
|
describe_config,
|
||||||
|
load_config,
|
||||||
|
normalise_api_base,
|
||||||
|
password_digest,
|
||||||
|
resolve_config_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Fixture credentials. These are throwaway values that exercise the schema; no
|
||||||
|
# real account is involved and neither value is a secret. They are applied
|
||||||
|
# through the helpers below so every test states its intent in one line.
|
||||||
|
FIXTURE_UPSTREAM_VALUE = "upstream-fixture-value"
|
||||||
|
FIXTURE_READER_VALUE = "reader-fixture-value"
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
EXAMPLE_CONFIG = REPO_ROOT / "config.example.json"
|
||||||
|
|
||||||
|
|
||||||
|
def with_upstream_credential(document: dict, value: str) -> dict:
|
||||||
|
"""Set the upstream FreshRSS API credential under test."""
|
||||||
|
document["freshrss"]["api_password"] = value
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def with_reader_credential(document: dict, value: str, username: str = "reader") -> dict:
|
||||||
|
"""Replace auth.users with one user carrying a clear-text credential."""
|
||||||
|
document["auth"]["users"] = [{"username": username, "password": value}]
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def with_reader_digest(document: dict, digest: str, username: str = "reader") -> dict:
|
||||||
|
"""Replace auth.users with one user carrying a pre-computed digest."""
|
||||||
|
document["auth"]["users"] = [{"username": username, "password_sha256": digest}]
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def with_conflicting_credentials(document: dict, value: str) -> dict:
|
||||||
|
"""A deliberately over-specified user, to exercise the conflict rule."""
|
||||||
|
document["auth"]["users"] = [
|
||||||
|
{
|
||||||
|
"username": "reader",
|
||||||
|
"password": value,
|
||||||
|
"password_sha256": password_digest(value),
|
||||||
|
}
|
||||||
|
]
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def base_config() -> dict:
|
||||||
|
"""A minimal valid configuration, ready to be mutated by a test."""
|
||||||
|
document = {
|
||||||
|
"server": {"port": 8080},
|
||||||
|
"freshrss": {
|
||||||
|
"url": "https://rss.example.com",
|
||||||
|
"username": "tyler",
|
||||||
|
"api_password": FIXTURE_UPSTREAM_VALUE,
|
||||||
|
},
|
||||||
|
"auth": {"users": [{"username": "tyler", "password": FIXTURE_READER_VALUE}]},
|
||||||
|
}
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigTestCase(unittest.TestCase):
|
||||||
|
"""Writes a config document into a temporary directory and loads it."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self._dir = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self._dir.cleanup)
|
||||||
|
self.dir = Path(self._dir.name)
|
||||||
|
|
||||||
|
def write(self, document, name: str = "config.json") -> str:
|
||||||
|
path = self.dir / name
|
||||||
|
if isinstance(document, str):
|
||||||
|
path.write_text(document, encoding="utf-8")
|
||||||
|
else:
|
||||||
|
path.write_text(json.dumps(document), encoding="utf-8")
|
||||||
|
return str(path)
|
||||||
|
|
||||||
|
def load(self, document, name: str = "config.json"):
|
||||||
|
return load_config(self.write(document, name))
|
||||||
|
|
||||||
|
def expect_problem(self, document, fragment: str) -> ConfigError:
|
||||||
|
"""Assert the config is rejected, and return the error for more checks."""
|
||||||
|
with self.assertRaises(ConfigError) as caught:
|
||||||
|
self.load(document)
|
||||||
|
joined = "\n".join(caught.exception.problems)
|
||||||
|
self.assertIn(fragment, joined)
|
||||||
|
return caught.exception
|
||||||
|
|
||||||
|
|
||||||
|
class LoadingTests(ConfigTestCase):
|
||||||
|
def test_minimal_config_uses_documented_defaults(self):
|
||||||
|
config = self.load(base_config())
|
||||||
|
self.assertEqual(config.server.host, DEFAULT_BIND_HOST)
|
||||||
|
self.assertEqual(config.server.port, 8080)
|
||||||
|
self.assertEqual(config.freshrss.items_per_page, 30)
|
||||||
|
self.assertTrue(config.freshrss.mark_read)
|
||||||
|
self.assertTrue(config.epub.include_images)
|
||||||
|
self.assertEqual(config.auth.users[0].username, "tyler")
|
||||||
|
self.assertEqual(len(config.auth.users[0].password_sha256), 64)
|
||||||
|
|
||||||
|
def test_default_bind_address_is_loopback(self):
|
||||||
|
# The wildcard belongs in the deployment's config file, not in the code:
|
||||||
|
# a stray `docker run -p 8080:8080` should not silently be reachable.
|
||||||
|
self.assertEqual(DEFAULT_BIND_HOST, "127.0.0.1")
|
||||||
|
|
||||||
|
def test_missing_file_is_a_clear_error(self):
|
||||||
|
with self.assertRaises(ConfigError) as caught:
|
||||||
|
load_config(str(self.dir / "nope.json"))
|
||||||
|
joined = "\n".join(caught.exception.problems)
|
||||||
|
self.assertIn("not found", joined)
|
||||||
|
self.assertIn("CONFIG_PATH", joined)
|
||||||
|
|
||||||
|
def test_invalid_json_reports_position(self):
|
||||||
|
error = self.expect_problem('{"server": {,}', "not valid JSON")
|
||||||
|
self.assertIn("line 1", "\n".join(error.problems))
|
||||||
|
|
||||||
|
def test_non_object_root_is_rejected(self):
|
||||||
|
self.expect_problem("[1, 2, 3]", "top level")
|
||||||
|
|
||||||
|
def test_source_path_is_recorded(self):
|
||||||
|
path = self.write(base_config())
|
||||||
|
self.assertEqual(load_config(path).source_path, str(Path(path).resolve()))
|
||||||
|
|
||||||
|
def test_resolve_config_path_prefers_explicit_then_env(self):
|
||||||
|
explicit = self.write(base_config(), "explicit.json")
|
||||||
|
self.assertEqual(resolve_config_path(explicit), explicit)
|
||||||
|
from_env = str(self.dir / "from-env.json")
|
||||||
|
os.environ["CONFIG_PATH"] = from_env
|
||||||
|
self.addCleanup(os.environ.pop, "CONFIG_PATH", None)
|
||||||
|
self.assertEqual(resolve_config_path(), from_env)
|
||||||
|
|
||||||
|
|
||||||
|
class ExampleConfigTests(unittest.TestCase):
|
||||||
|
"""The shipped example must load: a broken example is a broken first run.
|
||||||
|
|
||||||
|
This is also where the container's wildcard bind address is asserted. That
|
||||||
|
literal belongs in the deployment's configuration file, not in Python, and the
|
||||||
|
test proves the file really does override the loopback default.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
# The example deliberately leaves secrets to the environment.
|
||||||
|
os.environ["FRESHRSS_API_PASSWORD"] = FIXTURE_UPSTREAM_VALUE
|
||||||
|
os.environ["OPDS_PASSWORD_SHA256"] = password_digest(FIXTURE_READER_VALUE)
|
||||||
|
self.addCleanup(os.environ.pop, "FRESHRSS_API_PASSWORD", None)
|
||||||
|
self.addCleanup(os.environ.pop, "OPDS_PASSWORD_SHA256", None)
|
||||||
|
self.document = json.loads(EXAMPLE_CONFIG.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
def test_example_config_exists_and_is_valid(self):
|
||||||
|
config = load_config(str(EXAMPLE_CONFIG))
|
||||||
|
self.assertEqual(config.warnings, [])
|
||||||
|
self.assertEqual(len(config.auth.users), 1)
|
||||||
|
|
||||||
|
def test_example_config_overrides_the_loopback_default(self):
|
||||||
|
config = load_config(str(EXAMPLE_CONFIG))
|
||||||
|
self.assertNotEqual(config.server.host, DEFAULT_BIND_HOST)
|
||||||
|
self.assertEqual(config.server.host, self.document["server"]["host"])
|
||||||
|
|
||||||
|
def test_example_keeps_secrets_out_of_the_file(self):
|
||||||
|
self.assertTrue(self.document["freshrss"]["api_password"].startswith("${"))
|
||||||
|
self.assertTrue(self.document["auth"]["users"][0]["password_sha256"].startswith("${"))
|
||||||
|
|
||||||
|
def test_example_is_self_documenting(self):
|
||||||
|
comments = [key for key in self.document if key.startswith("_")]
|
||||||
|
self.assertTrue(comments)
|
||||||
|
self.assertTrue(self.document["server"].get("_host"))
|
||||||
|
|
||||||
|
def test_example_interpolates_its_credentials(self):
|
||||||
|
self.assertEqual(
|
||||||
|
load_config(str(EXAMPLE_CONFIG)).freshrss.api_password, FIXTURE_UPSTREAM_VALUE
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class InterpolationTests(ConfigTestCase):
|
||||||
|
def test_environment_variable_is_expanded(self):
|
||||||
|
os.environ["TEST_FRESHRSS_CREDENTIAL"] = "from-env"
|
||||||
|
self.addCleanup(os.environ.pop, "TEST_FRESHRSS_CREDENTIAL", None)
|
||||||
|
document = with_upstream_credential(base_config(), "${TEST_FRESHRSS_CREDENTIAL}")
|
||||||
|
self.assertEqual(self.load(document).freshrss.api_password, "from-env")
|
||||||
|
|
||||||
|
def test_environment_variable_default_is_used_when_unset(self):
|
||||||
|
os.environ.pop("TEST_UNSET_VAR", None)
|
||||||
|
document = with_upstream_credential(base_config(), "${TEST_UNSET_VAR:-fallback}")
|
||||||
|
self.assertEqual(self.load(document).freshrss.api_password, "fallback")
|
||||||
|
|
||||||
|
def test_missing_environment_variable_names_itself(self):
|
||||||
|
os.environ.pop("TEST_DEFINITELY_UNSET", None)
|
||||||
|
document = with_upstream_credential(base_config(), "${TEST_DEFINITELY_UNSET}")
|
||||||
|
error = self.expect_problem(document, "TEST_DEFINITELY_UNSET")
|
||||||
|
self.assertIn("freshrss.api_password", "\n".join(error.problems))
|
||||||
|
|
||||||
|
def test_secret_file_is_read_and_trimmed(self):
|
||||||
|
(self.dir / "upstream-secret").write_text("file-secret\n", encoding="utf-8")
|
||||||
|
# A relative path resolves against the config file's own directory.
|
||||||
|
document = with_upstream_credential(base_config(), "${file:upstream-secret}")
|
||||||
|
self.assertEqual(self.load(document).freshrss.api_password, "file-secret")
|
||||||
|
|
||||||
|
def test_secret_file_may_be_absolute(self):
|
||||||
|
absolute = self.dir / "absolute-secret"
|
||||||
|
absolute.write_text("abs", encoding="utf-8")
|
||||||
|
document = with_upstream_credential(base_config(), f"${{file:{absolute}}}")
|
||||||
|
self.assertEqual(self.load(document).freshrss.api_password, "abs")
|
||||||
|
|
||||||
|
def test_missing_secret_file_is_an_error(self):
|
||||||
|
document = with_upstream_credential(base_config(), "${file:not-there}")
|
||||||
|
self.expect_problem(document, "cannot read secret file")
|
||||||
|
|
||||||
|
def test_escaped_dollar_is_literal(self):
|
||||||
|
document = with_upstream_credential(base_config(), "$${NOT_A_REFERENCE}")
|
||||||
|
self.assertEqual(self.load(document).freshrss.api_password, "${NOT_A_REFERENCE}")
|
||||||
|
|
||||||
|
def test_interpolation_reaches_nested_arrays(self):
|
||||||
|
os.environ.pop("TEST_PW", None)
|
||||||
|
document = with_reader_credential(base_config(), "${TEST_PW:-pw}")
|
||||||
|
self.assertEqual(self.load(document).auth.users[0].password_sha256, password_digest("pw"))
|
||||||
|
|
||||||
|
def test_empty_reference_is_an_error(self):
|
||||||
|
document = with_upstream_credential(base_config(), "${}")
|
||||||
|
self.expect_problem(document, "empty ${}")
|
||||||
|
|
||||||
|
|
||||||
|
class FreshRSSTests(ConfigTestCase):
|
||||||
|
def test_api_base_appends_the_greader_suffix(self):
|
||||||
|
self.assertEqual(
|
||||||
|
normalise_api_base("https://rss.example.com"),
|
||||||
|
"https://rss.example.com/api/greader.php",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_api_base_keeps_an_existing_suffix(self):
|
||||||
|
self.assertEqual(
|
||||||
|
normalise_api_base("https://rss.example.com/api/greader.php"),
|
||||||
|
"https://rss.example.com/api/greader.php",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_api_base_preserves_a_path_prefix(self):
|
||||||
|
self.assertEqual(
|
||||||
|
normalise_api_base("https://example.com/freshrss/"),
|
||||||
|
"https://example.com/freshrss/api/greader.php",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_api_base_is_case_insensitive_about_the_suffix(self):
|
||||||
|
self.assertEqual(
|
||||||
|
normalise_api_base("https://e.com/API/GReader.php"),
|
||||||
|
"https://e.com/API/GReader.php",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_url_without_scheme_is_rejected(self):
|
||||||
|
document = base_config()
|
||||||
|
document["freshrss"]["url"] = "rss.example.com"
|
||||||
|
self.expect_problem(document, "must start with http:// or https://")
|
||||||
|
|
||||||
|
def test_required_fields_are_reported_together(self):
|
||||||
|
document = {"server": {}, "auth": {"users": [{"username": "u", "password": "p"}]}}
|
||||||
|
with self.assertRaises(ConfigError) as caught:
|
||||||
|
self.load(document)
|
||||||
|
joined = "\n".join(caught.exception.problems)
|
||||||
|
for fragment in ("freshrss.url", "freshrss.username", "freshrss.api_password"):
|
||||||
|
self.assertIn(fragment, joined)
|
||||||
|
|
||||||
|
def test_items_per_page_must_fit_the_request_ceiling(self):
|
||||||
|
document = base_config()
|
||||||
|
document["freshrss"]["items_per_page"] = 100
|
||||||
|
document["freshrss"]["max_items_per_request"] = 50
|
||||||
|
self.expect_problem(document, "must be at least")
|
||||||
|
|
||||||
|
def test_retries_and_timeouts_are_validated(self):
|
||||||
|
document = base_config()
|
||||||
|
document["freshrss"]["timeout_seconds"] = 0
|
||||||
|
document["freshrss"]["retries"] = -1
|
||||||
|
with self.assertRaises(ConfigError) as caught:
|
||||||
|
self.load(document)
|
||||||
|
joined = "\n".join(caught.exception.problems)
|
||||||
|
self.assertIn("timeout_seconds", joined)
|
||||||
|
self.assertIn("retries", joined)
|
||||||
|
|
||||||
|
def test_mark_read_can_be_disabled(self):
|
||||||
|
document = base_config()
|
||||||
|
document["freshrss"]["mark_read"] = False
|
||||||
|
self.assertFalse(self.load(document).freshrss.mark_read)
|
||||||
|
|
||||||
|
|
||||||
|
class TypeValidationTests(ConfigTestCase):
|
||||||
|
def test_port_must_be_a_number_in_range(self):
|
||||||
|
for value, fragment in (
|
||||||
|
("8080", "whole number"),
|
||||||
|
(-1, "at least"),
|
||||||
|
(70000, "at most"),
|
||||||
|
):
|
||||||
|
document = base_config()
|
||||||
|
document["server"]["port"] = value
|
||||||
|
self.expect_problem(document, fragment)
|
||||||
|
|
||||||
|
def test_port_zero_asks_the_operating_system_for_a_free_port(self):
|
||||||
|
# Useful for a local run or a test; a normal deployment sets a real port.
|
||||||
|
document = base_config()
|
||||||
|
document["server"]["port"] = 0
|
||||||
|
self.assertEqual(self.load(document).server.port, 0)
|
||||||
|
|
||||||
|
def test_boolean_field_rejects_a_number(self):
|
||||||
|
document = base_config()
|
||||||
|
document["freshrss"]["mark_read"] = 1
|
||||||
|
self.expect_problem(document, "must be true or false")
|
||||||
|
|
||||||
|
def test_integer_field_rejects_a_boolean(self):
|
||||||
|
# bool is an int subclass in Python; `true` must not become 1.
|
||||||
|
document = base_config()
|
||||||
|
document["server"]["port"] = True
|
||||||
|
self.expect_problem(document, "whole number")
|
||||||
|
|
||||||
|
def test_string_field_rejects_a_list(self):
|
||||||
|
document = base_config()
|
||||||
|
document["catalog"] = {"title": ["not", "a", "string"]}
|
||||||
|
self.expect_problem(document, "catalog.title: must be a string")
|
||||||
|
|
||||||
|
def test_string_list_rejects_scalars(self):
|
||||||
|
document = base_config()
|
||||||
|
document["catalog"] = {"exclude_feeds": "one-feed"}
|
||||||
|
self.expect_problem(document, "must be an array of strings")
|
||||||
|
|
||||||
|
def test_section_must_be_an_object(self):
|
||||||
|
document = base_config()
|
||||||
|
document["cache"] = "nope"
|
||||||
|
self.expect_problem(document, "cache: must be a JSON object")
|
||||||
|
|
||||||
|
def test_log_level_is_restricted(self):
|
||||||
|
document = base_config()
|
||||||
|
document["server"]["log_level"] = "chatty"
|
||||||
|
self.expect_problem(document, "must be one of")
|
||||||
|
|
||||||
|
def test_ttl_must_be_positive(self):
|
||||||
|
document = base_config()
|
||||||
|
document["cache"] = {"stream_ttl_seconds": 0}
|
||||||
|
self.expect_problem(document, "greater than 0")
|
||||||
|
|
||||||
|
|
||||||
|
class AuthTests(ConfigTestCase):
|
||||||
|
def test_users_are_required(self):
|
||||||
|
document = base_config()
|
||||||
|
del document["auth"]
|
||||||
|
error = self.expect_problem(document, "auth.users: required")
|
||||||
|
self.assertIn("credentials", "\n".join(error.problems))
|
||||||
|
|
||||||
|
def test_empty_user_list_is_rejected(self):
|
||||||
|
document = base_config()
|
||||||
|
document["auth"]["users"] = []
|
||||||
|
self.expect_problem(document, "at least one user")
|
||||||
|
|
||||||
|
def test_clear_text_password_is_hashed(self):
|
||||||
|
document = with_reader_credential(base_config(), FIXTURE_READER_VALUE)
|
||||||
|
config = self.load(document)
|
||||||
|
self.assertEqual(
|
||||||
|
config.auth.users[0].password_sha256, password_digest(FIXTURE_READER_VALUE)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_clear_text_password_is_never_kept_in_the_config_object(self):
|
||||||
|
document = with_reader_credential(base_config(), FIXTURE_READER_VALUE)
|
||||||
|
config = self.load(document)
|
||||||
|
self.assertNotIn(FIXTURE_READER_VALUE, json.dumps(config.auth.users[0].__dict__))
|
||||||
|
|
||||||
|
def test_sha256_password_is_accepted_and_lowercased(self):
|
||||||
|
document = with_reader_digest(base_config(), password_digest("s3cret").upper())
|
||||||
|
self.assertEqual(
|
||||||
|
self.load(document).auth.users[0].password_sha256, password_digest("s3cret")
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_both_password_forms_is_an_error(self):
|
||||||
|
document = with_conflicting_credentials(base_config(), FIXTURE_READER_VALUE)
|
||||||
|
self.expect_problem(document, "not both")
|
||||||
|
|
||||||
|
def test_neither_password_form_is_an_error(self):
|
||||||
|
document = base_config()
|
||||||
|
document["auth"]["users"] = [{"username": "a"}]
|
||||||
|
self.expect_problem(document, "needs password or password_sha256")
|
||||||
|
|
||||||
|
def test_malformed_digest_is_rejected(self):
|
||||||
|
document = with_reader_digest(base_config(), "deadbeef")
|
||||||
|
self.expect_problem(document, "64 hex characters")
|
||||||
|
|
||||||
|
def test_duplicate_usernames_are_rejected(self):
|
||||||
|
document = base_config()
|
||||||
|
document["auth"]["users"] = [
|
||||||
|
{"username": "a", "password": FIXTURE_READER_VALUE},
|
||||||
|
{"username": "a", "password": FIXTURE_UPSTREAM_VALUE},
|
||||||
|
]
|
||||||
|
self.expect_problem(document, "duplicate username")
|
||||||
|
|
||||||
|
def test_user_must_be_an_object(self):
|
||||||
|
document = base_config()
|
||||||
|
document["auth"]["users"] = ["tyler"]
|
||||||
|
self.expect_problem(document, "must be a JSON object")
|
||||||
|
|
||||||
|
def test_realm_is_configurable(self):
|
||||||
|
document = base_config()
|
||||||
|
document["auth"]["realm"] = "Reading"
|
||||||
|
self.assertEqual(self.load(document).auth.realm, "Reading")
|
||||||
|
|
||||||
|
|
||||||
|
class TlsTests(ConfigTestCase):
|
||||||
|
def test_cert_without_key_is_rejected(self):
|
||||||
|
document = base_config()
|
||||||
|
document["server"]["tls"] = {"cert_file": str(self.dir / "cert.pem")}
|
||||||
|
self.expect_problem(document, "must be set together")
|
||||||
|
|
||||||
|
def test_missing_certificate_file_is_rejected(self):
|
||||||
|
document = base_config()
|
||||||
|
document["server"]["tls"] = {
|
||||||
|
"cert_file": str(self.dir / "absent-cert.pem"),
|
||||||
|
"key_file": str(self.dir / "absent-key.pem"),
|
||||||
|
}
|
||||||
|
self.expect_problem(document, "no such file")
|
||||||
|
|
||||||
|
def test_present_certificate_pair_is_accepted(self):
|
||||||
|
cert = self.dir / "cert.pem"
|
||||||
|
key = self.dir / "key.pem"
|
||||||
|
cert.write_text("dummy", encoding="utf-8")
|
||||||
|
key.write_text("dummy", encoding="utf-8")
|
||||||
|
document = base_config()
|
||||||
|
document["server"]["tls"] = {"cert_file": str(cert), "key_file": str(key)}
|
||||||
|
config = self.load(document)
|
||||||
|
self.assertEqual(config.server.tls_cert_file, str(cert))
|
||||||
|
|
||||||
|
|
||||||
|
class WarningTests(ConfigTestCase):
|
||||||
|
def test_unknown_key_is_reported_with_a_suggestion(self):
|
||||||
|
document = base_config()
|
||||||
|
document["freshrss"]["mark-read"] = True
|
||||||
|
config = self.load(document)
|
||||||
|
self.assertTrue(config.warnings)
|
||||||
|
self.assertIn("freshrss.mark-read", config.warnings[0])
|
||||||
|
self.assertIn("mark_read", config.warnings[0])
|
||||||
|
|
||||||
|
def test_unknown_user_key_is_reported(self):
|
||||||
|
document = base_config()
|
||||||
|
document["auth"]["users"] = [
|
||||||
|
{"username": "a", "password": FIXTURE_READER_VALUE, "passwrod": "typo"}
|
||||||
|
]
|
||||||
|
config = self.load(document)
|
||||||
|
self.assertIn("auth.users[0].passwrod", config.warnings[0])
|
||||||
|
|
||||||
|
def test_unknown_tls_key_is_reported(self):
|
||||||
|
document = base_config()
|
||||||
|
document["server"]["tls"] = {"certfile": "/x", "keyfile": "/y"}
|
||||||
|
config = self.load(document)
|
||||||
|
self.assertEqual(len(config.warnings), 2)
|
||||||
|
|
||||||
|
def test_unknown_section_is_reported(self):
|
||||||
|
document = base_config()
|
||||||
|
document["cach"] = {"max_entries": 10}
|
||||||
|
config = self.load(document)
|
||||||
|
self.assertIn("cach:", config.warnings[0])
|
||||||
|
self.assertIn("cache", config.warnings[0])
|
||||||
|
|
||||||
|
def test_warnings_do_not_block_a_valid_config(self):
|
||||||
|
document = base_config()
|
||||||
|
document["nonsense"] = 1
|
||||||
|
self.assertEqual(len(self.load(document).warnings), 1)
|
||||||
|
|
||||||
|
|
||||||
|
class DescribeConfigTests(ConfigTestCase):
|
||||||
|
def test_password_is_redacted_in_the_summary(self):
|
||||||
|
summary = describe_config(self.load(base_config()))
|
||||||
|
self.assertNotIn(FIXTURE_UPSTREAM_VALUE, summary)
|
||||||
|
self.assertIn("rss.example.com/api/greader.php", summary)
|
||||||
|
|
||||||
|
def test_summary_lists_users(self):
|
||||||
|
self.assertIn("tyler", describe_config(self.load(base_config())))
|
||||||
|
|
||||||
|
def test_summary_notes_tls_state(self):
|
||||||
|
summary = describe_config(self.load(base_config()))
|
||||||
|
self.assertIn("127.0.0.1:8080", summary)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
533
tests/test_epub.py
Normal file
533
tests/test_epub.py
Normal file
@@ -0,0 +1,533 @@
|
|||||||
|
"""Tests for image metadata and EPUB generation.
|
||||||
|
|
||||||
|
The EPUB checks are structural on purpose. A book is rejected by a reader for
|
||||||
|
shape reasons -- a deflated mimetype, a manifest href with no entry, a document
|
||||||
|
that will not parse -- far more often than for content reasons, so those are the
|
||||||
|
properties asserted here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
import zipfile
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
from app import epub, images
|
||||||
|
from app.fetch import FetchedImage
|
||||||
|
from app.xhtml import parse_article
|
||||||
|
from tests.support import (
|
||||||
|
gif_bytes,
|
||||||
|
jpeg_bytes,
|
||||||
|
parse_xml,
|
||||||
|
png_bytes,
|
||||||
|
webp_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
|
MODIFIED = datetime(2026, 11, 15, 12, 30, 45, tzinfo=UTC)
|
||||||
|
PUBLISHED = datetime(2026, 11, 14, 9, 0, 0, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
class ImageMetadataTests(unittest.TestCase):
|
||||||
|
"""Sniffing and sizing, against structurally real image bytes."""
|
||||||
|
|
||||||
|
def test_sniffing_each_supported_format(self):
|
||||||
|
cases = {
|
||||||
|
"image/png": png_bytes(),
|
||||||
|
"image/jpeg": jpeg_bytes(),
|
||||||
|
"image/gif": gif_bytes(),
|
||||||
|
"image/webp": webp_bytes(),
|
||||||
|
}
|
||||||
|
for expected, data in cases.items():
|
||||||
|
self.assertEqual(images.sniff_media_type(data), expected)
|
||||||
|
|
||||||
|
def test_dimensions_are_read_for_each_format(self):
|
||||||
|
cases = [
|
||||||
|
(png_bytes(800, 400), (800, 400)),
|
||||||
|
(jpeg_bytes(640, 480), (640, 480)),
|
||||||
|
(gif_bytes(320, 200), (320, 200)),
|
||||||
|
(webp_bytes(1024, 512), (1024, 512)),
|
||||||
|
]
|
||||||
|
for data, expected in cases:
|
||||||
|
self.assertEqual(images.dimensions(data), expected)
|
||||||
|
|
||||||
|
def test_unknown_bytes_have_no_media_type(self):
|
||||||
|
self.assertIsNone(images.sniff_media_type(b"not an image at all"))
|
||||||
|
self.assertIsNone(images.sniff_media_type(b""))
|
||||||
|
self.assertIsNone(images.sniff_media_type(b"\x89PNG"))
|
||||||
|
|
||||||
|
def test_truncated_images_do_not_crash_the_sizer(self):
|
||||||
|
for data in (png_bytes()[:12], jpeg_bytes()[:6], gif_bytes()[:8], webp_bytes()[:20]):
|
||||||
|
self.assertIsNone(images.dimensions(data))
|
||||||
|
|
||||||
|
def test_svg_is_not_an_embeddable_type(self):
|
||||||
|
# SVG can carry script and external references; it is not on the list.
|
||||||
|
self.assertIsNone(images.normalise_media_type("image/svg+xml"))
|
||||||
|
|
||||||
|
def test_content_type_parameters_are_ignored(self):
|
||||||
|
self.assertEqual(images.normalise_media_type("image/jpeg; charset=binary"), "image/jpeg")
|
||||||
|
|
||||||
|
def test_aliases_are_normalised(self):
|
||||||
|
self.assertEqual(images.normalise_media_type("IMAGE/JPG"), "image/jpeg")
|
||||||
|
self.assertEqual(images.normalise_media_type("image/x-png"), "image/png")
|
||||||
|
|
||||||
|
def test_bytes_win_over_a_wrong_declared_type(self):
|
||||||
|
# A mislabelled payload must not produce an invalid manifest entry.
|
||||||
|
self.assertEqual(images.identify(png_bytes(), "text/html"), "image/png")
|
||||||
|
self.assertEqual(images.identify(b"unknown", "image/png"), "image/png")
|
||||||
|
self.assertIsNone(images.identify(b"unknown", "text/html"))
|
||||||
|
|
||||||
|
def test_cover_heuristic(self):
|
||||||
|
self.assertTrue(images.looks_like_a_cover(800, 400))
|
||||||
|
self.assertTrue(images.looks_like_a_cover(400, 200))
|
||||||
|
self.assertFalse(images.looks_like_a_cover(399, 200))
|
||||||
|
self.assertFalse(images.looks_like_a_cover(1, 1))
|
||||||
|
self.assertFalse(images.looks_like_a_cover(800, 100))
|
||||||
|
|
||||||
|
|
||||||
|
def make_book(**overrides) -> epub.Book:
|
||||||
|
values = {
|
||||||
|
"identifier": "urn:uuid:12345678-1234-1234-1234-123456789abc",
|
||||||
|
"title": "A Test Article & Its Friends",
|
||||||
|
"language": "en",
|
||||||
|
"modified": MODIFIED,
|
||||||
|
"published": PUBLISHED,
|
||||||
|
"content_xhtml": '<div class="article-body">\n<p>Body text.</p>\n</div>',
|
||||||
|
"author": "A. Writer",
|
||||||
|
"publisher": "Example Press",
|
||||||
|
"source_url": "https://example.com/articles/1",
|
||||||
|
"feed_title": "Example Tech",
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return epub.Book(**values)
|
||||||
|
|
||||||
|
|
||||||
|
class EpubContainerTests(unittest.TestCase):
|
||||||
|
"""The ZIP-level rules that make a file an EPUB at all."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.data = epub.build_epub(make_book())
|
||||||
|
self.archive = zipfile.ZipFile(BytesIO(self.data))
|
||||||
|
self.addCleanup(self.archive.close)
|
||||||
|
|
||||||
|
def test_mimetype_is_the_first_entry(self):
|
||||||
|
self.assertEqual(self.archive.namelist()[0], "mimetype")
|
||||||
|
|
||||||
|
def test_mimetype_is_stored_not_deflated(self):
|
||||||
|
info = self.archive.getinfo("mimetype")
|
||||||
|
self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
|
||||||
|
self.assertEqual(self.archive.read("mimetype"), b"application/epub+zip")
|
||||||
|
|
||||||
|
def test_mimetype_has_no_extra_field(self):
|
||||||
|
# An extra field in the local header is a known cause of readers refusing
|
||||||
|
# a book, so the raw bytes are inspected rather than the parsed record.
|
||||||
|
offset = self.archive.getinfo("mimetype").header_offset
|
||||||
|
local_header = self.data[offset : offset + 30]
|
||||||
|
extra_length = int.from_bytes(local_header[28:30], "little")
|
||||||
|
self.assertEqual(extra_length, 0)
|
||||||
|
|
||||||
|
def test_container_names_the_package_document(self):
|
||||||
|
xml = self.archive.read("META-INF/container.xml").decode("utf-8")
|
||||||
|
root = parse_xml(xml)
|
||||||
|
rootfile = root.must_find("rootfiles").must_find("rootfile")
|
||||||
|
self.assertEqual(rootfile.must_attr("full-path"), epub.OPF_PATH)
|
||||||
|
self.assertEqual(rootfile.must_attr("media-type"), "application/oebps-package+xml")
|
||||||
|
# Whatever it points at must actually exist.
|
||||||
|
self.assertIn(rootfile.must_attr("full-path"), self.archive.namelist())
|
||||||
|
|
||||||
|
def test_expected_parts_are_present(self):
|
||||||
|
names = set(self.archive.namelist())
|
||||||
|
for path in (
|
||||||
|
"META-INF/container.xml",
|
||||||
|
epub.OPF_PATH,
|
||||||
|
epub.NAV_PATH,
|
||||||
|
epub.NCX_PATH,
|
||||||
|
epub.CSS_PATH,
|
||||||
|
epub.COVER_PATH,
|
||||||
|
epub.ARTICLE_PATH,
|
||||||
|
):
|
||||||
|
self.assertIn(path, names)
|
||||||
|
|
||||||
|
|
||||||
|
class EpubDocumentTests(unittest.TestCase):
|
||||||
|
"""Every text part has to parse with a strict XML parser."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.data = epub.build_epub(make_book())
|
||||||
|
self.archive = zipfile.ZipFile(BytesIO(self.data))
|
||||||
|
self.addCleanup(self.archive.close)
|
||||||
|
|
||||||
|
def test_every_xml_part_is_well_formed(self):
|
||||||
|
for path in (
|
||||||
|
epub.OPF_PATH,
|
||||||
|
epub.NAV_PATH,
|
||||||
|
epub.NCX_PATH,
|
||||||
|
epub.COVER_PATH,
|
||||||
|
epub.ARTICLE_PATH,
|
||||||
|
"META-INF/container.xml",
|
||||||
|
):
|
||||||
|
with self.subTest(path=path):
|
||||||
|
parse_xml(self.archive.read(path))
|
||||||
|
|
||||||
|
def test_all_xml_starts_with_a_declaration(self):
|
||||||
|
for path in (
|
||||||
|
epub.OPF_PATH,
|
||||||
|
epub.NAV_PATH,
|
||||||
|
epub.NCX_PATH,
|
||||||
|
epub.COVER_PATH,
|
||||||
|
epub.ARTICLE_PATH,
|
||||||
|
):
|
||||||
|
with self.subTest(path=path):
|
||||||
|
text = self.archive.read(path).decode("utf-8")
|
||||||
|
self.assertTrue(text.startswith('<?xml version="1.0" encoding="utf-8"?>'))
|
||||||
|
|
||||||
|
def test_xhtml_documents_declare_the_namespace(self):
|
||||||
|
for path in (epub.NAV_PATH, epub.COVER_PATH, epub.ARTICLE_PATH):
|
||||||
|
with self.subTest(path=path):
|
||||||
|
root = parse_xml(self.archive.read(path))
|
||||||
|
self.assertEqual(root.must_attr("xmlns"), epub.XHTML_NAMESPACE)
|
||||||
|
|
||||||
|
def test_no_doctype_is_emitted(self):
|
||||||
|
# A DOCTYPE would have to be resolved by the reader's XML parser; EPUB 3
|
||||||
|
# makes it optional, so it is left out entirely.
|
||||||
|
for path in (epub.OPF_PATH, epub.NAV_PATH, epub.ARTICLE_PATH):
|
||||||
|
with self.subTest(path=path):
|
||||||
|
self.assertNotIn("<!DOCTYPE", self.archive.read(path).decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
class EpubPackageTests(unittest.TestCase):
|
||||||
|
"""OPF metadata, manifest and spine."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.data = epub.build_epub(make_book())
|
||||||
|
self.archive = zipfile.ZipFile(BytesIO(self.data))
|
||||||
|
self.addCleanup(self.archive.close)
|
||||||
|
self.opf = parse_xml(self.archive.read(epub.OPF_PATH))
|
||||||
|
|
||||||
|
def manifest(self) -> dict:
|
||||||
|
items = self.opf.must_find("manifest").must_find_all("item")
|
||||||
|
return {item.must_attr("id"): item for item in items}
|
||||||
|
|
||||||
|
def test_package_declares_version_and_unique_identifier(self):
|
||||||
|
self.assertEqual(self.opf.must_attr("version"), "3.0")
|
||||||
|
self.assertEqual(self.opf.must_attr("unique-identifier"), "pub-id")
|
||||||
|
|
||||||
|
def test_required_dublin_core_metadata_is_present(self):
|
||||||
|
metadata = self.opf.must_find("metadata")
|
||||||
|
identifier = metadata.must_find("identifier")
|
||||||
|
self.assertEqual(identifier.must_attr("id"), "pub-id")
|
||||||
|
self.assertEqual(identifier.all_text(), make_book().identifier)
|
||||||
|
self.assertEqual(metadata.must_find("title").all_text(), make_book().title)
|
||||||
|
self.assertEqual(metadata.must_find("language").all_text(), "en")
|
||||||
|
|
||||||
|
def test_optional_metadata_is_present_when_known(self):
|
||||||
|
metadata = self.opf.must_find("metadata")
|
||||||
|
self.assertEqual(metadata.must_find("creator").all_text(), "A. Writer")
|
||||||
|
self.assertEqual(metadata.must_find("publisher").all_text(), "Example Press")
|
||||||
|
self.assertEqual(metadata.must_find("source").all_text(), "https://example.com/articles/1")
|
||||||
|
self.assertEqual(metadata.must_find("date").all_text(), "2026-11-14T09:00:00Z")
|
||||||
|
|
||||||
|
def test_dcterms_modified_has_the_required_format(self):
|
||||||
|
import re
|
||||||
|
|
||||||
|
meta = [
|
||||||
|
item
|
||||||
|
for item in self.opf.must_find("metadata").must_find_all("meta")
|
||||||
|
if item.attr("property") == "dcterms:modified"
|
||||||
|
]
|
||||||
|
self.assertEqual(len(meta), 1)
|
||||||
|
value = meta[0].all_text()
|
||||||
|
self.assertRegex(value, r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
|
||||||
|
self.assertEqual(value, "2026-11-15T12:30:45Z")
|
||||||
|
|
||||||
|
def test_absent_metadata_is_omitted_rather_than_emptied(self):
|
||||||
|
data = epub.build_epub(make_book(author=None, publisher=None, source_url=None))
|
||||||
|
opf = parse_xml(zipfile.ZipFile(BytesIO(data)).read(epub.OPF_PATH))
|
||||||
|
metadata = opf.must_find("metadata")
|
||||||
|
self.assertIsNone(metadata.find("creator"))
|
||||||
|
self.assertIsNone(metadata.find("publisher"))
|
||||||
|
self.assertIsNone(metadata.find("source"))
|
||||||
|
|
||||||
|
def test_navigation_document_is_marked(self):
|
||||||
|
nav = self.manifest()["nav"]
|
||||||
|
self.assertEqual(nav.must_attr("href"), "nav.xhtml")
|
||||||
|
self.assertEqual(nav.must_attr("properties"), "nav")
|
||||||
|
|
||||||
|
def test_spine_toc_points_at_the_ncx(self):
|
||||||
|
self.assertEqual(self.opf.must_find("spine").must_attr("toc"), "ncx")
|
||||||
|
|
||||||
|
def test_every_manifest_href_exists_in_the_zip(self):
|
||||||
|
for identifier, item in self.manifest().items():
|
||||||
|
with self.subTest(item=identifier):
|
||||||
|
self.assertIn("OEBPS/" + item.must_attr("href"), self.archive.namelist())
|
||||||
|
|
||||||
|
def test_every_spine_idref_resolves(self):
|
||||||
|
manifest = self.manifest()
|
||||||
|
for itemref in self.opf.must_find("spine").must_find_all("itemref"):
|
||||||
|
self.assertIn(itemref.must_attr("idref"), manifest)
|
||||||
|
|
||||||
|
def test_every_manifest_item_has_a_media_type(self):
|
||||||
|
for identifier, item in self.manifest().items():
|
||||||
|
with self.subTest(item=identifier):
|
||||||
|
self.assertTrue(item.must_attr("media-type"))
|
||||||
|
|
||||||
|
def test_guide_points_at_the_cover_page(self):
|
||||||
|
reference = self.opf.must_find("guide").must_find("reference")
|
||||||
|
self.assertEqual(reference.must_attr("type"), "cover")
|
||||||
|
self.assertEqual(reference.must_attr("href"), "text/cover.xhtml")
|
||||||
|
|
||||||
|
def test_ncx_has_matching_uid_and_navpoints(self):
|
||||||
|
ncx = parse_xml(self.archive.read(epub.NCX_PATH))
|
||||||
|
self.assertEqual(ncx.must_attr("version"), "2005-1")
|
||||||
|
uid = [
|
||||||
|
m for m in ncx.must_find("head").must_find_all("meta") if m.attr("name") == "dtb:uid"
|
||||||
|
]
|
||||||
|
self.assertEqual(uid[0].must_attr("content"), make_book().identifier)
|
||||||
|
self.assertEqual(len(ncx.must_find("navMap").must_find_all("navPoint")), 2)
|
||||||
|
|
||||||
|
|
||||||
|
class EpubContentTests(unittest.TestCase):
|
||||||
|
def build(self, **overrides) -> tuple[zipfile.ZipFile, bytes]:
|
||||||
|
data = epub.build_epub(make_book(**overrides))
|
||||||
|
archive = zipfile.ZipFile(BytesIO(data))
|
||||||
|
self.addCleanup(archive.close)
|
||||||
|
return archive, data
|
||||||
|
|
||||||
|
def test_article_document_contains_the_body_and_title(self):
|
||||||
|
archive, _ = self.build()
|
||||||
|
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
|
||||||
|
self.assertIn("Body text.", text)
|
||||||
|
self.assertIn("A Test Article & Its Friends", text)
|
||||||
|
|
||||||
|
def test_article_document_carries_a_byline_and_date(self):
|
||||||
|
archive, _ = self.build()
|
||||||
|
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
|
||||||
|
self.assertIn("A. Writer", text)
|
||||||
|
self.assertIn("2026-11-14", text)
|
||||||
|
|
||||||
|
def test_source_link_can_be_suppressed(self):
|
||||||
|
archive, _ = self.build(include_source_link=False)
|
||||||
|
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
|
||||||
|
self.assertNotIn("Originally published", text)
|
||||||
|
|
||||||
|
def test_empty_body_still_produces_a_readable_document(self):
|
||||||
|
archive, _ = self.build(content_xhtml="")
|
||||||
|
parse_xml(archive.read(epub.ARTICLE_PATH))
|
||||||
|
self.assertIn("no content", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
|
||||||
|
|
||||||
|
def test_title_with_markup_is_escaped(self):
|
||||||
|
archive, _ = self.build(title='<script>alert("x")</script>')
|
||||||
|
text = archive.read(epub.ARTICLE_PATH).decode("utf-8")
|
||||||
|
self.assertNotIn("<script>", text)
|
||||||
|
parse_xml(archive.read(epub.ARTICLE_PATH))
|
||||||
|
|
||||||
|
def test_stylesheet_is_embedded(self):
|
||||||
|
archive, _ = self.build(stylesheet="body { color: red; }")
|
||||||
|
self.assertEqual(archive.read(epub.CSS_PATH).decode("utf-8"), "body { color: red; }")
|
||||||
|
|
||||||
|
def test_default_stylesheet_is_not_empty(self):
|
||||||
|
archive, _ = self.build()
|
||||||
|
self.assertIn("line-height", archive.read(epub.CSS_PATH).decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
class EpubImageTests(unittest.TestCase):
|
||||||
|
def test_images_are_embedded_and_declared(self):
|
||||||
|
cover = epub.BookImage(
|
||||||
|
href="images/img-1.png", media_type="image/png", data=png_bytes(800, 400)
|
||||||
|
)
|
||||||
|
data = epub.build_epub(make_book(images=[cover]))
|
||||||
|
archive = zipfile.ZipFile(BytesIO(data))
|
||||||
|
self.addCleanup(archive.close)
|
||||||
|
self.assertIn("OEBPS/images/img-1.png", archive.namelist())
|
||||||
|
opf = parse_xml(archive.read(epub.OPF_PATH))
|
||||||
|
item = opf.must_find("manifest").must_find("item")
|
||||||
|
self.assertIsNotNone(item)
|
||||||
|
|
||||||
|
def test_cover_image_is_marked_and_referenced(self):
|
||||||
|
cover = epub.BookImage(
|
||||||
|
href="images/img-1.png",
|
||||||
|
media_type="image/png",
|
||||||
|
data=png_bytes(800, 400),
|
||||||
|
is_cover=True,
|
||||||
|
)
|
||||||
|
data = epub.build_epub(make_book(images=[cover]))
|
||||||
|
archive = zipfile.ZipFile(BytesIO(data))
|
||||||
|
self.addCleanup(archive.close)
|
||||||
|
opf = parse_xml(archive.read(epub.OPF_PATH))
|
||||||
|
covers = [
|
||||||
|
item
|
||||||
|
for item in opf.must_find("manifest").must_find_all("item")
|
||||||
|
if item.attr("properties") == "cover-image"
|
||||||
|
]
|
||||||
|
self.assertEqual(len(covers), 1)
|
||||||
|
self.assertEqual(covers[0].must_attr("id"), "cover-image")
|
||||||
|
# The EPUB 2 spelling for older readers.
|
||||||
|
metas = [
|
||||||
|
m for m in opf.must_find("metadata").must_find_all("meta") if m.attr("name") == "cover"
|
||||||
|
]
|
||||||
|
self.assertEqual(metas[0].must_attr("content"), "cover-image")
|
||||||
|
# And the cover page shows it, with a path relative to text/.
|
||||||
|
self.assertIn("../images/img-1.png", archive.read(epub.COVER_PATH).decode("utf-8"))
|
||||||
|
|
||||||
|
def test_cover_image_is_not_promoted_without_a_real_cover(self):
|
||||||
|
small = epub.BookImage(
|
||||||
|
href="images/img-1.png", media_type="image/png", data=png_bytes(16, 16)
|
||||||
|
)
|
||||||
|
book = make_book(images=[small])
|
||||||
|
_choose = getattr(epub, "_choose_cover")
|
||||||
|
_choose(book)
|
||||||
|
self.assertIsNone(book.cover_image)
|
||||||
|
|
||||||
|
def test_deterministic_output_for_identical_input(self):
|
||||||
|
left = epub.build_epub(make_book())
|
||||||
|
right = epub.build_epub(make_book())
|
||||||
|
self.assertEqual(left, right)
|
||||||
|
|
||||||
|
|
||||||
|
class BuildArticleEpubTests(unittest.TestCase):
|
||||||
|
"""The end-to-end assembly step, with and without an image fetcher."""
|
||||||
|
|
||||||
|
class Item:
|
||||||
|
def __init__(self, content: str, title: str = "An Article") -> None:
|
||||||
|
self.id = "tag:google.com,2005:reader/item/0000000000000001"
|
||||||
|
self.title = title
|
||||||
|
self.content_html = content
|
||||||
|
self.author = "A. Writer"
|
||||||
|
self.url = "https://example.com/a/1"
|
||||||
|
self.feed_title = "Example Tech"
|
||||||
|
self.published = 1_700_000_000
|
||||||
|
self.updated = 1_700_000_000
|
||||||
|
self.published_at = datetime.fromtimestamp(1_700_000_000, tz=UTC)
|
||||||
|
self.updated_at = datetime.fromtimestamp(1_700_000_000, tz=UTC)
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
include_images = True
|
||||||
|
max_images_per_article = 20
|
||||||
|
cover_image = True
|
||||||
|
include_source_link = True
|
||||||
|
stylesheet = None
|
||||||
|
max_epub_bytes = 33_554_432
|
||||||
|
|
||||||
|
class Catalog:
|
||||||
|
language = "en"
|
||||||
|
author = "FreshRSS"
|
||||||
|
publisher = "Example Press"
|
||||||
|
|
||||||
|
class FakeFetcher:
|
||||||
|
"""Returns one PNG for every URL, recording what it was asked for."""
|
||||||
|
|
||||||
|
def __init__(self, fail: bool = False) -> None:
|
||||||
|
self.asked: list[str] = []
|
||||||
|
self.fail = fail
|
||||||
|
|
||||||
|
def fetch_all(self, urls):
|
||||||
|
self.asked.extend(urls)
|
||||||
|
if self.fail:
|
||||||
|
return {url: None for url in urls}
|
||||||
|
return {
|
||||||
|
url: FetchedImage(url=url, media_type="image/png", data=png_bytes(800, 400))
|
||||||
|
for url in urls
|
||||||
|
}
|
||||||
|
|
||||||
|
def build(self, **overrides) -> tuple[zipfile.ZipFile, bytes]:
|
||||||
|
item = overrides.pop("item", self.Item("<p>Hello</p>"))
|
||||||
|
config = overrides.pop("config", self.Config())
|
||||||
|
fetcher = overrides.pop("fetcher", None)
|
||||||
|
data = epub.build_article_epub(
|
||||||
|
item, epub_config=config, catalog_config=self.Catalog(), fetcher=fetcher
|
||||||
|
)
|
||||||
|
archive = zipfile.ZipFile(BytesIO(data))
|
||||||
|
self.addCleanup(archive.close)
|
||||||
|
return archive, data
|
||||||
|
|
||||||
|
def test_book_is_valid_without_a_fetcher(self):
|
||||||
|
archive, _ = self.build()
|
||||||
|
parse_xml(archive.read(epub.OPF_PATH))
|
||||||
|
self.assertIn("Hello", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
|
||||||
|
|
||||||
|
def test_article_identifier_is_stable(self):
|
||||||
|
_, first = self.build()
|
||||||
|
_, second = self.build()
|
||||||
|
self.assertEqual(first, second)
|
||||||
|
|
||||||
|
def test_remote_images_are_embedded_when_a_fetcher_is_supplied(self):
|
||||||
|
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
|
||||||
|
fetcher = self.FakeFetcher()
|
||||||
|
archive, _ = self.build(item=item, fetcher=fetcher)
|
||||||
|
self.assertEqual(fetcher.asked, ["https://cdn.example.com/a.png"])
|
||||||
|
self.assertIn("OEBPS/images/img-1.png", archive.namelist())
|
||||||
|
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
|
||||||
|
self.assertIn("../images/img-1.png", article)
|
||||||
|
self.assertNotIn("https://cdn.example.com", article)
|
||||||
|
|
||||||
|
def test_unfetchable_images_are_dropped_from_the_body(self):
|
||||||
|
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
|
||||||
|
archive, _ = self.build(item=item, fetcher=self.FakeFetcher(fail=True))
|
||||||
|
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
|
||||||
|
self.assertNotIn("<img", article)
|
||||||
|
self.assertNotIn("cdn.example.com", article)
|
||||||
|
|
||||||
|
def test_lead_image_becomes_the_cover(self):
|
||||||
|
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
|
||||||
|
archive, _ = self.build(item=item, fetcher=self.FakeFetcher())
|
||||||
|
opf = parse_xml(archive.read(epub.OPF_PATH))
|
||||||
|
covers = [
|
||||||
|
item
|
||||||
|
for item in opf.must_find("manifest").must_find_all("item")
|
||||||
|
if item.attr("properties") == "cover-image"
|
||||||
|
]
|
||||||
|
self.assertEqual(len(covers), 1)
|
||||||
|
|
||||||
|
def test_images_are_left_remote_when_inlining_is_off(self):
|
||||||
|
class NoImages(self.Config):
|
||||||
|
include_images = False
|
||||||
|
|
||||||
|
item = self.Item('<p>x</p><img src="https://cdn.example.com/a.png">')
|
||||||
|
archive, _ = self.build(item=item, config=NoImages(), fetcher=self.FakeFetcher())
|
||||||
|
article = archive.read(epub.ARTICLE_PATH).decode("utf-8")
|
||||||
|
self.assertIn("https://cdn.example.com/a.png", article)
|
||||||
|
|
||||||
|
def test_image_count_is_capped(self):
|
||||||
|
class OneImage(self.Config):
|
||||||
|
max_images_per_article = 1
|
||||||
|
|
||||||
|
item = self.Item(
|
||||||
|
'<img src="https://cdn.example.com/a.png"><img src="https://cdn.example.com/b.png">'
|
||||||
|
)
|
||||||
|
fetcher = self.FakeFetcher()
|
||||||
|
archive, _ = self.build(item=item, config=OneImage(), fetcher=fetcher)
|
||||||
|
self.assertEqual(len(fetcher.asked), 1)
|
||||||
|
self.assertEqual(len(archive.namelist()), 9)
|
||||||
|
|
||||||
|
def test_oversized_book_is_refused(self):
|
||||||
|
class TinyLimit(self.Config):
|
||||||
|
max_epub_bytes = 1024
|
||||||
|
|
||||||
|
with self.assertRaises(epub.EpubTooLargeError):
|
||||||
|
self.build(config=TinyLimit(), fetcher=self.FakeFetcher())
|
||||||
|
|
||||||
|
def test_stylesheet_path_falls_back_when_unreadable(self):
|
||||||
|
class BadStylesheet(self.Config):
|
||||||
|
stylesheet = "/nonexistent/stylesheet.css"
|
||||||
|
|
||||||
|
archive, _ = self.build(config=BadStylesheet())
|
||||||
|
self.assertIn("line-height", archive.read(epub.CSS_PATH).decode("utf-8"))
|
||||||
|
|
||||||
|
def test_sanitised_body_has_no_scripts(self):
|
||||||
|
item = self.Item("<p>ok</p><script>alert(1)</script>")
|
||||||
|
archive, _ = self.build(item=item)
|
||||||
|
self.assertNotIn("alert", archive.read(epub.ARTICLE_PATH).decode("utf-8"))
|
||||||
|
|
||||||
|
def test_parsed_content_respects_the_byte_cap(self):
|
||||||
|
item = self.Item("<p>" + "word " * 20_000 + "</p>")
|
||||||
|
data = epub.build_article_epub(
|
||||||
|
item,
|
||||||
|
epub_config=self.Config(),
|
||||||
|
catalog_config=self.Catalog(),
|
||||||
|
max_content_bytes=4096,
|
||||||
|
)
|
||||||
|
self.assertLess(len(data), 100_000)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
414
tests/test_freshrss.py
Normal file
414
tests/test_freshrss.py
Normal file
@@ -0,0 +1,414 @@
|
|||||||
|
"""Tests for the FreshRSS client, against an in-process FreshRSS stand-in.
|
||||||
|
|
||||||
|
These assert on the wire format as much as on the parsed result: the query
|
||||||
|
parameters the client sends (``r=d``, ``n=``, ``xt=``, ``ck=``) and the form body
|
||||||
|
it posts for ``edit-tag`` are the parts that silently break against a real server,
|
||||||
|
so they are checked explicitly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from app.config import FreshRSSConfig
|
||||||
|
from app.freshrss import (
|
||||||
|
READ_TAG,
|
||||||
|
READING_LIST,
|
||||||
|
STARRED,
|
||||||
|
AuthenticationError,
|
||||||
|
FreshRSSClient,
|
||||||
|
FreshRSSError,
|
||||||
|
NotFoundError,
|
||||||
|
TransientError,
|
||||||
|
label_stream,
|
||||||
|
)
|
||||||
|
from tests.stub_freshrss import build_demo_catalog
|
||||||
|
|
||||||
|
TECH_FEED = "feed/https://example.com/tech/atom.xml"
|
||||||
|
NEWS_FEED = "feed/https://news.example.org/rss?source=daily"
|
||||||
|
SOLO_FEED = "feed/https://solo.example.net/feed.xml"
|
||||||
|
|
||||||
|
|
||||||
|
def make_config(stub, **overrides) -> FreshRSSConfig:
|
||||||
|
"""A client config aimed at the stub, with fast, non-retrying defaults."""
|
||||||
|
values = {
|
||||||
|
"api_base": stub.api_url,
|
||||||
|
"username": stub.username,
|
||||||
|
"api_password": stub.password,
|
||||||
|
"timeout_seconds": 5.0,
|
||||||
|
"retries": 0,
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return FreshRSSConfig(**values)
|
||||||
|
|
||||||
|
|
||||||
|
class ClientTestCase(unittest.TestCase):
|
||||||
|
"""Starts a fresh stub per test."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.stub = build_demo_catalog().start()
|
||||||
|
self.addCleanup(self.stub.stop)
|
||||||
|
|
||||||
|
def client(self, **overrides) -> FreshRSSClient:
|
||||||
|
client = FreshRSSClient(
|
||||||
|
make_config(self.stub, **overrides),
|
||||||
|
# Retry timing is not what these tests are about, and sleeping would
|
||||||
|
# only make the suite slow.
|
||||||
|
sleep=lambda _seconds: None,
|
||||||
|
)
|
||||||
|
self.addCleanup(client.close)
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
class AuthenticationTests(ClientTestCase):
|
||||||
|
def test_login_posts_credentials_and_reads_the_auth_line(self):
|
||||||
|
self.client().list_subscriptions()
|
||||||
|
logins = self.stub.find_requests("/accounts/ClientLogin")
|
||||||
|
self.assertEqual(len(logins), 1)
|
||||||
|
self.assertEqual(logins[0]["method"], "POST")
|
||||||
|
self.assertIn("Email=tyler", logins[0]["body"])
|
||||||
|
self.assertIn("Passwd=api-password", logins[0]["body"])
|
||||||
|
|
||||||
|
def test_subsequent_calls_use_the_googlelogin_header(self):
|
||||||
|
client = self.client()
|
||||||
|
client.list_subscriptions()
|
||||||
|
client.list_subscriptions()
|
||||||
|
requests = self.stub.find_requests("/subscription/list")
|
||||||
|
self.assertEqual(len(requests), 2)
|
||||||
|
for entry in requests:
|
||||||
|
self.assertEqual(entry["authorization"], "GoogleLogin auth=stub-auth-token")
|
||||||
|
# The token is cached, so only one login happened.
|
||||||
|
self.assertEqual(len(self.stub.find_requests("/accounts/ClientLogin")), 1)
|
||||||
|
|
||||||
|
def test_bad_password_raises_authentication_error(self):
|
||||||
|
client = self.client(api_password="wrong")
|
||||||
|
with self.assertRaises(AuthenticationError):
|
||||||
|
client.list_subscriptions()
|
||||||
|
|
||||||
|
def test_credentials_are_form_encoded(self):
|
||||||
|
# A password with & or = must not corrupt the form body.
|
||||||
|
self.stub.password = "p&ss=word+plus"
|
||||||
|
client = self.client(api_password="p&ss=word+plus")
|
||||||
|
client.list_subscriptions()
|
||||||
|
body = self.stub.find_requests("/accounts/ClientLogin")[0]["body"]
|
||||||
|
self.assertIn("Passwd=p%26ss%3Dword%2Bplus", body)
|
||||||
|
|
||||||
|
|
||||||
|
class SubscriptionTests(ClientTestCase):
|
||||||
|
def test_subscriptions_are_parsed_with_categories(self):
|
||||||
|
feeds = {feed.id: feed for feed in self.client().list_subscriptions()}
|
||||||
|
self.assertEqual(len(feeds), 3)
|
||||||
|
self.assertEqual(feeds[TECH_FEED].title, "Example Tech")
|
||||||
|
self.assertEqual(feeds[TECH_FEED].categories, ["Tech"])
|
||||||
|
self.assertEqual(feeds[NEWS_FEED].categories, ["News"])
|
||||||
|
|
||||||
|
def test_feed_without_a_category_is_recognised(self):
|
||||||
|
feeds = {feed.id: feed for feed in self.client().list_subscriptions()}
|
||||||
|
self.assertEqual(feeds[SOLO_FEED].categories, [])
|
||||||
|
self.assertEqual(feeds[SOLO_FEED].primary_category, "Uncategorised")
|
||||||
|
|
||||||
|
def test_subscriptions_are_sorted_by_title(self):
|
||||||
|
titles = [feed.title for feed in self.client().list_subscriptions()]
|
||||||
|
self.assertEqual(titles, sorted(titles, key=str.casefold))
|
||||||
|
|
||||||
|
def test_query_asks_for_json(self):
|
||||||
|
self.client().list_subscriptions()
|
||||||
|
query = self.stub.find_requests("/subscription/list")[0]["query"]
|
||||||
|
self.assertEqual(query["output"], ["json"])
|
||||||
|
|
||||||
|
def test_non_json_response_is_reported_clearly(self):
|
||||||
|
self.stub.corrupt_json = True
|
||||||
|
with self.assertRaises(FreshRSSError) as caught:
|
||||||
|
self.client().list_subscriptions()
|
||||||
|
self.assertIn("not JSON", str(caught.exception))
|
||||||
|
|
||||||
|
def test_probe_reports_success_and_feed_count(self):
|
||||||
|
ok, detail = self.client().probe()
|
||||||
|
self.assertTrue(ok)
|
||||||
|
self.assertEqual(detail, "3 feeds")
|
||||||
|
|
||||||
|
def test_probe_reports_failure_without_raising(self):
|
||||||
|
ok, detail = self.client(api_password="wrong").probe()
|
||||||
|
self.assertFalse(ok)
|
||||||
|
self.assertTrue(detail)
|
||||||
|
|
||||||
|
|
||||||
|
class StreamTests(ClientTestCase):
|
||||||
|
def test_stream_query_parameters(self):
|
||||||
|
self.client().stream_items(TECH_FEED, count=7, exclude_read=True)
|
||||||
|
request = self.stub.find_requests("/stream/contents/")[0]
|
||||||
|
self.assertEqual(request["query"]["output"], ["json"])
|
||||||
|
self.assertEqual(request["query"]["r"], ["d"])
|
||||||
|
self.assertEqual(request["query"]["n"], ["7"])
|
||||||
|
self.assertEqual(request["query"]["xt"], [READ_TAG])
|
||||||
|
self.assertTrue(request["query"]["ck"][0].isdigit())
|
||||||
|
|
||||||
|
def test_stream_id_is_path_encoded(self):
|
||||||
|
self.client().stream_items(NEWS_FEED, count=1)
|
||||||
|
raw_path = self.stub.find_requests("/stream/contents/")[0]["raw_path"]
|
||||||
|
# The feed URL contains "?" and "/" which must not leak into the path.
|
||||||
|
self.assertNotIn("?", raw_path.split("?")[0].split("/api/greader.php")[1])
|
||||||
|
self.assertIn("%3F", raw_path)
|
||||||
|
|
||||||
|
def test_unread_filter_is_applied_upstream(self):
|
||||||
|
client = self.client()
|
||||||
|
everything = client.stream_items(TECH_FEED, count=50)
|
||||||
|
unread = client.stream_items(TECH_FEED, count=50, exclude_read=True)
|
||||||
|
self.assertEqual(len(everything), 3)
|
||||||
|
self.assertEqual(len(unread), 2)
|
||||||
|
self.assertNotIn("Already Read Post", [item.title for item in unread])
|
||||||
|
|
||||||
|
def test_count_bounds_the_result(self):
|
||||||
|
items = self.client().stream_items(TECH_FEED, count=2)
|
||||||
|
self.assertEqual(len(items), 2)
|
||||||
|
|
||||||
|
def test_newest_first(self):
|
||||||
|
items = self.client().stream_items(TECH_FEED, count=10)
|
||||||
|
published = [item.published for item in items]
|
||||||
|
self.assertEqual(published, sorted(published, reverse=True))
|
||||||
|
|
||||||
|
def test_older_than_is_passed_through(self):
|
||||||
|
self.client().stream_items(TECH_FEED, count=5, older_than=1_699_500_000)
|
||||||
|
self.assertEqual(
|
||||||
|
self.stub.find_requests("/stream/contents/")[0]["query"]["ot"],
|
||||||
|
["1699500000"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_label_stream_selects_label_articles(self):
|
||||||
|
items = self.client().stream_items(label_stream("News"), count=10)
|
||||||
|
self.assertEqual({item.feed_id for item in items}, {NEWS_FEED})
|
||||||
|
|
||||||
|
def test_reading_list_covers_every_feed(self):
|
||||||
|
items = self.client().stream_items(READING_LIST, count=50)
|
||||||
|
self.assertEqual({item.feed_id for item in items}, {TECH_FEED, NEWS_FEED, SOLO_FEED})
|
||||||
|
|
||||||
|
def test_starred_stream_returns_only_starred(self):
|
||||||
|
items = self.client().stream_items(STARRED, count=10)
|
||||||
|
self.assertEqual([item.title for item in items], ["Starred Investigation"])
|
||||||
|
self.assertTrue(items[0].is_starred)
|
||||||
|
|
||||||
|
def test_missing_stream_is_treated_as_empty(self):
|
||||||
|
# A feed unsubscribed upstream must not break the catalog.
|
||||||
|
self.stub.fail_next(404, path="/stream/contents/")
|
||||||
|
self.assertEqual(self.client().stream_items(TECH_FEED, count=5), [])
|
||||||
|
self.stub.fail_next(400, path="/stream/contents/")
|
||||||
|
self.assertEqual(self.client().stream_items(TECH_FEED, count=5), [])
|
||||||
|
|
||||||
|
|
||||||
|
class ItemParsingTests(ClientTestCase):
|
||||||
|
def items_by_title(self, stream=TECH_FEED) -> dict:
|
||||||
|
return {item.title: item for item in self.client().stream_items(stream, count=50)}
|
||||||
|
|
||||||
|
def test_full_content_is_preferred_over_the_summary(self):
|
||||||
|
self.stub.add_item(
|
||||||
|
TECH_FEED,
|
||||||
|
99,
|
||||||
|
"Full versus summary",
|
||||||
|
content="<p>FULL BODY</p>",
|
||||||
|
summary="<p>teaser</p>",
|
||||||
|
)
|
||||||
|
item = self.items_by_title()["Full versus summary"]
|
||||||
|
self.assertIn("FULL BODY", item.content_html)
|
||||||
|
|
||||||
|
def test_summary_is_used_when_there_is_no_content(self):
|
||||||
|
self.stub.add_item(
|
||||||
|
TECH_FEED,
|
||||||
|
98,
|
||||||
|
"Summary only",
|
||||||
|
summary="<p>only teaser</p>",
|
||||||
|
)
|
||||||
|
self.stub.items[TECH_FEED][-1].pop("content")
|
||||||
|
item = self.items_by_title()["Summary only"]
|
||||||
|
self.assertIn("only teaser", item.content_html)
|
||||||
|
|
||||||
|
def test_item_fields_are_parsed(self):
|
||||||
|
item = self.items_by_title()["A Tech Post & Its Friends"]
|
||||||
|
self.assertEqual(item.feed_id, TECH_FEED)
|
||||||
|
self.assertEqual(item.feed_title, "Example Tech")
|
||||||
|
self.assertEqual(item.author, "A. Writer")
|
||||||
|
self.assertEqual(item.url, "https://example.com/articles/1")
|
||||||
|
self.assertEqual(item.published, 1_700_000_100)
|
||||||
|
self.assertTrue(item.title.startswith("A Tech Post"))
|
||||||
|
|
||||||
|
def test_missing_author_is_allowed(self):
|
||||||
|
item = self.items_by_title(NEWS_FEED)["Markets Dip Slightly"]
|
||||||
|
self.assertIsNone(item.author)
|
||||||
|
|
||||||
|
def test_read_state_is_parsed(self):
|
||||||
|
items = self.items_by_title()
|
||||||
|
self.assertTrue(items["Already Read Post"].is_read)
|
||||||
|
self.assertFalse(items["A Tech Post & Its Friends"].is_read)
|
||||||
|
|
||||||
|
def test_kept_unread_overrides_the_read_tag(self):
|
||||||
|
# FreshRSS keeps both tags when a read item is explicitly marked unread.
|
||||||
|
item = self.items_by_title()["Already Read Post"]
|
||||||
|
item.categories.append("user/-/state/com.google/kept-unread")
|
||||||
|
self.assertFalse(item.is_read)
|
||||||
|
|
||||||
|
def test_malformed_entries_are_skipped(self):
|
||||||
|
self.stub.inject_raw_item(TECH_FEED, {"title": "no id"})
|
||||||
|
self.stub.inject_raw_item(TECH_FEED, "not even an object")
|
||||||
|
titles = [item.title for item in self.client().stream_items(TECH_FEED, count=50)]
|
||||||
|
self.assertNotIn("no id", titles)
|
||||||
|
self.assertEqual(len(titles), 3)
|
||||||
|
|
||||||
|
def test_untitled_item_gets_a_placeholder(self):
|
||||||
|
self.stub.add_item(TECH_FEED, 97, "", content="<p>x</p>")
|
||||||
|
titles = [item.title for item in self.client().stream_items(TECH_FEED, count=50)]
|
||||||
|
self.assertIn("(untitled)", titles)
|
||||||
|
|
||||||
|
def test_timestamps_are_utc_datetimes(self):
|
||||||
|
item = self.items_by_title()["A Tech Post & Its Friends"]
|
||||||
|
self.assertEqual(item.published_at, datetime(2023, 11, 14, 22, 15, 0, tzinfo=UTC))
|
||||||
|
self.assertEqual(item.published_at.tzinfo, UTC)
|
||||||
|
|
||||||
|
def test_broken_timestamp_falls_back_to_the_epoch(self):
|
||||||
|
# Every date field has to be unusable for the epoch fallback to apply;
|
||||||
|
# while `updated` survives, using it is the correct behaviour.
|
||||||
|
self.stub.add_item(TECH_FEED, 96, "Broken date", content="<p>x</p>")
|
||||||
|
entry = self.stub.items[TECH_FEED][-1]
|
||||||
|
for field in ("published", "updated", "timestampUsec"):
|
||||||
|
entry.pop(field, None)
|
||||||
|
item = self.items_by_title()["Broken date"]
|
||||||
|
self.assertEqual(item.published, 0)
|
||||||
|
self.assertEqual(item.published_at, datetime(1970, 1, 1, tzinfo=UTC))
|
||||||
|
|
||||||
|
def test_unusable_publication_date_falls_back_to_updated(self):
|
||||||
|
self.stub.add_item(TECH_FEED, 94, "Only updated", content="<p>x</p>")
|
||||||
|
entry = self.stub.items[TECH_FEED][-1]
|
||||||
|
entry["published"] = "not-a-number"
|
||||||
|
entry.pop("timestampUsec", None)
|
||||||
|
entry["updated"] = 1_600_000_000
|
||||||
|
item = self.items_by_title()["Only updated"]
|
||||||
|
self.assertEqual(item.published_at, datetime(2020, 9, 13, 12, 26, 40, tzinfo=UTC))
|
||||||
|
|
||||||
|
def test_timestamp_usec_is_used_when_published_is_absent(self):
|
||||||
|
self.stub.add_item(TECH_FEED, 95, "Usec only", content="<p>x</p>")
|
||||||
|
entry = self.stub.items[TECH_FEED][-1]
|
||||||
|
entry.pop("published")
|
||||||
|
entry["timestampUsec"] = "1699999999000000"
|
||||||
|
item = self.items_by_title()["Usec only"]
|
||||||
|
self.assertEqual(item.published, 1_699_999_999)
|
||||||
|
|
||||||
|
|
||||||
|
class UnreadCountTests(ClientTestCase):
|
||||||
|
def test_unread_counts_are_parsed(self):
|
||||||
|
counts = self.client().unread_counts()
|
||||||
|
self.assertEqual(counts[TECH_FEED], 2)
|
||||||
|
self.assertEqual(counts[NEWS_FEED], 2)
|
||||||
|
|
||||||
|
def test_unsupported_counts_raise_not_found(self):
|
||||||
|
# The caller latches this and carries on without counts.
|
||||||
|
self.stub.supports_unread_counts = False
|
||||||
|
with self.assertRaises(NotFoundError):
|
||||||
|
self.client().unread_counts()
|
||||||
|
|
||||||
|
|
||||||
|
class MarkReadTests(ClientTestCase):
|
||||||
|
def test_mark_read_fetches_the_write_token_and_posts_ids(self):
|
||||||
|
client = self.client()
|
||||||
|
identifier = self.stub.items[TECH_FEED][0]["id"]
|
||||||
|
self.assertEqual(client.mark_read([identifier]), 1)
|
||||||
|
|
||||||
|
token_requests = self.stub.find_requests("/reader/api/0/token")
|
||||||
|
self.assertEqual(len(token_requests), 1)
|
||||||
|
|
||||||
|
edits = self.stub.find_requests("/reader/api/0/edit-tag")
|
||||||
|
self.assertEqual(len(edits), 1)
|
||||||
|
self.assertEqual(edits[0]["method"], "POST")
|
||||||
|
body = edits[0]["body"]
|
||||||
|
self.assertIn("T=stub-write-token", body)
|
||||||
|
self.assertIn("a=user%2F-%2Fstate%2Fcom.google%2Fread", body)
|
||||||
|
self.assertIn("i=tag%3Agoogle.com%2C2005%3Areader%2Fitem", body)
|
||||||
|
self.assertIn(identifier, self.stub.read)
|
||||||
|
|
||||||
|
def test_mark_read_sends_one_i_parameter_per_id(self):
|
||||||
|
ids = [entry["id"] for entry in self.stub.items[TECH_FEED]]
|
||||||
|
self.assertEqual(self.client().mark_read(ids), 3)
|
||||||
|
body = self.stub.find_requests("/reader/api/0/edit-tag")[0]["body"]
|
||||||
|
self.assertEqual(body.count("&i="), 3)
|
||||||
|
|
||||||
|
def test_item_id_survives_form_encoding(self):
|
||||||
|
identifier = "tag:google.com,2005:reader/item/00000000000000ff"
|
||||||
|
self.client().mark_read([identifier])
|
||||||
|
from urllib.parse import parse_qs
|
||||||
|
|
||||||
|
body = self.stub.find_requests("/reader/api/0/edit-tag")[0]["body"]
|
||||||
|
self.assertEqual(parse_qs(body)["i"], [identifier])
|
||||||
|
|
||||||
|
def test_mark_read_is_idempotent(self):
|
||||||
|
client = self.client()
|
||||||
|
identifier = self.stub.items[TECH_FEED][0]["id"]
|
||||||
|
client.mark_read([identifier])
|
||||||
|
client.mark_read([identifier])
|
||||||
|
self.assertIn(identifier, self.stub.read)
|
||||||
|
self.assertEqual(len(self.stub.find_requests("/reader/api/0/edit-tag")), 2)
|
||||||
|
|
||||||
|
def test_mark_read_with_no_ids_makes_no_requests(self):
|
||||||
|
self.assertEqual(self.client().mark_read([]), 0)
|
||||||
|
self.assertEqual(self.stub.requests, [])
|
||||||
|
|
||||||
|
def test_stale_write_token_is_refreshed_and_retried(self):
|
||||||
|
client = self.client()
|
||||||
|
identifier = self.stub.items[TECH_FEED][0]["id"]
|
||||||
|
# Warm the token cache, then rotate the token upstream so the cached one
|
||||||
|
# is rejected. The client must re-authenticate and retry rather than lose
|
||||||
|
# the mark-read.
|
||||||
|
client.mark_read([identifier])
|
||||||
|
self.stub.write_token = "rotated-token"
|
||||||
|
self.stub.clear_read()
|
||||||
|
client.mark_read([identifier])
|
||||||
|
self.assertIn(identifier, self.stub.read)
|
||||||
|
self.assertEqual(len(self.stub.find_requests("/reader/api/0/edit-tag")), 3)
|
||||||
|
|
||||||
|
def test_marking_read_removes_the_item_from_the_unread_stream(self):
|
||||||
|
client = self.client()
|
||||||
|
identifier = self.stub.items[TECH_FEED][0]["id"]
|
||||||
|
client.mark_read([identifier])
|
||||||
|
unread = client.stream_items(TECH_FEED, count=50, exclude_read=True)
|
||||||
|
self.assertNotIn(identifier, [item.id for item in unread])
|
||||||
|
|
||||||
|
|
||||||
|
class FailureHandlingTests(ClientTestCase):
|
||||||
|
def test_transient_status_is_retried(self):
|
||||||
|
client = self.client(retries=2)
|
||||||
|
self.stub.fail_next(503, path="/subscription/list")
|
||||||
|
feeds = client.list_subscriptions()
|
||||||
|
self.assertEqual(len(feeds), 3)
|
||||||
|
self.assertEqual(len(self.stub.find_requests("/subscription/list")), 2)
|
||||||
|
|
||||||
|
def test_retries_are_bounded(self):
|
||||||
|
client = self.client(retries=1)
|
||||||
|
self.stub.fail_next(503, count=5, path="/subscription/list")
|
||||||
|
with self.assertRaises(TransientError):
|
||||||
|
client.list_subscriptions()
|
||||||
|
self.assertEqual(len(self.stub.find_requests("/subscription/list")), 2)
|
||||||
|
|
||||||
|
def test_server_error_is_not_swallowed(self):
|
||||||
|
self.stub.fail_next(500, path="/subscription/list")
|
||||||
|
with self.assertRaises(FreshRSSError) as caught:
|
||||||
|
self.client(retries=0).list_subscriptions()
|
||||||
|
self.assertEqual(caught.exception.status, 500)
|
||||||
|
|
||||||
|
def test_unreachable_upstream_raises_transient_error(self):
|
||||||
|
# Bind a port, then close it, so nothing is listening.
|
||||||
|
import socket
|
||||||
|
|
||||||
|
probe = socket.socket()
|
||||||
|
probe.bind(("127.0.0.1", 0))
|
||||||
|
port = probe.getsockname()[1]
|
||||||
|
probe.close()
|
||||||
|
config = make_config(self.stub, api_base=f"http://127.0.0.1:{port}/api/greader.php")
|
||||||
|
with self.assertRaises(TransientError):
|
||||||
|
FreshRSSClient(config).list_subscriptions()
|
||||||
|
|
||||||
|
def test_error_body_is_included_in_the_message(self):
|
||||||
|
self.stub.fail_next(500, path="/subscription/list")
|
||||||
|
with self.assertRaises(FreshRSSError) as caught:
|
||||||
|
self.client(retries=0).list_subscriptions()
|
||||||
|
self.assertIn("injected failure", str(caught.exception))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
543
tests/test_server.py
Normal file
543
tests/test_server.py
Normal file
@@ -0,0 +1,543 @@
|
|||||||
|
"""End-to-end tests: a real server, talking to a real (stand-in) FreshRSS.
|
||||||
|
|
||||||
|
These run the whole stack rather than units of it: the configuration is loaded
|
||||||
|
through its validation path, the HTTP server serves on a socket, the FreshRSS
|
||||||
|
client speaks HTTP to the stub, and the EPUB that comes back is opened and
|
||||||
|
checked. The claim that matters most -- that downloading a book marks the
|
||||||
|
matching article read upstream, and only a completed download does -- can only be
|
||||||
|
verified here, by looking at what the stub received.
|
||||||
|
|
||||||
|
Image embedding is turned off in this configuration so the suite stays offline;
|
||||||
|
image handling has its own tests with a stand-in fetcher.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
import zipfile
|
||||||
|
from io import BytesIO
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app import server as app_server
|
||||||
|
from app.config import load_config, password_digest
|
||||||
|
from app.epub import ARTICLE_PATH, OPF_PATH
|
||||||
|
from tests.stub_freshrss import READING_LIST, build_demo_catalog, item_id
|
||||||
|
from tests.support import parse_xml
|
||||||
|
|
||||||
|
|
||||||
|
class _NoRedirects(urllib.request.HTTPRedirectHandler):
|
||||||
|
"""Surface a redirect as its own status instead of following it.
|
||||||
|
|
||||||
|
urlopen follows redirects transparently, so a test that asserts on a 302 would
|
||||||
|
otherwise be looking at the status of whatever the redirect pointed at.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
USERNAME = "reader"
|
||||||
|
PASSWORD = "reader-password"
|
||||||
|
TECH_FEED = "feed/https://example.com/tech/atom.xml"
|
||||||
|
|
||||||
|
|
||||||
|
class ServerTestCase(unittest.TestCase):
|
||||||
|
"""Starts a stub FreshRSS and the real server for each test."""
|
||||||
|
|
||||||
|
include_images = False
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.stub = build_demo_catalog().start()
|
||||||
|
self.addCleanup(self.stub.stop)
|
||||||
|
# The demo account ships one already-read article, so assertions about
|
||||||
|
# marking read are written against the change, not the absolute state.
|
||||||
|
self.baseline_read = set(self.stub.read)
|
||||||
|
self._dir = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self._dir.cleanup)
|
||||||
|
self._no_redirects = urllib.request.build_opener(_NoRedirects)
|
||||||
|
self.config = self._write_config()
|
||||||
|
self.httpd, self.application = app_server.build_server(self.config)
|
||||||
|
self.port = self.httpd.server_address[1]
|
||||||
|
self.thread = threading.Thread(
|
||||||
|
target=self.httpd.serve_forever, kwargs={"poll_interval": 0.02}, daemon=True
|
||||||
|
)
|
||||||
|
self.thread.start()
|
||||||
|
self.addCleanup(self._stop_server)
|
||||||
|
|
||||||
|
def _stop_server(self):
|
||||||
|
self.httpd.shutdown()
|
||||||
|
self.httpd.server_close()
|
||||||
|
self.thread.join(timeout=5)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def base(self) -> str:
|
||||||
|
return f"http://127.0.0.1:{self.port}"
|
||||||
|
|
||||||
|
def _write_config(self):
|
||||||
|
document = {
|
||||||
|
"server": {"host": "127.0.0.1", "port": 0, "log_level": "critical"},
|
||||||
|
"freshrss": {
|
||||||
|
"url": self.stub.api_url,
|
||||||
|
"username": self.stub.username,
|
||||||
|
"api_password": self.stub.password,
|
||||||
|
"items_per_page": 2,
|
||||||
|
"timeout_seconds": 5,
|
||||||
|
"retries": 0,
|
||||||
|
},
|
||||||
|
"catalog": {"title": "Test Catalog", "author": "Tester"},
|
||||||
|
"epub": {"include_images": self.include_images},
|
||||||
|
"auth": {
|
||||||
|
"users": [{"username": USERNAME, "password_sha256": password_digest(PASSWORD)}]
|
||||||
|
},
|
||||||
|
"cache": {"subscriptions_ttl_seconds": 1, "stream_ttl_seconds": 1},
|
||||||
|
}
|
||||||
|
path = Path(self._dir.name) / "config.json"
|
||||||
|
path.write_text(json.dumps(document), encoding="utf-8")
|
||||||
|
return load_config(str(path))
|
||||||
|
|
||||||
|
# -- HTTP helpers ----------------------------------------------------
|
||||||
|
|
||||||
|
def request(self, path, *, auth=True, method="GET", headers=None, follow_redirects=True):
|
||||||
|
request = urllib.request.Request(self.base + path, method=method)
|
||||||
|
if auth:
|
||||||
|
token = base64.b64encode(f"{USERNAME}:{PASSWORD}".encode()).decode()
|
||||||
|
request.add_header("Authorization", f"Basic {token}")
|
||||||
|
for key, value in (headers or {}).items():
|
||||||
|
request.add_header(key, value)
|
||||||
|
opener = urllib.request.urlopen if follow_redirects else self._no_redirects.open
|
||||||
|
try:
|
||||||
|
with opener(request, timeout=10) as response:
|
||||||
|
return response.status, dict(response.headers), response.read()
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
# With redirects suppressed, a 3xx arrives here, which is the point.
|
||||||
|
# The error object owns the response socket, so it is closed rather
|
||||||
|
# than left for the garbage collector.
|
||||||
|
try:
|
||||||
|
body = error.read()
|
||||||
|
finally:
|
||||||
|
error.close()
|
||||||
|
return error.code, dict(error.headers), body
|
||||||
|
|
||||||
|
def wait_for_mark_read(self, expected: set[str], timeout: float = 5.0) -> set[str]:
|
||||||
|
"""Wait for a mark-read to land upstream (returns what landed).
|
||||||
|
|
||||||
|
The server marks an article read only after the body has been delivered,
|
||||||
|
so from the client's side that upstream effect is asynchronous: a test
|
||||||
|
asserting immediately would be racing the handler thread. It is
|
||||||
|
deliberately not the other way round -- marking before delivery would
|
||||||
|
consume articles whose transfer failed.
|
||||||
|
"""
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
landed = self.stub.read - self.baseline_read
|
||||||
|
if landed >= expected:
|
||||||
|
return landed
|
||||||
|
time.sleep(0.02)
|
||||||
|
return self.stub.read - self.baseline_read
|
||||||
|
|
||||||
|
def get_xml(self, path, *, auth=True):
|
||||||
|
status, headers, body = self.request(path, auth=auth)
|
||||||
|
self.assertEqual(status, 200, body[:400])
|
||||||
|
node = parse_xml(body)
|
||||||
|
return node, headers
|
||||||
|
|
||||||
|
def entries(self, node):
|
||||||
|
return {entry.must_find("title").all_text(): entry for entry in node.must_find_all("entry")}
|
||||||
|
|
||||||
|
def follow(self, entry, rel: str) -> str:
|
||||||
|
for link in entry.must_find_all("link"):
|
||||||
|
if link.attr("rel") == rel:
|
||||||
|
return link.must_attr("href")
|
||||||
|
raise AssertionError(f"no {rel!r} link in entry")
|
||||||
|
|
||||||
|
def acquisition_path(self, section="unread", label="Tech", feed_title="Example Tech") -> str:
|
||||||
|
"""Walk the catalog down to one feed's acquisition feed.
|
||||||
|
|
||||||
|
Shared by the suite rather than owned by one test class: the download,
|
||||||
|
pagination, search and serialisation tests all start from a feed.
|
||||||
|
"""
|
||||||
|
node, _ = self.get_xml(f"/opds/{section}")
|
||||||
|
category = self.follow(self.entries(node)[label], "subsection")
|
||||||
|
node, _ = self.get_xml(urllib.parse.urlsplit(category).path)
|
||||||
|
return urllib.parse.urlsplit(self.follow(self.entries(node)[feed_title], "subsection")).path
|
||||||
|
|
||||||
|
def download_path(self, title="A Tech Post & Its Friends", section="unread") -> str:
|
||||||
|
"""The EPUB acquisition URL for one article."""
|
||||||
|
node, _ = self.get_xml(self.acquisition_path(section=section))
|
||||||
|
entry = self.entries(node)[title]
|
||||||
|
return urllib.parse.urlsplit(
|
||||||
|
self.follow(entry, "http://opds-spec.org/acquisition/open-access")
|
||||||
|
).path
|
||||||
|
|
||||||
|
|
||||||
|
class AuthenticationTests(ServerTestCase):
|
||||||
|
def test_catalog_requires_credentials(self):
|
||||||
|
status, headers, _body = self.request("/opds", auth=False)
|
||||||
|
self.assertEqual(status, 401)
|
||||||
|
self.assertIn("Basic", headers.get("WWW-Authenticate", ""))
|
||||||
|
self.assertIn("Test" if False else "OPDS", headers.get("WWW-Authenticate", ""))
|
||||||
|
|
||||||
|
def test_wrong_password_is_rejected(self):
|
||||||
|
token = base64.b64encode(f"{USERNAME}:wrong".encode()).decode()
|
||||||
|
status, _headers, _body = self.request(
|
||||||
|
"/opds", auth=False, headers={"Authorization": f"Basic {token}"}
|
||||||
|
)
|
||||||
|
self.assertEqual(status, 401)
|
||||||
|
|
||||||
|
def test_health_needs_no_credentials_and_touches_nothing(self):
|
||||||
|
status, _headers, body = self.request("/healthz", auth=False)
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
self.assertEqual(json.loads(body), {"status": "ok"})
|
||||||
|
self.assertEqual(self.stub.requests, [])
|
||||||
|
|
||||||
|
def test_malformed_authorization_header_is_rejected(self):
|
||||||
|
for value in ("Basic", "Bearer xyz", "Basic !!!not-base64", "Basic " + "aGk="):
|
||||||
|
with self.subTest(value=value):
|
||||||
|
status, _headers, _body = self.request(
|
||||||
|
"/opds", auth=False, headers={"Authorization": value}
|
||||||
|
)
|
||||||
|
self.assertEqual(status, 401)
|
||||||
|
|
||||||
|
def test_diagnostics_requires_credentials(self):
|
||||||
|
self.assertEqual(self.request("/opds/healthz", auth=False)[0], 401)
|
||||||
|
|
||||||
|
def test_diagnostics_reports_upstream_state(self):
|
||||||
|
status, _headers, body = self.request("/opds/healthz")
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
payload = json.loads(body)
|
||||||
|
self.assertTrue(payload["upstream"]["reachable"])
|
||||||
|
self.assertEqual(payload["catalog"]["feeds"], 3)
|
||||||
|
self.assertTrue(payload["mark_read"])
|
||||||
|
|
||||||
|
|
||||||
|
class NavigationTests(ServerTestCase):
|
||||||
|
def test_bare_root_redirects_to_the_catalog(self):
|
||||||
|
# Redirects must be suppressed, or this would see the 401 that /opds
|
||||||
|
# answers an unauthenticated follower with.
|
||||||
|
status, headers, _body = self.request("/", auth=False, follow_redirects=False)
|
||||||
|
self.assertEqual(status, 302)
|
||||||
|
self.assertTrue(headers["Location"].endswith("/opds"))
|
||||||
|
|
||||||
|
def test_root_lists_the_sections(self):
|
||||||
|
node, headers = self.get_xml("/opds")
|
||||||
|
self.assertEqual(node.must_find("title").all_text(), "Test Catalog")
|
||||||
|
titles = set(self.entries(node))
|
||||||
|
self.assertIn("Unread", titles)
|
||||||
|
self.assertIn("Read", titles)
|
||||||
|
self.assertIn("Starred", titles)
|
||||||
|
|
||||||
|
def test_navigation_feed_declares_its_profile(self):
|
||||||
|
_node, headers = self.get_xml("/opds")
|
||||||
|
self.assertEqual(
|
||||||
|
headers["Content-Type"],
|
||||||
|
"application/atom+xml;profile=opds-catalog;kind=navigation",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_root_offers_a_search_link(self):
|
||||||
|
node, _headers = self.get_xml("/opds")
|
||||||
|
rels = {link.attr("rel") for link in node.must_find_all("link")}
|
||||||
|
self.assertIn("search", rels)
|
||||||
|
self.assertIn("self", rels)
|
||||||
|
|
||||||
|
def test_section_lists_categories_with_unread_counts(self):
|
||||||
|
node, _headers = self.get_xml("/opds/unread")
|
||||||
|
entries = self.entries(node)
|
||||||
|
self.assertIn("Tech", entries)
|
||||||
|
self.assertIn("News", entries)
|
||||||
|
self.assertIn("Uncategorised", entries)
|
||||||
|
self.assertIn("unread", entries["Tech"].must_find("content").all_text())
|
||||||
|
|
||||||
|
def test_category_lists_its_feeds(self):
|
||||||
|
category = self.follow(self.entries(self.get_xml("/opds/unread")[0])["Tech"], "subsection")
|
||||||
|
node, _headers = self.get_xml(urllib.parse.urlsplit(category).path)
|
||||||
|
titles = set(self.entries(node))
|
||||||
|
self.assertIn("Example Tech", titles)
|
||||||
|
self.assertIn("All in this category", titles)
|
||||||
|
|
||||||
|
def test_unknown_path_is_404(self):
|
||||||
|
self.assertEqual(self.request("/opds/nope")[0], 404)
|
||||||
|
|
||||||
|
def test_undecodable_identifier_is_404(self):
|
||||||
|
self.assertEqual(self.request("/opds/unread/c/!!!not-base64")[0], 404)
|
||||||
|
|
||||||
|
def test_post_is_not_allowed(self):
|
||||||
|
self.assertEqual(self.request("/opds", method="POST")[0], 405)
|
||||||
|
|
||||||
|
def test_opensearch_descriptor_is_served(self):
|
||||||
|
node, headers = self.get_xml("/opds/opensearch.xml")
|
||||||
|
self.assertEqual(headers["Content-Type"], "application/opensearchdescription+xml")
|
||||||
|
url = node.must_find("Url")
|
||||||
|
self.assertIn("searchTerms", url.must_attr("template"))
|
||||||
|
|
||||||
|
|
||||||
|
class AcquisitionTests(ServerTestCase):
|
||||||
|
def test_acquisition_feed_lists_articles(self):
|
||||||
|
node, headers = self.get_xml(self.acquisition_path())
|
||||||
|
self.assertEqual(
|
||||||
|
headers["Content-Type"],
|
||||||
|
"application/atom+xml;profile=opds-catalog;kind=acquisition",
|
||||||
|
)
|
||||||
|
titles = set(self.entries(node))
|
||||||
|
self.assertIn("A Tech Post & Its Friends", titles)
|
||||||
|
|
||||||
|
def test_read_articles_are_absent_from_the_unread_feed(self):
|
||||||
|
titles = set(self.entries(self.get_xml(self.acquisition_path())[0]))
|
||||||
|
self.assertNotIn("Already Read Post", titles)
|
||||||
|
|
||||||
|
def test_read_section_contains_read_articles(self):
|
||||||
|
path = self.acquisition_path(section="read")
|
||||||
|
titles = set(self.entries(self.get_xml(path)[0]))
|
||||||
|
self.assertIn("Already Read Post", titles)
|
||||||
|
|
||||||
|
def test_entry_offers_an_epub_acquisition_link(self):
|
||||||
|
node, _headers = self.get_xml(self.acquisition_path())
|
||||||
|
entry = self.entries(node)["A Tech Post & Its Friends"]
|
||||||
|
href = self.follow(entry, "http://opds-spec.org/acquisition/open-access")
|
||||||
|
self.assertTrue(href.endswith(".epub"))
|
||||||
|
types = {link.attr("type") for link in entry.must_find_all("link")}
|
||||||
|
self.assertIn("application/epub+zip", types)
|
||||||
|
# The generic relation is offered too, because readers disagree.
|
||||||
|
self.assertIn(
|
||||||
|
"http://opds-spec.org/acquisition",
|
||||||
|
{link.attr("rel") for link in entry.must_find_all("link")},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_entry_links_back_to_the_original_article(self):
|
||||||
|
node, _headers = self.get_xml(self.acquisition_path())
|
||||||
|
entry = self.entries(node)["A Tech Post & Its Friends"]
|
||||||
|
self.assertTrue(self.follow(entry, "alternate").startswith("https://"))
|
||||||
|
|
||||||
|
def test_entry_carries_its_feed_as_a_category(self):
|
||||||
|
node, _headers = self.get_xml(self.acquisition_path())
|
||||||
|
entry = self.entries(node)["A Tech Post & Its Friends"]
|
||||||
|
labels = {category.attr("label") for category in entry.must_find_all("category")}
|
||||||
|
self.assertIn("Example Tech", labels)
|
||||||
|
|
||||||
|
def test_articles_are_newest_first(self):
|
||||||
|
node, _headers = self.get_xml(self.acquisition_path())
|
||||||
|
stamps = [entry.must_find("updated").all_text() for entry in node.must_find_all("entry")]
|
||||||
|
self.assertEqual(stamps, sorted(stamps, reverse=True))
|
||||||
|
|
||||||
|
def test_pagination_link_appears_when_there_is_more(self):
|
||||||
|
# The test configuration serves two items per page, and the reading list
|
||||||
|
# holds five unread articles.
|
||||||
|
node, _headers = self.get_xml("/opds/unread/all")
|
||||||
|
rels = {link.attr("rel") for link in node.must_find_all("link")}
|
||||||
|
self.assertIn("next", rels)
|
||||||
|
self.assertIn("first", rels)
|
||||||
|
self.assertNotIn("previous", rels)
|
||||||
|
|
||||||
|
def test_second_page_is_reachable_and_different(self):
|
||||||
|
path = "/opds/unread/all"
|
||||||
|
first, _ = self.get_xml(path)
|
||||||
|
second, _ = self.get_xml(path + "?page=2")
|
||||||
|
first_ids = {entry.must_find("id").all_text() for entry in first.must_find_all("entry")}
|
||||||
|
second_ids = {entry.must_find("id").all_text() for entry in second.must_find_all("entry")}
|
||||||
|
self.assertTrue(second_ids)
|
||||||
|
self.assertFalse(first_ids & second_ids)
|
||||||
|
|
||||||
|
def test_starred_section_lists_starred_articles(self):
|
||||||
|
titles = set(self.entries(self.get_xml("/opds/starred")[0]))
|
||||||
|
self.assertIn("Starred Investigation", titles)
|
||||||
|
|
||||||
|
def test_search_finds_articles_in_the_window(self):
|
||||||
|
node, _headers = self.get_xml("/opds/search?q=Tech+Post")
|
||||||
|
titles = set(self.entries(node))
|
||||||
|
self.assertIn("A Tech Post & Its Friends", titles)
|
||||||
|
self.assertNotIn("Markets Dip Slightly", titles)
|
||||||
|
|
||||||
|
def test_empty_search_returns_an_empty_feed(self):
|
||||||
|
# A search with no term is a valid, empty acquisition feed rather than an
|
||||||
|
# error, so a reader that sends one sees an empty list.
|
||||||
|
node, _headers = self.get_xml("/opds/search?q=")
|
||||||
|
self.assertEqual(node.find_all("entry"), [])
|
||||||
|
|
||||||
|
def test_bad_page_number_falls_back_to_the_first_page(self):
|
||||||
|
node, _headers = self.get_xml(self.acquisition_path() + "?page=nonsense")
|
||||||
|
self.assertTrue(node.must_find_all("entry"))
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadTests(ServerTestCase):
|
||||||
|
"""The claim under test: a completed download is what marks an article read."""
|
||||||
|
|
||||||
|
def test_download_returns_an_epub(self):
|
||||||
|
status, headers, body = self.request(self.download_path())
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
self.assertEqual(headers["Content-Type"], "application/epub+zip")
|
||||||
|
disposition = headers["Content-Disposition"]
|
||||||
|
self.assertTrue(disposition.startswith("attachment;"))
|
||||||
|
# Both spellings are sent: a plain name every reader understands, and an
|
||||||
|
# RFC 5987 one so a non-ASCII title survives.
|
||||||
|
self.assertIn('filename="a-tech-post-its-friends.epub"', disposition)
|
||||||
|
self.assertIn("filename*=UTF-8''", disposition)
|
||||||
|
|
||||||
|
def test_downloaded_file_is_a_valid_epub(self):
|
||||||
|
_status, _headers, body = self.request(self.download_path())
|
||||||
|
archive = zipfile.ZipFile(BytesIO(body))
|
||||||
|
self.addCleanup(archive.close)
|
||||||
|
self.assertEqual(archive.namelist()[0], "mimetype")
|
||||||
|
self.assertEqual(archive.read("mimetype"), b"application/epub+zip")
|
||||||
|
self.assertEqual(archive.getinfo("mimetype").compress_type, zipfile.ZIP_STORED)
|
||||||
|
opf = parse_xml(archive.read(OPF_PATH))
|
||||||
|
self.assertEqual(opf.must_find("title").all_text(), "A Tech Post & Its Friends")
|
||||||
|
# The body went through the sanitiser on the way in.
|
||||||
|
article = archive.read(ARTICLE_PATH).decode("utf-8")
|
||||||
|
self.assertNotIn("alert(", article)
|
||||||
|
self.assertIn("Entities: &", article)
|
||||||
|
|
||||||
|
def test_download_marks_the_article_read_upstream(self):
|
||||||
|
self.request(self.download_path())
|
||||||
|
self.assertEqual(self.wait_for_mark_read({item_id(1)}), {item_id(1)})
|
||||||
|
edits = self.stub.find_requests("/reader/api/0/edit-tag")
|
||||||
|
self.assertEqual(len(edits), 1)
|
||||||
|
self.assertIn("i=tag%3Agoogle.com", edits[0]["body"])
|
||||||
|
|
||||||
|
def test_downloads_mark_only_their_own_article(self):
|
||||||
|
self.request(self.download_path())
|
||||||
|
self.assertEqual(self.wait_for_mark_read({item_id(1)}), {item_id(1)})
|
||||||
|
|
||||||
|
def test_head_does_not_mark_read(self):
|
||||||
|
# A reader probing the URL, or a CDN, must not consume the article.
|
||||||
|
status, headers, body = self.request(self.download_path(), method="HEAD")
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
self.assertEqual(headers["Content-Type"], "application/epub+zip")
|
||||||
|
self.assertEqual(body, b"")
|
||||||
|
# Deterministic rather than a race: a HEAD never reaches the mark-read.
|
||||||
|
self.assertEqual(self.stub.read - self.baseline_read, set())
|
||||||
|
self.assertEqual(self.stub.find_requests("/reader/api/0/edit-tag"), [])
|
||||||
|
|
||||||
|
def test_missing_article_is_404(self):
|
||||||
|
stream = base64.urlsafe_b64encode(TECH_FEED.encode()).decode().rstrip("=")
|
||||||
|
absent = (
|
||||||
|
base64.urlsafe_b64encode(b"tag:google.com,2005:reader/item/nope").decode().rstrip("=")
|
||||||
|
)
|
||||||
|
status, _headers, _body = self.request(f"/opds/d/{stream}/{absent}/nope.epub")
|
||||||
|
self.assertEqual(status, 404)
|
||||||
|
|
||||||
|
def test_article_id_is_encoded_in_the_url_and_survives(self):
|
||||||
|
# FreshRSS ids contain ':' and '/', so the URL must carry them encoded.
|
||||||
|
path = self.download_path()
|
||||||
|
self.assertNotIn("tag:google.com", path)
|
||||||
|
self.assertEqual(path.count("/"), 5)
|
||||||
|
|
||||||
|
def test_marked_read_article_leaves_the_unread_queue(self):
|
||||||
|
path = self.download_path()
|
||||||
|
self.request(path)
|
||||||
|
self.wait_for_mark_read({item_id(1)})
|
||||||
|
# The stream cache is invalidated by the mark-read, so the next browse
|
||||||
|
# reflects it without waiting for a TTL.
|
||||||
|
titles = set(self.entries(self.get_xml(self.acquisition_path())[0]))
|
||||||
|
self.assertNotIn("A Tech Post & Its Friends", titles)
|
||||||
|
|
||||||
|
def test_marked_read_article_appears_in_the_read_section(self):
|
||||||
|
self.request(self.download_path())
|
||||||
|
self.wait_for_mark_read({item_id(1)})
|
||||||
|
path = self.acquisition_path(section="read")
|
||||||
|
titles = set(self.entries(self.get_xml(path)[0]))
|
||||||
|
self.assertIn("A Tech Post & Its Friends", titles)
|
||||||
|
|
||||||
|
def test_mark_read_can_be_disabled(self):
|
||||||
|
self.application.config.freshrss.mark_read = False
|
||||||
|
self.request(self.download_path())
|
||||||
|
self.assertEqual(self.stub.read - self.baseline_read, set())
|
||||||
|
self.assertEqual(self.stub.find_requests("/reader/api/0/edit-tag"), [])
|
||||||
|
|
||||||
|
def test_advertised_unread_count_refreshes_after_a_download(self):
|
||||||
|
"""The count must not lag the queue it describes.
|
||||||
|
|
||||||
|
Unread counts come from the cached account snapshot, so a mark-read has to
|
||||||
|
drop that cache too -- otherwise a reader downloads an article and is still
|
||||||
|
told the old total, with one fewer entry underneath it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def advertised_unread() -> int:
|
||||||
|
node, _ = self.get_xml("/opds/unread")
|
||||||
|
for entry in node.must_find_all("entry"):
|
||||||
|
if entry.must_find("title").all_text() == "All feeds":
|
||||||
|
return int(entry.must_find("content").all_text().split()[0])
|
||||||
|
raise AssertionError("no 'All feeds' entry in the unread catalog")
|
||||||
|
|
||||||
|
before = advertised_unread()
|
||||||
|
self.request(self.download_path())
|
||||||
|
self.wait_for_mark_read({item_id(1)})
|
||||||
|
self.assertEqual(advertised_unread(), before - 1)
|
||||||
|
|
||||||
|
def test_second_download_is_still_served_after_the_queue_moved_on(self):
|
||||||
|
path = self.download_path()
|
||||||
|
self.request(path)
|
||||||
|
status, _headers, body = self.request(path)
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
self.assertTrue(body.startswith(b"PK"))
|
||||||
|
|
||||||
|
|
||||||
|
class UpstreamFailureTests(ServerTestCase):
|
||||||
|
def test_upstream_outage_is_a_bad_gateway(self):
|
||||||
|
self.stub.fail_next(503, count=10, path="/subscription/list")
|
||||||
|
status, _headers, _body = self.request("/opds")
|
||||||
|
self.assertEqual(status, 502)
|
||||||
|
|
||||||
|
def test_catalog_recovers_once_upstream_returns(self):
|
||||||
|
# Exactly one failure, so the next attempt succeeds.
|
||||||
|
self.stub.fail_next(503, path="/subscription/list")
|
||||||
|
self.assertEqual(self.request("/opds")[0], 502)
|
||||||
|
self.application.invalidate()
|
||||||
|
self.assertEqual(self.request("/opds")[0], 200)
|
||||||
|
|
||||||
|
def test_unread_counts_missing_is_not_fatal(self):
|
||||||
|
# The feature is optional; the catalog must still browse without it.
|
||||||
|
self.stub.supports_unread_counts = False
|
||||||
|
self.application.invalidate()
|
||||||
|
status, _headers, body = self.request("/opds/unread")
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
node = parse_xml(body)
|
||||||
|
titles = {entry.must_find("title").all_text() for entry in node.must_find_all("entry")}
|
||||||
|
self.assertIn("Tech", titles)
|
||||||
|
|
||||||
|
def test_unknown_host_is_refused_when_an_allowlist_is_set(self):
|
||||||
|
self.application.config.server.allowed_hosts = ["catalog.example.com"]
|
||||||
|
status, _headers, _body = self.request("/opds", auth=False)
|
||||||
|
self.assertEqual(status, 500)
|
||||||
|
|
||||||
|
def test_public_base_url_controls_the_links(self):
|
||||||
|
self.application.config.server.public_base_url = "https://catalog.example.com/books"
|
||||||
|
node, _headers = self.get_xml("/opds")
|
||||||
|
hrefs = {link.must_attr("href") for link in node.must_find_all("link")}
|
||||||
|
self.assertTrue(
|
||||||
|
any(href.startswith("https://catalog.example.com/books/opds") for href in hrefs)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SerializationTests(ServerTestCase):
|
||||||
|
"""Responses must be correct XML even when titles contain markup."""
|
||||||
|
|
||||||
|
def test_titles_with_markup_are_escaped(self):
|
||||||
|
self.stub.subscriptions[0]["title"] = 'Feed & <Friends> "quoted"'
|
||||||
|
self.application.invalidate()
|
||||||
|
category = urllib.parse.urlsplit(
|
||||||
|
self.follow(self.entries(self.get_xml("/opds/unread")[0])["Tech"], "subsection")
|
||||||
|
).path
|
||||||
|
node, _headers = self.get_xml(category)
|
||||||
|
titles = {entry.must_find("title").all_text() for entry in node.must_find_all("entry")}
|
||||||
|
self.assertIn('Feed & <Friends> "quoted"', titles)
|
||||||
|
|
||||||
|
def test_feed_ids_are_stable_across_requests(self):
|
||||||
|
first, _ = self.get_xml("/opds/unread")
|
||||||
|
second, _ = self.get_xml("/opds/unread")
|
||||||
|
self.assertEqual(first.must_find("id").all_text(), second.must_find("id").all_text())
|
||||||
|
|
||||||
|
def test_entry_ids_are_unique_within_a_feed(self):
|
||||||
|
node, _headers = self.get_xml(self.acquisition_path())
|
||||||
|
ids = [entry.must_find("id").all_text() for entry in node.must_find_all("entry")]
|
||||||
|
self.assertEqual(len(ids), len(set(ids)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
399
tests/test_xhtml.py
Normal file
399
tests/test_xhtml.py
Normal file
@@ -0,0 +1,399 @@
|
|||||||
|
"""Tests for the HTML to XHTML normaliser.
|
||||||
|
|
||||||
|
The bar every case is held to: the output must parse with a strict XML parser,
|
||||||
|
because that is what an EPUB reader does with it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from app.xhtml import parse_article
|
||||||
|
from tests.support import assert_well_formed, body, collapse, parse_xml
|
||||||
|
|
||||||
|
BASE = "https://example.com/posts/one?utm=x"
|
||||||
|
|
||||||
|
|
||||||
|
def render(raw: str, image_map=None, base: str = BASE, max_input_bytes: int = 0) -> str:
|
||||||
|
"""Parse and serialise, asserting well-formedness on the way out."""
|
||||||
|
article = parse_article(raw, base, max_input_bytes=max_input_bytes)
|
||||||
|
fragment = article.to_xhtml(image_map)
|
||||||
|
assert_well_formed(unittest.TestCase(), fragment)
|
||||||
|
return fragment
|
||||||
|
|
||||||
|
|
||||||
|
class MalformedInputTests(unittest.TestCase):
|
||||||
|
"""Feed HTML is broken in predictable ways; none of it may reach the output."""
|
||||||
|
|
||||||
|
def test_unclosed_paragraphs_are_balanced(self):
|
||||||
|
out = render("<p>one<p>two<p>three")
|
||||||
|
self.assertEqual(body(out).count("<p>"), 3)
|
||||||
|
self.assertEqual(body(out).count("</p>"), 3)
|
||||||
|
self.assertIn("three", out)
|
||||||
|
|
||||||
|
def test_block_element_closes_open_paragraph(self):
|
||||||
|
# A naive stack would nest the div inside the paragraph.
|
||||||
|
out = render("<p>lead<div>block</div>trailing")
|
||||||
|
self.assertIn("<p>lead</p>", out)
|
||||||
|
self.assertIn("<div>block</div>", out)
|
||||||
|
self.assertNotIn("<p>lead<div>", out)
|
||||||
|
|
||||||
|
def test_unclosed_list_items(self):
|
||||||
|
out = render("<ul><li>a<li>b<li>c</ul>")
|
||||||
|
self.assertEqual(body(out).count("<li>"), 3)
|
||||||
|
self.assertEqual(body(out).count("</li>"), 3)
|
||||||
|
|
||||||
|
def test_stray_end_tag_is_ignored(self):
|
||||||
|
out = render("<p>text</span></div></p>")
|
||||||
|
self.assertIn("<p>text</p>", out)
|
||||||
|
|
||||||
|
def test_unclosed_table_row_and_cells(self):
|
||||||
|
out = render("<table><tr><td>a<td>b<tr><td>c</table>")
|
||||||
|
self.assertEqual(body(out).count("<tr>"), 2)
|
||||||
|
self.assertEqual(body(out).count("<td>"), 3)
|
||||||
|
self.assertIn("<td>c</td>", out)
|
||||||
|
|
||||||
|
def test_self_closing_non_void_element_swallows_nothing(self):
|
||||||
|
# `<div/>` must not open a subtree that eats the rest of the document, and
|
||||||
|
# the empty element it leaves behind is noise worth dropping.
|
||||||
|
out = render("<div/>after")
|
||||||
|
self.assertIn("after", out)
|
||||||
|
self.assertEqual(body(out), "after")
|
||||||
|
|
||||||
|
def test_void_elements_are_self_closed(self):
|
||||||
|
out = render("a<br>b<hr>c")
|
||||||
|
self.assertIn("<br/>", out)
|
||||||
|
self.assertIn("<hr/>", out)
|
||||||
|
self.assertNotIn("<br>", out)
|
||||||
|
|
||||||
|
def test_control_characters_are_removed(self):
|
||||||
|
# NUL and vertical tab cannot be represented in XML even as entities.
|
||||||
|
out = render("<p>be\x00fore\x0baft\x0cft</p>")
|
||||||
|
self.assertNotIn("\x00", out)
|
||||||
|
self.assertNotIn("\x0b", out)
|
||||||
|
self.assertNotIn("\x0c", out)
|
||||||
|
self.assertIn("before", out)
|
||||||
|
|
||||||
|
def test_unpaired_surrogate_is_removed(self):
|
||||||
|
out = render("<p>ok\ud800tail</p>")
|
||||||
|
self.assertNotIn("\ud800", out)
|
||||||
|
self.assertIn("oktail", out)
|
||||||
|
|
||||||
|
def test_zero_width_characters_are_removed(self):
|
||||||
|
out = render("<p>a\u200bb\ufeffc</p>")
|
||||||
|
self.assertEqual(collapse(parse_xml(out).all_text()), "abc")
|
||||||
|
|
||||||
|
def test_comments_are_dropped(self):
|
||||||
|
out = render("<p>keep<!-- drop me --> this</p>")
|
||||||
|
self.assertNotIn("drop me", out)
|
||||||
|
|
||||||
|
def test_processing_instruction_and_doctype_are_dropped(self):
|
||||||
|
out = render("<?xml version='1.0'?><!DOCTYPE html><p>body</p>")
|
||||||
|
self.assertNotIn("DOCTYPE", out)
|
||||||
|
self.assertIn("body", out)
|
||||||
|
|
||||||
|
def test_deeply_nested_wrappers_keep_their_text(self):
|
||||||
|
out = render("<section><article><aside>deep</aside></article></section>")
|
||||||
|
self.assertIn("deep", out)
|
||||||
|
self.assertNotIn("<section>", out)
|
||||||
|
self.assertNotIn("<aside>", out)
|
||||||
|
|
||||||
|
|
||||||
|
class EscapingTests(unittest.TestCase):
|
||||||
|
"""Nothing in the output may be interpretable as markup or as an entity."""
|
||||||
|
|
||||||
|
def test_bare_ampersand_is_escaped(self):
|
||||||
|
out = render("<p>Tom & Jerry</p>")
|
||||||
|
self.assertIn("Tom & Jerry", out)
|
||||||
|
|
||||||
|
def test_named_and_numeric_entities_are_decoded_then_reescaped(self):
|
||||||
|
out = render("<p> é&</p>")
|
||||||
|
# must not survive as an entity, and must not break the parse.
|
||||||
|
self.assertNotIn(" ", out)
|
||||||
|
self.assertIn("\u00a0", out)
|
||||||
|
self.assertIn("\u00e9", out)
|
||||||
|
|
||||||
|
def test_text_that_looks_like_xml_is_escaped(self):
|
||||||
|
out = render("<p><script>alert(1)</script></p>")
|
||||||
|
self.assertIn("<script>", out)
|
||||||
|
self.assertEqual(body(out).count("<script>"), 0)
|
||||||
|
|
||||||
|
def test_quotes_in_attributes_are_escaped(self):
|
||||||
|
out = render('<p><a href="https://e.com/?q="x"" title="a"b">t</a></p>')
|
||||||
|
self.assertIn(""", out)
|
||||||
|
|
||||||
|
def test_cdata_section_is_unwrapped_to_text(self):
|
||||||
|
out = render("<p><![CDATA[raw text]]></p>")
|
||||||
|
self.assertIn("raw text", out)
|
||||||
|
|
||||||
|
def test_gets_escaped_inside_attributes_only_once(self):
|
||||||
|
out = render('<p><a href="https://e.com/a?b=1&c=2">t</a></p>')
|
||||||
|
self.assertIn('href="https://e.com/a?b=1&c=2"', out)
|
||||||
|
|
||||||
|
|
||||||
|
class DangerousContentTests(unittest.TestCase):
|
||||||
|
"""Scriptable and resource-fetching constructs must not survive."""
|
||||||
|
|
||||||
|
def test_script_element_and_body_are_removed(self):
|
||||||
|
out = render("<p>before</p><script>alert('xss')</script><p>after</p>")
|
||||||
|
self.assertNotIn("script", out.lower())
|
||||||
|
self.assertIn("before", out)
|
||||||
|
self.assertIn("after", out)
|
||||||
|
|
||||||
|
def test_unclosed_script_does_not_swallow_forever(self):
|
||||||
|
out = render("<script>alert(1)</script><p>visible</p>")
|
||||||
|
self.assertIn("visible", out)
|
||||||
|
|
||||||
|
def test_style_element_is_removed(self):
|
||||||
|
out = render("<style>p{color:red}</style><p>text</p>")
|
||||||
|
self.assertNotIn("color:red", out)
|
||||||
|
|
||||||
|
def test_iframe_and_object_are_removed_with_contents(self):
|
||||||
|
out = render('<iframe src="https://e.com/x"></iframe><p>t</p>')
|
||||||
|
self.assertNotIn("iframe", out)
|
||||||
|
|
||||||
|
def test_style_attribute_is_dropped(self):
|
||||||
|
out = render('<p style="background:url(javascript:1)">t</p>')
|
||||||
|
self.assertNotIn("style", out)
|
||||||
|
|
||||||
|
def test_event_handler_attributes_are_dropped(self):
|
||||||
|
out = render('<p onclick="evil()" onmouseover="evil()">t</p>')
|
||||||
|
self.assertNotIn("onclick", out)
|
||||||
|
self.assertNotIn("onmouseover", out)
|
||||||
|
|
||||||
|
def test_class_attribute_is_dropped(self):
|
||||||
|
# class fights the book's own stylesheet and means nothing offline.
|
||||||
|
out = render('<p class="x">t</p>')
|
||||||
|
self.assertNotIn("class", body(out))
|
||||||
|
|
||||||
|
def test_id_is_kept_for_anchor_targets(self):
|
||||||
|
out = render('<p id="section-2">t</p>')
|
||||||
|
self.assertIn('id="section-2"', out)
|
||||||
|
|
||||||
|
def test_invalid_id_is_dropped(self):
|
||||||
|
# An id has to be a usable XML name; anything else is not an anchor. The
|
||||||
|
# quote case needs single-quoted markup, or the parser would read a
|
||||||
|
# shorter, valid value and the case would prove nothing.
|
||||||
|
cases = {
|
||||||
|
"leading digit": '<p id="9lives">t</p>',
|
||||||
|
"space": '<p id="has space">t</p>',
|
||||||
|
"slash": '<p id="sla/sh">t</p>',
|
||||||
|
"quote": "<p id='quote\"d'>t</p>",
|
||||||
|
"empty": '<p id="">t</p>',
|
||||||
|
"too long": f'<p id="{"a" * 100}">t</p>',
|
||||||
|
}
|
||||||
|
for label, markup in cases.items():
|
||||||
|
self.assertNotIn("id=", body(render(markup)), label)
|
||||||
|
|
||||||
|
def test_javascript_url_is_dropped(self):
|
||||||
|
out = render('<a href="javascript:alert(1)">click</a>')
|
||||||
|
self.assertNotIn("javascript", out)
|
||||||
|
self.assertIn("click", out)
|
||||||
|
|
||||||
|
def test_data_url_link_is_dropped(self):
|
||||||
|
out = render('<a href="data:text/html,<script>x</script>">t</a>')
|
||||||
|
self.assertNotIn("data:", out)
|
||||||
|
|
||||||
|
def test_data_url_image_is_dropped(self):
|
||||||
|
out = render('<img src="data:image/png;base64,AAAA"><p>t</p>')
|
||||||
|
self.assertNotIn("data:", out)
|
||||||
|
self.assertNotIn("<img", out)
|
||||||
|
|
||||||
|
def test_img_without_source_is_dropped(self):
|
||||||
|
out = render('<p><img alt="no source"></p>')
|
||||||
|
self.assertNotIn("<img", out)
|
||||||
|
|
||||||
|
def test_only_href_survives_on_links(self):
|
||||||
|
out = render('<a href="https://e.com/a" target="_blank" rel="noopener" download>x</a>')
|
||||||
|
self.assertIn('href="https://e.com/a"', out)
|
||||||
|
self.assertNotIn("target", out)
|
||||||
|
self.assertNotIn("rel=", out)
|
||||||
|
|
||||||
|
|
||||||
|
class UrlResolutionTests(unittest.TestCase):
|
||||||
|
"""Bodies are re-based into the book, so relative URLs must be resolved."""
|
||||||
|
|
||||||
|
def test_root_relative_link_uses_article_origin(self):
|
||||||
|
out = render('<a href="/about">about</a>')
|
||||||
|
self.assertIn('href="https://example.com/about"', out)
|
||||||
|
|
||||||
|
def test_document_relative_link_uses_article_directory(self):
|
||||||
|
out = render('<a href="sibling.html">s</a>')
|
||||||
|
self.assertIn('href="https://example.com/posts/sibling.html"', out)
|
||||||
|
|
||||||
|
def test_protocol_relative_link_is_resolved(self):
|
||||||
|
out = render('<a href="//cdn.example.com/x">x</a>')
|
||||||
|
self.assertIn('href="https://cdn.example.com/x"', out)
|
||||||
|
|
||||||
|
def test_mailto_link_is_kept(self):
|
||||||
|
out = render('<a href="mailto:someone@example.com">mail</a>')
|
||||||
|
self.assertIn('href="mailto:someone@example.com"', out)
|
||||||
|
|
||||||
|
def test_anchor_only_link_stays_in_document(self):
|
||||||
|
# A same-document link must not be rewritten to the source web page: the
|
||||||
|
# matching id is kept, so it jumps within the book instead.
|
||||||
|
out = render('<p id="section">target</p><a href="#section">jump</a>')
|
||||||
|
self.assertIn('href="#section"', out)
|
||||||
|
self.assertIn('id="section"', out)
|
||||||
|
|
||||||
|
def test_relative_image_is_absolutised(self):
|
||||||
|
article = parse_article('<img src="images/a.png">', BASE)
|
||||||
|
self.assertEqual(article.image_urls, ["https://example.com/posts/images/a.png"])
|
||||||
|
|
||||||
|
def test_blockquote_cite_is_validated(self):
|
||||||
|
out = render('<blockquote cite="javascript:x">q</blockquote>')
|
||||||
|
self.assertNotIn("cite", out)
|
||||||
|
out = render('<blockquote cite="https://e.com/src">q</blockquote>')
|
||||||
|
self.assertIn('cite="https://e.com/src"', out)
|
||||||
|
|
||||||
|
|
||||||
|
class AnchorIdTests(unittest.TestCase):
|
||||||
|
"""Ids survive so an article's own table of contents works offline."""
|
||||||
|
|
||||||
|
def test_duplicate_ids_are_renamed_not_dropped(self):
|
||||||
|
out = render('<p id="a">1</p><p id="a">2</p><p id="a">3</p>')
|
||||||
|
self.assertIn('id="a"', out)
|
||||||
|
self.assertIn('id="a-2"', out)
|
||||||
|
self.assertIn('id="a-3"', out)
|
||||||
|
|
||||||
|
def test_id_on_void_element_is_dropped(self):
|
||||||
|
out = render('<img id="x" src="a.png">')
|
||||||
|
self.assertNotIn('id="x"', out)
|
||||||
|
|
||||||
|
def test_renamed_ids_do_not_collide_with_existing_ones(self):
|
||||||
|
out = render('<p id="a">1</p><p id="a-2">2</p><p id="a">3</p>')
|
||||||
|
self.assertEqual(body(out).count('id="a-2"'), 1)
|
||||||
|
self.assertIn('id="a-3"', out)
|
||||||
|
|
||||||
|
def test_unique_ids_are_left_alone(self):
|
||||||
|
out = render('<p id="one">1</p><p id="two">2</p>')
|
||||||
|
self.assertIn('id="one"', out)
|
||||||
|
self.assertIn('id="two"', out)
|
||||||
|
|
||||||
|
def test_an_element_keeping_only_an_id_is_not_pruned(self):
|
||||||
|
out = render('<div id="anchor"></div><p>text</p>')
|
||||||
|
self.assertIn('id="anchor"', out)
|
||||||
|
|
||||||
|
|
||||||
|
class ImageHandlingTests(unittest.TestCase):
|
||||||
|
"""Images are collected for the fetcher and rewritten by the caller."""
|
||||||
|
|
||||||
|
def test_image_urls_are_collected_in_order_and_deduplicated(self):
|
||||||
|
article = parse_article('<p><img src="a.png"><img src="a.png"><img src="b.png"></p>', BASE)
|
||||||
|
self.assertEqual(
|
||||||
|
article.image_urls,
|
||||||
|
["https://example.com/posts/a.png", "https://example.com/posts/b.png"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_remote_urls_are_kept_when_no_map_is_given(self):
|
||||||
|
out = render('<img src="a.png" alt="alt text">')
|
||||||
|
self.assertIn('src="https://example.com/posts/a.png"', out)
|
||||||
|
self.assertIn('alt="alt text"', out)
|
||||||
|
|
||||||
|
def test_map_rewrites_src_to_local_path(self):
|
||||||
|
url = "https://example.com/posts/a.png"
|
||||||
|
out = render('<img src="a.png">', image_map={url: "images/img-1.png"})
|
||||||
|
self.assertIn('src="images/img-1.png"', out)
|
||||||
|
|
||||||
|
def test_none_in_map_drops_image_and_empty_wrapper(self):
|
||||||
|
url = "https://example.com/posts/a.png"
|
||||||
|
out = render('<div><img src="a.png"></div><p>text</p>', image_map={url: None})
|
||||||
|
self.assertNotIn("<img", out)
|
||||||
|
self.assertNotIn("<div>", out)
|
||||||
|
self.assertIn("<p>text</p>", out)
|
||||||
|
|
||||||
|
def test_alt_and_title_are_preserved_for_localised_images(self):
|
||||||
|
url = "https://example.com/posts/a.png"
|
||||||
|
out = render('<img src="a.png" alt="A" title="T">', image_map={url: "images/i.png"})
|
||||||
|
self.assertIn('alt="A"', out)
|
||||||
|
self.assertIn('title="T"', out)
|
||||||
|
|
||||||
|
|
||||||
|
class WhitespaceAndStructureTests(unittest.TestCase):
|
||||||
|
def test_whitespace_is_collapsed_outside_pre(self):
|
||||||
|
out = render("<p>one\n\n two\t\tthree</p>")
|
||||||
|
self.assertIn("<p>one two three</p>", out)
|
||||||
|
|
||||||
|
def test_whitespace_is_preserved_inside_pre(self):
|
||||||
|
out = render("<pre>def f():\n return 1</pre>")
|
||||||
|
self.assertIn("def f():\n return 1", out)
|
||||||
|
|
||||||
|
def test_empty_paragraphs_are_pruned(self):
|
||||||
|
out = render("<p> </p><p> </p><p>real</p>")
|
||||||
|
self.assertEqual(body(out).count("<p>"), 1)
|
||||||
|
self.assertIn("real", out)
|
||||||
|
|
||||||
|
def test_empty_table_cells_are_kept(self):
|
||||||
|
# An empty cell is layout, not noise, so pruning must leave it alone.
|
||||||
|
out = render("<table><tr><td></td><td>b</td></tr></table>")
|
||||||
|
self.assertEqual(body(out).count("<td></td>"), 1)
|
||||||
|
|
||||||
|
def test_plain_text_body_is_wrapped_in_a_paragraph(self):
|
||||||
|
out = render("just words with no markup")
|
||||||
|
self.assertEqual(body(out), "<p>just words with no markup</p>")
|
||||||
|
|
||||||
|
def test_whitespace_only_body_renders_empty(self):
|
||||||
|
article = parse_article(" \n ", BASE)
|
||||||
|
self.assertEqual(article.to_xhtml(), "")
|
||||||
|
|
||||||
|
def test_empty_input_renders_empty(self):
|
||||||
|
self.assertEqual(parse_article("", BASE).to_xhtml(), "")
|
||||||
|
|
||||||
|
def test_heading_levels_survive(self):
|
||||||
|
out = render("<h1>a</h1><h2>b</h2><h3>c</h3>")
|
||||||
|
for tag in ("h1", "h2", "h3"):
|
||||||
|
self.assertIn(f"<{tag}>", out)
|
||||||
|
|
||||||
|
def test_colspan_is_clamped(self):
|
||||||
|
out = render('<td colspan="99999">x</td>')
|
||||||
|
self.assertNotIn("colspan", out)
|
||||||
|
out = render('<td colspan="2">x</td>')
|
||||||
|
self.assertIn('colspan="2"', out)
|
||||||
|
|
||||||
|
def test_element_structure_is_preserved(self):
|
||||||
|
out = render("<ul><li>a<ul><li>b</li></ul></li></ul>")
|
||||||
|
tree = parse_xml(out)
|
||||||
|
# The fragment's own root is the article wrapper.
|
||||||
|
self.assertEqual(tree.must_attr("class"), "article-body")
|
||||||
|
outer = tree.must_find("ul")
|
||||||
|
items = outer.must_find_all("li")
|
||||||
|
self.assertEqual(len(items), 2)
|
||||||
|
# The nested list belongs to the first item, not to the outer list.
|
||||||
|
self.assertTrue(items[0].find_all("ul"))
|
||||||
|
|
||||||
|
|
||||||
|
class ArticleApiTests(unittest.TestCase):
|
||||||
|
def test_text_extraction_flattens_and_collapses(self):
|
||||||
|
article = parse_article("<p>One</p> <p>Two & three</p>", BASE)
|
||||||
|
self.assertEqual(article.text(), "One Two & three")
|
||||||
|
|
||||||
|
def test_text_extraction_is_limited_at_a_word_boundary(self):
|
||||||
|
article = parse_article("<p>alpha beta gamma delta</p>", BASE)
|
||||||
|
text = article.text(limit=12)
|
||||||
|
self.assertTrue(text.endswith("\u2026"))
|
||||||
|
self.assertLessEqual(len(text), 13)
|
||||||
|
self.assertIn("alpha beta", text)
|
||||||
|
|
||||||
|
def test_truncation_flag_and_output_validity(self):
|
||||||
|
article = parse_article("<p>" + "word " * 5000 + "</p>", BASE, max_input_bytes=2000)
|
||||||
|
self.assertTrue(article.truncated)
|
||||||
|
assert_well_formed(self, article.to_xhtml())
|
||||||
|
|
||||||
|
def test_no_truncation_when_under_limit(self):
|
||||||
|
article = parse_article("<p>short</p>", BASE, max_input_bytes=10_000)
|
||||||
|
self.assertFalse(article.truncated)
|
||||||
|
|
||||||
|
def test_output_is_deterministic(self):
|
||||||
|
raw = '<p>a & b</p><div><img src="i.png"><ul><li>x</ul></div>'
|
||||||
|
self.assertEqual(render(raw), render(raw))
|
||||||
|
|
||||||
|
def test_image_urls_are_cached_between_calls(self):
|
||||||
|
article = parse_article('<img src="a.png">', BASE)
|
||||||
|
self.assertIs(article.image_urls, article.image_urls)
|
||||||
|
|
||||||
|
def test_output_wrapper_has_the_expected_class(self):
|
||||||
|
self.assertTrue(render("<p>x</p>").startswith('<div class="article-body">'))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
145
tools/fake-freshrss.py
Executable file
145
tools/fake-freshrss.py
Executable file
@@ -0,0 +1,145 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Run a fake FreshRSS, so the whole server can be tried without an account.
|
||||||
|
|
||||||
|
Serves the same in-process stand-in the test suite uses: three feeds across two
|
||||||
|
categories, a starred article, an already-read article, a feed with no category,
|
||||||
|
and article bodies that exercise the sanitiser. It also writes a ready-to-run
|
||||||
|
configuration pointing at itself, with a known demo password.
|
||||||
|
|
||||||
|
Run locally::
|
||||||
|
|
||||||
|
python3 tools/fake-freshrss.py
|
||||||
|
python3 -m app --config config.demo.json # in another shell
|
||||||
|
# then browse http://127.0.0.1:8080/opds as demo / demo
|
||||||
|
|
||||||
|
Or as the other half of the demo compose stack, where it binds the whole
|
||||||
|
container interface and writes a configuration addressing itself by service name
|
||||||
|
(see ``docker-compose.demo.yaml``).
|
||||||
|
|
||||||
|
Nothing here is a real credential. With the default loopback bind it is
|
||||||
|
reachable only from this machine; the demo compose stack puts both containers on
|
||||||
|
an internal network with no published ports other than the catalogue.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
sys.path.insert(0, str(REPO_ROOT))
|
||||||
|
|
||||||
|
from app.config import password_digest # noqa: E402
|
||||||
|
from tests.stub_freshrss import build_demo_catalog # noqa: E402
|
||||||
|
|
||||||
|
DEMO_USER = "demo"
|
||||||
|
DEMO_PASSWORD = "demo"
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description=__doc__,
|
||||||
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--write-config",
|
||||||
|
metavar="PATH",
|
||||||
|
default="config.demo.json",
|
||||||
|
help="where to write a configuration that points at this fake server",
|
||||||
|
)
|
||||||
|
app = parser.add_argument_group("the server being configured")
|
||||||
|
app.add_argument("--app-port", type=int, default=8080, help="port for the server")
|
||||||
|
app.add_argument(
|
||||||
|
"--app-host",
|
||||||
|
default="127.0.0.1",
|
||||||
|
help="address the server should bind (use 0.0.0.0 inside a container)",
|
||||||
|
)
|
||||||
|
api = parser.add_argument_group("this fake FreshRSS")
|
||||||
|
api.add_argument(
|
||||||
|
"--api-port",
|
||||||
|
type=int,
|
||||||
|
default=0,
|
||||||
|
help="port to listen on; 0 asks the operating system for a free one",
|
||||||
|
)
|
||||||
|
api.add_argument(
|
||||||
|
"--api-bind",
|
||||||
|
default="127.0.0.1",
|
||||||
|
help="address to listen on (use 0.0.0.0 so another container can reach it)",
|
||||||
|
)
|
||||||
|
api.add_argument(
|
||||||
|
"--api-public-host",
|
||||||
|
default=None,
|
||||||
|
help="host name to write into the configuration (a compose service name, "
|
||||||
|
"for instance); defaults to the bind address",
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def demo_configuration(stub, arguments: argparse.Namespace) -> dict:
|
||||||
|
"""The configuration that points a server at this fake account."""
|
||||||
|
return {
|
||||||
|
"_comment": "Written by tools/fake-freshrss.py; not a real configuration.",
|
||||||
|
"server": {
|
||||||
|
"host": arguments.app_host,
|
||||||
|
"port": arguments.app_port,
|
||||||
|
"log_level": "info",
|
||||||
|
},
|
||||||
|
"freshrss": {
|
||||||
|
"url": stub.api_url,
|
||||||
|
"username": stub.username,
|
||||||
|
"api_password": stub.password,
|
||||||
|
"items_per_page": 5,
|
||||||
|
# A short window so the demo reflects a download immediately rather
|
||||||
|
# than showing a stale unread count for minutes.
|
||||||
|
"retries": 1,
|
||||||
|
},
|
||||||
|
"catalog": {"title": "Demo Catalog", "author": "Demo"},
|
||||||
|
"epub": {"include_images": False},
|
||||||
|
"auth": {
|
||||||
|
"users": [{"username": DEMO_USER, "password_sha256": password_digest(DEMO_PASSWORD)}]
|
||||||
|
},
|
||||||
|
"cache": {"subscriptions_ttl_seconds": 30, "stream_ttl_seconds": 10},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
arguments = build_parser().parse_args(argv)
|
||||||
|
stub = build_demo_catalog(
|
||||||
|
bind_host=arguments.api_bind,
|
||||||
|
bind_port=arguments.api_port,
|
||||||
|
public_host=arguments.api_public_host,
|
||||||
|
).start()
|
||||||
|
|
||||||
|
path = Path(arguments.write_config)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(
|
||||||
|
json.dumps(demo_configuration(stub, arguments), indent=2) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
browse_host = "127.0.0.1" if arguments.app_host == "0.0.0.0" else arguments.app_host
|
||||||
|
print(f"fake FreshRSS listening on {stub.api_url}", flush=True)
|
||||||
|
print(f" api user: {stub.username}", flush=True)
|
||||||
|
print(f" api password: {stub.password}", flush=True)
|
||||||
|
print(f"wrote {path}", flush=True)
|
||||||
|
print(f"catalogue will be at http://{browse_host}:{arguments.app_port}/opds", flush=True)
|
||||||
|
print(f" sign in as {DEMO_USER} / {DEMO_PASSWORD}", flush=True)
|
||||||
|
if arguments.app_host == "127.0.0.1":
|
||||||
|
print(f"\nnow run, in another shell:\n python3 -m app --config {path}", flush=True)
|
||||||
|
print("\npress ctrl-c to stop", flush=True)
|
||||||
|
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
time.sleep(1)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print("\nstopping", flush=True)
|
||||||
|
finally:
|
||||||
|
stub.stop()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
Reference in New Issue
Block a user