340 lines
14 KiB
Python
340 lines
14 KiB
Python
"""Fetching article images for embedding, with an SSRF guard.
|
|
|
|
Image URLs come from feed content, which means they are attacker-controlled by
|
|
anyone who can publish to a feed this account follows. That makes this the one
|
|
place in the server that makes an outbound request to a host the operator did not
|
|
choose, so the guard is the point of the module rather than an extra.
|
|
|
|
What is enforced, and why:
|
|
|
|
*Only http and https*, so ``file:``, ``gopher:`` and friends are unreachable.
|
|
*Every address a name resolves to is checked* against the private, loopback,
|
|
link-local, multicast and reserved ranges. If a name resolves to a mixture, the
|
|
whole fetch is refused rather than picking the acceptable address -- a mixed
|
|
answer is itself the attack.
|
|
*The connection is dialled to the validated address, not to the name.* This is
|
|
what closes the DNS-rebinding window: a name that answered with a public address
|
|
during validation cannot answer with ``169.254.169.254`` a millisecond later,
|
|
because the name is never resolved a second time. TLS still verifies the name
|
|
the URL used, so a valid certificate for the real host is required.
|
|
*Redirects are followed manually*, so each hop is validated again; ``urllib``'s
|
|
automatic redirect handling would happily follow the first hop to
|
|
``http://127.0.0.1:8080/``.
|
|
*Bodies are capped*, per image and across the whole article, so one hostile feed
|
|
cannot exhaust memory.
|
|
|
|
A refusal is a normal outcome: the image is dropped from the book and the article
|
|
is still built.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import http.client
|
|
import ipaddress
|
|
import logging
|
|
import socket
|
|
import ssl
|
|
import threading
|
|
import urllib.parse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from dataclasses import dataclass
|
|
|
|
from app import images
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
|
|
USER_AGENT = "odps-from-rss/1.0 (+image fetch for EPUB embedding)"
|
|
|
|
_ALLOWED_SCHEMES = ("http", "https")
|
|
_MAX_REDIRECTS = 3
|
|
_READ_CHUNK = 64 * 1024
|
|
# Refuse to even start reading a body larger than the cap, when the server tells
|
|
# us the size up front. A little slack covers a mislabelled Content-Length.
|
|
_CONTENT_LENGTH_SLACK = 4096
|
|
|
|
|
|
class ImageFetchError(Exception):
|
|
"""The URL was refused by the guard, or the fetch failed."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FetchedImage:
|
|
"""An image that was fetched, validated and identified."""
|
|
|
|
url: str
|
|
media_type: str
|
|
data: bytes
|
|
|
|
@property
|
|
def filename(self) -> str:
|
|
"""A safe file name for the EPUB, with the extension from the media type."""
|
|
return "image" + images.MEDIA_TYPES.get(self.media_type, ".bin")
|
|
|
|
|
|
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
|
|
"""HTTPS to a validated address, verifying the hostname from the URL.
|
|
|
|
``HTTPSConnection`` would use its own host for SNI and certificate
|
|
verification, which here is a bare IP address and would always fail. Dialling
|
|
the address but verifying the name keeps both properties: no second DNS
|
|
lookup, and a real certificate check.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
address: str,
|
|
port: int,
|
|
*,
|
|
server_hostname: str,
|
|
context: ssl.SSLContext,
|
|
timeout: float,
|
|
) -> None:
|
|
super().__init__(address, port, timeout=timeout, context=context)
|
|
self._verify_hostname = server_hostname
|
|
|
|
def connect(self) -> None:
|
|
http.client.HTTPConnection.connect(self)
|
|
self.sock = self._context.wrap_socket( # type: ignore[union-attr]
|
|
self.sock, server_hostname=self._verify_hostname
|
|
)
|
|
|
|
|
|
class ImageFetcher:
|
|
"""Fetches article images concurrently, refusing anything unsafe."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
allowed_hosts: list[str] | None = None,
|
|
blocked_hosts: list[str] | None = None,
|
|
max_image_bytes: int = 2_097_152,
|
|
max_total_bytes: int = 16_777_216,
|
|
timeout: float = 10.0,
|
|
max_workers: int = 4,
|
|
allow_private_networks: bool = False,
|
|
) -> None:
|
|
self._allowed_hosts = [host.lower() for host in (allowed_hosts or [])]
|
|
self._blocked_hosts = [host.lower() for host in (blocked_hosts or [])]
|
|
self._max_image_bytes = max_image_bytes
|
|
self._max_total_bytes = max_total_bytes
|
|
self._timeout = timeout
|
|
self._max_workers = max(1, max_workers)
|
|
# The upstream FreshRSS may be on a LAN, but article images never have a
|
|
# legitimate reason to point into one. Kept as a parameter so a test can
|
|
# exercise the guard, not as a configuration option.
|
|
self._allow_private_networks = allow_private_networks
|
|
self._tls_context = ssl.create_default_context()
|
|
self._budget_lock = threading.Lock()
|
|
self._budget_used = 0
|
|
|
|
def fetch_all(self, urls: list[str]) -> dict[str, FetchedImage | None]:
|
|
"""Fetch each URL, mapping it to an image or to None when refused.
|
|
|
|
Order is preserved by the caller, which walks the document's images; this
|
|
method only has to answer "did it arrive?". Concurrent, because a article
|
|
with a dozen images should not cost a dozen sequential round trips.
|
|
"""
|
|
unique: list[str] = []
|
|
for url in urls:
|
|
if url not in unique:
|
|
unique.append(url)
|
|
if not unique:
|
|
return {}
|
|
with self._budget_lock:
|
|
self._budget_used = 0
|
|
workers = min(self._max_workers, len(unique))
|
|
results: dict[str, FetchedImage | None] = {}
|
|
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="image") as pool:
|
|
for url, result in zip(unique, pool.map(self._fetch_safely, unique), strict=True):
|
|
results[url] = result
|
|
return results
|
|
|
|
# -- internals -------------------------------------------------------
|
|
|
|
def _fetch_safely(self, url: str) -> FetchedImage | None:
|
|
"""Fetch one image, turning every failure into a dropped image."""
|
|
try:
|
|
return self._fetch(url)
|
|
except ImageFetchError as exc:
|
|
LOGGER.info("not embedding image %s: %s", url, exc)
|
|
except (OSError, http.client.HTTPException, ssl.SSLError) as exc:
|
|
LOGGER.info("could not fetch image %s: %s", url, exc)
|
|
except Exception:
|
|
# One bad image must never break the book it was going into.
|
|
LOGGER.exception("unexpected failure fetching image %s", url)
|
|
return None
|
|
|
|
def _fetch(self, url: str) -> FetchedImage:
|
|
target = url
|
|
for _ in range(_MAX_REDIRECTS + 1):
|
|
parsed = urllib.parse.urlsplit(target)
|
|
scheme = parsed.scheme.lower()
|
|
host = parsed.hostname
|
|
if scheme not in _ALLOWED_SCHEMES:
|
|
raise ImageFetchError(f"scheme {scheme or '(none)'!r} is not allowed")
|
|
if not host:
|
|
raise ImageFetchError("no host in URL")
|
|
if parsed.username or parsed.password:
|
|
raise ImageFetchError("credentials in an image URL are not followed")
|
|
self._check_host_allowed(host)
|
|
address = self._resolve(host)
|
|
port = parsed.port or (443 if scheme == "https" else 80)
|
|
status, headers, body = self._request_once(scheme, address, host, port, parsed)
|
|
if status in (301, 302, 303, 307, 308):
|
|
location = headers.get("location")
|
|
if not location:
|
|
raise ImageFetchError("redirect without a location")
|
|
# Resolve relative redirects against the URL that produced them.
|
|
target = urllib.parse.urljoin(target, location)
|
|
LOGGER.debug("following image redirect to %s", target)
|
|
continue
|
|
if status != 200:
|
|
raise ImageFetchError(f"server answered {status}")
|
|
declared = headers.get("content-type")
|
|
media_type = images.identify(body, declared)
|
|
if media_type is None:
|
|
raise ImageFetchError(f"not an embeddable image ({declared or 'no type'})")
|
|
return FetchedImage(url=url, media_type=media_type, data=body)
|
|
raise ImageFetchError("too many redirects")
|
|
|
|
def _check_host_allowed(self, host: str) -> None:
|
|
lowered = host.lower()
|
|
if any(_host_matches(lowered, blocked) for blocked in self._blocked_hosts):
|
|
raise ImageFetchError("host is on the blocked list")
|
|
if self._allowed_hosts and not any(
|
|
_host_matches(lowered, allowed) for allowed in self._allowed_hosts
|
|
):
|
|
raise ImageFetchError("host is not on the allowed list")
|
|
|
|
def _resolve(self, host: str) -> str:
|
|
"""Validate every address for a host and return the one to dial."""
|
|
try:
|
|
infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
|
|
except socket.gaierror as exc:
|
|
raise ImageFetchError(f"cannot resolve host ({exc.strerror or exc})") from exc
|
|
addresses: list[str] = []
|
|
for info in infos:
|
|
address = str(info[4][0])
|
|
if address not in addresses:
|
|
addresses.append(address)
|
|
if not addresses:
|
|
raise ImageFetchError("host resolved to no addresses")
|
|
for address in addresses:
|
|
reason = self._address_refusal(address)
|
|
if reason is not None:
|
|
# Refuse the whole host rather than picking a good address: a
|
|
# mixed answer is the signature of a rebinding attempt.
|
|
raise ImageFetchError(f"{address} is refused ({reason})")
|
|
return addresses[0]
|
|
|
|
def _address_refusal(self, address: str) -> str | None:
|
|
try:
|
|
parsed = ipaddress.ip_address(address)
|
|
except ValueError:
|
|
return "unparseable address"
|
|
if self._allow_private_networks:
|
|
return None
|
|
# A v4-mapped v6 address is judged as the v4 address it is, and this has to
|
|
# come first: ::ffff:127.0.0.1 is neither loopback nor private when it is
|
|
# tested as a v6 address, so the checks below would let it through.
|
|
if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None:
|
|
return self._address_refusal(str(parsed.ipv4_mapped))
|
|
if parsed.is_loopback:
|
|
return "loopback"
|
|
if parsed.is_link_local:
|
|
return "link-local"
|
|
if parsed.is_private:
|
|
return "private network"
|
|
if parsed.is_multicast:
|
|
return "multicast"
|
|
if parsed.is_reserved or parsed.is_unspecified:
|
|
return "reserved"
|
|
return None
|
|
|
|
def _request_once(
|
|
self,
|
|
scheme: str,
|
|
address: str,
|
|
host: str,
|
|
port: int,
|
|
parsed: urllib.parse.SplitResult,
|
|
) -> tuple[int, dict[str, str], bytes]:
|
|
"""One request, to a pinned address, reading at most the byte cap."""
|
|
if scheme == "https":
|
|
connection: http.client.HTTPConnection = _PinnedHTTPSConnection(
|
|
address,
|
|
port,
|
|
server_hostname=host,
|
|
context=self._tls_context,
|
|
timeout=self._timeout,
|
|
)
|
|
else:
|
|
connection = http.client.HTTPConnection(address, port, timeout=self._timeout)
|
|
path = urllib.parse.urlunsplit(("", "", parsed.path or "/", parsed.query, ""))
|
|
headers = {
|
|
"Host": _host_header(host, port, scheme),
|
|
"User-Agent": USER_AGENT,
|
|
"Accept": "image/*",
|
|
"Connection": "close",
|
|
}
|
|
try:
|
|
connection.request("GET", path, headers=headers)
|
|
response = connection.getresponse()
|
|
status = response.status
|
|
response_headers = {key.lower(): value for key, value in response.getheaders()}
|
|
if status != 200:
|
|
response.read(_READ_CHUNK)
|
|
return status, response_headers, b""
|
|
return status, response_headers, self._read_capped(response, response_headers)
|
|
finally:
|
|
connection.close()
|
|
|
|
def _read_capped(self, response: http.client.HTTPResponse, headers: dict[str, str]) -> bytes:
|
|
"""Read a response body, refusing one that exceeds the budget."""
|
|
limit = self._image_limit(headers)
|
|
chunks: list[bytes] = []
|
|
total = 0
|
|
while True:
|
|
chunk = response.read(_READ_CHUNK)
|
|
if not chunk:
|
|
break
|
|
total += len(chunk)
|
|
if total > limit:
|
|
raise ImageFetchError(f"image is larger than {limit} bytes")
|
|
chunks.append(chunk)
|
|
self._spend(total)
|
|
return b"".join(chunks)
|
|
|
|
def _image_limit(self, headers: dict[str, str]) -> int:
|
|
"""The smaller of the per-image cap and what is left of the total budget."""
|
|
declared = headers.get("content-length")
|
|
limit = self._max_image_bytes
|
|
if declared and declared.isdigit():
|
|
size = int(declared)
|
|
if size > self._max_image_bytes:
|
|
raise ImageFetchError(f"image declares {size} bytes")
|
|
with self._budget_lock:
|
|
remaining = max(0, self._max_total_bytes - self._budget_used)
|
|
if remaining == 0:
|
|
# The article's total allowance is spent; the rest of its images are
|
|
# dropped rather than growing the book further.
|
|
raise ImageFetchError("the article's image budget is exhausted")
|
|
return min(limit, remaining)
|
|
|
|
def _spend(self, size: int) -> None:
|
|
with self._budget_lock:
|
|
self._budget_used += size
|
|
|
|
|
|
def _host_matches(host: str, pattern: str) -> bool:
|
|
"""Exact host or subdomain match, so ``example.com`` covers ``cdn.example.com``."""
|
|
return host == pattern or host.endswith("." + pattern)
|
|
|
|
|
|
def _host_header(host: str, port: int, scheme: str) -> str:
|
|
"""A Host header, including the port only when it is not the default."""
|
|
default = 443 if scheme == "https" else 80
|
|
if port == default:
|
|
return host
|
|
return f"{host}:{port}"
|