"""Fetching article images for embedding, with an SSRF guard. Image URLs come from feed content, which means they are attacker-controlled by anyone who can publish to a feed this account follows. That makes this the one place in the server that makes an outbound request to a host the operator did not choose, so the guard is the point of the module rather than an extra. What is enforced, and why: *Only http and https*, so ``file:``, ``gopher:`` and friends are unreachable. *Every address a name resolves to is checked* against the private, loopback, link-local, multicast and reserved ranges. If a name resolves to a mixture, the whole fetch is refused rather than picking the acceptable address -- a mixed answer is itself the attack. *The connection is dialled to the validated address, not to the name.* This is what closes the DNS-rebinding window: a name that answered with a public address during validation cannot answer with ``169.254.169.254`` a millisecond later, because the name is never resolved a second time. TLS still verifies the name the URL used, so a valid certificate for the real host is required. *Redirects are followed manually*, so each hop is validated again; ``urllib``'s automatic redirect handling would happily follow the first hop to ``http://127.0.0.1:8080/``. *Bodies are capped*, per image and across the whole article, so one hostile feed cannot exhaust memory. A refusal is a normal outcome: the image is dropped from the book and the article is still built. """ from __future__ import annotations import http.client import ipaddress import logging import socket import ssl import threading import urllib.parse from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from app import images LOGGER = logging.getLogger(__name__) USER_AGENT = "odps-from-rss/1.0 (+image fetch for EPUB embedding)" _ALLOWED_SCHEMES = ("http", "https") _MAX_REDIRECTS = 3 _READ_CHUNK = 64 * 1024 # Refuse to even start reading a body larger than the cap, when the server tells # us the size up front. A little slack covers a mislabelled Content-Length. _CONTENT_LENGTH_SLACK = 4096 class ImageFetchError(Exception): """The URL was refused by the guard, or the fetch failed.""" @dataclass(frozen=True) class FetchedImage: """An image that was fetched, validated and identified.""" url: str media_type: str data: bytes @property def filename(self) -> str: """A safe file name for the EPUB, with the extension from the media type.""" return "image" + images.MEDIA_TYPES.get(self.media_type, ".bin") class _PinnedHTTPSConnection(http.client.HTTPSConnection): """HTTPS to a validated address, verifying the hostname from the URL. ``HTTPSConnection`` would use its own host for SNI and certificate verification, which here is a bare IP address and would always fail. Dialling the address but verifying the name keeps both properties: no second DNS lookup, and a real certificate check. """ def __init__( self, address: str, port: int, *, server_hostname: str, context: ssl.SSLContext, timeout: float, ) -> None: super().__init__(address, port, timeout=timeout, context=context) self._verify_hostname = server_hostname def connect(self) -> None: http.client.HTTPConnection.connect(self) self.sock = self._context.wrap_socket( # type: ignore[union-attr] self.sock, server_hostname=self._verify_hostname ) class ImageFetcher: """Fetches article images concurrently, refusing anything unsafe.""" def __init__( self, *, allowed_hosts: list[str] | None = None, blocked_hosts: list[str] | None = None, max_image_bytes: int = 2_097_152, max_total_bytes: int = 16_777_216, timeout: float = 10.0, max_workers: int = 4, allow_private_networks: bool = False, ) -> None: self._allowed_hosts = [host.lower() for host in (allowed_hosts or [])] self._blocked_hosts = [host.lower() for host in (blocked_hosts or [])] self._max_image_bytes = max_image_bytes self._max_total_bytes = max_total_bytes self._timeout = timeout self._max_workers = max(1, max_workers) # The upstream FreshRSS may be on a LAN, but article images never have a # legitimate reason to point into one. Kept as a parameter so a test can # exercise the guard, not as a configuration option. self._allow_private_networks = allow_private_networks self._tls_context = ssl.create_default_context() self._budget_lock = threading.Lock() self._budget_used = 0 def fetch_all(self, urls: list[str]) -> dict[str, FetchedImage | None]: """Fetch each URL, mapping it to an image or to None when refused. Order is preserved by the caller, which walks the document's images; this method only has to answer "did it arrive?". Concurrent, because a article with a dozen images should not cost a dozen sequential round trips. """ unique: list[str] = [] for url in urls: if url not in unique: unique.append(url) if not unique: return {} with self._budget_lock: self._budget_used = 0 workers = min(self._max_workers, len(unique)) results: dict[str, FetchedImage | None] = {} with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="image") as pool: for url, result in zip(unique, pool.map(self._fetch_safely, unique), strict=True): results[url] = result return results # -- internals ------------------------------------------------------- def _fetch_safely(self, url: str) -> FetchedImage | None: """Fetch one image, turning every failure into a dropped image.""" try: return self._fetch(url) except ImageFetchError as exc: LOGGER.info("not embedding image %s: %s", url, exc) except (OSError, http.client.HTTPException, ssl.SSLError) as exc: LOGGER.info("could not fetch image %s: %s", url, exc) except Exception: # One bad image must never break the book it was going into. LOGGER.exception("unexpected failure fetching image %s", url) return None def _fetch(self, url: str) -> FetchedImage: target = url for _ in range(_MAX_REDIRECTS + 1): parsed = urllib.parse.urlsplit(target) scheme = parsed.scheme.lower() host = parsed.hostname if scheme not in _ALLOWED_SCHEMES: raise ImageFetchError(f"scheme {scheme or '(none)'!r} is not allowed") if not host: raise ImageFetchError("no host in URL") if parsed.username or parsed.password: raise ImageFetchError("credentials in an image URL are not followed") self._check_host_allowed(host) address = self._resolve(host) port = parsed.port or (443 if scheme == "https" else 80) status, headers, body = self._request_once(scheme, address, host, port, parsed) if status in (301, 302, 303, 307, 308): location = headers.get("location") if not location: raise ImageFetchError("redirect without a location") # Resolve relative redirects against the URL that produced them. target = urllib.parse.urljoin(target, location) LOGGER.debug("following image redirect to %s", target) continue if status != 200: raise ImageFetchError(f"server answered {status}") declared = headers.get("content-type") media_type = images.identify(body, declared) if media_type is None: raise ImageFetchError(f"not an embeddable image ({declared or 'no type'})") return FetchedImage(url=url, media_type=media_type, data=body) raise ImageFetchError("too many redirects") def _check_host_allowed(self, host: str) -> None: lowered = host.lower() if any(_host_matches(lowered, blocked) for blocked in self._blocked_hosts): raise ImageFetchError("host is on the blocked list") if self._allowed_hosts and not any( _host_matches(lowered, allowed) for allowed in self._allowed_hosts ): raise ImageFetchError("host is not on the allowed list") def _resolve(self, host: str) -> str: """Validate every address for a host and return the one to dial.""" try: infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP) except socket.gaierror as exc: raise ImageFetchError(f"cannot resolve host ({exc.strerror or exc})") from exc addresses: list[str] = [] for info in infos: address = str(info[4][0]) if address not in addresses: addresses.append(address) if not addresses: raise ImageFetchError("host resolved to no addresses") for address in addresses: reason = self._address_refusal(address) if reason is not None: # Refuse the whole host rather than picking a good address: a # mixed answer is the signature of a rebinding attempt. raise ImageFetchError(f"{address} is refused ({reason})") return addresses[0] def _address_refusal(self, address: str) -> str | None: try: parsed = ipaddress.ip_address(address) except ValueError: return "unparseable address" if self._allow_private_networks: return None # A v4-mapped v6 address is judged as the v4 address it is, and this has to # come first: ::ffff:127.0.0.1 is neither loopback nor private when it is # tested as a v6 address, so the checks below would let it through. if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None: return self._address_refusal(str(parsed.ipv4_mapped)) if parsed.is_loopback: return "loopback" if parsed.is_link_local: return "link-local" if parsed.is_private: return "private network" if parsed.is_multicast: return "multicast" if parsed.is_reserved or parsed.is_unspecified: return "reserved" return None def _request_once( self, scheme: str, address: str, host: str, port: int, parsed: urllib.parse.SplitResult, ) -> tuple[int, dict[str, str], bytes]: """One request, to a pinned address, reading at most the byte cap.""" if scheme == "https": connection: http.client.HTTPConnection = _PinnedHTTPSConnection( address, port, server_hostname=host, context=self._tls_context, timeout=self._timeout, ) else: connection = http.client.HTTPConnection(address, port, timeout=self._timeout) path = urllib.parse.urlunsplit(("", "", parsed.path or "/", parsed.query, "")) headers = { "Host": _host_header(host, port, scheme), "User-Agent": USER_AGENT, "Accept": "image/*", "Connection": "close", } try: connection.request("GET", path, headers=headers) response = connection.getresponse() status = response.status response_headers = {key.lower(): value for key, value in response.getheaders()} if status != 200: response.read(_READ_CHUNK) return status, response_headers, b"" return status, response_headers, self._read_capped(response, response_headers) finally: connection.close() def _read_capped(self, response: http.client.HTTPResponse, headers: dict[str, str]) -> bytes: """Read a response body, refusing one that exceeds the budget.""" limit = self._image_limit(headers) chunks: list[bytes] = [] total = 0 while True: chunk = response.read(_READ_CHUNK) if not chunk: break total += len(chunk) if total > limit: raise ImageFetchError(f"image is larger than {limit} bytes") chunks.append(chunk) self._spend(total) return b"".join(chunks) def _image_limit(self, headers: dict[str, str]) -> int: """The smaller of the per-image cap and what is left of the total budget.""" declared = headers.get("content-length") limit = self._max_image_bytes if declared and declared.isdigit(): size = int(declared) if size > self._max_image_bytes: raise ImageFetchError(f"image declares {size} bytes") with self._budget_lock: remaining = max(0, self._max_total_bytes - self._budget_used) if remaining == 0: # The article's total allowance is spent; the rest of its images are # dropped rather than growing the book further. raise ImageFetchError("the article's image budget is exhausted") return min(limit, remaining) def _spend(self, size: int) -> None: with self._budget_lock: self._budget_used += size def _host_matches(host: str, pattern: str) -> bool: """Exact host or subdomain match, so ``example.com`` covers ``cdn.example.com``.""" return host == pattern or host.endswith("." + pattern) def _host_header(host: str, port: int, scheme: str) -> str: """A Host header, including the port only when it is not the default.""" default = 443 if scheme == "https" else 80 if port == default: return host return f"{host}:{port}"