Microscope camera kiosk
Touchscreen kiosk for the Raspberry Pi HQ camera (IMX477) used as a microscope: live preview from boot, raw frame recording, full-resolution stills, and automatic capture-SSD mounting. - app.py: GTK3 fullscreen UI over cage — preview, REC/STOP, PHOTO, 90-degree rotate button, tap-to-open exposure/gain/settings sliders, and an "SSD NOT MOUNTED" badge - camera.py: picamera2 wrapper with separate video (ISP preview + packed 12-bit raw) and still (4056x3040) configurations - recorder.py: writer thread turning the raw stream into one Bayer frame per file plus index.jsonl/session.json, with dropped/missed accounting - photos.py: still capture to DNG + JPEG, and a background worker that renders a viewable 16-bit sRGB TIFF from the 12-bit Bayer frame - storage.py: SSD mount state and the date/time/format session layout - tools/unpack_raw.py: desktop-side unpacker/demosaicer - system/ + install.sh: systemd units, udev rule, mount helper, sudoers Note: raw frames are written byte-for-byte as the sensor produced them, so they stay decodeable regardless of how the stills pipeline evolves.
This commit is contained in:
Microscope Camera
443
photos.py
Normal file
443
photos.py
Normal file
@@ -0,0 +1,443 @@
|
||||
"""Full-resolution still capture: DNG + 16-bit TIFF + JPEG.
|
||||
|
||||
The IMX477 is a 12-bit sensor and the Pi's ISP only outputs 8-bit processed
|
||||
images, so a genuinely 16-bit TIFF cannot come from the ISP. Instead the TIFF is
|
||||
built from the raw Bayer frame through the same chain a raw converter would use:
|
||||
|
||||
unpack 12-bit MIPI -> black-level subtract -> white balance
|
||||
-> colour-correction matrix -> bilinear demosaic -> sRGB gamma -> 16-bit
|
||||
|
||||
The result is a normal, viewable 16-bit image rather than linear sensor data.
|
||||
The untouched sensor data stays in the DNG next to it, so nothing is lost for
|
||||
quantitative work.
|
||||
|
||||
The slow part (demosaic + 74 MB TIFF write) runs on a worker thread so the live
|
||||
preview resumes as soon as the capture itself is done.
|
||||
"""
|
||||
|
||||
import datetime as _dt
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
log = logging.getLogger("photos")
|
||||
|
||||
_PATTERN_OFFSETS = {
|
||||
# pattern -> (row parity, col parity) of the red sample in the 2x2 tile
|
||||
"RGGB": (0, 0),
|
||||
"BGGR": (1, 1),
|
||||
"GRBG": (0, 1),
|
||||
"GBRG": (1, 0),
|
||||
}
|
||||
|
||||
|
||||
def pattern_from_format(fmt: str) -> str:
|
||||
"""'SRGGB12_CSI2P' -> 'RGGB'."""
|
||||
if not fmt:
|
||||
return "RGGB"
|
||||
text = fmt.upper()
|
||||
if text.startswith("S"):
|
||||
text = text[1:]
|
||||
return text[:4] if text[:4] in _PATTERN_OFFSETS else "RGGB"
|
||||
|
||||
|
||||
def unpack_mipi12(packed: np.ndarray, width: int, height: int, stride: int) -> np.ndarray:
|
||||
"""Unpack MIPI CSI-2 packed 12-bit Bayer into a uint16 (height, width) array.
|
||||
|
||||
Two pixels live in three bytes with the *first* pixel in the high bits:
|
||||
|
||||
byte0 byte1 byte2
|
||||
p0[11:4] p0[3:0] p1[11:8] p1[7:0]
|
||||
|
||||
so even pixels are ``(b0 << 4) | (b1 >> 4)`` and odd pixels are
|
||||
``((b1 & 0x0F) << 8) | b2``. Getting this order wrong still yields values in
|
||||
the 0-4095 range, but it destroys the image: the tell-tale is that the two
|
||||
green sites of the Bayer tile stop matching each other.
|
||||
"""
|
||||
rows = packed.reshape(height, stride)
|
||||
useful = rows[:, : (width * 3) // 2]
|
||||
triplets = useful.reshape(height, width // 2, 3).astype(np.uint16)
|
||||
b0 = triplets[:, :, 0]
|
||||
b1 = triplets[:, :, 1]
|
||||
b2 = triplets[:, :, 2]
|
||||
even = (b0 << 4) | (b1 >> 4)
|
||||
odd = ((b1 & 0x0F) << 8) | b2
|
||||
out = np.empty((height, width), dtype=np.uint16)
|
||||
out[:, 0::2] = even
|
||||
out[:, 1::2] = odd
|
||||
return out
|
||||
|
||||
|
||||
def demosaic(raw: np.ndarray, pattern: str = "BGGR") -> np.ndarray:
|
||||
"""Bilinear demosaic of a Bayer plane into a uint16 (h, w, 3) RGB image.
|
||||
|
||||
Quantisation-free for 12-bit input (max 4095) and cheap enough (~1 s at
|
||||
12 MP) to run on the Pi.
|
||||
"""
|
||||
rp, cp = _PATTERN_OFFSETS.get(pattern, (0, 0))
|
||||
h, w = raw.shape
|
||||
tiles = raw.reshape(h // 2, 2, w // 2, 2).transpose(0, 2, 1, 3)
|
||||
red = tiles[..., rp, cp].astype(np.uint32)
|
||||
blue = tiles[..., 1 - rp, 1 - cp].astype(np.uint32)
|
||||
green_r = tiles[..., rp, 1 - cp].astype(np.uint32)
|
||||
green_b = tiles[..., 1 - rp, cp].astype(np.uint32)
|
||||
|
||||
def padded(plane):
|
||||
return np.pad(plane, 1, mode="edge")
|
||||
|
||||
def upsample(plane):
|
||||
p = padded(plane)
|
||||
center = p[1:-1, 1:-1]
|
||||
left = p[1:-1, :-2]
|
||||
right = p[1:-1, 2:]
|
||||
up = p[:-2, 1:-1]
|
||||
down = p[2:, 1:-1]
|
||||
horiz = (center + (right if cp == 0 else left)) // 2
|
||||
vert = (center + (down if rp == 0 else up)) // 2
|
||||
col_n = right if cp == 0 else left
|
||||
row_n = down if rp == 0 else up
|
||||
if cp == 0 and rp == 0:
|
||||
corner = p[2:, 2:]
|
||||
elif cp == 0 and rp == 1:
|
||||
corner = p[2:, :-2]
|
||||
elif cp == 1 and rp == 0:
|
||||
corner = p[:-2, 2:]
|
||||
else:
|
||||
corner = p[:-2, :-2]
|
||||
diag = (center + col_n + row_n + corner) // 4
|
||||
full = np.empty((h, w), dtype=np.uint16)
|
||||
full[rp::2, cp::2] = center
|
||||
full[rp::2, (1 - cp)::2] = horiz
|
||||
full[(1 - rp)::2, cp::2] = vert
|
||||
full[(1 - rp)::2, (1 - cp)::2] = diag
|
||||
return full
|
||||
|
||||
red_full = upsample(red)
|
||||
blue_full = upsample(blue)
|
||||
|
||||
g1 = padded(green_r)
|
||||
g2 = padded(green_b)
|
||||
g1_left = (g1[1:-1, 1:-1] + g1[1:-1, :-2]) // 2
|
||||
g2_right = (g2[1:-1, 1:-1] + g2[1:-1, 2:]) // 2
|
||||
green_full = np.empty((h, w), dtype=np.uint16)
|
||||
green_full[rp::2, 1 - cp::2] = green_r
|
||||
green_full[1 - rp::2, cp::2] = green_b
|
||||
green_full[rp::2, cp::2] = g1_left
|
||||
green_full[1 - rp::2, 1 - cp::2] = g2_right
|
||||
|
||||
rgb = np.empty((h, w, 3), dtype=np.uint16)
|
||||
rgb[..., 0] = red_full
|
||||
rgb[..., 1] = green_full
|
||||
rgb[..., 2] = blue_full
|
||||
return rgb
|
||||
|
||||
|
||||
def srgb_encode(linear: np.ndarray) -> np.ndarray:
|
||||
"""Apply the sRGB transfer function (linear light -> display values)."""
|
||||
out = np.empty_like(linear)
|
||||
low = linear <= 0.0031308
|
||||
np.multiply(linear, 12.92, out=out, where=low)
|
||||
high = ~low
|
||||
np.power(np.clip(linear, 1e-8, None), 1.0 / 2.4, out=out, where=high)
|
||||
np.multiply(out, 1.055, out=out, where=high)
|
||||
np.subtract(out, 0.055, out=out, where=high)
|
||||
return out
|
||||
|
||||
|
||||
def render_still(raw, pattern, black=256.0, white=4095.0, gains=None, ccm=None,
|
||||
gamma=True) -> np.ndarray:
|
||||
"""Turn a 12-bit Bayer frame into a viewable 16-bit RGB image.
|
||||
|
||||
Mirrors what a raw converter does, in the order the ISP does it:
|
||||
black-level subtract, white balance, colour matrix, demosaic, gamma.
|
||||
|
||||
``gains`` is picamera2's ``ColourGains`` (red, blue; green is 1.0) and
|
||||
``ccm`` is its ``ColourCorrectionMatrix`` (3x3, camera RGB -> sRGB). Either
|
||||
may be None, in which case that step is skipped.
|
||||
"""
|
||||
rgb = demosaic(raw, pattern)
|
||||
img = rgb.astype(np.float32)
|
||||
del rgb
|
||||
|
||||
scale = 1.0 / float(white - black)
|
||||
img *= scale
|
||||
img -= black * scale
|
||||
np.clip(img, 0.0, None, out=img)
|
||||
|
||||
if gains is not None:
|
||||
img[..., 0] *= float(gains[0])
|
||||
img[..., 2] *= float(gains[1])
|
||||
|
||||
if ccm is not None:
|
||||
matrix = np.asarray(ccm, dtype=np.float32).reshape(3, 3)
|
||||
img = img @ matrix.T
|
||||
np.clip(img, 0.0, None, out=img)
|
||||
|
||||
# Highlight clipping happens here, as it does in the ISP.
|
||||
np.clip(img, 0.0, 1.0, out=img)
|
||||
if gamma:
|
||||
img = srgb_encode(img)
|
||||
|
||||
return np.clip(img * 65535.0 + 0.5, 0, 65535).astype(np.uint16)
|
||||
|
||||
|
||||
def _rotate_array(rgb, rotation):
|
||||
"""Rotate an (h, w, 3) array clockwise by 0/90/180/270 degrees."""
|
||||
rotation = int(rotation) % 360
|
||||
if rotation == 0:
|
||||
return rgb
|
||||
if rotation == 90:
|
||||
return np.ascontiguousarray(np.rot90(rgb, k=3))
|
||||
if rotation == 180:
|
||||
return np.ascontiguousarray(np.rot90(rgb, k=2))
|
||||
if rotation == 270:
|
||||
return np.ascontiguousarray(np.rot90(rgb, k=1))
|
||||
return rgb
|
||||
|
||||
|
||||
def _pil_transpose(rotation):
|
||||
"""PIL constant that rotates an image clockwise by *rotation* degrees.
|
||||
|
||||
PIL's ROTATE_* constants turn counter-clockwise, hence the inversion.
|
||||
"""
|
||||
from PIL import Image
|
||||
|
||||
transpose = getattr(Image, "Transpose", Image)
|
||||
return {90: transpose.ROTATE_270, 180: transpose.ROTATE_180,
|
||||
270: transpose.ROTATE_90}[int(rotation) % 360]
|
||||
|
||||
|
||||
def rotate_jpeg(path, rotation):
|
||||
"""Rotate a JPEG on disk in place (used for the ISP-written still)."""
|
||||
rotation = int(rotation) % 360
|
||||
if rotation == 0:
|
||||
return False
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(path) as image:
|
||||
turned = image.transpose(_pil_transpose(rotation))
|
||||
turned.save(path, quality=95, subsampling=0)
|
||||
return True
|
||||
|
||||
|
||||
def set_dng_orientation(path, rotation):
|
||||
"""Write the DNG/TIFF Orientation tag (274) in place.
|
||||
|
||||
Rotating the Bayer mosaic itself would change the sensor pattern, so the raw
|
||||
DNG keeps its pixels and records the rotation the way every raw converter
|
||||
expects: as metadata. Orientation 1/3/6/8 = 0/180/90/270 degrees.
|
||||
"""
|
||||
value = {0: 1, 90: 6, 180: 3, 270: 8}.get(int(rotation) % 360)
|
||||
if value in (None, 1):
|
||||
return False
|
||||
try:
|
||||
with open(path, "r+b") as handle:
|
||||
header = handle.read(8)
|
||||
if header[:2] not in (b"II", b"MM"):
|
||||
return False
|
||||
order = "little" if header[:2] == b"II" else "big"
|
||||
ifd_offset = int.from_bytes(header[4:8], order)
|
||||
handle.seek(ifd_offset)
|
||||
count = int.from_bytes(handle.read(2), order)
|
||||
entries = handle.read(12 * count)
|
||||
for index in range(count):
|
||||
entry = entries[index * 12:(index + 1) * 12]
|
||||
tag = int.from_bytes(entry[0:2], order)
|
||||
if tag == 274:
|
||||
handle.seek(ifd_offset + 2 + index * 12 + 8)
|
||||
handle.write(value.to_bytes(2, order) + b"\x00\x00")
|
||||
return True
|
||||
except OSError:
|
||||
log.exception("could not set DNG orientation on %s", path)
|
||||
return False
|
||||
|
||||
|
||||
class TiffWorker(threading.Thread):
|
||||
"""Serial background worker for the expensive demosaic + TIFF write."""
|
||||
|
||||
def __init__(self, on_done=None):
|
||||
super().__init__(name="tiff-writer", daemon=True)
|
||||
self._jobs = queue.Queue()
|
||||
self._on_done = on_done
|
||||
|
||||
def submit(self, job):
|
||||
self._jobs.put(job)
|
||||
|
||||
def stop(self, timeout=90):
|
||||
"""Finish any queued job, then end the thread."""
|
||||
if self.is_alive():
|
||||
self._jobs.put(None)
|
||||
self.join(timeout=timeout)
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
job = self._jobs.get()
|
||||
if job is None:
|
||||
log.info("TIFF worker stopping")
|
||||
return
|
||||
try:
|
||||
self._do(job)
|
||||
except Exception:
|
||||
log.exception("TIFF job failed")
|
||||
if self._on_done:
|
||||
self._on_done(job, None, "TIFF failed (see log)")
|
||||
|
||||
def _do(self, job):
|
||||
import tifffile
|
||||
|
||||
started = time.monotonic()
|
||||
raw = unpack_mipi12(job["packed"], job["width"], job["height"], job["stride"])
|
||||
rgb = render_still(
|
||||
raw,
|
||||
job["pattern"],
|
||||
black=job.get("black_level", 256.0),
|
||||
white=job.get("white_level", 4095.0),
|
||||
gains=job.get("gains") if job.get("apply_white_balance", True) else None,
|
||||
ccm=job.get("ccm") if job.get("apply_ccm", True) else None,
|
||||
gamma=job.get("gamma", True),
|
||||
)
|
||||
del raw
|
||||
rotation = int(job.get("rotation", 0) or 0)
|
||||
if rotation:
|
||||
rgb = _rotate_array(rgb, rotation)
|
||||
|
||||
description = (
|
||||
"16-bit RGB from the 12-bit Bayer frame; "
|
||||
f"pattern={job['pattern']} black={job.get('black_level')} "
|
||||
f"white={job.get('white_level')} "
|
||||
f"white_balance={job.get('gains') if job.get('apply_white_balance', True) else None} "
|
||||
f"colour_matrix={'applied' if job.get('apply_ccm', True) and job.get('ccm') else 'none'} "
|
||||
f"transfer={'sRGB' if job.get('gamma', True) else 'linear'}; "
|
||||
"raw sensor data preserved in the DNG alongside."
|
||||
)
|
||||
tifffile.imwrite(
|
||||
job["path"], rgb, photometric="rgb",
|
||||
description=description, compression=None,
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
log.info("TIFF written in %.1fs: %s", elapsed, job["path"])
|
||||
if self._on_done:
|
||||
self._on_done(job, elapsed, None)
|
||||
|
||||
|
||||
def capture_still(camera, storage, cfg, tiff_worker, on_tiff_done=None, rotation=0):
|
||||
"""Capture one full-resolution still. Must run on the camera thread.
|
||||
|
||||
*rotation* (clockwise degrees) is applied to the saved output: the JPEG is
|
||||
turned on disk, the TIFF is turned before it is written, and the DNG carries
|
||||
it as an Orientation tag (its Bayer data must stay sensor-native).
|
||||
"""
|
||||
photo_cfg = cfg["photo"]
|
||||
rotation = int(rotation or 0) % 360
|
||||
session = storage.new_session("photo")
|
||||
raw_dir = storage.format_dir(session, "raw")
|
||||
serial = storage.next_serial(raw_dir, "still")
|
||||
stem = "still_%04d" % serial
|
||||
|
||||
t_switch = time.monotonic()
|
||||
camera.switch_to_still()
|
||||
settle = max(0, int(photo_cfg.get("settle_frames", 3)))
|
||||
for _ in range(settle):
|
||||
stale = camera.capture_request()
|
||||
stale.release()
|
||||
switch_seconds = time.monotonic() - t_switch
|
||||
|
||||
t_capture = time.monotonic()
|
||||
request = camera.capture_request()
|
||||
metadata = request.get_metadata()
|
||||
written = {}
|
||||
|
||||
try:
|
||||
if photo_cfg.get("write_dng", True):
|
||||
path = raw_dir / (stem + ".dng")
|
||||
request.save_dng(str(path), "raw")
|
||||
if rotation:
|
||||
set_dng_orientation(str(path), rotation)
|
||||
written["dng"] = str(path)
|
||||
|
||||
if photo_cfg.get("write_jpeg", True):
|
||||
jpeg_dir = storage.format_dir(session, "jpeg")
|
||||
path = jpeg_dir / (stem + ".jpg")
|
||||
request.save("main", str(path))
|
||||
if rotation:
|
||||
rotate_jpeg(str(path), rotation)
|
||||
written["jpeg"] = str(path)
|
||||
|
||||
packed = None
|
||||
if photo_cfg.get("write_tiff", True):
|
||||
raw_cfg = camera.stream_config("raw") or {}
|
||||
packed = np.array(request.make_buffer("raw"), dtype=np.uint8)
|
||||
width = int(raw_cfg.get("size", (photo_cfg["width"], photo_cfg["height"]))[0])
|
||||
height = int(raw_cfg.get("size", (photo_cfg["width"], photo_cfg["height"]))[1])
|
||||
stride = int(raw_cfg.get("stride") or (width * 3 // 2))
|
||||
pattern = pattern_from_format(raw_cfg.get("format", "SRGGB12"))
|
||||
ccm = metadata.get("ColourCorrectionMatrix")
|
||||
tiff_dir = storage.format_dir(session, "tiff")
|
||||
job = {
|
||||
"packed": packed,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"stride": stride,
|
||||
"pattern": pattern,
|
||||
"gains": metadata.get("ColourGains"),
|
||||
"apply_white_balance": photo_cfg.get("tiff_apply_white_balance", True),
|
||||
"ccm": list(ccm) if ccm is not None else None,
|
||||
"apply_ccm": photo_cfg.get("tiff_apply_colour_matrix", True),
|
||||
"gamma": photo_cfg.get("tiff_apply_gamma", True),
|
||||
"black_level": photo_cfg.get("black_level", 256),
|
||||
"white_level": photo_cfg.get("white_level", 4095),
|
||||
"rotation": rotation,
|
||||
"path": str(tiff_dir / (stem + ".tiff")),
|
||||
"stem": stem,
|
||||
"session": str(session),
|
||||
}
|
||||
tiff_worker.submit(job)
|
||||
written["tiff"] = job["path"]
|
||||
finally:
|
||||
request.release()
|
||||
camera.switch_to_video()
|
||||
|
||||
summary = {
|
||||
"kind": "photo",
|
||||
"stem": stem,
|
||||
"session": str(session),
|
||||
"taken": _dt.datetime.now().astimezone().isoformat(timespec="milliseconds"),
|
||||
"files": {k: storage.relative(v) for k, v in written.items()},
|
||||
"mode": "4056x3040",
|
||||
"rotation": rotation,
|
||||
"capture_ms": round((time.monotonic() - t_capture) * 1000),
|
||||
"switch_ms": round(switch_seconds * 1000),
|
||||
"exposure_us": metadata.get("ExposureTime"),
|
||||
"gain": metadata.get("AnalogueGain"),
|
||||
"awb": list(metadata.get("ColourGains")) if metadata.get("ColourGains") else None,
|
||||
"colour_correction_matrix": list(metadata["ColourCorrectionMatrix"])
|
||||
if metadata.get("ColourCorrectionMatrix") is not None else None,
|
||||
"black_level": photo_cfg.get("black_level", 256),
|
||||
"white_level": photo_cfg.get("white_level", 4095),
|
||||
"sensor_temperature": metadata.get("SensorTemperature"),
|
||||
"notes": [
|
||||
"DNG holds the untouched 12-bit Bayer frame.",
|
||||
"TIFF is 16-bit sRGB, demosaiced from that Bayer frame with white balance",
|
||||
"and the colour-correction matrix applied.",
|
||||
"JPEG is the ISP-processed 8-bit image.",
|
||||
"rotation is the clockwise display/export rotation in degrees.",
|
||||
],
|
||||
}
|
||||
_write_json(session / "photo.json", summary)
|
||||
return summary
|
||||
|
||||
|
||||
def _write_json(path, payload):
|
||||
tmp = str(path) + ".tmp"
|
||||
try:
|
||||
with open(tmp, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, indent=2, default=str)
|
||||
handle.write("\n")
|
||||
os.replace(tmp, path)
|
||||
except OSError:
|
||||
log.exception("could not write %s", path)
|
||||
Reference in New Issue
Block a user