Multiframe scrubbing improvements

scripts/sentinelDownload/README.md

@@ -27,13 +27,41 @@ The script accepts command-line options via `click`:
 - `--start-date` _(str, default `2025-01-01`)_ - Start date in `YYYY-MM-DD` format.
 - `--end-date` _(str, default = today)_ - End date in `YYYY-MM-DD` format.
 - `--max-results` _(int, default `5`)_ - Maximum number of images to download.
-- `--output-dir` _(path, default `sequentialTestRasters`)_ - Directory to save clipped files and JSON file.
+- `--output-name` _(str, default `sequentialTestRasters`)_ - Base name for output. Rasters are saved under `downloads/<output-name>/`; ingest JSON is written as `<output-name>.json` next to the script.
 - `--cloud-cover` _(float, default `30.0`)_ - Maximum allowed cloud cover percentage of files found.
 - `--size-km` _(float, default `10.0`)_ - Size of square window (in kilometers) to clip around the point.
+- `--single-file` _(flag, default off)_ - Combine all downloaded frames into one multiframe GeoTIFF instead of writing separate files per date. The generated ingest JSON uses `frame_property: "frame"` for multiframe ingest.
+
+### `--single-file` GDAL requirement
+
+The `--single-file` option shells out to `gdal_translate` to append each clipped frame as a subdataset in one multiframe GeoTIFF. GDAL must be installed and `gdal_translate` must be on your `PATH`.
+
+If GDAL is not available, run without `--single-file` (the default writes one GeoTIFF per frame).
 
 ---
 
 ## Outputs
 
-- **GeoTIFF files** - Clipped Sentinel-2 visual images (RGB).
-- **`sample.json`** - JSON metadata describing datasets, layers, and frames, useful for ingestion into GeoDatalytics.
+Files are written relative to this script directory:
+
+```
+scripts/sentinelDownload/
+  sentinel2Download.py
+  <output-name>.json          # ingest manifest (sibling to the script)
+  downloads/
+    <output-name>/
+      *.tif                   # clipped GeoTIFF(s)
+```
+
+- **GeoTIFF files** - Clipped Sentinel-2 visual images (RGB). With `--single-file`, one multiframe GeoTIFF is written instead of separate per-date files.
+- **`<output-name>.json`** - Ingest manifest describing the project, dataset, layers, and frames.
+
+## Ingesting into GeoDatalytics
+
+Then ingest the sequential data from the project root:
+
+```bash
+./manage.py ingest <output-name>.json --replace
+```
+
+Use `--replace` if you have previously ingested the same project or dataset and need to refresh it.

scripts/sentinelDownload/sentinel2Download.py

@@ -14,6 +14,9 @@
 from datetime import UTC, datetime
 import json
 from pathlib import Path
+import shutil
+import subprocess
+import tempfile
 
 import click
 import numpy as np
@@ -26,6 +29,8 @@
 
 # STAC API from AWS Earth Search
 STAC_API_URL = "https://earth-search.aws.element84.com/v1"
+SCRIPT_DIR = Path(__file__).resolve().parent
+DOWNLOADS_DIR = SCRIPT_DIR / "downloads"
 
 
 def default_end_date():
@@ -101,6 +106,46 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10):
         return data, meta
 
 
+def _run_checked_command(cmd: list[str]) -> None:
+    subprocess.run(cmd, check=True)  # noqa: S603
+
+
+def combine_frames_to_multiframe(frame_paths, output_path):
+    """
+    Combine single-frame GeoTIFFs into one multiframe GeoTIFF.
+
+    Each appended page becomes a scrubbable frame when imported with
+    frame_property: "frame".
+    """
+    if not frame_paths:
+        return
+
+    gdal_translate = shutil.which("gdal_translate")
+    if gdal_translate is None:
+        msg = (
+            "gdal_translate is required for --single-file but was not found on PATH. "
+            "Install GDAL or run without --single-file."
+        )
+        raise RuntimeError(msg)
+
+    output_path = Path(output_path)
+    creation_options = ["-co", "COMPRESS=LZW"]
+    _run_checked_command(
+        [gdal_translate, *creation_options, str(frame_paths[0]), str(output_path)],
+    )
+    for frame_path in frame_paths[1:]:
+        _run_checked_command(
+            [
+                gdal_translate,
+                *creation_options,
+                "-co",
+                "APPEND_SUBDATASET=YES",
+                str(frame_path),
+                str(output_path),
+            ],
+        )
+
+
 @click.command()
 @click.option(
     "--lat", default=43.135763, type=float, required=True, help="Latitude of the location."
@@ -130,11 +175,14 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10):
     help="Maximum number of images to download.",
 )
 @click.option(
-    "--output-dir",
-    type=click.Path(),
+    "--output-name",
+    type=str,
     default="sequentialTestRasters",
     show_default=True,
-    help="Directory to save the downloaded files.",
+    help=(
+        "Base name for output: writes rasters to downloads/<output-name>/ and "
+        "a sibling ingest JSON named <output-name>.json next to this script."
+    ),
 )
 @click.option(
     "--cloud-cover", type=float, default=30.0, show_default=True, help="Max cloud cover percentage."
@@ -146,11 +194,21 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10):
     show_default=True,
     help="Size of square window to clip around the point in kilometers.",
 )
-def download_stac_sentinel(  # noqa: PLR0913, PLR0915
-    lat, lon, start_date, end_date, max_results, output_dir, cloud_cover, size_km
+@click.option(
+    "--single-file",
+    is_flag=True,
+    default=False,
+    help=(
+        "Write all frames into one multiframe GeoTIFF instead of separate files. "
+        'The generated output JSON will use frame_property: "frame".'
+    ),
+)
+def download_stac_sentinel(  # noqa: C901, PLR0912, PLR0913, PLR0915
+    lat, lon, start_date, end_date, max_results, output_name, cloud_cover, size_km, single_file
 ):
     """Download clipped Sentinel-2 L1C visual images from AWS via STAC API."""
-    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    output_dir = DOWNLOADS_DIR / output_name
+    output_dir.mkdir(parents=True, exist_ok=True)
 
     catalog = Client.open(STAC_API_URL)
 
@@ -166,7 +224,7 @@ def download_stac_sentinel(  # noqa: PLR0913, PLR0915
         limit=max_results,
     )
 
-    items = list(search.get_items())
+    items = list(search.items())
 
     if not items:
         click.echo("⚠️  No Sentinel-2 images found.")
@@ -187,40 +245,59 @@ def download_stac_sentinel(  # noqa: PLR0913, PLR0915
     )
 
     downloaded_files = []
-
-    for i, item in enumerate(items):
-        if i >= max_results:
-            break
-        date_str = item.datetime.strftime("%Y-%m-%d")
-        item_id = item.id
-        click.echo(f"[{i + 1}/{len(items)}] {item_id} from {date_str}")
-
-        visual_asset = item.assets.get("visual")
-        if visual_asset:
-            url = visual_asset.href
-            filename = f"{item_id}_visual_clip_{int(size_km)}km.tif"
-            filepath = Path(output_dir) / filename
-
-            click.echo(f"  - Reading {size_km}km x {size_km}km window around point")
-            try:
-                data, meta = read_cog_window_rgb(url, lon, lat, size_km=size_km)
-                with rasterio.open(filepath, "w", **meta) as dst:
-                    dst.write(data)
-            except (RasterioError, RasterioIOError) as e:
-                click.echo(f"  - ⚠️ Failed to read or save clipped image: {e}")
+    downloaded_frame_paths = []
+    multiframe_filename = None
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir)
+
+        for i, item in enumerate(items):
+            if i >= max_results:
+                break
+            date_str = item.datetime.strftime("%Y-%m-%d")
+            item_id = item.id
+            click.echo(f"[{i + 1}/{len(items)}] {item_id} from {date_str}")
+
+            visual_asset = item.assets.get("visual")
+            if visual_asset:
+                url = visual_asset.href
+                filename = f"{item_id}_visual_clip_{int(size_km)}km.tif"
+                filepath = output_dir / filename
+                write_path = temp_path / filename if single_file else filepath
+
+                click.echo(f"  - Reading {size_km}km x {size_km}km window around point")
+                try:
+                    data, meta = read_cog_window_rgb(url, lon, lat, size_km=size_km)
+                    with rasterio.open(write_path, "w", **meta) as dst:
+                        dst.write(data)
+                except (RasterioError, RasterioIOError) as e:
+                    click.echo(f"  - ⚠️ Failed to read or save clipped image: {e}")
+                else:
+                    if single_file:
+                        click.echo(f"  - Buffered frame {len(downloaded_frame_paths)}")
+                        downloaded_frame_paths.append(write_path)
+                    else:
+                        click.echo(f"  - Saved clipped image to {filename}")
+                    downloaded_files.append(filename)
             else:
-                click.echo(f"  - Saved clipped image to {filename}")
-                downloaded_files.append(filename)
-        else:
-            click.echo(f"  - ⚠️ Visual asset not available in item {item_id}")
+                click.echo(f"  - ⚠️ Visual asset not available in item {item_id}")
+
+        if single_file and downloaded_frame_paths:
+            multiframe_filename = f"sentinel_visual_clip_{int(size_km)}km_multiframe.tif"
+            multiframe_path = output_dir / multiframe_filename
+            click.echo(
+                f"Combining {len(downloaded_frame_paths)} frames into {multiframe_filename}..."
+            )
+            combine_frames_to_multiframe(downloaded_frame_paths, multiframe_path)
+            click.echo(f"  - Saved multiframe image to {multiframe_filename}")
 
     click.echo("✅ Download complete.")
-    # Generate dataset.json
     dataset_json = {
         "type": "Dataset",
         "name": "Sequential Test Rasters",
         "description": "Clipped Sentinel-2 images downloaded and clipped around point",
         "category": "imagery",
+        "tags": ["sentinel-2", "imagery", "sequential"],
         "files": [],
         "layers": [],
     }
@@ -229,22 +306,40 @@ def download_stac_sentinel(  # noqa: PLR0913, PLR0915
         "type": "Project",
         "name": "Sentinel-2 Clipped Images",
         "datasets": ["Sequential Test Rasters"],
-        "default_map_center": [lat, lon],
+        "default_map_center": [lon, lat],
         "default_map_zoom": 11,
     }
 
-    # Add each file as its own layer
-    layer_frames = []
-    for idx, f in enumerate(downloaded_files):
-        dataset_json["files"].append({"path": f"{output_dir}/{f}", "name": f"Frame {idx}"})
-        layer_frames.append({"name": f"Sequential Layer {idx}", "index": idx, "data": f})
-
-    layer = {"name": "Sequential Test Layers", "frames": layer_frames}
-    dataset_json["layers"].append(layer)
-
-    json_path = Path(output_dir, "sample.json")
+    if single_file and multiframe_filename:
+        dataset_json["files"].append(
+            {"path": f"{output_name}/{multiframe_filename}", "name": multiframe_filename}
+        )
+        dataset_json["layers"].append(
+            {
+                "name": "Sequential Test Layers",
+                "frame_property": "frame",
+                "data": multiframe_filename,
+            }
+        )
+    else:
+        layer_frames = []
+        for idx, f in enumerate(downloaded_files):
+            dataset_json["files"].append({"path": f"{output_name}/{f}", "name": f"Frame {idx}"})
+            layer_frames.append(
+                {
+                    "name": f"Sequential Layer {idx}",
+                    "index": idx,
+                    "data": f,
+                }
+            )
+
+        dataset_json["layers"].append({"name": "Sequential Test Layers", "frames": layer_frames})
+
+    json_path = SCRIPT_DIR / f"{output_name}.json"
     with json_path.open("w") as jf:
         json.dump([project_json, dataset_json], jf, indent=4)
+    click.echo(f"  - Wrote ingest JSON to {json_path}")
+    click.echo(f"  - Rasters saved under {output_dir}")
 
 
 if __name__ == "__main__":

uvdat/core/admin.py

@@ -19,6 +19,7 @@
     NetworkNode,
     Project,
     RasterData,
+    RasterFramePreview,
     Region,
     SizeConfig,
     SizeRangeConfig,
@@ -69,6 +70,12 @@ class LayerStyleAdmin(admin.ModelAdmin):
     list_display = ["id", "name", "layer"]
 
 
+@admin.register(RasterFramePreview)
+class RasterFramePreviewAdmin(admin.ModelAdmin):
+    list_display = ["id", "layer_style", "layer_frame", "width", "height"]
+    list_select_related = ["layer_style", "layer_frame"]
+
+
 @admin.register(Colormap)
 class ColormapAdmin(admin.ModelAdmin):
     list_display = ["id", "name"]

uvdat/core/frame_previews/fingerprint.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from uvdat.core.models import LayerStyle
+
+
+def style_fingerprint(layer_style: LayerStyle) -> str:
+    payload = json.dumps(layer_style.repr_style_configs(), sort_keys=True, default=str)
+    return hashlib.sha256(payload.encode()).hexdigest()