diff --git a/pyproject.toml b/pyproject.toml index e32c055f..cc819ee2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ tasks = [ "pyvips==3.1.1.8.18.1", "uvdat-flood-sim[large-image-writer]==1.0.4", "xdg-base-dirs==6.0.2", + "huggingface-hub==1.14.0", ] [dependency-groups] diff --git a/terraform/django.tf b/terraform/django.tf index cb41cd9d..a7e5c9c9 100644 --- a/terraform/django.tf +++ b/terraform/django.tf @@ -20,9 +20,14 @@ module "django" { ec2_worker_ssh_public_key = file("${path.module}/ssh-key.pub") additional_django_vars = { - DJANGO_UVDAT_WEB_URL = "https://www.geodatalytics.kitware.com/" - DJANGO_DATABASE_POOL_MAX_SIZE = "12" - DJANGO_SENTRY_DSN = "https://5302701c88f1fa6ec056e0c269071191@o267860.ingest.us.sentry.io/4510620385804288" + DJANGO_UVDAT_WEB_URL = "https://www.geodatalytics.kitware.com/" + DJANGO_DATABASE_POOL_MAX_SIZE = "12" + DJANGO_SENTRY_DSN = "https://5302701c88f1fa6ec056e0c269071191@o267860.ingest.us.sentry.io/4510620385804288" + DJANGO_UVDAT_HF_NAMESPACE = "Kitware" + DJANGO_UVDAT_HF_ENDPOINT_NAMES = "qwen=qwen3-5-9b-gguf-ulh," + } + additional_sensitive_django_vars = { + DJANGO_UVDAT_HF_TOKEN = var.DJANGO_UVDAT_HF_TOKEN } django_cors_allowed_origins = [ # Can't make this use "aws_route53_record.www.fqdn" because of a circular dependency diff --git a/terraform/variables.tf b/terraform/variables.tf index a597a3e3..24172426 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -3,3 +3,9 @@ variable "SENTRY_AUTH_TOKEN" { nullable = false sensitive = true } + +variable "DJANGO_UVDAT_HF_TOKEN" { + type = string + nullable = true + sensitive = true +} diff --git a/uv.lock b/uv.lock index bd21712b..70fd9162 100644 --- a/uv.lock +++ b/uv.lock @@ -1874,6 +1874,7 @@ development = [ ] tasks = [ { name = "geoai-py" }, + { name = "huggingface-hub" }, { name = "osmnx" }, { name = "pyvips" }, { name = "uvdat-flood-sim", extra = ["large-image-writer"] }, @@ -1940,6 +1941,7 @@ requires-dist = [ { name = "gdal", specifier = "==3.12.3.1", index = "https://girder.github.io/large_image_wheels/" }, { name = "geoai-py", marker = "extra == 'tasks'", specifier = "==0.37.2" }, { name = "geopandas", specifier = "==1.1.3" }, + { name = "huggingface-hub", marker = "extra == 'tasks'", specifier = "==1.14.0" }, { name = "ipython", marker = "extra == 'development'", specifier = "==9.13.0" }, { name = "large-image", extras = ["gdal"], specifier = "==1.34.1" }, { name = "large-image-converter", specifier = "==1.34.1" }, diff --git a/uvdat/core/tasks/analytics/__init__.py b/uvdat/core/tasks/analytics/__init__.py index 3bf07b30..4f1ed2b1 100644 --- a/uvdat/core/tasks/analytics/__init__.py +++ b/uvdat/core/tasks/analytics/__init__.py @@ -6,6 +6,7 @@ from .flood_network_failure import FloodNetworkFailure from .flood_simulation import FloodSimulation from .geoai_segmentation import GeoAISegmentation +from .imagery_ask_qwen import ImageryAskQwen from .network_recovery import NetworkRecovery from .uncertainty_quantification import UncertaintyQuantification @@ -15,6 +16,7 @@ analysis_types: list[type[AnalysisType]] = [ FloodSimulation, FloodNetworkFailure, + ImageryAskQwen, NetworkRecovery, UncertaintyQuantification, GeoAISegmentation, diff --git a/uvdat/core/tasks/analytics/imagery_ask_qwen.py b/uvdat/core/tasks/analytics/imagery_ask_qwen.py new file mode 100644 index 00000000..6459eae3 --- /dev/null +++ b/uvdat/core/tasks/analytics/imagery_ask_qwen.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import base64 + +from celery import shared_task +from django.conf import settings +from django_large_image import utilities +import large_image + +from uvdat.core.models import RasterData, TaskResult + +from .analysis_type import AnalysisInputError, AnalysisTask, AnalysisType + +MODEL_CARD_URL = "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF" +SYSTEM_PROMPT = ( + "You are an urban planning and geospatial analysis expert specializing in " + "land use patterns, hydrology, transportation networks, and municipal policy. " + "Analyze the provided imagery to answer the user's question. In your answer, " + "assume that the user is also a geospatial analyst with the same expertise." +) +TOKEN_RANGE = {"min": 1000, "max": 10000, "step": 1000} +MAX_PROMPT_LENGTH = 4000 +THUMBNAIL_SIZE = 4000 +MAX_STARTUP_WAIT = 300 + + +class ImageryAskQwen(AnalysisType): + def __init__(self): + super().__init__() + self.name = "Imagery: Ask Qwen" + self.description = "Select an imagery layer and ask Qwen 3.5 about it." + self.details = ( + "Inferencing with unsloth/Qwen3.5-9B-GGUF provided by a " + "Kitware-hosted Huggingface Inference Endpoint. " + f"See the model card at {MODEL_CARD_URL}. " + "Responses may cut off mid-sentence if max_tokens is reached." + ) + self.db_value = "imagery_ask_qwen" + self.input_types = { + "imagery": "RasterData", + "text_prompt": "string", + "max_tokens": "number", + } + self.output_types = { + "response": "markdown", + } + self.attribution = "Unsloth AI, Kitware Inc." + + @classmethod + def is_enabled(cls) -> bool: + return ( + settings.UVDAT_ENABLE_IMAGERY_ASK_QWEN + and settings.UVDAT_HF_TOKEN is not None + and settings.UVDAT_HF_NAMESPACE is not None + and settings.UVDAT_HF_ENDPOINT_NAMES.get("qwen") is not None + ) + + def get_input_options(self): + return { + "imagery": RasterData.objects.filter(dataset__category="imagery"), + "text_prompt": [], + "max_tokens": [TOKEN_RANGE], + } + + def validate_inputs(self, inputs): + super().validate_inputs(inputs) + try: + imagery = RasterData.objects.get(id=inputs.get("imagery")) + except RasterData.DoesNotExist as e: + err_msg = "Imagery raster does not exist." + raise AnalysisInputError(err_msg) from e + if imagery.dataset.category != "imagery": + err_msg = 'Selected raster is not categorized as "imagery".' + raise AnalysisInputError(err_msg) + text_prompt = str(inputs.get("text_prompt")) + if len(text_prompt) > MAX_PROMPT_LENGTH: + err_msg = f"Prompt too long. Provide a prompt with <{MAX_PROMPT_LENGTH} characters." + raise AnalysisInputError(err_msg) + max_tokens = int(inputs.get("max_tokens")) + if max_tokens < TOKEN_RANGE["min"] or max_tokens > TOKEN_RANGE["max"]: + err_msg = f"max_tokens must be between {TOKEN_RANGE['min']} and {TOKEN_RANGE['max']}." + raise AnalysisInputError(err_msg) + + def run_task(self, *, project, **inputs): + text_prompt = inputs.get("text_prompt") + result = TaskResult.objects.create( + name=text_prompt[:250], + task_type=self.db_value, + inputs=inputs, + project=project, + status="Initializing Task...", + ) + imagery_ask_qwen.delay(result.id) + return result + + def finalize(self, result): + pass + + +@shared_task(base=AnalysisTask) +def imagery_ask_qwen(result_id): + # Only available with [tasks] extra + from huggingface_hub import ( # noqa: PLC0415 + InferenceEndpointTimeoutError, + get_inference_endpoint, + ) + + result = TaskResult.objects.get(id=result_id) + imagery = RasterData.objects.get(id=result.inputs.get("imagery")) + text_prompt = result.inputs.get("text_prompt") + max_tokens = int(result.inputs.get("max_tokens")) + + result.write_status("Encoding imagery...") + imagery_path = utilities.field_file_to_local_path(imagery.cloud_optimized_geotiff) + src = large_image.open(imagery_path) + thumbnail_bytes, _ = src.getThumbnail(THUMBNAIL_SIZE, THUMBNAIL_SIZE, encoding="PNG") + thumbnail_b64 = base64.b64encode(thumbnail_bytes).decode("utf-8") + thumbnail_uri = f"data:image/jpeg;base64,{thumbnail_b64}" + + result.write_status("Starting inference endpoint...") + endpoint = get_inference_endpoint( + name=settings.UVDAT_HF_ENDPOINT_NAMES["qwen"], + namespace=settings.UVDAT_HF_NAMESPACE, + token=settings.UVDAT_HF_TOKEN, + ) + endpoint.resume() + try: + endpoint.wait(timeout=MAX_STARTUP_WAIT) + except InferenceEndpointTimeoutError: + result.write_error("Endpoint failed to start in 5 minutes. Try again later.") + return + + result.write_status("Sending question to Qwen...") + messages = [ + { + "role": "system", + "content": SYSTEM_PROMPT, + }, + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": thumbnail_uri}}, + {"type": "text", "text": text_prompt}, + ], + }, + ] + + result.write_status("Awaiting Qwen's response...") + chat = endpoint.client.chat_completion( + model="unsloth/Qwen3.5-9B-GGUF", + messages=messages, + max_tokens=max_tokens, + ) + response = "" + for choice in chat.choices: + if choice.finish_reason == "length": + # max tokens exceeded, use reasoning content + response += choice.message.reasoning_content + else: + response += choice.message.content + result.write_outputs({"response": response}) diff --git a/uvdat/core/tests/test_analytics.py b/uvdat/core/tests/test_analytics.py index 8ac4fb1a..f8c75757 100644 --- a/uvdat/core/tests/test_analytics.py +++ b/uvdat/core/tests/test_analytics.py @@ -19,9 +19,10 @@ def test_rest_list_analysis_types(user, authenticated_api_client, project): user.is_superuser = True user.save() - analysis_type_instances = [at() for at in analysis_types] + analysis_type_instances = [at() for at in analysis_types if at.is_enabled()] resp = authenticated_api_client.get(f"/api/v1/analytics/project/{project.id}/types/") data = resp.json() + assert len(data) == len(analysis_type_instances) assert {type_info.get("name") for type_info in data} == { i.name for i in analysis_type_instances diff --git a/uvdat/settings/base.py b/uvdat/settings/base.py index 92f5d3bb..97cd4460 100644 --- a/uvdat/settings/base.py +++ b/uvdat/settings/base.py @@ -158,6 +158,10 @@ } UVDAT_WEB_URL: str = env.url("DJANGO_UVDAT_WEB_URL").geturl() +UVDAT_HF_TOKEN: str | None = env.str("DJANGO_UVDAT_HF_TOKEN", default=None) +UVDAT_HF_NAMESPACE: str | None = env.str("DJANGO_UVDAT_HF_NAMESPACE", default=None) +UVDAT_HF_ENDPOINT_NAMES: dict = env.dict("DJANGO_UVDAT_HF_ENDPOINT_NAMES", default={}) + UVDAT_ENABLE_FLOOD_SIMULATION: bool = env.bool("DJANGO_UVDAT_ENABLE_FLOOD_SIMULATION", default=True) UVDAT_ENABLE_FLOOD_NETWORK_FAILURE: bool = env.bool( "DJANGO_UVDAT_ENABLE_FLOOD_NETWORK_FAILURE", default=True @@ -172,6 +176,7 @@ UVDAT_ENABLE_UNCERTAINTY_QUANTIFICATION: bool = env.bool( "DJANGO_UVDAT_ENABLE_UNCERTAINTY_QUANTIFICATION", default=True ) +UVDAT_ENABLE_IMAGERY_ASK_QWEN: bool = env.bool("DJANGO_UVDAT_ENABLE_IMAGERY_ASK_QWEN", default=True) logging.getLogger("pyvips").setLevel(logging.ERROR) logging.getLogger("rasterio").setLevel(logging.ERROR) diff --git a/web/package-lock.json b/web/package-lock.json index 29959e61..2b9217f7 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -27,6 +27,7 @@ "vue": "3.5.34", "vue-chartjs": "5.3.3", "vue-maplibre-compare": "1.0.27", + "vue-markdown-render": "^2.3.1", "vue-router": "5.0.6", "vuedraggable": "4.1.0", "vuetify": "4.0.7" @@ -4381,6 +4382,12 @@ "node": ">=0.4.0" } }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, "node_modules/aria-query": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.1.tgz", @@ -6223,6 +6230,25 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/linkify-it": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.1.tgz", + "integrity": "sha512-wVoTjP4Q6R0NW5hiZkVJaFZPWgtXfoGF+6LucL3/FtiNjmcHhYjEr5f1Kqjirc1nBW07J/ZuRFumqr2oqccEWg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/markdown-it" + } + ], + "license": "MIT", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, "node_modules/local-pkg": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-1.1.2.tgz", @@ -6383,6 +6409,45 @@ "integrity": "sha512-gRa9gwYU3ECmQYv3lslts5hxuIa90veaEcxDYuu3QGOIAEM2mOZkVHp48ANJuu1CURtRdHKUBY5Lm1tHV+sD4g==", "license": "ISC" }, + "node_modules/markdown-it": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.2.0.tgz", + "integrity": "sha512-1TGiQiJVRQ3NPmZH6sx5Cfnmg6GQm9jvC1ch4TK511NjSJvjzKLzn5pPfZRNZkRPZP0HqCioSndqH8v2nRaWVQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/markdown-it" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.1", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, + "node_modules/markdown-it/node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -6392,6 +6457,12 @@ "node": ">= 0.4" } }, + "node_modules/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", + "license": "MIT" + }, "node_modules/memoize-one": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-6.0.0.tgz", @@ -7044,6 +7115,15 @@ "node": ">=6" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/quansync": { "version": "0.2.11", "resolved": "https://registry.npmjs.org/quansync/-/quansync-0.2.11.tgz", @@ -7549,6 +7629,12 @@ "typescript": ">=4.8.4 <6.1.0" } }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "license": "MIT" + }, "node_modules/ufo": { "version": "1.6.4", "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.4.tgz", @@ -7964,6 +8050,18 @@ "vue": "^3.0.0" } }, + "node_modules/vue-markdown-render": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/vue-markdown-render/-/vue-markdown-render-2.3.1.tgz", + "integrity": "sha512-NzQfxEYLlSx+IYlb/p7ATucPrZb8BTfQWG8fEqa2V3DjLdu/0BjYoiEL5ar9nM6FjGv30c3geYoF8pitbENMnw==", + "license": "MIT", + "dependencies": { + "markdown-it": "^14.2.0" + }, + "peerDependencies": { + "vue": "^3.3.4" + } + }, "node_modules/vue-router": { "version": "5.0.6", "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-5.0.6.tgz", diff --git a/web/package.json b/web/package.json index 878e47bc..b6cf6379 100644 --- a/web/package.json +++ b/web/package.json @@ -35,6 +35,7 @@ "vue": "3.5.34", "vue-chartjs": "5.3.3", "vue-maplibre-compare": "1.0.27", + "vue-markdown-render": "^2.3.1", "vue-router": "5.0.6", "vuedraggable": "4.1.0", "vuetify": "4.0.7" diff --git a/web/src/components/sidebars/AnalyticsPanel.vue b/web/src/components/sidebars/AnalyticsPanel.vue index 092840cf..3c08ceac 100644 --- a/web/src/components/sidebars/AnalyticsPanel.vue +++ b/web/src/components/sidebars/AnalyticsPanel.vue @@ -8,6 +8,7 @@ import { getNetwork, subscribeToTaskResult, } from "@/api/rest"; +import VueMarkdown from "vue-markdown-render"; import NodeAnimation from "./NodeAnimation.vue"; import SliderNumericInput from "../SliderNumericInput.vue"; @@ -145,6 +146,7 @@ async function getFullObject(type: string, value: any) { } else { value = { name: value, + type: type, }; } return value; @@ -480,6 +482,11 @@ watch(
Show network to view animation.
+