Makefile

BASE:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
SHELL=/bin/sh
NAMESPACE=demo

.PHONY: setup-rhoai
setup-rhoai: add-gpu-operator add-nfs-provisioner
	oc apply -f $(BASE)/yaml/rhoai/kueue.yaml
	
	@$(BASE)/scripts/check-operator-install-status.sh kueue-operator openshift-kueue-operator
	
	oc apply -f $(BASE)/yaml/rhoai/kueue-cr.yaml
	
	@echo "Set Red Hat build of Kueue operator to be upgraded manually instead of automatic"
	@oc patch subscription kueue-operator \
	-n openshift-kueue-operator \
	--type=merge \
	-p '{"spec": {"installPlanApproval": "Manual"}}'

	oc apply -f ${BASE}/yaml/rhoai/rhoai.yaml
	@until oc get DSCInitialization/default-dsci -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' | grep -q "True"; do \
		echo "Waiting for DSCInitialization to be ready..."; \
		sleep 10; \
	done
	
	oc apply -f ${BASE}/yaml/rhoai/rhoai-cr.yaml	
	@until oc get DataScienceCluster/default-dsc -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' | grep -q "True"; do \
		echo "Waiting for DataScienceCluster to be ready..."; \
		sleep 10; \
	done

	@echo "Set RHOAI operator to be upgraded manually instead of automatic"
	@oc patch subscription rhods-operator \
	-n redhat-ods-operator \
	--type=merge \
	-p '{"spec": {"installPlanApproval": "Manual"}}'

	oc apply -f ${BASE}/yaml/rhoai/odhdashboardconfig.yaml

	oc delete pods -l app=rhods-dashboard -n redhat-ods-applications
	oc rollout status deployment/rhods-dashboard -n redhat-ods-applications

	#oc apply -f ${BASE}/yaml/rhoai/group.yaml
	oc apply -f ${BASE}/yaml/rhoai/template-rhaiis.yaml	
	oc apply -f ${BASE}/yaml/rhoai/hardwareprofile.yaml
	oc apply -f ${BASE}/yaml/rhoai/uwm.yaml

	@echo "Installing grafana operator"
	@oc apply -f ${BASE}/yaml/rhoai/grafana.yaml
	@$(BASE)/scripts/check-operator-install-status.sh grafana user-grafana

	@echo "Configuring the NVIDIA DCGM Exporter Dashboard"
	@curl -L https://raw.githubusercontent.com/NVIDIA/dcgm-exporter/main/grafana/dcgm-exporter-dashboard.json \
	| oc create configmap nvidia-dcgm-exporter-dashboard \
		-n openshift-config-managed \
		--from-file=dcgm-exporter-dashboard.json=/dev/fd/0 \
		--dry-run=client -o yaml \
	| oc apply -f -

	@oc label configmap nvidia-dcgm-exporter-dashboard -n openshift-config-managed \
	  console.openshift.io/dashboard=true --overwrite	

.PHONY: setup-llmd
setup-llmd:
	# https://docs.redhat.com/en/documentation/red_hat_openshift_ai_self-managed/3.0/html/deploying_models/deploying_models#deploying-models-using-distributed-inference_rhoai-user
	@CLUSTER_DOMAIN=$$(oc get ingresscontroller default -n openshift-ingress-operator -o jsonpath='{.status.domain}')  2>/dev/null \
		envsubst < $(BASE)/yaml/rhoai/gateway.yaml.tmpl | oc apply -f -
	oc apply -f $(BASE)/yaml/rhoai/lws.yaml

	@$(BASE)/scripts/check-operator-install-status.sh leader-worker-set openshift-lws-operator 
	@echo "Set LeaderWorkerSet operator to be upgraded manually instead of automatic"
	@oc patch subscription leader-worker-set \
	-n openshift-lws-operator   \
	--type=merge \
	-p '{"spec": {"installPlanApproval": "Manual"}}'

	oc apply -f $(BASE)/yaml/rhoai/lws-cr.yaml

	oc apply -f $(BASE)/yaml/rhoai/kuadrant.yaml

	@$(BASE)/scripts/check-operator-install-status.sh rhcl-operator openshift-operators	
	@$(BASE)/scripts/check-operator-install-status.sh authorino-operator-stable-redhat-operators-openshift-marketplace openshift-operators
	@$(BASE)/scripts/check-operator-install-status.sh dns-operator-stable-redhat-operators-openshift-marketplace openshift-operators
	@$(BASE)/scripts/check-operator-install-status.sh limitador-operator-stable-redhat-operators-openshift-marketplace openshift-operators

	@echo "Set Red Hat Connectivity Link operator to be upgraded manually instead of automatic"
	@oc patch subscription rhcl-operator \
	-n openshift-operators   \
	--type=merge \
	-p '{"spec": {"installPlanApproval": "Manual"}}'

	@echo "Set Authorino operator to be upgraded manually instead of automatic"
	@oc patch subscription authorino-operator-stable-redhat-operators-openshift-marketplace \
	-n openshift-operators   \
	--type=merge \
	-p '{"spec": {"installPlanApproval": "Manual"}}'

	@echo "Set DNS operator to be upgraded manually instead of automatic"
	@oc patch subscription dns-operator-stable-redhat-operators-openshift-marketplace \
		-n openshift-operators \
		--type=merge \
		-p '{"spec": {"installPlanApproval": "Manual"}}'

	@echo "Set Limitador operator to be upgraded manually instead of automatic"
	@oc patch subscription limitador-operator-stable-redhat-operators-openshift-marketplace \
		-n openshift-operators \
		--type=merge \
		-p '{"spec": {"installPlanApproval": "Manual"}}'
		
	oc apply -f $(BASE)/yaml/rhoai/kuadrant-cr.yaml
	oc wait Kuadrant -n kuadrant-system kuadrant --for=condition=Ready --timeout=10m

	oc annotate svc/authorino-authorino-authorization service.beta.openshift.io/serving-cert-secret-name=authorino-server-cert -n kuadrant-system

	@until oc get secret/authorino-server-cert -n kuadrant-system >/dev/null 2>&1; do \
		echo "Wait until secret/authorino-server-cert is ready..."; \
		sleep 10; \
	done	
	
	oc apply -f $(BASE)/yaml/rhoai/authorino.yaml
	oc wait --for=condition=ready pod -l authorino-resource=authorino -n kuadrant-system --timeout 150s

	oc delete pod -n redhat-ods-applications -l app=odh-model-controller
	oc delete pod -n redhat-ods-applications -l control-plane=kserve-controller-manager
	oc wait --for=condition=ready pod -n redhat-ods-applications -l app=odh-model-controller --timeout 150s
	oc wait --for=condition=ready pod -n redhat-ods-applications -l control-plane=kserve-controller-manager --timeout 150s

	oc apply -k "github.com/pierdipi/kserve//config/dashboards-odc?ref=example-dashboards"
	oc apply -f $(BASE)/yaml/rhoai/llmd-grafana-dashboard.yaml

.PHONY: teardown-llmd
teardown-llmd:

	-oc delete -f $(BASE)/yaml/rhoai/authorino.yaml
	-oc delete -f $(BASE)/yaml/rhoai/kuadrant-cr.yaml
	-oc delete -f $(BASE)/yaml/rhoai/gateway.yaml.tmpl

	-oc delete subscription authorino-operator-stable-redhat-operators-openshift-marketplace -n openshift-operators
	-oc delete subscription dns-operator-stable-redhat-operators-openshift-marketplace -n  openshift-operators
	-oc delete subscription limitador-operator-stable-redhat-operators-openshift-marketplace -n  openshift-operators
	-oc delete -f $(BASE)/yaml/rhoai/kuadrant.yaml

	-oc delete secret/authorino-server-cert -n kuadrant-system 
	-oc delete csv rhcl-operator.v1.2.0 -n openshift-operators
	-oc delete csv dns-operator.v1.2.0 -n openshift-operators
	-oc delete csv limitador-operator.v1.2.0 -n openshift-operators
	-oc delete csv authorino-operator.v1.2.4 -n openshift-operators

	-oc delete -f $(BASE)/yaml/rhoai/lws-cr.yaml
	-oc delete -f $(BASE)/yaml/rhoai/lws.yaml

	oc delete pod -n redhat-ods-applications -l app=odh-model-controller
	oc delete pod -n redhat-ods-applications -l control-plane=kserve-controller-manager

	-oc delete -k "github.com/pierdipi/kserve//config/dashboards-odc?ref=example-dashboards"
	-oc delete -f $(BASE)/yaml/rhoai/llmd-grafana-dashboard.yaml

	oc wait --for=condition=ready pod -n redhat-ods-applications -l app=odh-model-controller --timeout 150s
	oc wait --for=condition=ready pod -n redhat-ods-applications -l control-plane=kserve-controller-manager --timeout 150s

.PHONY: add-nfs-provisioner
add-nfs-provisioner:
	@$(BASE)/scripts/install-nfs-provisioner.sh

.PHONY: add-gpu-operator
add-gpu-operator:
	oc apply -f $(BASE)/yaml/rhoai/nfd.yaml

	@$(BASE)/scripts/check-operator-install-status.sh nfd openshift-nfd
	
	oc apply -f $(BASE)/yaml/rhoai/nfd-cr.yaml
	oc apply -f $(BASE)/yaml/rhoai/nvidia.yaml

	@$(BASE)/scripts/check-operator-install-status.sh gpu-operator-certified nvidia-gpu-operator

	oc apply -f $(BASE)/yaml/rhoai/nvidia-cr.yaml

.PHONY: setup-demo
setup-demo: setup-namespace deploy-minio setup-odh-tec deploy-pipline

	@oc apply -f $(BASE)/yaml/infra/model-pvc.yaml
	@oc apply -f $(BASE)/yaml/infra/llmcompressor-is.yaml
	#@oc apply -f $(BASE)/yaml/demo/anythingllm-wb.yaml 
	#@oc apply -f $(BASE)/yaml/demo/llama-cpp-wb.yaml
	@oc apply -f $(BASE)/yaml/demo/guidellm.yaml  -n ${NAMESPACE}
	@oc apply -f $(BASE)/yaml/demo/benchmark-arena.yaml -n ${NAMESPACE}
	@oc apply -f $(BASE)/yaml/demo/ai-toolkit.yaml -n ${NAMESPACE}
	@oc apply -f https://raw.githubusercontent.com/tsailiming/openshift-open-webui/refs/heads/main/open-webui.yaml -n ${NAMESPACE}
	@oc set env deploy/open-webui ENABLE_PERSISTENT_CONFIG=False -n ${NAMESPACE}

	@oc apply -f $(BASE)/yaml/demo/custom-model-catalog.yaml

	@oc delete pods -l app.kubernetes.io/name=model-catalog -n rhoai-model-registries

.PHONY: setup-ai-playground
setup-ai-playground:
	@echo "Serving llama-32-3b-instruct"
	@$(BASE)/scripts/serve-model.sh oci llama-32-3b-instruct oci://quay.io/redhat-ai-services/modelcar-catalog:llama-3.2-3b-instruct "--max-model-len 32768 --enable-auto-tool-choice --tool-call-parser=llama3_json --chat-template=/opt/app-root/template/tool_chat_template_llama3.2_json.jinja"
	
	@echo "Downloading and deploying Qwen/Qwen3-30B-A3B-Thinking-2507-FP8"
	@$(BASE)/scripts/download-model.sh pvc Qwen/Qwen3-30B-A3B-Thinking-2507-FP8
	@$(BASE)/scripts/serve-model.sh pvc qwen3-30b-a3b-thinking-2507-fp8 Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 "--max-model-len 32768 --enable-auto-tool-choice --reasoning-parser=deepseek_r1 --tool-call-parser=hermes"

	@oc apply -f $(BASE)/yaml/demo/mcp-kubernetes.yaml -n ${NAMESPACE}
	@oc apply -f $(BASE)/yaml/demo/mcp-weather.yaml -n ${NAMESPACE}
	@oc apply -f $(BASE)/yaml/demo/lsd-mcp-cm.yaml
	@oc apply -f $(BASE)/yaml/demo/llama-stack-cm.yaml -n ${NAMESPACE}
	@oc apply -f $(BASE)/yaml/demo/lsd.yaml -n ${NAMESPACE}

	@oc delete pod -l app=llama-stack -n ${NAMESPACE} --ignore-not-found  
	@oc rollout status deployment/lsd-genai-playground -n ${NAMESPACE}

.PHONY: download-models
download-models:
	@echo "Downloading Qwen/Qwen3-VL-8B-Instruct"
	@$(BASE)/scripts/download-model.sh s3 Qwen/Qwen3-VL-8B-Instruct

	@echo "Downloading openai/gpt-oss-20b"
	@$(BASE)/scripts/download-model.sh s3 openai/gpt-oss-20b

	@echo "Downloading Qwen/Qwen3-30B-A3B-Thinking-2507-FP8"
	@$(BASE)/scripts/download-model.sh pvc Qwen/Qwen3-30B-A3B-Thinking-2507-FP8
	
.PHONY: teardown-namespace
teardown-namespace:
	-oc delete project $(NAMESPACE)

.PHONY: setup-namespace
setup-namespace:
	-oc new-project $(NAMESPACE)
	@oc label namespace $(NAMESPACE) \
		maistra.io/member-of=istio-system \
		modelmesh-enabled=false \
		opendatahub.io/dashboard=true

.PHONY: setup-odh-tec
setup-odh-tec:
	@oc apply -f $(BASE)/yaml/infra/odh-tec.yaml -n $(NAMESPACE)
	
	@ODH_ROUTE=$$(oc get route odh-tec -n $(NAMESPACE) -o jsonpath='{.spec.host}') && \
	echo "S3 Browser: $${ODH_ROUTE}"

.PHONY: show-odh-tec
show-odh-tec:
	@ODH_ROUTE=$$(oc get route odh-tec -n $(NAMESPACE) -o jsonpath='{.spec.host}') && open https://$${ODH_ROUTE}

.PHONY: teardown-odh-tec
teardown-odh-tec:
	@oc delete -f $(BASE)/yaml/infra/odh-tec.yaml -n $(NAMESPACE)
	
.PHONY: teardown-all
teardown-all: teardown-minio teardown-odh-tec teardown-namespace
		
.PHONY: teardown-minio
teardown-minio:
	-oc delete -f $(BASE)/yaml/infra/minio.yaml -n $(NAMESPACE)
	
	@PV_NAME=$$(oc get pvc data-minio-0 -n $(NAMESPACE) -o jsonpath='{.spec.volumeName}' 2>/dev/null); \
	oc delete pvc data-minio-0 -n $(NAMESPACE); \
	if [ -z "$$PV_NAME" ]; then \
		echo "PVC data-minio-0 already deleted or has no PV bound."; \
	else \
		echo "Waiting for PV $$PV_NAME to be deleted..."; \
		until ! oc get pv $$PV_NAME >/dev/null 2>&1; do \
			echo "PV $$PV_NAME still exists..."; \
			sleep 2; \
		done; \
		echo "PV $$PV_NAME deleted."; \
	fi

.PHONY: deploy-minio
deploy-minio: teardown-minio
	@oc apply -f $(BASE)/yaml/infra/minio.yaml -n $(NAMESPACE)

	@until oc get statefulset minio -n $(NAMESPACE) -o jsonpath='{.status.readyReplicas}' | grep -q '1'; do \
		echo "Waiting for StatefulSet minio to have 1 ready replica..."; \
		sleep 10; \
	done
	@echo "StatefulSet minio has 1 ready replica."

	-oc delete secret aws-connection-my-storage -n $(NAMESPACE)

	@AWS_ACCESS_KEY_ID=$$(oc extract secret/minio  --to=- --keys=MINIO_ROOT_USER -n $(NAMESPACE) 2>/dev/null | tr -d '\n' | base64 ) \
	AWS_SECRET_ACCESS_KEY=$$(oc extract secret/minio  --to=- --keys=MINIO_ROOT_PASSWORD -n $(NAMESPACE) 2>/dev/null | tr -d '\n' | base64) \
	AWS_S3_ENDPOINT=minio.$(NAMESPACE).svc.cluster.local \
	AWS_ENDPOINT_URL=minio.$(NAMESPACE).svc.cluster.local \
		envsubst < $(BASE)/yaml/infra/data-connection-s3.yaml.tmpl | oc apply -n $(NAMESPACE) -f -	
		
	@$(BASE)/scripts/run-job.sh $(BASE)/yaml/infra/setup-s3.yaml.tmpl $(NAMESPACE) setup-s3-job aws-connection-my-storage

.PHONY: teardown-pipeline
teardown-pipeline: 
	-oc delete -f $(BASE)/yaml/infra/dashboard-dspa-secret.yaml -n $(NAMESPACE)
	-oc delete -f $(BASE)/yaml/infra/dspa.yaml -n $(NAMESPACE)

.PHONY: deploy-pipeline
deploy-pipline: teardown-pipeline	
	@AWS_ACCESS_KEY_ID=$$(oc extract secret/minio  --to=- --keys=MINIO_ROOT_USER -n $(NAMESPACE) 2>/dev/null | tr -d '\n' | base64 ) \
	AWS_SECRET_ACCESS_KEY=$$(oc extract secret/minio  --to=- --keys=MINIO_ROOT_PASSWORD -n $(NAMESPACE) 2>/dev/null | tr -d '\n' | base64) \
	AWS_S3_ENDPOINT=minio.$(NAMESPACE).svc.cluster.local \
	AWS_ENDPOINT_URL=minio.$(NAMESPACE).svc.cluster.local \
	  envsubst < $(BASE)/yaml/infra/pipeline-connection-s3.yaml.tmpl | oc apply -n $(NAMESPACE) -f -	

	@oc apply -f $(BASE)/yaml/infra/dashboard-dspa-secret.yaml -n $(NAMESPACE)
	@oc apply -f $(BASE)/yaml/infra/dspa.yaml -n $(NAMESPACE)

.PHONY: setup-kueue-demo
setup-kueue-demo:
	@oc apply -k "github.com/tsailiming/openshift-ai-kueue//yaml?ref=main"

.PHONY: setup-multi-user
setup-multi-user:
	@HASH=$$(oc get secret kubeadmin -n kube-system \
		-o jsonpath='{.data.kubeadmin}' | base64 --decode); \
	\
	echo "Using same kubeadmin bcrypt hash $$HASH"; \
	rm -f /tmp/users.htpasswd; \
	echo "user1:$$HASH" >> /tmp/users.htpasswd; \
	echo "user2:$$HASH" >> /tmp/users.htpasswd; \
	\
	oc create secret generic htpasswd-secret \
		--from-file=htpasswd=/tmp/users.htpasswd \
		-n openshift-config --dry-run=client -o yaml | oc apply -f -

	@oc apply -f $(BASE)/yaml/infra/oauth.yaml

	-oc new-project user1
	@oc label namespace user1 \
		maistra.io/member-of=istio-system \
		modelmesh-enabled=false \
		opendatahub.io/dashboard=true

	-oc new-project user2
	@oc label namespace user2 \
		maistra.io/member-of=istio-system \
		modelmesh-enabled=false \
		opendatahub.io/dashboard=true

	@oc adm policy add-role-to-user edit user1 -n user1
	@oc adm policy add-role-to-user edit user2 -n user2