Skip to content

Commit 8935008

Browse files
Juno13340Juno13340
committed
Add Kimi K2.5 generation vLLM runtime and K2.6/K2.7-Code models
1 parent e91ed23 commit 8935008

5 files changed

Lines changed: 219 additions & 0 deletions

File tree

‎config/models/kustomization.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ resources:
116116

117117
# moonshotai
118118
- moonshotai/Kimi-K2-Instruct.yaml
119+
- moonshotai/Kimi-K2.6.yaml
120+
- moonshotai/Kimi-K2.7-Code.yaml
119121
- moonshotai/Kimi-VL-A3B-Instruct.yaml
120122

121123
# nvidia

‎config/models/moonshotai/Kimi-K2.6.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: ome.io/v1beta1
2+
kind: ClusterBaseModel
3+
metadata:
4+
name: kimi-k2-6
5+
spec:
6+
modelCapabilities:
7+
- IMAGE_TEXT_TO_TEXT
8+
vendor: moonshotai
9+
displayName: moonshotai.kimi-k2-6
10+
disabled: false
11+
version: "1.0.0"
12+
storage:
13+
storageUri: hf://moonshotai/Kimi-K2.6
14+
path: /raid/models/moonshotai/Kimi-K2.6

‎config/models/moonshotai/Kimi-K2.7-Code.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: ome.io/v1beta1
2+
kind: ClusterBaseModel
3+
metadata:
4+
name: kimi-k2-7-code
5+
spec:
6+
modelCapabilities:
7+
- IMAGE_TEXT_TO_TEXT
8+
vendor: moonshotai
9+
displayName: moonshotai.kimi-k2-7-code
10+
disabled: false
11+
version: "1.0.0"
12+
storage:
13+
storageUri: hf://moonshotai/Kimi-K2.7-Code
14+
path: /raid/models/moonshotai/Kimi-K2.7-Code

‎config/runtimes/kustomization.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,4 @@ resources:
5252
- vllm/mixtral-8x7b-instruct-rt.yaml
5353
- vllm/deepseek-ai/deepseek-v4-flash-rt.yaml
5454
- vllm/deepseek-ai/deepseek-v4-pro-rt.yaml
55+
- vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml

‎config/runtimes/vllm/moonshotai/kimi-k25-single-node-8gpu-rt.yaml‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
apiVersion: ome.io/v1beta1
2+
kind: ClusterServingRuntime
3+
metadata:
4+
name: vllm-kimi-k25-single-node-8gpu
5+
annotations:
6+
ome.io/fit-quantization: int4
7+
ome.io/engine-ready-timeout-sec: "3600"
8+
spec:
9+
acceleratorRequirements:
10+
acceleratorClasses:
11+
- nvidia-b200-8
12+
- nvidia-h200-8
13+
# - nvidia-b300-8
14+
# - nvidia-h100-8
15+
# - nvidia-a100-80gb-8
16+
disabled: false
17+
routerConfig:
18+
annotations:
19+
prometheus.io/path: /metrics
20+
prometheus.io/port: '29000'
21+
prometheus.io/scrape: 'true'
22+
labels:
23+
logging-forward: enabled
24+
runner:
25+
name: router
26+
image: docker.io/lightseekorg/smg:1.5.0
27+
ports:
28+
- containerPort: 8080
29+
name: http
30+
resources:
31+
limits:
32+
cpu: "1"
33+
memory: 2Gi
34+
args:
35+
- --host
36+
- 0.0.0.0
37+
- --port
38+
- "8080"
39+
- --service-discovery
40+
- --service-discovery-namespace
41+
- $(NAMESPACE)
42+
- --service-discovery-port
43+
- "8080"
44+
- --selector
45+
- component=engine ome.io/inferenceservice=$(INFERENCESERVICE_NAME)
46+
- --enable-igw
47+
- --request-id-headers
48+
- opc-request-id
49+
- --log-json
50+
- --disable-retries
51+
- --disable-circuit-breaker
52+
- --disable-tokenizer-autoload
53+
env:
54+
- name: NAMESPACE
55+
valueFrom:
56+
fieldRef:
57+
fieldPath: metadata.namespace
58+
- name: INFERENCESERVICE_NAME
59+
valueFrom:
60+
fieldRef:
61+
fieldPath: metadata.labels['ome.io/inferenceservice']
62+
readinessProbe:
63+
httpGet:
64+
path: /readiness
65+
port: 8080
66+
failureThreshold: 5
67+
periodSeconds: 30
68+
timeoutSeconds: 10
69+
livenessProbe:
70+
httpGet:
71+
path: /liveness
72+
port: 8080
73+
failureThreshold: 5
74+
periodSeconds: 30
75+
timeoutSeconds: 10
76+
startupProbe:
77+
httpGet:
78+
path: /readiness
79+
port: 8080
80+
failureThreshold: 177
81+
periodSeconds: 20
82+
timeoutSeconds: 10
83+
initialDelaySeconds: 60
84+
supportedModelFormats:
85+
- modelFramework:
86+
name: transformers
87+
version: "4.57.1"
88+
modelFormat:
89+
name: safetensors
90+
version: "1.0.0"
91+
modelArchitecture: KimiK25ForConditionalGeneration
92+
autoSelect: true
93+
priority: 1
94+
acceleratorConfig:
95+
nvidia-b200-8:
96+
tensorParallelismOverride:
97+
tensorParallelSize: 8
98+
nvidia-h200-8:
99+
tensorParallelismOverride:
100+
tensorParallelSize: 8
101+
# nvidia-b300-8:
102+
# tensorParallelismOverride:
103+
# tensorParallelSize: 8
104+
# nvidia-h100-8:
105+
# tensorParallelismOverride:
106+
# tensorParallelSize: 8
107+
# nvidia-a100-80gb-8:
108+
# tensorParallelismOverride:
109+
# tensorParallelSize: 8
110+
modelSizeRange:
111+
min: 150B
112+
max: 300B
113+
protocolVersions:
114+
- openAI
115+
engineConfig:
116+
annotations:
117+
prometheus.io/scrape: "true"
118+
prometheus.io/port: "8080"
119+
prometheus.io/path: "/metrics"
120+
labels:
121+
logging-forward: enabled
122+
volumes:
123+
- name: dshm
124+
emptyDir:
125+
medium: Memory
126+
runner:
127+
name: ome-container
128+
image: docker.io/vllm/vllm-openai:v0.23.0-cu129
129+
ports:
130+
- containerPort: 8080
131+
name: http1
132+
protocol: TCP
133+
args:
134+
- $(MODEL_PATH)
135+
- --port=8080
136+
- --max-log-len=0
137+
- --served-model-name=vllm-model
138+
- --trust-remote-code
139+
- --enable-expert-parallel
140+
- --tensor-parallel-size=8
141+
- --max-model-len=262144
142+
- '--limit-mm-per-prompt={"image":5,"video":1}'
143+
- --tool-call-parser=kimi_k2
144+
- --enable-auto-tool-choice
145+
- --reasoning-parser=kimi_k2
146+
- --mm-encoder-tp-mode=data
147+
env:
148+
- name: VLLM_ENGINE_READY_TIMEOUT_S
149+
value: '3600'
150+
- name: VLLM_RPC_TIMEOUT
151+
value: '600000'
152+
volumeMounts:
153+
- mountPath: /dev/shm
154+
name: dshm
155+
resources:
156+
requests:
157+
cpu: 64
158+
memory: 512Gi
159+
nvidia.com/gpu: 8
160+
limits:
161+
cpu: 64
162+
memory: 512Gi
163+
nvidia.com/gpu: 8
164+
readinessProbe:
165+
httpGet:
166+
path: /health
167+
port: 8080
168+
failureThreshold: 3
169+
successThreshold: 1
170+
periodSeconds: 90
171+
timeoutSeconds: 60
172+
livenessProbe:
173+
httpGet:
174+
path: /health
175+
port: 8080
176+
failureThreshold: 5
177+
successThreshold: 1
178+
periodSeconds: 60
179+
timeoutSeconds: 60
180+
startupProbe:
181+
httpGet:
182+
path: /health
183+
port: 8080
184+
failureThreshold: 348
185+
successThreshold: 1
186+
periodSeconds: 10
187+
initialDelaySeconds: 120
188+
timeoutSeconds: 30

0 commit comments

Comments
 (0)