feat(BA-6883): expose kernel scheduling history over GraphQL v2

changes/12989.feature.md

@@ -0,0 +1 @@
+Add `adminKernelSchedulingHistories` and `scopedKernelSchedulingHistories` GraphQL queries, and a `KernelV2.schedulingHistories` field.

docs/manager/graphql-reference/supergraph.graphql

@@ -8508,6 +8508,100 @@ enum KernelResourceAllocationOrderField
   USED @join__enumValue(graph: STRAWBERRY)
 }
 
+"""Added in 26.8.0. Kernel scheduling history record."""
+type KernelSchedulingHistory implements Node
+  @join__implements(graph: STRAWBERRY, interface: "Node")
+  @join__type(graph: STRAWBERRY)
+{
+  """The Globally Unique ID of this object"""
+  id: ID!
+  kernelId: ID!
+  sessionId: ID!
+  phase: String!
+  fromStatus: String
+  toStatus: String
+  result: SchedulingResult!
+  errorCode: String
+  message: String
+  attempts: Int!
+  createdAt: DateTime!
+  updatedAt: DateTime!
+}
+
+"""Added in 26.8.0. Kernel scheduling history connection."""
+type KernelSchedulingHistoryConnection
+  @join__type(graph: STRAWBERRY)
+{
+  """Pagination data for this connection"""
+  pageInfo: PageInfo!
+
+  """Contains the nodes in this connection"""
+  edges: [KernelSchedulingHistoryEdge!]!
+  count: Int!
+}
+
+"""An edge in a connection."""
+type KernelSchedulingHistoryEdge
+  @join__type(graph: STRAWBERRY)
+{
+  """A cursor for use in pagination"""
+  cursor: String!
+
+  """The item at the end of the edge"""
+  node: KernelSchedulingHistory!
+}
+
+"""Added in 26.8.0. Filter for kernel scheduling history"""
+input KernelSchedulingHistoryFilter
+  @join__type(graph: STRAWBERRY)
+{
+  id: UUIDFilter = null
+  kernelId: UUIDFilter = null
+  sessionId: UUIDFilter = null
+  phase: StringFilter = null
+  fromStatus: [String!] = null
+  toStatus: [String!] = null
+  result: SchedulingResultFilter = null
+  errorCode: StringFilter = null
+  message: StringFilter = null
+  createdAt: DateTimeFilter = null
+  updatedAt: DateTimeFilter = null
+  AND: [KernelSchedulingHistoryFilter!] = null
+  OR: [KernelSchedulingHistoryFilter!] = null
+  NOT: [KernelSchedulingHistoryFilter!] = null
+}
+
+"""Added in 26.8.0. Order by specification for kernel scheduling history"""
+input KernelSchedulingHistoryOrderBy
+  @join__type(graph: STRAWBERRY)
+{
+  field: KernelSchedulingHistoryOrderField!
+  direction: OrderDirection! = DESC
+}
+
+"""
+Added in 26.8.0. Fields available for ordering kernel scheduling history
+"""
+enum KernelSchedulingHistoryOrderField
+  @join__type(graph: STRAWBERRY)
+{
+  CREATED_AT @join__enumValue(graph: STRAWBERRY)
+  UPDATED_AT @join__enumValue(graph: STRAWBERRY)
+  PHASE @join__enumValue(graph: STRAWBERRY)
+  FROM_STATUS @join__enumValue(graph: STRAWBERRY)
+  TO_STATUS @join__enumValue(graph: STRAWBERRY)
+  RESULT @join__enumValue(graph: STRAWBERRY)
+  ATTEMPTS @join__enumValue(graph: STRAWBERRY)
+}
+
+"""Added in 26.8.0. Scope for kernel scheduling history query"""
+input KernelScope
+  @join__type(graph: STRAWBERRY)
+{
+  """Kernel ID to get history for"""
+  kernelId: UUID!
+}
+
 """
 Added in 26.2.0. Represents a kernel (compute container) in Backend.AI.
 """
@@ -8572,6 +8666,11 @@ type KernelV2 implements Node
 
   """Added in 26.3.0. Per-slot resource allocation for this kernel."""
   resourceAllocations(filter: KernelResourceAllocationFilter = null, orderBy: [KernelResourceAllocationOrderBy!] = null, first: Int = null, after: String = null, last: Int = null, before: String = null, limit: Int = null, offset: Int = null): ResourceAllocationConnection
+
+  """
+  Added in 26.8.0. Scheduling history of this kernel with pagination support.
+  """
+  schedulingHistories(filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection
 }
 
 """
@@ -14709,6 +14808,9 @@ type Query
   """Added in 26.4.2. List session scheduling history (admin only)"""
   adminSessionSchedulingHistories(filter: SessionSchedulingHistoryFilter = null, orderBy: [SessionSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): SessionSchedulingHistoryConnection @join__field(graph: STRAWBERRY)
 
+  """Added in 26.8.0. List kernel scheduling history (superadmin only)"""
+  adminKernelSchedulingHistories(filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection @join__field(graph: STRAWBERRY)
+
   """Added in 25.16.0. List all deployments (superadmin only)."""
   adminDeployments(filter: DeploymentFilter = null, orderBy: [DeploymentOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): ModelDeploymentConnection @join__field(graph: STRAWBERRY)
 
@@ -14990,6 +15092,9 @@ type Query
   """Added in 26.2.0. Get scheduling history for a specific session."""
   sessionScopedSchedulingHistories(scope: SessionScope!, filter: SessionSchedulingHistoryFilter = null, orderBy: [SessionSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): SessionSchedulingHistoryConnection @join__field(graph: STRAWBERRY)
 
+  """Added in 26.8.0. Get scheduling history for a specific kernel."""
+  scopedKernelSchedulingHistories(scope: KernelScope!, filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection @join__field(graph: STRAWBERRY)
+
   """Added in 26.2.0. Get scheduling history for a specific deployment."""
   deploymentScopedSchedulingHistories(scope: DeploymentScope!, filter: DeploymentHistoryFilter = null, orderBy: [DeploymentHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): DeploymentHistoryConnection @join__field(graph: STRAWBERRY)
 

docs/manager/graphql-reference/v2-schema.graphql

@@ -5632,6 +5632,85 @@ enum KernelResourceAllocationOrderField {
   USED
 }
 
+"""Added in 26.8.0. Kernel scheduling history record."""
+type KernelSchedulingHistory implements Node {
+  """The Globally Unique ID of this object"""
+  id: ID!
+  kernelId: ID!
+  sessionId: ID!
+  phase: String!
+  fromStatus: String
+  toStatus: String
+  result: SchedulingResult!
+  errorCode: String
+  message: String
+  attempts: Int!
+  createdAt: DateTime!
+  updatedAt: DateTime!
+}
+
+"""Added in 26.8.0. Kernel scheduling history connection."""
+type KernelSchedulingHistoryConnection {
+  """Pagination data for this connection"""
+  pageInfo: PageInfo!
+
+  """Contains the nodes in this connection"""
+  edges: [KernelSchedulingHistoryEdge!]!
+  count: Int!
+}
+
+"""An edge in a connection."""
+type KernelSchedulingHistoryEdge {
+  """A cursor for use in pagination"""
+  cursor: String!
+
+  """The item at the end of the edge"""
+  node: KernelSchedulingHistory!
+}
+
+"""Added in 26.8.0. Filter for kernel scheduling history"""
+input KernelSchedulingHistoryFilter {
+  id: UUIDFilter = null
+  kernelId: UUIDFilter = null
+  sessionId: UUIDFilter = null
+  phase: StringFilter = null
+  fromStatus: [String!] = null
+  toStatus: [String!] = null
+  result: SchedulingResultFilter = null
+  errorCode: StringFilter = null
+  message: StringFilter = null
+  createdAt: DateTimeFilter = null
+  updatedAt: DateTimeFilter = null
+  AND: [KernelSchedulingHistoryFilter!] = null
+  OR: [KernelSchedulingHistoryFilter!] = null
+  NOT: [KernelSchedulingHistoryFilter!] = null
+}
+
+"""Added in 26.8.0. Order by specification for kernel scheduling history"""
+input KernelSchedulingHistoryOrderBy {
+  field: KernelSchedulingHistoryOrderField!
+  direction: OrderDirection! = DESC
+}
+
+"""
+Added in 26.8.0. Fields available for ordering kernel scheduling history
+"""
+enum KernelSchedulingHistoryOrderField {
+  CREATED_AT
+  UPDATED_AT
+  PHASE
+  FROM_STATUS
+  TO_STATUS
+  RESULT
+  ATTEMPTS
+}
+
+"""Added in 26.8.0. Scope for kernel scheduling history query"""
+input KernelScope {
+  """Kernel ID to get history for"""
+  kernelId: UUID!
+}
+
 """
 Added in 26.2.0. Represents a kernel (compute container) in Backend.AI.
 """
@@ -5693,6 +5772,11 @@ type KernelV2 implements Node {
 
   """Added in 26.3.0. Per-slot resource allocation for this kernel."""
   resourceAllocations(filter: KernelResourceAllocationFilter = null, orderBy: [KernelResourceAllocationOrderBy!] = null, first: Int = null, after: String = null, last: Int = null, before: String = null, limit: Int = null, offset: Int = null): ResourceAllocationConnection
+
+  """
+  Added in 26.8.0. Scheduling history of this kernel with pagination support.
+  """
+  schedulingHistories(filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection
 }
 
 """
@@ -9748,6 +9832,9 @@ type Query {
   """Added in 26.4.2. List session scheduling history (admin only)"""
   adminSessionSchedulingHistories(filter: SessionSchedulingHistoryFilter = null, orderBy: [SessionSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): SessionSchedulingHistoryConnection
 
+  """Added in 26.8.0. List kernel scheduling history (superadmin only)"""
+  adminKernelSchedulingHistories(filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection
+
   """Added in 25.16.0. List all deployments (superadmin only)."""
   adminDeployments(filter: DeploymentFilter = null, orderBy: [DeploymentOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): ModelDeploymentConnection
 
@@ -10029,6 +10116,9 @@ type Query {
   """Added in 26.2.0. Get scheduling history for a specific session."""
   sessionScopedSchedulingHistories(scope: SessionScope!, filter: SessionSchedulingHistoryFilter = null, orderBy: [SessionSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): SessionSchedulingHistoryConnection
 
+  """Added in 26.8.0. Get scheduling history for a specific kernel."""
+  scopedKernelSchedulingHistories(scope: KernelScope!, filter: KernelSchedulingHistoryFilter = null, orderBy: [KernelSchedulingHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): KernelSchedulingHistoryConnection
+
   """Added in 26.2.0. Get scheduling history for a specific deployment."""
   deploymentScopedSchedulingHistories(scope: DeploymentScope!, filter: DeploymentHistoryFilter = null, orderBy: [DeploymentHistoryOrderBy!] = null, before: String = null, after: String = null, first: Int = null, last: Int = null, limit: Int = null, offset: Int = null): DeploymentHistoryConnection
 

docs/manager/rest-reference/openapi.json

@@ -25127,7 +25127,12 @@
         "description": "Fields available for ordering kernel scheduling history.",
         "enum": [
           "created_at",
-          "updated_at"
+          "updated_at",
+          "phase",
+          "from_status",
+          "to_status",
+          "result",
+          "attempts"
         ],
         "title": "KernelHistoryOrderField",
         "type": "string"

src/ai/backend/common/dto/manager/v2/scheduling_history/types.py

@@ -52,6 +52,11 @@ class KernelHistoryOrderField(StrEnum):
 
     CREATED_AT = "created_at"
     UPDATED_AT = "updated_at"
+    PHASE = "phase"
+    FROM_STATUS = "from_status"
+    TO_STATUS = "to_status"
+    RESULT = "result"
+    ATTEMPTS = "attempts"
 
 
 class DeploymentHistoryOrderField(StrEnum):

src/ai/backend/manager/api/adapters/scheduling_history/adapter.py

@@ -31,6 +31,7 @@
     SessionHistoryNode,
 )
 from ai.backend.common.dto.manager.v2.scheduling_history.types import SubStepResultInfo
+from ai.backend.common.identifier.kernel_scheduling_history import KernelSchedulingHistoryID
 from ai.backend.common.identifier.replica import ReplicaID
 from ai.backend.common.types import KernelId
 from ai.backend.manager.api.adapter_options.pagination.pagination import PaginationSpec
@@ -162,6 +163,27 @@ async def batch_load_session_histories_by_ids(
         history_map = {h.id: self._session_data_to_dto(h) for h in action_result.histories}
         return [history_map.get(history_id) for history_id in ids]
 
+    async def batch_load_kernel_histories_by_ids(
+        self, ids: Sequence[KernelSchedulingHistoryID]
+    ) -> list[KernelHistoryNode | None]:
+        """Batch load kernel scheduling histories by their IDs for DataLoader use.
+
+        Returns KernelHistoryNode DTOs in the same order as the input ids list.
+        """
+        if not ids:
+            return []
+        querier = BatchQuerier(
+            pagination=OffsetPagination(limit=len(ids)),
+            conditions=[KernelSchedulingHistoryConditions.by_ids(ids)],
+        )
+        action_result = (
+            await self._processors.scheduling_history.search_kernel_history.wait_for_complete(
+                SearchKernelHistoryAction(querier=querier)
+            )
+        )
+        history_map = {h.id: self._kernel_data_to_dto(h) for h in action_result.items}
+        return [history_map.get(history_id) for history_id in ids]
+
     async def batch_load_deployment_histories_by_ids(
         self, ids: Sequence[UUID]
     ) -> list[DeploymentHistoryNode | None]:

src/ai/backend/manager/api/gql/data_loader/data_loaders.py

@@ -9,6 +9,7 @@
 from ai.backend.common.identifier.app_config_allow_list import AppConfigAllowListID
 from ai.backend.common.identifier.app_config_definition import AppConfigDefinitionID
 from ai.backend.common.identifier.deployment import DeploymentID
+from ai.backend.common.identifier.kernel_scheduling_history import KernelSchedulingHistoryID
 from ai.backend.common.types import AgentId, ImageID, KernelId, SessionId
 from ai.backend.manager.data.permission.id import ObjectId
 
@@ -102,6 +103,7 @@
     )
     from ai.backend.manager.api.gql.scheduling_history.types import (  # pants: no-infer-dep
         DeploymentHistory,
+        KernelSchedulingHistoryGQL,
         RouteHistory,
         SessionSchedulingHistory,
     )
@@ -718,6 +720,24 @@ async def load_fn(ids: list[uuid.UUID]) -> list[SessionSchedulingHistory | None]
 
         return DataLoader(load_fn=load_fn)
 
+    @cached_property
+    def kernel_history_loader(
+        self,
+    ) -> DataLoader[KernelSchedulingHistoryID, KernelSchedulingHistoryGQL | None]:
+        adapter = self._adapters.scheduling_history
+
+        async def load_fn(
+            ids: list[KernelSchedulingHistoryID],
+        ) -> list[KernelSchedulingHistoryGQL | None]:
+            from ai.backend.manager.api.gql.scheduling_history.types import (  # pants: no-infer-dep
+                KernelSchedulingHistoryGQL as KSH,
+            )
+
+            dtos = await adapter.batch_load_kernel_histories_by_ids(ids)
+            return [KSH.from_pydantic(dto) if dto is not None else None for dto in dtos]
+
+        return DataLoader(load_fn=load_fn)
+
     @cached_property
     def deployment_history_loader(
         self,