contrast/.github/workflows/bm_maintenance_platform.yml at main · edgelesssys/contrast · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
name: bare metal maintenance per platform

on:
  workflow_call:
    inputs:
      platform-name:
        type: string
        required: true
      runner:
        type: string
        required: true
      image:
        type: string
        required: true
      is-gpu:
        type: boolean
        default: false
    outputs:
      update-resources:
        description: "Result of the update-resources job"
        value: ${{ jobs.update-resources.outputs.result }}
      cleanup:
        description: "Result of the cleanup job"
        value: ${{ jobs.cleanup.outputs.result }}
      cleanup-containerd:
        description: "Result of the cleanup-containerd job"
        value: ${{ jobs.cleanup-containerd.outputs.result }}
      nix-gc:
        description: "Result of the nix-gc job"
        value: ${{ jobs.nix-gc.outputs.result }}

jobs:
  update-resources:
    name: "Update resources ${{ inputs.platform-name }}"
    runs-on: ${{ inputs.runner }}
    outputs:
      result: ${{ steps.report.outputs.result }}
    env:
      SET: base
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: nicknovitski/nix-develop@9be7cfb4b10451d3390a75dc18ad0465bed4932a # v1.2.1
      - name: Update storageclass
        run: |
          nix build ".#${SET}.csi-driver-host-path"
          kubectl apply -k result
      - name: Clean stale GPU leases
        if: ${{ inputs.is-gpu }}
        run: |
          kubectl delete leases --namespace default --selector ci.contrast.edgeless.systems/fifo-lease=gpu-sync
          kubectl delete leases --namespace default gpu-sync || true
      - name: Update namespace cleanup cronjob
        env:
          IMAGE: ${{ inputs.image }}
        run: |
          sed -i "s#@@REPLACE_IMAGE@@#${IMAGE}#g" ./tools/bm-maintenance/cleanup-namespaces.yml
          kubectl apply -f ./tools/bm-maintenance/cleanup-namespaces.yml
      - name: Update gpu operator
        if: ${{ inputs.is-gpu }}
        env:
          PLATFORM_NAME: ${{ inputs.platform-name }}
        run: |
          GPU_OPERATOR_VERSION=$(jq -r '."gpu-operator".currentValue' ./tools/bm-maintenance/versions.json)
          OPTS=(--version "$GPU_OPERATOR_VERSION")
          if [[ "${PLATFORM_NAME}" != "Metal-QEMU-TDX-GPU" ]]; then
            nix run ".#${SET}.scripts.upgrade-gpu-operator" -- "${OPTS[@]}"
          fi
      - name: Report success
        id: report
        run: echo "result=success" >> "$GITHUB_OUTPUT"

  cleanup:
    name: "Cleanup ${{ inputs.platform-name }}"
    runs-on: ${{ inputs.runner }}
    timeout-minutes: 15
    outputs:
      result: ${{ steps.update.outputs.result }}
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: nicknovitski/nix-develop@9be7cfb4b10451d3390a75dc18ad0465bed4932a # v1.2.1
      - name: Apply resources
        env:
          IMAGE: ${{ inputs.image }}
        run: |
          K3S_VERSION=$(jq -r '.k3s.currentValue' ./tools/bm-maintenance/versions.json)
          sed -i "s#@@REPLACE_IMAGE@@#${IMAGE}#g" ./tools/bm-maintenance/cleanup.yml
          sed -i "s#@@REPLACE_K3S_VERSION@@#${K3S_VERSION}#g" ./tools/bm-maintenance/cleanup.yml
          kubectl apply -f ./tools/bm-maintenance/cleanup.yml
      - name: Wait for cleanup job
        id: update
        run: |
          kubectl wait -n maintenance-cleanup --for=condition=complete --timeout=600s job/cleanup-maintenance
          echo "result=success" >> "$GITHUB_OUTPUT"
      - name: Collect logs and cleanup
        if: always()
        run: |
          kubectl logs -n maintenance-cleanup job/cleanup-maintenance || true
          kubectl delete -f ./tools/bm-maintenance/cleanup.yml || true

  cleanup-containerd:
    name: "Cleanup Containerd ${{ inputs.platform-name }}"
    runs-on: ${{ inputs.runner }}
    timeout-minutes: 15
    outputs:
      result: ${{ steps.update.outputs.result }}
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: nicknovitski/nix-develop@9be7cfb4b10451d3390a75dc18ad0465bed4932a # v1.2.1
      - name: Apply resources
        env:
          IMAGE: ${{ inputs.image }}
        run: |
          sed -i "s#@@REPLACE_IMAGE@@#${IMAGE}#g" ./tools/bm-maintenance/cleanup-containerd.yml
          kubectl apply -f ./tools/bm-maintenance/cleanup-containerd.yml
      - name: Wait for cleanup job
        id: update
        run: |
          kubectl wait -n maintenance-containerd-cleanup --for=condition=complete --timeout=600s job/containerd-cleanup-maintenance
          echo "result=success" >> "$GITHUB_OUTPUT"
      - name: Collect logs and cleanup
        if: always()
        run: |
          kubectl logs -n maintenance-containerd-cleanup job/containerd-cleanup-maintenance || true
          kubectl delete -f ./tools/bm-maintenance/cleanup-containerd.yml || true

  nix-gc:
    name: "Nix gc ${{ inputs.platform-name }}"
    runs-on: ${{ inputs.runner }}
    timeout-minutes: 15
    outputs:
      result: ${{ steps.update.outputs.result }}
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - uses: nicknovitski/nix-develop@9be7cfb4b10451d3390a75dc18ad0465bed4932a # v1.2.1
      - name: Apply resources
        env:
          IMAGE: ${{ inputs.image }}
        run: |
          sed -i "s#@@REPLACE_IMAGE@@#${IMAGE}#g" ./tools/bm-maintenance/nix-gc.yml
          kubectl apply -f ./tools/bm-maintenance/nix-gc.yml
      - name: Wait for nix garbage collection job
        id: update
        run: |
          kubectl wait -n maintenance-nix-gc --for=condition=complete --timeout=600s job/nix-garbage-collection
          echo "result=success" >> "$GITHUB_OUTPUT"
      - name: Collect logs and cleanup
        if: always()
        run: |
          kubectl logs -n maintenance-nix-gc job/nix-garbage-collection || true
          kubectl delete -f ./tools/bm-maintenance/nix-gc.yml || true