Skip to content

Commit b543a85

Browse files
nusantara-selfnusantara-self
authored
Merge pull request #1380 from TheHive-Project/ci-improvements-slim
CI - Build Improvements
2 parents 1d65d99 + aa13482 commit b543a85

2 files changed

Lines changed: 131 additions & 97 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 130 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
responders_matrix: ${{ steps.set-matrix.outputs.responders_matrix }}
2121
steps:
2222
- name: Checkout repository
23-
uses: actions/checkout@v4
23+
uses: actions/checkout@v5
2424
with:
2525
fetch-depth: 0
2626

@@ -83,7 +83,7 @@ jobs:
8383
matrix: ${{ fromJson(needs.generate-matrix.outputs.analyzers_matrix) }}
8484
steps:
8585
- name: Checkout repository
86-
uses: actions/checkout@v4
86+
uses: actions/checkout@v5
8787
with:
8888
fetch-depth: 0
8989

@@ -149,66 +149,82 @@ jobs:
149149
150150
if [ ! -f "$dockerfile_path" ]; then
151151
echo "Dockerfile not found in $dockerfile_path. Creating one..."
152-
#echo "FROM python:3-alpine" > "$dockerfile_path"
153-
#echo "RUN apk add --no-cache openssl ca-certificates" >> "$dockerfile_path"
154-
echo "FROM python:3-slim" > "$dockerfile_path"
155-
# echo "RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*" >> "$dockerfile_path"
152+
153+
# Multi-stage build - Builder stage
154+
echo "# Builder stage" > "$dockerfile_path"
155+
echo "FROM python:3-slim as builder" >> "$dockerfile_path"
156+
echo "WORKDIR /build" >> "$dockerfile_path"
156157
157-
# Check if current worker is among special alpine workers
158+
# Check if current worker is among special alpine workers and install build dependencies
159+
if echo "$special_alpine_workers" | grep -qw "$matrix_directory"; then
160+
echo "RUN apt-get update && apt-get install -y --no-install-recommends libmagic-dev build-essential && rm -rf /var/lib/apt/lists/*" >> "$dockerfile_path"
161+
fi
162+
163+
echo "COPY requirements.txt ." >> "$dockerfile_path"
164+
echo "RUN test ! -e requirements.txt || pip install --user --no-cache-dir -r requirements.txt" >> "$dockerfile_path"
165+
echo "" >> "$dockerfile_path"
166+
167+
# Runtime stage
168+
echo "# Runtime stage" >> "$dockerfile_path"
169+
echo "FROM python:3-slim" >> "$dockerfile_path"
170+
171+
# Check if current worker needs runtime libraries
158172
if echo "$special_alpine_workers" | grep -qw "$matrix_directory"; then
159-
# echo "RUN apk add --no-cache file-dev && rm -rf /var/cache/apk/*" >> "$dockerfile_path"
160173
echo "RUN apt-get update && apt-get install -y --no-install-recommends libmagic1 && rm -rf /var/lib/apt/lists/*" >> "$dockerfile_path"
161174
fi
162175
163176
echo "WORKDIR /worker" >> "$dockerfile_path"
164-
echo "COPY requirements.txt ${matrix_directory}/" >> "$dockerfile_path"
165-
echo "RUN test ! -e ${matrix_directory}/requirements.txt || pip install --no-cache-dir -r ${matrix_directory}/requirements.txt" >> "$dockerfile_path"
177+
echo "COPY --from=builder /root/.local /root/.local" >> "$dockerfile_path"
166178
echo "COPY . ${matrix_directory}/" >> "$dockerfile_path"
179+
echo "ENV PATH=/root/.local/bin:\$PATH" >> "$dockerfile_path"
167180
echo "ENTRYPOINT [\"python\", \"${command_value}\"]" >> "$dockerfile_path"
168181
else
169182
echo "Dockerfile exists: $dockerfile_path"
170183
fi
171184
172185
- name: Check if image needs rebuild
173186
id: check-rebuild
187+
env:
188+
IMAGE_NAME: ghcr.io/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}:${{ env.VERSION }}
189+
CURRENT_SHA: ${{ github.sha }}
190+
COMPONENT_DIR: analyzers/${{ matrix.directory }}
174191
run: |
175-
image="ghcr.io/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}:${{ env.VERSION }}"
176-
current_sha="${{ github.sha }}"
177-
178-
token="${{ secrets.GITHUB_TOKEN }}"
179-
180-
# Fetch image manifest from GHCR
181-
manifest_response=$(curl -sSL \
182-
-H "Authorization: Bearer $token" \
183-
-H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
184-
"https://ghcr.io/v2/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}/manifests/${{ env.VERSION }}")
185-
186-
# Check if the manifest response contains a valid digest
187-
labels=$(echo "$manifest_response" | jq -r '.config.digest // empty')
188-
189-
if [[ -z "$labels" ]]; then
190-
echo "No existing image or unable to fetch manifest. rebuild needed"
192+
set +x # Disable command echoing for security
193+
194+
# Force rebuild on scheduled runs (cron)
195+
if [ "${{ github.event_name }}" = "schedule" ]; then
196+
echo "Scheduled run detected - forcing rebuild"
191197
echo "rebuild=true" >> $GITHUB_OUTPUT
192198
exit 0
193199
fi
194-
195-
# Fetch image config blob to extract labels
196-
config_response=$(curl -sSL \
197-
-H "Authorization: Bearer $token" \
198-
"https://ghcr.io/v2/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}/blobs/$labels")
199-
200-
# Extract image label safely
201-
image_labels=$(echo "$config_response" | jq -r '.config.Labels["org.opencontainers.image.revision"] // empty')
202-
203-
# Debugging: print values
204-
echo "current_sha: $current_sha"
205-
echo "image_labels: $image_labels"
206-
207-
if [[ "$image_labels" == "$current_sha" ]]; then
208-
echo "No rebuild needed. SHA matches: $current_sha"
209-
echo "rebuild=false" >> $GITHUB_OUTPUT
200+
201+
# Try to pull the image to check if it exists
202+
if docker pull --platform linux/amd64 "$IMAGE_NAME" >/dev/null 2>&1; then
203+
# Image exists, check if it has the current SHA
204+
EXISTING_SHA=$(docker inspect "$IMAGE_NAME" --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' 2>/dev/null || echo "")
205+
206+
if [[ "$EXISTING_SHA" == "$CURRENT_SHA" ]]; then
207+
echo "Image is up to date with current commit. No rebuild needed."
208+
echo "rebuild=false" >> $GITHUB_OUTPUT
209+
else
210+
# Check if this specific component folder has changes since the image was built
211+
if [[ -n "$EXISTING_SHA" ]] && git rev-parse --verify "$EXISTING_SHA" >/dev/null 2>&1; then
212+
# Check if there are changes in the component directory since the existing SHA
213+
if git diff --quiet "$EXISTING_SHA" "$CURRENT_SHA" -- "$COMPONENT_DIR"; then
214+
echo "No changes in $COMPONENT_DIR since last build. No rebuild needed."
215+
echo "rebuild=false" >> $GITHUB_OUTPUT
216+
else
217+
echo "Changes detected in $COMPONENT_DIR. Rebuild needed."
218+
echo "rebuild=true" >> $GITHUB_OUTPUT
219+
fi
220+
else
221+
echo "Cannot verify existing SHA. Rebuild needed."
222+
echo "rebuild=true" >> $GITHUB_OUTPUT
223+
fi
224+
fi
210225
else
211-
echo "SHA mismatch or missing label. rebuild needed"
226+
# Image doesn't exist or can't be pulled
227+
echo "Image not found or inaccessible. Rebuild needed."
212228
echo "rebuild=true" >> $GITHUB_OUTPUT
213229
fi
214230
@@ -222,8 +238,8 @@ jobs:
222238
# Keep one image tag for import testing
223239
echo "IMAGE_TAG=${{ env.VERSION_SPLIT }}" >> $GITHUB_ENV
224240
# Produce three tags:
225-
# 1) :VERSION (e.g. 2.0)
226-
# 2) :VERSION_SPLIT (e.g. 2)
241+
# 1) :VERSION (eg 2.0)
242+
# 2) :VERSION_SPLIT (eg 2)
227243
# 3) :latest
228244
echo "IMAGE_TAGS=ghcr.io/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}:${{ env.VERSION }},ghcr.io/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}:${{ env.VERSION_SPLIT }}" >> $GITHUB_ENV
229245
else
@@ -258,6 +274,8 @@ jobs:
258274
platforms: ${{ env.PLATFORMS }}
259275
push: true
260276
tags: ${{ env.IMAGE_TAGS }}
277+
cache-from: type=gha
278+
cache-to: type=gha,mode=max,scope=shared
261279
labels: |
262280
org.opencontainers.image.created=${{ steps.build_date.outputs.date }}
263281
org.opencontainers.image.title=${{ env.LOWERCASE_NAME }}
@@ -276,7 +294,7 @@ jobs:
276294
org.opencontainers.image.version=${{ env.VERSION }}
277295
278296
- name: Scan image for vulnerabilities (Trivy)
279-
uses: aquasecurity/trivy-action@0.32.0
297+
uses: aquasecurity/trivy-action@0.33.1
280298
with:
281299
image-ref: ghcr.io/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}:${{ env.IMAGE_TAG }}
282300
format: table
@@ -504,7 +522,7 @@ jobs:
504522
matrix: ${{ fromJson(needs.generate-matrix.outputs.responders_matrix) }}
505523
steps:
506524
- name: Checkout repository
507-
uses: actions/checkout@v4
525+
uses: actions/checkout@v5
508526
with:
509527
fetch-depth: 0
510528

@@ -570,68 +588,84 @@ jobs:
570588
571589
if [ ! -f "$dockerfile_path" ]; then
572590
echo "Dockerfile not found in $dockerfile_path. Creating one..."
573-
# echo "FROM python:3-alpine" > "$dockerfile_path"
574-
# echo "RUN apk add --no-cache openssl ca-certificates bind-tools" >> "$dockerfile_path"
575-
echo "FROM python:3-slim" > "$dockerfile_path"
591+
592+
# Multi-stage build - Builder stage
593+
echo "# Builder stage" > "$dockerfile_path"
594+
echo "FROM python:3-slim as builder" >> "$dockerfile_path"
595+
echo "WORKDIR /build" >> "$dockerfile_path"
576596
577-
# Check if current worker is among special alpine workers
597+
# Check if current worker is among special alpine workers and install build dependencies
598+
if echo "$special_alpine_workers" | grep -qw "$matrix_directory"; then
599+
echo "RUN apt-get update && apt-get install -y --no-install-recommends libmagic-dev build-essential && rm -rf /var/lib/apt/lists/*" >> "$dockerfile_path"
600+
fi
601+
602+
echo "COPY requirements.txt ." >> "$dockerfile_path"
603+
echo "RUN test ! -e requirements.txt || pip install --user --no-cache-dir -r requirements.txt" >> "$dockerfile_path"
604+
echo "" >> "$dockerfile_path"
605+
606+
# Runtime stage
607+
echo "# Runtime stage" >> "$dockerfile_path"
608+
echo "FROM python:3-slim" >> "$dockerfile_path"
609+
610+
# Check if current worker needs runtime libraries
578611
if echo "$special_alpine_workers" | grep -qw "$matrix_directory"; then
579-
# echo "RUN apk add --no-cache file-dev && rm -rf /var/cache/apk/*" >> "$dockerfile_path"
580612
echo "RUN apt-get update && apt-get install -y --no-install-recommends libmagic1 && rm -rf /var/lib/apt/lists/*" >> "$dockerfile_path"
581613
fi
582614
583615
echo "WORKDIR /worker" >> "$dockerfile_path"
584-
echo "COPY requirements.txt ${matrix_directory}/" >> "$dockerfile_path"
585-
echo "RUN test ! -e ${matrix_directory}/requirements.txt || pip install --no-cache-dir -r ${matrix_directory}/requirements.txt" >> "$dockerfile_path"
616+
echo "COPY --from=builder /root/.local /root/.local" >> "$dockerfile_path"
586617
echo "COPY . ${matrix_directory}/" >> "$dockerfile_path"
618+
echo "ENV PATH=/root/.local/bin:\$PATH" >> "$dockerfile_path"
587619
echo "ENTRYPOINT [\"python\", \"${command_value}\"]" >> "$dockerfile_path"
588620
else
589621
echo "Dockerfile exists: $dockerfile_path"
590622
fi
591623
592624
- name: Check if image needs rebuild
593625
id: check-rebuild
626+
env:
627+
IMAGE_NAME: ghcr.io/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}:${{ env.VERSION }}
628+
CURRENT_SHA: ${{ github.sha }}
629+
COMPONENT_DIR: responders/${{ matrix.directory }}
594630
run: |
595-
image="ghcr.io/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}:${{ env.VERSION }}"
596-
current_sha="${{ github.sha }}"
597-
598-
token="${{ secrets.GITHUB_TOKEN }}"
599-
600-
# Fetch image manifest from GHCR
601-
manifest_response=$(curl -sSL \
602-
-H "Authorization: Bearer $token" \
603-
-H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
604-
"https://ghcr.io/v2/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}/manifests/${{ env.VERSION }}")
605-
606-
# Check if the manifest response contains a valid digest
607-
labels=$(echo "$manifest_response" | jq -r '.config.digest // empty')
608-
609-
if [[ -z "$labels" ]]; then
610-
echo "No existing image or unable to fetch manifest. rebuild needed"
631+
set +x # Disable command echoing for security
632+
633+
# Force rebuild on scheduled runs (cron)
634+
if [ "${{ github.event_name }}" = "schedule" ]; then
635+
echo "Scheduled run detected - forcing rebuild"
611636
echo "rebuild=true" >> $GITHUB_OUTPUT
612637
exit 0
613638
fi
614-
615-
# Fetch image config blob to extract labels
616-
config_response=$(curl -sSL \
617-
-H "Authorization: Bearer $token" \
618-
"https://ghcr.io/v2/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}/blobs/$labels")
619-
620-
# Extract image label safely
621-
image_labels=$(echo "$config_response" | jq -r '.config.Labels["org.opencontainers.image.revision"] // empty')
622-
623-
# Debugging: print values
624-
echo "current_sha: $current_sha"
625-
echo "image_labels: $image_labels"
626-
627-
if [[ "$image_labels" == "$current_sha" ]]; then
628-
echo "No rebuild needed. SHA matches: $current_sha"
629-
echo "rebuild=false" >> $GITHUB_OUTPUT
639+
640+
# Try to pull the image to check if it exists
641+
if docker pull --platform linux/amd64 "$IMAGE_NAME" >/dev/null 2>&1; then
642+
# Image exists, check if it has the current SHA
643+
EXISTING_SHA=$(docker inspect "$IMAGE_NAME" --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' 2>/dev/null || echo "")
644+
645+
if [[ "$EXISTING_SHA" == "$CURRENT_SHA" ]]; then
646+
echo "Image is up to date with current commit. No rebuild needed."
647+
echo "rebuild=false" >> $GITHUB_OUTPUT
648+
else
649+
# Check if this specific component folder has changes since the image was built
650+
if [[ -n "$EXISTING_SHA" ]] && git rev-parse --verify "$EXISTING_SHA" >/dev/null 2>&1; then
651+
# Check if there are changes in the component directory since the existing SHA
652+
if git diff --quiet "$EXISTING_SHA" "$CURRENT_SHA" -- "$COMPONENT_DIR"; then
653+
echo "No changes in $COMPONENT_DIR since last build. No rebuild needed."
654+
echo "rebuild=false" >> $GITHUB_OUTPUT
655+
else
656+
echo "Changes detected in $COMPONENT_DIR. Rebuild needed."
657+
echo "rebuild=true" >> $GITHUB_OUTPUT
658+
fi
659+
else
660+
echo "Cannot verify existing SHA. Rebuild needed."
661+
echo "rebuild=true" >> $GITHUB_OUTPUT
662+
fi
663+
fi
630664
else
631-
echo "SHA mismatch or missing label. rebuild needed"
665+
# Image doesn't exist or can't be pulled
666+
echo "Image not found or inaccessible. Rebuild needed."
632667
echo "rebuild=true" >> $GITHUB_OUTPUT
633668
fi
634-
635669
636670
- name: Set build date
637671
id: build_date
@@ -643,8 +677,8 @@ jobs:
643677
# Keep one image tag for import testing
644678
echo "IMAGE_TAG=${{ env.VERSION_SPLIT }}" >> $GITHUB_ENV
645679
# Produce three tags:
646-
# 1) :VERSION (e.g. 2.0)
647-
# 2) :VERSION_SPLIT (e.g. 2)
680+
# 1) :VERSION (eg 2.0)
681+
# 2) :VERSION_SPLIT (eg 2)
648682
# 3) :latest
649683
echo "IMAGE_TAGS=ghcr.io/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}:${{ env.VERSION }},ghcr.io/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}:${{ env.VERSION_SPLIT }}" >> $GITHUB_ENV
650684
else
@@ -679,6 +713,8 @@ jobs:
679713
platforms: ${{ env.PLATFORMS }}
680714
push: true
681715
tags: ${{ env.IMAGE_TAGS }}
716+
cache-from: type=gha
717+
cache-to: type=gha,mode=max,scope=shared
682718
labels: |
683719
org.opencontainers.image.created=${{ steps.build_date.outputs.date }}
684720
org.opencontainers.image.title=${{ env.LOWERCASE_NAME }}
@@ -697,7 +733,7 @@ jobs:
697733
org.opencontainers.image.version=${{ env.VERSION }}
698734
699735
- name: Scan image for vulnerabilities (Trivy)
700-
uses: aquasecurity/trivy-action@0.32.0
736+
uses: aquasecurity/trivy-action@0.33.1
701737
with:
702738
image-ref: ghcr.io/${{ env.LOWER_REPO_OWNER }}/${{ env.LOWERCASE_NAME }}:${{ env.IMAGE_TAG }}
703739
format: table
@@ -924,7 +960,7 @@ jobs:
924960
if: always()
925961
steps:
926962
- name: Checkout repository
927-
uses: actions/checkout@v4
963+
uses: actions/checkout@v5
928964
- name: Set lowercase repository owner
929965
run: |
930966
owner="${{ github.repository_owner }}"
@@ -982,13 +1018,13 @@ jobs:
9821018
needs: [ build_analyzers, build_responders ]
9831019
if: startsWith(github.ref, 'refs/tags/') && always()
9841020
steps:
985-
- uses: actions/checkout@v4
1021+
- uses: actions/checkout@v5
9861022
- name: Prepare documentation files
9871023
uses: docker://thehiveproject/doc-builder
9881024
with:
9891025
args: --type Cortex-Neurons
9901026
- name: Set up Python
991-
uses: actions/setup-python@v5
1027+
uses: actions/setup-python@v6
9921028
with:
9931029
python-version: "3.x"
9941030
architecture: x64

‎analyzers/Jupyter_Analyzer/Dockerfile‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# syntax=docker/dockerfile:1.3-labs
2-
FROM python:3.9
2+
FROM python:3.12-slim-bookworm
33
WORKDIR /worker
44
COPY . Jupyter_Analyzer
55
RUN test ! -e Jupyter_Analyzer/requirements.txt || pip install --no-cache-dir -r Jupyter_Analyzer/requirements.txt
@@ -9,5 +9,3 @@ RUN apt update
99
RUN apt install patch
1010
RUN patch $(python3 -c "from papermill import iorw; print(iorw.__file__)") /patches/papermill_iorw.patch
1111
ENTRYPOINT Jupyter_Analyzer/jupyter.py
12-
13-

0 commit comments

Comments
 (0)