Bug fix 1.2.0 missing auth consistent data push

.asf.yaml

@@ -17,8 +17,26 @@
 # under the License.
 #
 
+# Please refer https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features
+
 github:
   description: "Apache Pinot - A realtime distributed OLAP datastore"
   homepage: https://pinot.apache.org/
   labels:
     - java
+  del_branch_on_merge: true
+  enabled_merge_buttons:
+    squash: true
+    merge: false
+    rebase: false
+  collaborators:
+    - sullis
+    - shenyu0127
+    - tibrewalpratik17
+    - abhioncbr
+    - zhtaoxiang
+    - shounakmk219
+    - itschrispeck
+    - soumitra-st
+    - swaminathanmanish
+    - yashmayya

.github/dependabot.yml

@@ -24,6 +24,7 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    open-pull-requests-limit: 20
 
   - package-ecosystem: "npm"
     directory: "/pinot-controller/src/main/resources"

.github/workflows/build-multi-arch-pinot-docker-image.yml

@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Multi-Arch Platform Docker Image Build and Publish
+
+on:
+  workflow_dispatch:
+    inputs:
+      gitUrl:
+        description: "The Pinot git repo to check out to build, use https."
+        default: "https://github.com/apache/pinot.git"
+        required: true
+      commit:
+        description: "The branch/commit to check out to build Pinot image."
+        default: "master"
+        required: true
+      dockerImageName:
+        description: "The docker image name, default to 'apachepinot/pinot'."
+        default: "apachepinot/pinot"
+        required: true
+      tags:
+        description: "Tags to push of the image, comma separated, e.g. tag1,tag2,tag3"
+        default: ""
+
+jobs:
+  generate-build-info:
+    name: Generate Build Info
+    runs-on: ubuntu-latest
+    outputs:
+      commit-id: ${{ steps.generate-build-info.outputs.commit-id }}
+      tags: ${{ steps.generate-build-info.outputs.tags }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Generate Build Info
+        id: generate-build-info
+        env:
+          PINOT_GIT_URL: ${{ github.event.inputs.gitUrl }}
+          PINOT_BRANCH: ${{ github.event.inputs.commit }}
+          TAGS: ${{ github.event.inputs.tags }}
+        run: |
+          .github/workflows/scripts/docker/.pinot_build_info_gen.sh
+  build-pinot-docker-image:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [ "amd64", "arm64" ]
+        base-image-tag: [ "11-amazoncorretto", "11-ms-openjdk" ]
+    name: Build Pinot Docker Image on ${{ matrix.arch }} with base image ${{ matrix.base-image-tag }}
+    needs: [ generate-build-info ]
+    steps:
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: docker/setup-qemu-action@v3
+        name: Set up QEMU
+        with:
+          platforms: linux/${{ matrix.arch }}
+      - uses: docker/setup-buildx-action@v3
+        name: Set up Docker Buildx
+      - uses: actions/checkout@v4
+      - name: Build and push the Docker image
+        env:
+          DOCKER_FILE_BASE_DIR: "docker/images/pinot"
+          DOCKER_IMAGE_NAME: ${{ github.event.inputs.dockerImageName }}
+          BUILD_PLATFORM: "linux/${{ matrix.arch }}"
+          BASE_IMAGE_TAG: ${{ matrix.base-image-tag }}
+          PINOT_GIT_URL: ${{ github.event.inputs.gitUrl }}
+          PINOT_BRANCH: "${{needs.generate-build-info.outputs.commit-id}}"
+          TAGS: "${{needs.generate-build-info.outputs.tags}}"
+        run: .github/workflows/scripts/docker/.pinot_single_platform_docker_image_build.sh
+  create-multi-arch-manifest:
+    name: Create Multi-Arch Manifest
+    runs-on: ubuntu-latest
+    needs: [ generate-build-info, build-pinot-docker-image ]
+    steps:
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: docker/setup-qemu-action@v3
+        name: Set up QEMU
+      - uses: docker/setup-buildx-action@v3
+        name: Set up Docker Buildx
+      - uses: actions/checkout@v4
+      - name: Create Multi-Arch Manifest
+        env:
+          TAGS: "${{needs.generate-build-info.outputs.tags}}"
+          BUILD_PLATFORM: "linux/arm64,linux/amd64"
+          BASE_IMAGE_TAGS: "11-amazoncorretto,11-ms-openjdk"
+        run: .github/workflows/scripts/docker/.pinot_multi_arch_docker_image_manifest_package.sh

.github/workflows/build-pinot-base-docker-image.yml

@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Base Docker Image Build and Publish
+
+on:
+  workflow_dispatch:
+    inputs: { }
+
+jobs:
+  build-pinot-build-docker-image:
+    name: Build Pinot Base Docker Image
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        baseImageType: [ "build", "runtime" ]
+        openJdkDist: [ "amazoncorretto", "ms-openjdk" ]
+    steps:
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: docker/setup-qemu-action@v3
+        name: Set up QEMU
+      - uses: docker/setup-buildx-action@v3
+        name: Set up Docker Buildx
+      - uses: actions/checkout@v4
+      - name: Build and push the Docker image
+        env:
+          OPEN_JDK_DIST: ${{ matrix.openJdkDist }}
+          BASE_IMAGE_TYPE: ${{ matrix.baseImageType }}
+          BUILD_PLATFORM: "linux/amd64,linux/arm64"
+          TAG: "11-${{ matrix.openJdkDist }}"
+        run: .github/workflows/scripts/docker/.pinot_base_docker_image_build_and_push.sh

.github/workflows/build-pinot-docker-image.yml

@@ -47,15 +47,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Login to DockerHub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v1
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
-    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-buildx-action@v3
       name: Set up Docker Buildx
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build and push the Docker image
       env:
         DOCKER_FILE_BASE_DIR: "docker/images/pinot"

.github/workflows/build-presto-docker-image.yml

@@ -47,15 +47,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Login to DockerHub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v1
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
-    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-buildx-action@v3
       name: Set up Docker Buildx
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build and push the Docker image
       env:
         DOCKER_FILE_BASE_DIR: "docker/images/pinot-presto"

.github/workflows/build-superset-docker-image.yml

@@ -43,15 +43,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Login to DockerHub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v1
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
-    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-buildx-action@v3
       name: Set up Docker Buildx
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build and push the Docker image
       env:
         DOCKER_FILE_BASE_DIR: "docker/images/pinot-superset"

.github/workflows/pinot_compatibility_tests.yml

@@ -36,11 +36,23 @@ jobs:
         test_suite: [ "compatibility-verifier/sample-test-suite" ]
     name: Pinot Compatibility Regression Testing against ${{ github.event.inputs.oldCommit }} and ${{ github.event.inputs.newCommit }} on ${{ matrix.test_suite }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up JDK 11
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
           java-version: 11
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: v16.15.0
+          cache: 'npm'
+          cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
+      - name: Install npm
+        run: |
+          npm install -g npm@8.5.5
+          npm --version
       - name: Pinot Compatibility Regression Testing
         if : ${{github.event_name == 'workflow_dispatch'}}
         env:
@@ -50,7 +62,7 @@ jobs:
           TEST_SUITE: ${{ matrix.test_suite }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-            -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -62,7 +74,7 @@ jobs:
         if: always()
         run: |
           zip -1 -r artifacts.zip /tmp/compatibility-verifier/*
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         name: Store compatibility verifier work directory
         if: always()
         with:

.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml

@@ -0,0 +1,84 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Multi-Stage Query Engine Compatibility Regression Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      oldCommit:
+        description: "Git hash (or tag) for old commit. (required)"
+        required: true
+      newCommit:
+        description: "Git hash (or tag) for new commit. (required)"
+        required: true
+
+jobs:
+  compatibility-verifier:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test_suite: [ "compatibility-verifier/multi-stage-query-engine-test-suite" ]
+    name: Pinot Multi-Stage Query Engine Compatibility Regression Testing against ${{ github.event.inputs.oldCommit }} and ${{ github.event.inputs.newCommit }} on ${{ matrix.test_suite }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: v16.15.0
+          cache: 'npm'
+          cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
+      - name: Install npm
+        run: |
+          npm install -g npm@8.5.5
+          npm --version
+      - name: Pinot Multi-Stage Query Engine Compatibility Regression Testing
+        if : ${{github.event_name == 'workflow_dispatch'}}
+        env:
+          OLD_COMMIT: ${{ github.event.inputs.oldCommit }}
+          NEW_COMMIT: ${{ github.event.inputs.newCommit }}
+          WORKING_DIR: /tmp/multi-stage-compatibility-verifier
+          TEST_SUITE: ${{ matrix.test_suite }}
+          MAVEN_OPTS: >
+            -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -XX:+IgnoreUnrecognizedVMOptions
+            --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
+      - name: Archive artifacts into zip
+        if: always()
+        run: |
+          zip -1 -r artifacts.zip /tmp/multi-stage-compatibility-verifier/*
+      - uses: actions/upload-artifact@v4
+        name: Store multi-stage compatibility verifier work directory
+        if: always()
+        with:
+          ## TODO: currently matrix.test_suite cannot be used as part of name due to invalid path character.
+          name: multi_stage_compatibility_verifier_work_dir
+          retention-days: 3
+          path: artifacts.zip