diff --git a/.github/workflows/cluster-it-1c1d.yml b/.github/workflows/cluster-it-1c1d.yml deleted file mode 100644 index 4ab201450e69..000000000000 --- a/.github/workflows/cluster-it-1c1d.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Cluster IT - 1C1D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Adjust network dynamic TCP ports range - if: ${{ runner.os == 'Windows' }} - shell: pwsh - run: | - netsh int ipv4 set dynamicport tcp start=32768 num=32768 - netsh int ipv4 set dynamicport udp start=32768 num=32768 - netsh int ipv6 set dynamicport tcp start=32768 num=32768 - netsh int ipv6 set dynamicport udp start=32768 num=32768 - - name: Adjust Linux kernel somaxconn - if: ${{ runner.os == 'Linux' }} - shell: bash - run: sudo sysctl -w net.core.somaxconn=65535 - # - name: Adjust Mac kernel somaxconn - # if: ${{ runner.os == 'macOS' }} - # shell: bash - # run: sudo sysctl -w kern.ipc.somaxconn=65535 - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=2 \ - -pl integration-test \ - -am - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: standalone-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/cluster-it-1c1d1a.yml b/.github/workflows/cluster-it-1c1d1a.yml deleted file mode 100644 index 7ee6ca6ace66..000000000000 --- a/.github/workflows/cluster-it-1c1d1a.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Cluster IT - 1C1D1A - -on: - push: - branches: - - master - - 'rel/*' - - 'rc/*' - paths-ignore: - - 'docs/**' - - 'site/**' - pull_request: - branches: - - master - - 'rel/*' - - 'rc/*' - - 'force_ci/**' - paths-ignore: - - 'docs/**' - - 'site/**' - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - AINode: - strategy: - fail-fast: false - max-parallel: 1 - matrix: - os: [ ubuntu-latest ] - runs-on: [self-hosted, gpu] - - steps: - - uses: actions/checkout@v5 - - name: IT Test - shell: bash - run: | - mvn clean verify \ - -P with-integration-tests,with-ainode \ - -DskipUTs \ - -DintegrationTest.forkCount=1 \ - -pl integration-test,iotdb-core/ainode \ - -am \ - -PAIClusterIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-ainode-${{ matrix.os }} - path: integration-test/target/*-logs - retention-days: 30 diff --git a/.github/workflows/cluster-it-1c3d.yml b/.github/workflows/cluster-it-1c3d.yml deleted file mode 100644 index 035403e1695f..000000000000 --- a/.github/workflows/cluster-it-1c3d.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Cluster IT - 1C3D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 20 - matrix: - java: [17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PClusterIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/compile-check.yml b/.github/workflows/compile-check.yml deleted file mode 100644 index 48e0571fc788..000000000000 --- a/.github/workflows/compile-check.yml +++ /dev/null @@ -1,55 +0,0 @@ -# This workflow will compile IoTDB under jdk8 to check for compatibility issues - -name: Compile Check - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - compile-check: - strategy: - fail-fast: false - matrix: - java: [8] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Compiler Test - shell: bash - run: | - mvn clean package -P with-integration-tests -DskipTests -ntp diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml deleted file mode 100644 index 54b502bcfa58..000000000000 --- a/.github/workflows/daily-it.yml +++ /dev/null @@ -1,1079 +0,0 @@ -name: Daily IT - -on: - schedule: - # Run at UTC 19:00 every day (CST 03:00 AM) - - cron: "0 19 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 20 - matrix: - java: [8, 17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PDailyIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 3 - SingleRegionTableModel: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [8, 17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 -DintegrationTest.dataRegionPerDataNode=1 \ - -pl integration-test \ - -am -PTableSimpleIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 3 - PipeSingle: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - ] - os: [ubuntu-latest] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2BatchMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2BatchMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2StreamMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2StreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT1 \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTreeAutoBasic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTreeAutoEnhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - ] - os: [ubuntu-latest] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2BatchMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2BatchMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2StreamMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2StreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTreeManual: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - ] - os: [ubuntu-latest] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2BatchMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2BatchMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2StreamMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2StreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeManual \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTreeArchVerification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - ScalableSingleNodeMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTableArchVerification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTableArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTreeRegressionConsumer: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: - [ - ScalableSingleNodeMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTreeRegressionMisc: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: - [ - ScalableSingleNodeMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTableManualBasic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTableManualEnhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 diff --git a/.github/workflows/daily-ut.yml b/.github/workflows/daily-ut.yml deleted file mode 100644 index 348276962d2b..000000000000 --- a/.github/workflows/daily-ut.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Daily UT - -on: - schedule: - # Run at UTC 19:00 every day (CST 03:00 AM) - - cron: "0 19 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - unit-test: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [8] - os: [ubuntu-latest, windows-latest] - it_task: ["others", "datanode"] - include: - - java: 17 - os: macos-latest - it_task: "datanode" - - java: 17 - os: macos-latest - it_task: "others" - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Test Datanode Module with Maven - shell: bash - if: ${{ matrix.it_task == 'datanode'}} - run: mvn clean integration-test -Dtest.port.closed=true -pl iotdb-core/datanode -am -DskipTests -Diotdb.test.only=true - - name: Test Other Modules with Maven - shell: bash - if: ${{ matrix.it_task == 'others'}} - run: | - mvn clean install -DskipTests - mvn -P get-jar-with-dependencies,with-integration-tests clean test -Dtest.port.closed=true -Diotdb.test.skip=true diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml deleted file mode 100644 index 4cf6cb202afc..000000000000 --- a/.github/workflows/dependency-check.yml +++ /dev/null @@ -1,62 +0,0 @@ -# This workflow will check if dependencies have changed (adding new dependencies or removing existing ones) - -name: Dependency Check - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - dependency-check: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Do the dependency check - shell: bash - run: mvn verify -Dmaven.test.skip=true -Dmdep.analyze.skip=true -P enable-sbom-check diff --git a/.github/workflows/greeting-ainode.yml b/.github/workflows/greeting-ainode.yml deleted file mode 100644 index 53deb59f1ce0..000000000000 --- a/.github/workflows/greeting-ainode.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: AINode Code Style Check - -on: - push: - branches: - - master - - "rc/*" - paths: - - 'iotdb-core/ainode/**' - pull_request: - branches: - - master - - "rc/*" - paths: - - 'iotdb-core/ainode/**' - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - -jobs: - check-style: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Set up Python 3.10 - uses: actions/setup-python@v6 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip3 install black==25.1.0 isort==6.0.1 - - name: Check code formatting (Black) - run: | - cd iotdb-core/ainode - black --check . - continue-on-error: false - - - name: Check import order (Isort) - run: | - cd iotdb-core/ainode - isort --check-only --profile black . - continue-on-error: false diff --git a/.github/workflows/multi-language-client.yml b/.github/workflows/multi-language-client.yml deleted file mode 100644 index 831e3e37639b..000000000000 --- a/.github/workflows/multi-language-client.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: Multi-Language Client -on: - push: - branches: - - master - - "rc/*" - paths: - - 'pom.xml' - - 'iotdb-client/pom.xml' - - 'iotdb-client/client-py/**' - - 'iotdb-client/client-cpp/**' - - 'example/client-cpp-example/**' - - 'iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift' - - 'iotdb-protocol/thrift-commons/src/main/thrift/common.thrift' - - '.github/workflows/multi-language-client.yml' - pull_request: - branches: - - master - - "rc/*" - - 'force_ci/**' - paths: - - 'pom.xml' - - 'iotdb-client/pom.xml' - - 'iotdb-client/client-py/**' - - 'iotdb-client/client-cpp/**' - - 'example/client-cpp-example/**' - - 'iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift' - - 'iotdb-protocol/thrift-commons/src/main/thrift/common.thrift' - - '.github/workflows/multi-language-client.yml' - # allow manually run the action: - workflow_dispatch: - - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - -jobs: - cpp: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - os: [ubuntu-22.04, ubuntu-24.04, windows-2022, windows-latest, windows-2025-vs2026, macos-latest] - runs-on: ${{ matrix.os}} - - steps: - - uses: actions/checkout@v5 - - name: Install CPP Dependencies (Ubuntu) - if: runner.os == 'Linux' - shell: bash - run: | - sudo apt-get update - sudo apt-get install libboost-all-dev - sudo apt-get install openssl libssl-dev - - name: Install CPP Dependencies (Mac) - # remove some xcode to release disk space - if: runner.os == 'macOS' - shell: bash - run: | - brew install boost - brew install openssl - sudo rm -rf /Applications/Xcode_14.3.1.app - sudo rm -rf /Applications/Xcode_15.0.1.app - sudo rm -rf /Applications/Xcode_15.1.app - sudo rm -rf /Applications/Xcode_15.2.app - sudo rm -rf /Applications/Xcode_15.3.app - - name: Install CPP Dependencies (Windows) - if: runner.os == 'Windows' - run: | - choco install winflexbison3 - choco install boost-msvc-14.3 - $boost_path = (Get-ChildItem -Path 'C:\local\' -Filter 'boost_*').FullName - echo $boost_path >> $env:GITHUB_PATH - - choco install openssl - $sslPath = (Get-ChildItem 'C:\Program Files\OpenSSL*' -Directory | Select-Object -First 1).FullName - echo "$sslPath\bin" >> $env:GITHUB_PATH - echo "OPENSSL_ROOT_DIR=$sslPath" >> $env:GITHUB_ENV - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Build IoTDB server - shell: bash - run: ./mvnw clean install -pl distribution -am -DskipTests - - name: Test with Maven - shell: bash - # Explicitly using mvnw here as the build requires maven 3.9 and the default installation is older - # Explicitly using "install" instead of package in order to be sure we're using libs built on this machine - # (was causing problems on windows, but could cause problem on linux, when updating the thrift module) - run: | - if [[ "${{ matrix.os }}" == "windows-2025-vs2026" ]]; then - ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am -Dcmake.generator="Visual Studio 18 2026" - else - ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am - fi - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cpp-IT-${{ runner.os }} - path: distribution/target/apache-iotdb-*-all-bin/apache-iotdb-*-all-bin/logs - retention-days: 1 - - go: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - with: - token: ${{secrets.GITHUB_TOKEN}} - submodules: recursive - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Compile IoTDB Server - run: mvn clean package -pl distribution -am -DskipTests - - name: Integration test - shell: bash - run: | - cd iotdb-client - git clone https://github.com/apache/iotdb-client-go.git - cd iotdb-client-go - make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo - - python: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - python: ['3.x'] - runs-on: ${{ 'ubuntu-latest' }} - - steps: - - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python }} - - uses: actions/checkout@v5 - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Cache pip packages - uses: actions/cache@v5 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: ${{ runner.os }}-pip- - - name: Build IoTDB server distribution zip and python client - run: mvn -B clean install -pl distribution,iotdb-client/client-py -am -DskipTests - - name: Build IoTDB server docker image - run: | - docker build . -f docker/src/main/Dockerfile-1c1d -t "iotdb:dev" - docker images - - name: Install IoTDB python client requirements - run: pip3 install -r iotdb-client/client-py/requirements_dev.txt - - name: Check code style - if: ${{ matrix.python == '3.x'}} - shell: bash - run: black iotdb-client/client-py/ --check --diff - - name: Integration test and test make package - shell: bash - run: | - cd iotdb-client/client-py/ && pytest . - ./release.sh diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 0968e7739a05..f834c884a257 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -33,263 +33,6 @@ env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: - single: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [HighPerformanceMode] - cluster2: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT1 \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-tree-auto-basic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-tree-auto-enhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [HighPerformanceMode] - cluster2: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 dual-tree-manual: strategy: fail-fast: false @@ -334,7 +77,10 @@ jobs: mvn clean verify \ -P with-integration-tests \ -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -Dtest=IoTDBPipePermissionIT#testIllegalPassword \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -DintegrationTest.forkCount=0 -DforkCount=0 \ + -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ -pl integration-test \ -am -PMultiClusterIT2DualTreeManual \ @@ -375,604 +121,4 @@ jobs: with: name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} path: integration-test/target/cluster-logs - retention-days: 30 - subscription-tree-arch-verification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-table-arch-verification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTableArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-tree-regression-consumer: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-tree-regression-misc: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-table-manual-basic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-table-manual-enhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - triple: - strategy: - fail-fast: false - max-parallel: 1 - matrix: - java: [ 17 ] - cluster1: [ ScalableSingleNodeMode ] - cluster2: [ ScalableSingleNodeMode ] - cluster3: [ ScalableSingleNodeMode ] - os: [ ubuntu-latest ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }},${{ matrix.cluster3 }} \ - -pl integration-test \ - -am -PMultiClusterIT3 \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-triple-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}-${{ matrix.cluster3 }} - path: integration-test/target/cluster-logs - retention-days: 30 + retention-days: 30 \ No newline at end of file diff --git a/.github/workflows/sonar-codecov.yml b/.github/workflows/sonar-codecov.yml deleted file mode 100644 index adb943a7f62e..000000000000 --- a/.github/workflows/sonar-codecov.yml +++ /dev/null @@ -1,89 +0,0 @@ -# This workflow will build a Java project with Maven -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: Sonar-Codecov - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "new_*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - PR_NUMBER: ${{ github.event.number }} - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - codecov: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'apache/iotdb' || github.event_name == 'push' - - steps: - - uses: actions/checkout@v5 - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Test - run: | - mvn -B -P with-code-coverage clean package -pl distribution,iotdb-client/cli,iotdb-client/session,iotdb-client/jdbc -am -Dtest.port.closed=true - mvn -B -P with-code-coverage post-integration-test -pl code-coverage - - name: Upload coverage reports to codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./code-coverage/target/jacoco-merged-reports/jacoco.xml - - sonar: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'apache/iotdb' || github.event_name == 'push' - steps: - - uses: actions/checkout@v5 - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: SonarCloud Report - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} - run: | - mvn -B -P with-integration-tests,with-code-coverage verify sonar:sonar \ - -Dsonar.organization=apache \ - -Dsonar.projectKey=apache_iotdb \ - -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.token=${{ secrets.SONARCLOUD_TOKEN }} \ - -DskipTests -pl '!distribution,!integration-test' -am diff --git a/.github/workflows/table-cluster-it-1c1d.yml b/.github/workflows/table-cluster-it-1c1d.yml deleted file mode 100644 index 782bafa4ddbe..000000000000 --- a/.github/workflows/table-cluster-it-1c1d.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Table Cluster IT - 1C1D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Adjust network dynamic TCP ports range - if: ${{ runner.os == 'Windows' }} - shell: pwsh - run: | - netsh int ipv4 set dynamicport tcp start=32768 num=32768 - netsh int ipv4 set dynamicport udp start=32768 num=32768 - netsh int ipv6 set dynamicport tcp start=32768 num=32768 - netsh int ipv6 set dynamicport udp start=32768 num=32768 - - name: Adjust Linux kernel somaxconn - if: ${{ runner.os == 'Linux' }} - shell: bash - run: sudo sysctl -w net.core.somaxconn=65535 - # - name: Adjust Mac kernel somaxconn - # if: ${{ runner.os == 'macOS' }} - # shell: bash - # run: sudo sysctl -w kern.ipc.somaxconn=65535 - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PTableSimpleIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/table-cluster-it-1c3d.yml b/.github/workflows/table-cluster-it-1c3d.yml deleted file mode 100644 index 99a7469500c9..000000000000 --- a/.github/workflows/table-cluster-it-1c3d.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Table Cluster IT - 1C3D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 20 - matrix: - java: [17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PTableClusterIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: table-cluster-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/todos-check.yml b/.github/workflows/todos-check.yml deleted file mode 100644 index fc33b12d3cb5..000000000000 --- a/.github/workflows/todos-check.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Check TODOs and FIXMEs in Changed Files - -on: - pull_request: - branches: - - master - - 'dev/*' - - 'rel/*' - - "rc/*" - - 'force_ci/**' - paths-ignore: - - 'docs/**' - - 'site/**' - # allow manually run the action: - workflow_dispatch: - -jobs: - todo-check: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Check for TODOs and FIXMEs in changed files - run: | - # Fetch the target branch - git fetch origin $GITHUB_BASE_REF - - git switch -c check_branch - - # Get the diff of the changes - echo Get the diff of the changes - DIFF=$(git diff origin/$GITHUB_BASE_REF check_branch -- . ':(exclude).github/workflows/todos-check.yml') - - if [ -z "$DIFF" ]; then - echo "No changes detected." - exit 0 - fi - - - # Check the diff for TODOs - - # Check the diff for TODOs - echo Check the diff for TODOs - TODOsCOUNT=$(echo "$DIFF" | grep -E '^\+.*(TODO|FIXME)' | wc -l) - if [ "$TODOsCOUNT" -eq 0 ]; then - echo "No TODOs or FIXMEs found in changed content."; - exit 0 - fi - - echo "TODO or FIXME found in the changes. Please resolve it before merging." - echo "$DIFF" | grep -E '^\+.*(TODO|FIXME)' | tee -a output.log - exit 1 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml deleted file mode 100644 index c53c8d89cfa9..000000000000 --- a/.github/workflows/unit-test.yml +++ /dev/null @@ -1,71 +0,0 @@ -# This workflow will build a Java project with Maven -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: Unit-Test - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xms2g -Xmx4g -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - unit-test: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - os: [ubuntu-latest, windows-latest] - it_task: ["others", "datanode"] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Test Datanode Module with Maven - shell: bash - if: ${{ matrix.it_task == 'datanode'}} - run: mvn clean integration-test -Dtest.port.closed=true -pl iotdb-core/datanode -am -DskipTests -Diotdb.test.only=true - - name: Test Other Modules with Maven - shell: bash - if: ${{ matrix.it_task == 'others'}} - run: | - mvn clean install -DskipTests - mvn -P get-jar-with-dependencies,with-integration-tests clean test -Dtest.port.closed=true -Diotdb.test.skip=true diff --git a/.github/workflows/vulnerability-check.yml b/.github/workflows/vulnerability-check.yml deleted file mode 100644 index c120bfa5291a..000000000000 --- a/.github/workflows/vulnerability-check.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: vulnerability-check -on: - schedule: - # Run at 16:00 UTC every Sunday (Monday 00:00 CST) - - cron: "0 16 * * 0" - workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - dependency-check: - if: ${{ github.event_name == 'workflow_dispatch' || github.repository == 'apache/iotdb' }} - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - - - name: Do Maven install - shell: bash - run: mvn $MAVEN_ARGS clean install -DskipTests - - - name: Do the dependency-check:aggregate - shell: bash - run: mvn $MAVEN_ARGS org.owasp:dependency-check-maven:aggregate -DossIndexUsername=${{ secrets.OSS_INDEX_USER }} -DossIndexPassword=${{ secrets.OSS_INDEX_TOKEN }} -DnvdApiKey=${{ secrets.NVD_API_KEY }} - - - name: Generate report date for artifact name - run: | - utc_time="${{ github.run_started_at }}" - target_time=$(TZ=Asia/Shanghai date -d "$utc_time" +"%Y-%m-%d") - echo "REPORT_DATE=$target_time" >> $GITHUB_ENV - - - name: Upload Artifact - uses: actions/upload-artifact@v6 - with: - name: vulnerability-check-result-${{ env.REPORT_DATE }} - path: target/dependency-check-report.html - retention-days: 15 diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java index 24807622cf6b..c4d6da211fa1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java @@ -311,7 +311,9 @@ public void testSourcePermission() { receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); fail("Shall fail if password is wrong."); } catch (final SQLException e) { - Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); + Assert.assertEquals( + "801: Failed to check password for pipe a2b, useEncryptedPassword: false", + e.getMessage()); } // Use current session, user is root @@ -538,7 +540,9 @@ public void testIllegalPassword() throws Exception { receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); fail(); } catch (final Exception e) { - Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); + Assert.assertEquals( + "801: Failed to check password for pipe a2b, useEncryptedPassword: false", + e.getMessage()); } try { @@ -591,7 +595,9 @@ public void testIllegalPassword() throws Exception { try { statement.execute("alter pipe a2b modify source ('password'='fake')"); } catch (final SQLException e) { - Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); + Assert.assertEquals( + "801: Failed to check password for pipe a2b, useEncryptedPassword: false", + e.getMessage()); } statement.execute("alter pipe a2b modify source ('password'='newST@ongPassword')"); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java index cf537d5c667e..e1a0dde846a3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java @@ -74,21 +74,15 @@ public AuthorPlanExecutor(IAuthorizer authorizer) { @Override public TPermissionInfoResp login( - String username, final String password, final boolean useEncryptedPassword) { - boolean status; - String loginMessage = null; - TSStatus tsStatus = new TSStatus(); + final String username, final String password, final boolean useEncryptedPassword) { + final String loginMessage; + final TSStatus tsStatus = new TSStatus(); TPermissionInfoResp result = new TPermissionInfoResp(); try { - status = authorizer.login(username, password, useEncryptedPassword); - if (status) { - result = getUserPermissionInfo(username, ModelType.ALL); - - result.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Login successfully")); - } else { - result = AuthUtils.generateEmptyPermissionInfoResp(); - } - } catch (AuthException e) { + authorizer.login(username, password, useEncryptedPassword); + result = getUserPermissionInfo(username, ModelType.ALL); + result.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Login successfully")); + } catch (final AuthException e) { LOGGER.error("meet error while logging in.", e); loginMessage = e.getMessage(); tsStatus.setCode(e.getCode().getStatusCode()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java index b90469b6f70f..15e56f5303c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java @@ -64,11 +64,13 @@ public void putUserCache(String userName, User user) { AuthorityChecker.setSuperUser(user.getName()); } userCache.put(userName, user); + LOGGER.info("Put user cache, user: {}", user); } @Override public void putRoleCache(String roleName, Role role) { roleCache.put(roleName, role); + LOGGER.info("Put role cache, role: {}", role); } /** @@ -85,8 +87,10 @@ public boolean invalidateCache(final String userName, final String roleName) { Set roleSet = userCache.getIfPresent(userName).getRoleSet(); if (!roleSet.isEmpty()) { roleCache.invalidateAll(roleSet); + LOGGER.info("Invalidated cache for roles {}", roleSet); } userCache.invalidate(userName); + LOGGER.info("Invalidated cache for user {}", userName); } if (userCache.getIfPresent(userName) != null) { LOGGER.error("datanode cache initialization failed"); @@ -96,6 +100,7 @@ public boolean invalidateCache(final String userName, final String roleName) { if (roleName != null) { if (roleCache.getIfPresent(roleName) != null) { roleCache.invalidate(roleName); + LOGGER.info("Invalidated cache for role {}", roleName); } if (roleCache.getIfPresent(roleName) != null) { LOGGER.error("datanode cache initialization failed"); @@ -109,5 +114,6 @@ public boolean invalidateCache(final String userName, final String roleName) { public void invalidAllCache() { userCache.invalidateAll(); roleCache.invalidateAll(); + LOGGER.info("Invalidated all users and roles cache."); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/ClusterAuthorityFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/ClusterAuthorityFetcher.java index 325476174c66..799607b953ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/ClusterAuthorityFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/ClusterAuthorityFetcher.java @@ -580,6 +580,7 @@ public TSStatus checkUser( } finally { if (status == null) { status = new TPermissionInfoResp(); + status.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR)); } } if (status.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index 32e5c300f7d1..5653b81b48e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -561,6 +561,7 @@ public void customize( @Override protected void login(final @Nonnull String password) { if (!pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + final boolean useEncryptedPassword = regionId >= 0; if (SessionManager.getInstance() .login( new InternalClientSession("Source_login_session_" + regionId), @@ -570,11 +571,13 @@ protected void login(final @Nonnull String password) { SessionManager.CURRENT_RPC_VERSION, IoTDBConstant.ClientVersion.V_1_0, IClientSession.SqlDialect.TREE, - regionId >= 0) + useEncryptedPassword) .getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new PipePasswordCheckException( - String.format("Failed to check password for pipe %s.", pipeName)); + String.format( + "Failed to check password for pipe %s, useEncryptedPassword: %s", + pipeName, useEncryptedPassword)); } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/authorizer/LocalFileAuthorizerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/authorizer/LocalFileAuthorizerTest.java index 194f33e8d67b..88decde4e216 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/authorizer/LocalFileAuthorizerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/authorizer/LocalFileAuthorizerTest.java @@ -64,7 +64,7 @@ public void tearDown() throws Exception { @Test public void testLogin() throws AuthException { - Assert.assertTrue(authorizer.login("root", "root", false)); + authorizer.login("root", "root", false); Assert.assertThrows(AuthException.class, () -> authorizer.login("root", "error", false)); } @@ -76,7 +76,7 @@ public void createAndDeleteUser() throws AuthException { } catch (AuthException e) { assertEquals("User user already exists", e.getMessage()); } - Assert.assertTrue(authorizer.login(userName, password, false)); + authorizer.login(userName, password, false); authorizer.deleteUser(userName); try { authorizer.deleteUser(userName); @@ -230,7 +230,7 @@ public void testUserRole() throws AuthException { public void testUpdatePassword() throws AuthException { authorizer.createUser(userName, password); authorizer.updateUserPassword(userName, "newPassword123456"); - Assert.assertTrue(authorizer.login(userName, "newPassword123456", false)); + authorizer.login(userName, "newPassword123456", false); } @Test diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java index 6ba8c5336b19..8cb569bfc829 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java @@ -107,7 +107,7 @@ private void checkAdmin(long userId, String errmsg) throws AuthException { } @Override - public boolean login( + public void login( final String username, final String password, final boolean useEncryptedPassword) throws AuthException { User user = userManager.getEntity(username); @@ -116,11 +116,14 @@ public boolean login( TSStatusCode.USER_NOT_EXIST, String.format("The user %s does not exist.", username)); } if (useEncryptedPassword) { - return password.equals(user.getPassword()); + if (password.equals(user.getPassword())) { + return; + } + throw new AuthException(TSStatusCode.WRONG_LOGIN_PASSWORD, "Incorrect password."); } if (AuthUtils.validatePassword( password, user.getPassword(), AsymmetricEncrypt.DigestAlgorithm.SHA_256)) { - return true; + return; } if (AuthUtils.validatePassword( password, user.getPassword(), AsymmetricEncrypt.DigestAlgorithm.MD5)) { @@ -128,7 +131,7 @@ public boolean login( forceUpdateUserPassword(username, password); } catch (AuthException ignore) { } - return true; + return; } throw new AuthException(TSStatusCode.WRONG_LOGIN_PASSWORD, "Incorrect password."); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java index 3ac33dbeddd4..d0baf07b6670 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java @@ -40,9 +40,8 @@ public interface IAuthorizer extends SnapshotProcessor { * * @param username The username of the user. * @param password The password of the user. - * @return True if such user exists and the given password is correct, else return false. */ - boolean login(String username, String password, final boolean useEncryptedPassword) + void login(String username, String password, final boolean useEncryptedPassword) throws AuthException; /**