From 5b3c01c12d920cc32fd43b4042e2a2f314ec0fd1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 21 Mar 2026 22:56:39 +0800 Subject: [PATCH 01/25] fix-1 --- .../confignode/persistence/auth/AuthorPlanExecutor.java | 1 - .../iotdb/commons/auth/authorizer/BasicAuthorizer.java | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java index cf537d5c667e..3c0095c91559 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java @@ -83,7 +83,6 @@ public TPermissionInfoResp login( status = authorizer.login(username, password, useEncryptedPassword); if (status) { result = getUserPermissionInfo(username, ModelType.ALL); - result.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Login successfully")); } else { result = AuthUtils.generateEmptyPermissionInfoResp(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java index 6ba8c5336b19..948feb8b250b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java @@ -116,7 +116,10 @@ public boolean login( TSStatusCode.USER_NOT_EXIST, String.format("The user %s does not exist.", username)); } if (useEncryptedPassword) { - return password.equals(user.getPassword()); + if (password.equals(user.getPassword())) { + return true; + } + throw new AuthException(TSStatusCode.WRONG_LOGIN_PASSWORD, "Incorrect password."); } if (AuthUtils.validatePassword( password, user.getPassword(), AsymmetricEncrypt.DigestAlgorithm.SHA_256)) { From bc9ec6e25ba41350d5ac6f16c0bad66e3f3fb403 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 21 Mar 2026 23:02:03 +0800 Subject: [PATCH 02/25] fix --- .../persistence/auth/AuthorPlanExecutor.java | 19 +++++++------------ .../db/auth/ClusterAuthorityFetcher.java | 1 + .../dataregion/IoTDBDataRegionSource.java | 5 +++-- .../authorizer/LocalFileAuthorizerTest.java | 6 +++--- .../auth/authorizer/BasicAuthorizer.java | 8 ++++---- .../commons/auth/authorizer/IAuthorizer.java | 3 +-- 6 files changed, 19 insertions(+), 23 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java index 3c0095c91559..e1a0dde846a3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java @@ -74,20 +74,15 @@ public AuthorPlanExecutor(IAuthorizer authorizer) { @Override public TPermissionInfoResp login( - String username, final String password, final boolean useEncryptedPassword) { - boolean status; - String loginMessage = null; - TSStatus tsStatus = new TSStatus(); + final String username, final String password, final boolean useEncryptedPassword) { + final String loginMessage; + final TSStatus tsStatus = new TSStatus(); TPermissionInfoResp result = new TPermissionInfoResp(); try { - status = authorizer.login(username, password, useEncryptedPassword); - if (status) { - result = getUserPermissionInfo(username, ModelType.ALL); - result.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Login successfully")); - } else { - result = AuthUtils.generateEmptyPermissionInfoResp(); - } - } catch (AuthException e) { + authorizer.login(username, password, useEncryptedPassword); + result = getUserPermissionInfo(username, ModelType.ALL); + result.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Login successfully")); + } catch (final AuthException e) { LOGGER.error("meet error while logging in.", e); loginMessage = e.getMessage(); tsStatus.setCode(e.getCode().getStatusCode()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/ClusterAuthorityFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/ClusterAuthorityFetcher.java index 325476174c66..799607b953ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/ClusterAuthorityFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/ClusterAuthorityFetcher.java @@ -580,6 +580,7 @@ public TSStatus checkUser( } finally { if (status == null) { status = new TPermissionInfoResp(); + status.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR)); } } if (status.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index 32e5c300f7d1..a6d47bc19752 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -561,6 +561,7 @@ public void customize( @Override protected void login(final @Nonnull String password) { if (!pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { + final boolean useEncryptedPassword = regionId >= 0; if (SessionManager.getInstance() .login( new InternalClientSession("Source_login_session_" + regionId), @@ -570,11 +571,11 @@ protected void login(final @Nonnull String password) { SessionManager.CURRENT_RPC_VERSION, IoTDBConstant.ClientVersion.V_1_0, IClientSession.SqlDialect.TREE, - regionId >= 0) + useEncryptedPassword) .getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new PipePasswordCheckException( - String.format("Failed to check password for pipe %s.", pipeName)); + String.format("Failed to check password for pipe %s, useEncryptedPassword: %s", pipeName, useEncryptedPassword)); } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/authorizer/LocalFileAuthorizerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/authorizer/LocalFileAuthorizerTest.java index 194f33e8d67b..88decde4e216 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/authorizer/LocalFileAuthorizerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/authorizer/LocalFileAuthorizerTest.java @@ -64,7 +64,7 @@ public void tearDown() throws Exception { @Test public void testLogin() throws AuthException { - Assert.assertTrue(authorizer.login("root", "root", false)); + authorizer.login("root", "root", false); Assert.assertThrows(AuthException.class, () -> authorizer.login("root", "error", false)); } @@ -76,7 +76,7 @@ public void createAndDeleteUser() throws AuthException { } catch (AuthException e) { assertEquals("User user already exists", e.getMessage()); } - Assert.assertTrue(authorizer.login(userName, password, false)); + authorizer.login(userName, password, false); authorizer.deleteUser(userName); try { authorizer.deleteUser(userName); @@ -230,7 +230,7 @@ public void testUserRole() throws AuthException { public void testUpdatePassword() throws AuthException { authorizer.createUser(userName, password); authorizer.updateUserPassword(userName, "newPassword123456"); - Assert.assertTrue(authorizer.login(userName, "newPassword123456", false)); + authorizer.login(userName, "newPassword123456", false); } @Test diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java index 948feb8b250b..8cb569bfc829 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java @@ -107,7 +107,7 @@ private void checkAdmin(long userId, String errmsg) throws AuthException { } @Override - public boolean login( + public void login( final String username, final String password, final boolean useEncryptedPassword) throws AuthException { User user = userManager.getEntity(username); @@ -117,13 +117,13 @@ public boolean login( } if (useEncryptedPassword) { if (password.equals(user.getPassword())) { - return true; + return; } throw new AuthException(TSStatusCode.WRONG_LOGIN_PASSWORD, "Incorrect password."); } if (AuthUtils.validatePassword( password, user.getPassword(), AsymmetricEncrypt.DigestAlgorithm.SHA_256)) { - return true; + return; } if (AuthUtils.validatePassword( password, user.getPassword(), AsymmetricEncrypt.DigestAlgorithm.MD5)) { @@ -131,7 +131,7 @@ public boolean login( forceUpdateUserPassword(username, password); } catch (AuthException ignore) { } - return true; + return; } throw new AuthException(TSStatusCode.WRONG_LOGIN_PASSWORD, "Incorrect password."); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java index 3ac33dbeddd4..d0baf07b6670 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java @@ -40,9 +40,8 @@ public interface IAuthorizer extends SnapshotProcessor { * * @param username The username of the user. * @param password The password of the user. - * @return True if such user exists and the given password is correct, else return false. */ - boolean login(String username, String password, final boolean useEncryptedPassword) + void login(String username, String password, final boolean useEncryptedPassword) throws AuthException; /** From 11018f1e3d1d49bc84865ceaf69118e2196cf5b8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 21 Mar 2026 23:02:20 +0800 Subject: [PATCH 03/25] spt --- .../db/pipe/source/dataregion/IoTDBDataRegionSource.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index a6d47bc19752..5653b81b48e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -561,7 +561,7 @@ public void customize( @Override protected void login(final @Nonnull String password) { if (!pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { - final boolean useEncryptedPassword = regionId >= 0; + final boolean useEncryptedPassword = regionId >= 0; if (SessionManager.getInstance() .login( new InternalClientSession("Source_login_session_" + regionId), @@ -575,7 +575,9 @@ protected void login(final @Nonnull String password) { .getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new PipePasswordCheckException( - String.format("Failed to check password for pipe %s, useEncryptedPassword: %s", pipeName, useEncryptedPassword)); + String.format( + "Failed to check password for pipe %s, useEncryptedPassword: %s", + pipeName, useEncryptedPassword)); } } } From da5134db875f8cd9083ccb1ca46ce5fab35f16f4 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 21 Mar 2026 23:07:31 +0800 Subject: [PATCH 04/25] added-logger --- .../java/org/apache/iotdb/db/auth/BasicAuthorityCache.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java index b90469b6f70f..ac1d4783613e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java @@ -64,11 +64,13 @@ public void putUserCache(String userName, User user) { AuthorityChecker.setSuperUser(user.getName()); } userCache.put(userName, user); + LOGGER.info("Put user cache, user: {}", user); } @Override public void putRoleCache(String roleName, Role role) { roleCache.put(roleName, role); + LOGGER.info("Put role cache, role: {}", role); } /** @@ -85,8 +87,10 @@ public boolean invalidateCache(final String userName, final String roleName) { Set roleSet = userCache.getIfPresent(userName).getRoleSet(); if (!roleSet.isEmpty()) { roleCache.invalidateAll(roleSet); + LOGGER.info("Invalidated cache for roles {}", roleSet); } userCache.invalidate(userName); + LOGGER.info("Invalidated cache for user {}", userName); } if (userCache.getIfPresent(userName) != null) { LOGGER.error("datanode cache initialization failed"); @@ -96,6 +100,7 @@ public boolean invalidateCache(final String userName, final String roleName) { if (roleName != null) { if (roleCache.getIfPresent(roleName) != null) { roleCache.invalidate(roleName); + LOGGER.info("Invalidated cache for role {}", roleName); } if (roleCache.getIfPresent(roleName) != null) { LOGGER.error("datanode cache initialization failed"); @@ -109,5 +114,6 @@ public boolean invalidateCache(final String userName, final String roleName) { public void invalidAllCache() { userCache.invalidateAll(); roleCache.invalidateAll(); + LOGGER.info("Invalidated all users and roles cache."); } } From 11a6e0a4070d258578e2b6d9f9e99c7c359c6e86 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 21 Mar 2026 23:11:16 +0800 Subject: [PATCH 05/25] sptls --- .../main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java index ac1d4783613e..15e56f5303c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/BasicAuthorityCache.java @@ -114,6 +114,6 @@ public boolean invalidateCache(final String userName, final String roleName) { public void invalidAllCache() { userCache.invalidateAll(); roleCache.invalidateAll(); - LOGGER.info("Invalidated all users and roles cache."); + LOGGER.info("Invalidated all users and roles cache."); } } From e27816d83cace9329de367e7e62455077c63e71f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 23 Mar 2026 09:49:51 +0800 Subject: [PATCH 06/25] fix --- .../it/dual/treemodel/manual/IoTDBPipePermissionIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java index 24807622cf6b..789df51254d0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java @@ -311,7 +311,7 @@ public void testSourcePermission() { receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); fail("Shall fail if password is wrong."); } catch (final SQLException e) { - Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); + Assert.assertEquals("801: Failed to check password for pipe a2b, useEncryptedPassword: false", e.getMessage()); } // Use current session, user is root @@ -538,7 +538,7 @@ public void testIllegalPassword() throws Exception { receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); fail(); } catch (final Exception e) { - Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); + Assert.assertEquals("801: Failed to check password for pipe a2b, useEncryptedPassword: false", e.getMessage()); } try { @@ -546,7 +546,7 @@ public void testIllegalPassword() throws Exception { "create pipe a2b ('sink'='write-back-sink', 'user'='thulab', 'password'='passwd')"); fail(); } catch (final Exception e) { - Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); + Assert.assertEquals("801: Failed to check password for pipe a2b", e.getMessage()); } statement.execute( From 2352c311e54e9c4ff6f135ac45814fc61be3973e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 23 Mar 2026 09:51:02 +0800 Subject: [PATCH 07/25] spt --- .../it/dual/treemodel/manual/IoTDBPipePermissionIT.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java index 789df51254d0..d82b2a36d0be 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java @@ -311,7 +311,9 @@ public void testSourcePermission() { receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); fail("Shall fail if password is wrong."); } catch (final SQLException e) { - Assert.assertEquals("801: Failed to check password for pipe a2b, useEncryptedPassword: false", e.getMessage()); + Assert.assertEquals( + "801: Failed to check password for pipe a2b, useEncryptedPassword: false", + e.getMessage()); } // Use current session, user is root @@ -538,7 +540,9 @@ public void testIllegalPassword() throws Exception { receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); fail(); } catch (final Exception e) { - Assert.assertEquals("801: Failed to check password for pipe a2b, useEncryptedPassword: false", e.getMessage()); + Assert.assertEquals( + "801: Failed to check password for pipe a2b, useEncryptedPassword: false", + e.getMessage()); } try { From 0ad093d8422a86df21b26222edbdb3258911d450 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 23 Mar 2026 10:25:58 +0800 Subject: [PATCH 08/25] . --- .../pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java index d82b2a36d0be..60c4623517f4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java @@ -550,7 +550,7 @@ public void testIllegalPassword() throws Exception { "create pipe a2b ('sink'='write-back-sink', 'user'='thulab', 'password'='passwd')"); fail(); } catch (final Exception e) { - Assert.assertEquals("801: Failed to check password for pipe a2b", e.getMessage()); + Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); } statement.execute( From fd62741a49816ec9b108d41e3ac396d9d712b2ff Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 23 Mar 2026 10:29:20 +0800 Subject: [PATCH 09/25] run --- .../pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java index 60c4623517f4..c4d6da211fa1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java @@ -595,7 +595,9 @@ public void testIllegalPassword() throws Exception { try { statement.execute("alter pipe a2b modify source ('password'='fake')"); } catch (final SQLException e) { - Assert.assertEquals("801: Failed to check password for pipe a2b.", e.getMessage()); + Assert.assertEquals( + "801: Failed to check password for pipe a2b, useEncryptedPassword: false", + e.getMessage()); } statement.execute("alter pipe a2b modify source ('password'='newST@ongPassword')"); From 6fa792aba327fe11b52cfdc026a429fe59df2d56 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 23 Mar 2026 10:36:03 +0800 Subject: [PATCH 10/25] ci-fun --- .github/workflows/cluster-it-1c1d.yml | 89 -- .github/workflows/cluster-it-1c1d1a.yml | 60 -- .github/workflows/cluster-it-1c3d.yml | 69 -- .github/workflows/compile-check.yml | 55 - .github/workflows/daily-it.yml | 1079 ------------------- .github/workflows/daily-ut.yml | 59 - .github/workflows/dependency-check.yml | 62 -- .github/workflows/greeting-ainode.yml | 52 - .github/workflows/multi-language-client.yml | 177 --- .github/workflows/sonar-codecov.yml | 89 -- .github/workflows/table-cluster-it-1c1d.yml | 90 -- .github/workflows/table-cluster-it-1c3d.yml | 69 -- .github/workflows/todos-check.yml | 54 - .github/workflows/unit-test.yml | 71 -- .github/workflows/vulnerability-check.yml | 50 - 15 files changed, 2125 deletions(-) delete mode 100644 .github/workflows/cluster-it-1c1d.yml delete mode 100644 .github/workflows/cluster-it-1c1d1a.yml delete mode 100644 .github/workflows/cluster-it-1c3d.yml delete mode 100644 .github/workflows/compile-check.yml delete mode 100644 .github/workflows/daily-it.yml delete mode 100644 .github/workflows/daily-ut.yml delete mode 100644 .github/workflows/dependency-check.yml delete mode 100644 .github/workflows/greeting-ainode.yml delete mode 100644 .github/workflows/multi-language-client.yml delete mode 100644 .github/workflows/sonar-codecov.yml delete mode 100644 .github/workflows/table-cluster-it-1c1d.yml delete mode 100644 .github/workflows/table-cluster-it-1c3d.yml delete mode 100644 .github/workflows/todos-check.yml delete mode 100644 .github/workflows/unit-test.yml delete mode 100644 .github/workflows/vulnerability-check.yml diff --git a/.github/workflows/cluster-it-1c1d.yml b/.github/workflows/cluster-it-1c1d.yml deleted file mode 100644 index 4ab201450e69..000000000000 --- a/.github/workflows/cluster-it-1c1d.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Cluster IT - 1C1D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Adjust network dynamic TCP ports range - if: ${{ runner.os == 'Windows' }} - shell: pwsh - run: | - netsh int ipv4 set dynamicport tcp start=32768 num=32768 - netsh int ipv4 set dynamicport udp start=32768 num=32768 - netsh int ipv6 set dynamicport tcp start=32768 num=32768 - netsh int ipv6 set dynamicport udp start=32768 num=32768 - - name: Adjust Linux kernel somaxconn - if: ${{ runner.os == 'Linux' }} - shell: bash - run: sudo sysctl -w net.core.somaxconn=65535 - # - name: Adjust Mac kernel somaxconn - # if: ${{ runner.os == 'macOS' }} - # shell: bash - # run: sudo sysctl -w kern.ipc.somaxconn=65535 - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=2 \ - -pl integration-test \ - -am - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: standalone-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/cluster-it-1c1d1a.yml b/.github/workflows/cluster-it-1c1d1a.yml deleted file mode 100644 index 7ee6ca6ace66..000000000000 --- a/.github/workflows/cluster-it-1c1d1a.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Cluster IT - 1C1D1A - -on: - push: - branches: - - master - - 'rel/*' - - 'rc/*' - paths-ignore: - - 'docs/**' - - 'site/**' - pull_request: - branches: - - master - - 'rel/*' - - 'rc/*' - - 'force_ci/**' - paths-ignore: - - 'docs/**' - - 'site/**' - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - AINode: - strategy: - fail-fast: false - max-parallel: 1 - matrix: - os: [ ubuntu-latest ] - runs-on: [self-hosted, gpu] - - steps: - - uses: actions/checkout@v5 - - name: IT Test - shell: bash - run: | - mvn clean verify \ - -P with-integration-tests,with-ainode \ - -DskipUTs \ - -DintegrationTest.forkCount=1 \ - -pl integration-test,iotdb-core/ainode \ - -am \ - -PAIClusterIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-ainode-${{ matrix.os }} - path: integration-test/target/*-logs - retention-days: 30 diff --git a/.github/workflows/cluster-it-1c3d.yml b/.github/workflows/cluster-it-1c3d.yml deleted file mode 100644 index 035403e1695f..000000000000 --- a/.github/workflows/cluster-it-1c3d.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Cluster IT - 1C3D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 20 - matrix: - java: [17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PClusterIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/compile-check.yml b/.github/workflows/compile-check.yml deleted file mode 100644 index 48e0571fc788..000000000000 --- a/.github/workflows/compile-check.yml +++ /dev/null @@ -1,55 +0,0 @@ -# This workflow will compile IoTDB under jdk8 to check for compatibility issues - -name: Compile Check - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - compile-check: - strategy: - fail-fast: false - matrix: - java: [8] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Compiler Test - shell: bash - run: | - mvn clean package -P with-integration-tests -DskipTests -ntp diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml deleted file mode 100644 index 54b502bcfa58..000000000000 --- a/.github/workflows/daily-it.yml +++ /dev/null @@ -1,1079 +0,0 @@ -name: Daily IT - -on: - schedule: - # Run at UTC 19:00 every day (CST 03:00 AM) - - cron: "0 19 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 20 - matrix: - java: [8, 17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PDailyIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 3 - SingleRegionTableModel: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [8, 17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 -DintegrationTest.dataRegionPerDataNode=1 \ - -pl integration-test \ - -am -PTableSimpleIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 3 - PipeSingle: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - ] - os: [ubuntu-latest] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2BatchMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2BatchMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2StreamMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2StreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT1 \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTreeAutoBasic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTreeAutoEnhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - ] - os: [ubuntu-latest] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2BatchMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2BatchMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2StreamMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2StreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTreeManual: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - ] - os: [ubuntu-latest] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2BatchMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2BatchMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2StreamMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2StreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeManual \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTreeArchVerification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - ScalableSingleNodeMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTableArchVerification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTableArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTreeRegressionConsumer: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: - [ - ScalableSingleNodeMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTreeRegressionMisc: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: - [ - ScalableSingleNodeMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTableManualBasic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTableManualEnhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 diff --git a/.github/workflows/daily-ut.yml b/.github/workflows/daily-ut.yml deleted file mode 100644 index 348276962d2b..000000000000 --- a/.github/workflows/daily-ut.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Daily UT - -on: - schedule: - # Run at UTC 19:00 every day (CST 03:00 AM) - - cron: "0 19 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - unit-test: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [8] - os: [ubuntu-latest, windows-latest] - it_task: ["others", "datanode"] - include: - - java: 17 - os: macos-latest - it_task: "datanode" - - java: 17 - os: macos-latest - it_task: "others" - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Test Datanode Module with Maven - shell: bash - if: ${{ matrix.it_task == 'datanode'}} - run: mvn clean integration-test -Dtest.port.closed=true -pl iotdb-core/datanode -am -DskipTests -Diotdb.test.only=true - - name: Test Other Modules with Maven - shell: bash - if: ${{ matrix.it_task == 'others'}} - run: | - mvn clean install -DskipTests - mvn -P get-jar-with-dependencies,with-integration-tests clean test -Dtest.port.closed=true -Diotdb.test.skip=true diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml deleted file mode 100644 index 4cf6cb202afc..000000000000 --- a/.github/workflows/dependency-check.yml +++ /dev/null @@ -1,62 +0,0 @@ -# This workflow will check if dependencies have changed (adding new dependencies or removing existing ones) - -name: Dependency Check - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - dependency-check: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Do the dependency check - shell: bash - run: mvn verify -Dmaven.test.skip=true -Dmdep.analyze.skip=true -P enable-sbom-check diff --git a/.github/workflows/greeting-ainode.yml b/.github/workflows/greeting-ainode.yml deleted file mode 100644 index 53deb59f1ce0..000000000000 --- a/.github/workflows/greeting-ainode.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: AINode Code Style Check - -on: - push: - branches: - - master - - "rc/*" - paths: - - 'iotdb-core/ainode/**' - pull_request: - branches: - - master - - "rc/*" - paths: - - 'iotdb-core/ainode/**' - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - -jobs: - check-style: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Set up Python 3.10 - uses: actions/setup-python@v6 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip3 install black==25.1.0 isort==6.0.1 - - name: Check code formatting (Black) - run: | - cd iotdb-core/ainode - black --check . - continue-on-error: false - - - name: Check import order (Isort) - run: | - cd iotdb-core/ainode - isort --check-only --profile black . - continue-on-error: false diff --git a/.github/workflows/multi-language-client.yml b/.github/workflows/multi-language-client.yml deleted file mode 100644 index 831e3e37639b..000000000000 --- a/.github/workflows/multi-language-client.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: Multi-Language Client -on: - push: - branches: - - master - - "rc/*" - paths: - - 'pom.xml' - - 'iotdb-client/pom.xml' - - 'iotdb-client/client-py/**' - - 'iotdb-client/client-cpp/**' - - 'example/client-cpp-example/**' - - 'iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift' - - 'iotdb-protocol/thrift-commons/src/main/thrift/common.thrift' - - '.github/workflows/multi-language-client.yml' - pull_request: - branches: - - master - - "rc/*" - - 'force_ci/**' - paths: - - 'pom.xml' - - 'iotdb-client/pom.xml' - - 'iotdb-client/client-py/**' - - 'iotdb-client/client-cpp/**' - - 'example/client-cpp-example/**' - - 'iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift' - - 'iotdb-protocol/thrift-commons/src/main/thrift/common.thrift' - - '.github/workflows/multi-language-client.yml' - # allow manually run the action: - workflow_dispatch: - - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - -jobs: - cpp: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - os: [ubuntu-22.04, ubuntu-24.04, windows-2022, windows-latest, windows-2025-vs2026, macos-latest] - runs-on: ${{ matrix.os}} - - steps: - - uses: actions/checkout@v5 - - name: Install CPP Dependencies (Ubuntu) - if: runner.os == 'Linux' - shell: bash - run: | - sudo apt-get update - sudo apt-get install libboost-all-dev - sudo apt-get install openssl libssl-dev - - name: Install CPP Dependencies (Mac) - # remove some xcode to release disk space - if: runner.os == 'macOS' - shell: bash - run: | - brew install boost - brew install openssl - sudo rm -rf /Applications/Xcode_14.3.1.app - sudo rm -rf /Applications/Xcode_15.0.1.app - sudo rm -rf /Applications/Xcode_15.1.app - sudo rm -rf /Applications/Xcode_15.2.app - sudo rm -rf /Applications/Xcode_15.3.app - - name: Install CPP Dependencies (Windows) - if: runner.os == 'Windows' - run: | - choco install winflexbison3 - choco install boost-msvc-14.3 - $boost_path = (Get-ChildItem -Path 'C:\local\' -Filter 'boost_*').FullName - echo $boost_path >> $env:GITHUB_PATH - - choco install openssl - $sslPath = (Get-ChildItem 'C:\Program Files\OpenSSL*' -Directory | Select-Object -First 1).FullName - echo "$sslPath\bin" >> $env:GITHUB_PATH - echo "OPENSSL_ROOT_DIR=$sslPath" >> $env:GITHUB_ENV - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Build IoTDB server - shell: bash - run: ./mvnw clean install -pl distribution -am -DskipTests - - name: Test with Maven - shell: bash - # Explicitly using mvnw here as the build requires maven 3.9 and the default installation is older - # Explicitly using "install" instead of package in order to be sure we're using libs built on this machine - # (was causing problems on windows, but could cause problem on linux, when updating the thrift module) - run: | - if [[ "${{ matrix.os }}" == "windows-2025-vs2026" ]]; then - ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am -Dcmake.generator="Visual Studio 18 2026" - else - ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am - fi - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cpp-IT-${{ runner.os }} - path: distribution/target/apache-iotdb-*-all-bin/apache-iotdb-*-all-bin/logs - retention-days: 1 - - go: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - with: - token: ${{secrets.GITHUB_TOKEN}} - submodules: recursive - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Compile IoTDB Server - run: mvn clean package -pl distribution -am -DskipTests - - name: Integration test - shell: bash - run: | - cd iotdb-client - git clone https://github.com/apache/iotdb-client-go.git - cd iotdb-client-go - make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo - - python: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - python: ['3.x'] - runs-on: ${{ 'ubuntu-latest' }} - - steps: - - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python }} - - uses: actions/checkout@v5 - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Cache pip packages - uses: actions/cache@v5 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: ${{ runner.os }}-pip- - - name: Build IoTDB server distribution zip and python client - run: mvn -B clean install -pl distribution,iotdb-client/client-py -am -DskipTests - - name: Build IoTDB server docker image - run: | - docker build . -f docker/src/main/Dockerfile-1c1d -t "iotdb:dev" - docker images - - name: Install IoTDB python client requirements - run: pip3 install -r iotdb-client/client-py/requirements_dev.txt - - name: Check code style - if: ${{ matrix.python == '3.x'}} - shell: bash - run: black iotdb-client/client-py/ --check --diff - - name: Integration test and test make package - shell: bash - run: | - cd iotdb-client/client-py/ && pytest . - ./release.sh diff --git a/.github/workflows/sonar-codecov.yml b/.github/workflows/sonar-codecov.yml deleted file mode 100644 index adb943a7f62e..000000000000 --- a/.github/workflows/sonar-codecov.yml +++ /dev/null @@ -1,89 +0,0 @@ -# This workflow will build a Java project with Maven -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: Sonar-Codecov - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "new_*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - PR_NUMBER: ${{ github.event.number }} - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - codecov: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'apache/iotdb' || github.event_name == 'push' - - steps: - - uses: actions/checkout@v5 - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Test - run: | - mvn -B -P with-code-coverage clean package -pl distribution,iotdb-client/cli,iotdb-client/session,iotdb-client/jdbc -am -Dtest.port.closed=true - mvn -B -P with-code-coverage post-integration-test -pl code-coverage - - name: Upload coverage reports to codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./code-coverage/target/jacoco-merged-reports/jacoco.xml - - sonar: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'apache/iotdb' || github.event_name == 'push' - steps: - - uses: actions/checkout@v5 - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: SonarCloud Report - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} - run: | - mvn -B -P with-integration-tests,with-code-coverage verify sonar:sonar \ - -Dsonar.organization=apache \ - -Dsonar.projectKey=apache_iotdb \ - -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.token=${{ secrets.SONARCLOUD_TOKEN }} \ - -DskipTests -pl '!distribution,!integration-test' -am diff --git a/.github/workflows/table-cluster-it-1c1d.yml b/.github/workflows/table-cluster-it-1c1d.yml deleted file mode 100644 index 782bafa4ddbe..000000000000 --- a/.github/workflows/table-cluster-it-1c1d.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Table Cluster IT - 1C1D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Adjust network dynamic TCP ports range - if: ${{ runner.os == 'Windows' }} - shell: pwsh - run: | - netsh int ipv4 set dynamicport tcp start=32768 num=32768 - netsh int ipv4 set dynamicport udp start=32768 num=32768 - netsh int ipv6 set dynamicport tcp start=32768 num=32768 - netsh int ipv6 set dynamicport udp start=32768 num=32768 - - name: Adjust Linux kernel somaxconn - if: ${{ runner.os == 'Linux' }} - shell: bash - run: sudo sysctl -w net.core.somaxconn=65535 - # - name: Adjust Mac kernel somaxconn - # if: ${{ runner.os == 'macOS' }} - # shell: bash - # run: sudo sysctl -w kern.ipc.somaxconn=65535 - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PTableSimpleIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/table-cluster-it-1c3d.yml b/.github/workflows/table-cluster-it-1c3d.yml deleted file mode 100644 index 99a7469500c9..000000000000 --- a/.github/workflows/table-cluster-it-1c3d.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Table Cluster IT - 1C3D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 20 - matrix: - java: [17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PTableClusterIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: table-cluster-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/todos-check.yml b/.github/workflows/todos-check.yml deleted file mode 100644 index fc33b12d3cb5..000000000000 --- a/.github/workflows/todos-check.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Check TODOs and FIXMEs in Changed Files - -on: - pull_request: - branches: - - master - - 'dev/*' - - 'rel/*' - - "rc/*" - - 'force_ci/**' - paths-ignore: - - 'docs/**' - - 'site/**' - # allow manually run the action: - workflow_dispatch: - -jobs: - todo-check: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Check for TODOs and FIXMEs in changed files - run: | - # Fetch the target branch - git fetch origin $GITHUB_BASE_REF - - git switch -c check_branch - - # Get the diff of the changes - echo Get the diff of the changes - DIFF=$(git diff origin/$GITHUB_BASE_REF check_branch -- . ':(exclude).github/workflows/todos-check.yml') - - if [ -z "$DIFF" ]; then - echo "No changes detected." - exit 0 - fi - - - # Check the diff for TODOs - - # Check the diff for TODOs - echo Check the diff for TODOs - TODOsCOUNT=$(echo "$DIFF" | grep -E '^\+.*(TODO|FIXME)' | wc -l) - if [ "$TODOsCOUNT" -eq 0 ]; then - echo "No TODOs or FIXMEs found in changed content."; - exit 0 - fi - - echo "TODO or FIXME found in the changes. Please resolve it before merging." - echo "$DIFF" | grep -E '^\+.*(TODO|FIXME)' | tee -a output.log - exit 1 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml deleted file mode 100644 index c53c8d89cfa9..000000000000 --- a/.github/workflows/unit-test.yml +++ /dev/null @@ -1,71 +0,0 @@ -# This workflow will build a Java project with Maven -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: Unit-Test - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xms2g -Xmx4g -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - unit-test: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - os: [ubuntu-latest, windows-latest] - it_task: ["others", "datanode"] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Test Datanode Module with Maven - shell: bash - if: ${{ matrix.it_task == 'datanode'}} - run: mvn clean integration-test -Dtest.port.closed=true -pl iotdb-core/datanode -am -DskipTests -Diotdb.test.only=true - - name: Test Other Modules with Maven - shell: bash - if: ${{ matrix.it_task == 'others'}} - run: | - mvn clean install -DskipTests - mvn -P get-jar-with-dependencies,with-integration-tests clean test -Dtest.port.closed=true -Diotdb.test.skip=true diff --git a/.github/workflows/vulnerability-check.yml b/.github/workflows/vulnerability-check.yml deleted file mode 100644 index c120bfa5291a..000000000000 --- a/.github/workflows/vulnerability-check.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: vulnerability-check -on: - schedule: - # Run at 16:00 UTC every Sunday (Monday 00:00 CST) - - cron: "0 16 * * 0" - workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - dependency-check: - if: ${{ github.event_name == 'workflow_dispatch' || github.repository == 'apache/iotdb' }} - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - - - name: Do Maven install - shell: bash - run: mvn $MAVEN_ARGS clean install -DskipTests - - - name: Do the dependency-check:aggregate - shell: bash - run: mvn $MAVEN_ARGS org.owasp:dependency-check-maven:aggregate -DossIndexUsername=${{ secrets.OSS_INDEX_USER }} -DossIndexPassword=${{ secrets.OSS_INDEX_TOKEN }} -DnvdApiKey=${{ secrets.NVD_API_KEY }} - - - name: Generate report date for artifact name - run: | - utc_time="${{ github.run_started_at }}" - target_time=$(TZ=Asia/Shanghai date -d "$utc_time" +"%Y-%m-%d") - echo "REPORT_DATE=$target_time" >> $GITHUB_ENV - - - name: Upload Artifact - uses: actions/upload-artifact@v6 - with: - name: vulnerability-check-result-${{ env.REPORT_DATE }} - path: target/dependency-check-report.html - retention-days: 15 From 4b275c1ff940c0f806f3f58a0916684c16bb7c22 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 23 Mar 2026 10:37:40 +0800 Subject: [PATCH 11/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 988 ++-------------------------------- 1 file changed, 58 insertions(+), 930 deletions(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 0968e7739a05..1c1958ad1dcd 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -1,31 +1,16 @@ -name: Multi-Cluster IT +name: Pipe IT Debug (Single Test Loop) on: push: branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - - "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**" #queryengine + - "**" # 只要有代码推送就跑，方便你在 PR 上触发 pull_request: branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - - "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**" #queryengine - # allow manually run the action: - workflow_dispatch: + - "**" concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: true # 如果有新的提交，取消旧的构建，节省资源 env: MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 @@ -33,104 +18,18 @@ env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: - single: + debug-specific-it: + name: Run IoTDBPipePermissionIT in Loop + runs-on: ubuntu-latest strategy: fail-fast: false - max-parallel: 15 matrix: java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. cluster1: [HighPerformanceMode] cluster2: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT1 \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-tree-auto-basic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v5 with: @@ -138,841 +37,70 @@ jobs: java-version: ${{ matrix.java }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-tree-auto-enhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [HighPerformanceMode] - cluster2: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Cache Maven packages uses: actions/cache@v5 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-tree-manual: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [HighPerformanceMode] - cluster2: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeManual \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-tree-arch-verification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test + - name: Run Test in Loop Until Failure shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ + # 确保日志目录存在 + mkdir -p integration-test/target/cluster-logs + + local -i attempt=1 + + while true; do + echo "====================================================" + echo "Starting attempt #$attempt" + echo "====================================================" + + # 注意：这里使用 mvn verify 或 test，重点是 -Dtest 参数 + # 请根据你的项目模块结构调整 -pl 参数，如果 integration-test 是根模块则不需要 -pl + # 我保留了你原有的 -PMultiClusterIT2DualTreeManual Profile，请确认这是正确的 Profile + + mvn clean verify \ + -P with-integration-tests,MultiClusterIT2DualTreeManual \ -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -Dtest=IoTDBPipePermissionIT#testIllegalPassword \ + -DintegrationTest.forkCount=1 \ + -DConfigNodeMaxHeapSize=256 \ + -DDataNodeMaxHeapSize=1024 \ + -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-table-arch-verification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTableArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-tree-regression-consumer: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-tree-regression-misc: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-table-manual-basic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-table-manual-enhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - triple: - strategy: - fail-fast: false - max-parallel: 1 - matrix: - java: [ 17 ] - cluster1: [ ScalableSingleNodeMode ] - cluster2: [ ScalableSingleNodeMode ] - cluster3: [ ScalableSingleNodeMode ] - os: [ ubuntu-latest ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }},${{ matrix.cluster3 }} \ - -pl integration-test \ - -am -PMultiClusterIT3 \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact + -am \ + -ntp + + EXIT_CODE=$? + + # 收集日志（每次跑完都把当前的日志存一下，防止 runner 被重置） + if [ -d "integration-test/target/cluster-logs" ]; then + # 简单的打包一下当前状态，防止文件被 clean 清空 + tar -czf integration-test/target/last-run-logs-$attempt.tar.gz integration-test/target/cluster-logs || true + fi + + if [ $EXIT_CODE -ne 0 ]; then + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "FAIL: Test failed on attempt #$attempt" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + exit 1 + else + echo "PASS: Attempt #$attempt succeeded. Looping again immediately..." + attempt=$((attempt + 1)) + # 如果你想让机器歇一会，可以在这加个 sleep 10 + fi + done + + - name: Upload Failure Logs if: failure() uses: actions/upload-artifact@v6 with: - name: cluster-log-triple-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}-${{ matrix.cluster3 }} - path: integration-test/target/cluster-logs - retention-days: 30 + name: failure-logs-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: | + integration-test/target/cluster-logs + integration-test/target/last-run-logs-*.tar.gz + retention-days: 30 \ No newline at end of file From 9579d8da80d2e128b538f86f3f57760a7085de57 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 23 Mar 2026 10:40:00 +0800 Subject: [PATCH 12/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 1c1958ad1dcd..1986114f3543 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -51,7 +51,7 @@ jobs: # 确保日志目录存在 mkdir -p integration-test/target/cluster-logs - local -i attempt=1 + attempt=1 while true; do echo "====================================================" From 680777639beca0abc03f5bf9a14e62a083f00429 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 23 Mar 2026 10:45:15 +0800 Subject: [PATCH 13/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 1986114f3543..4f1ad4a4b6f5 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -70,6 +70,7 @@ jobs: -DConfigNodeMaxHeapSize=256 \ -DDataNodeMaxHeapSize=1024 \ -DDataNodeMaxDirectMemorySize=768 \ + -Dsurefire.failIfNoSpecifiedTests=false \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ -pl integration-test \ -am \ From c238ea143114e631207c6be00ebdf719fdc544fb Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 23 Mar 2026 11:01:19 +0800 Subject: [PATCH 14/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 92 ++++++++++++----------------------- 1 file changed, 32 insertions(+), 60 deletions(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 4f1ad4a4b6f5..e997084cd6ef 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -1,107 +1,79 @@ -name: Pipe IT Debug (Single Test Loop) +name: Pipe IT Debug (Final Stable) on: push: - branches: - - "**" # 只要有代码推送就跑，方便你在 PR 上触发 + branches: [ "**" ] pull_request: - branches: - - "**" + branches: [ "**" ] concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true # 如果有新的提交，取消旧的构建，节省资源 + cancel-in-progress: true env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: debug-specific-it: - name: Run IoTDBPipePermissionIT in Loop runs-on: ubuntu-latest + name: Loop testIllegalPassword strategy: - fail-fast: false matrix: java: [17] cluster1: [HighPerformanceMode] cluster2: [HighPerformanceMode] + steps: - uses: actions/checkout@v5 - - - name: Set up JDK ${{ matrix.java }} + - name: Set up JDK uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: distribution: corretto, java-version: ${{ matrix.java }} - - name: Cache Maven packages + - name: Cache Maven uses: actions/cache@v5 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Run Test in Loop Until Failure - shell: bash + # ===== 关键修复 1/2：只在最开始做一次全量构建，生成 template-node ===== + - name: Build Once and Generate Template + run: | + mvn clean package \ + -P with-integration-tests,MultiClusterIT2DualTreeManual \ + -pl integration-test -am \ + -DskipTests -ntp + + # ===== 关键修复 2/2：循环里绝对不 clean，直接跑 test ===== + - name: Run Test in Loop run: | - # 确保日志目录存在 - mkdir -p integration-test/target/cluster-logs - attempt=1 - while true; do - echo "====================================================" - echo "Starting attempt #$attempt" - echo "====================================================" - - # 注意：这里使用 mvn verify 或 test，重点是 -Dtest 参数 - # 请根据你的项目模块结构调整 -pl 参数，如果 integration-test 是根模块则不需要 -pl - # 我保留了你原有的 -PMultiClusterIT2DualTreeManual Profile，请确认这是正确的 Profile + echo "--- Attempt $attempt ---" - mvn clean verify \ + # 注意：这里没有 clean，用的是 test 而不是 verify，速度更快 + mvn test \ -P with-integration-tests,MultiClusterIT2DualTreeManual \ - -DskipUTs \ -Dtest=IoTDBPipePermissionIT#testIllegalPassword \ -DintegrationTest.forkCount=1 \ -DConfigNodeMaxHeapSize=256 \ -DDataNodeMaxHeapSize=1024 \ -DDataNodeMaxDirectMemorySize=768 \ - -Dsurefire.failIfNoSpecifiedTests=false \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am \ - -ntp - - EXIT_CODE=$? + -pl integration-test -ntp - # 收集日志（每次跑完都把当前的日志存一下，防止 runner 被重置） - if [ -d "integration-test/target/cluster-logs" ]; then - # 简单的打包一下当前状态，防止文件被 clean 清空 - tar -czf integration-test/target/last-run-logs-$attempt.tar.gz integration-test/target/cluster-logs || true - fi - - if [ $EXIT_CODE -ne 0 ]; then - echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" - echo "FAIL: Test failed on attempt #$attempt" - echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + if [ $? -ne 0 ]; then + echo "FAILED at $attempt" exit 1 - else - echo "PASS: Attempt #$attempt succeeded. Looping again immediately..." - attempt=$((attempt + 1)) - # 如果你想让机器歇一会，可以在这加个 sleep 10 fi + + echo "PASSED $attempt" + attempt=$((attempt + 1)) done - - name: Upload Failure Logs + - name: Upload Logs if: failure() uses: actions/upload-artifact@v6 with: - name: failure-logs-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: | - integration-test/target/cluster-logs - integration-test/target/last-run-logs-*.tar.gz - retention-days: 30 \ No newline at end of file + name: failure-logs + path: integration-test/target/cluster-logs \ No newline at end of file From e5073f748e52b9ea69f873b63febec9627d55683 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 23 Mar 2026 11:04:33 +0800 Subject: [PATCH 15/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index e997084cd6ef..9c04f17c52fd 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -28,7 +28,9 @@ jobs: - uses: actions/checkout@v5 - name: Set up JDK uses: actions/setup-java@v5 - with: distribution: corretto, java-version: ${{ matrix.java }} + with: + distribution: corretto + java-version: ${{ matrix.java }} - name: Cache Maven uses: actions/cache@v5 From 7bf7c2993497e4d564b0efa150df80db39c7c82e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 24 Mar 2026 17:24:09 +0800 Subject: [PATCH 16/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 9c04f17c52fd..fb9a4d6b267a 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -38,31 +38,23 @@ jobs: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - # ===== 关键修复 1/2：只在最开始做一次全量构建，生成 template-node ===== - - name: Build Once and Generate Template - run: | - mvn clean package \ - -P with-integration-tests,MultiClusterIT2DualTreeManual \ - -pl integration-test -am \ - -DskipTests -ntp - - # ===== 关键修复 2/2：循环里绝对不 clean，直接跑 test ===== - name: Run Test in Loop run: | attempt=1 while true; do echo "--- Attempt $attempt ---" - - # 注意：这里没有 clean，用的是 test 而不是 verify，速度更快 - mvn test \ - -P with-integration-tests,MultiClusterIT2DualTreeManual \ + + mvn clean verify \ + -P with-integration-tests \ -Dtest=IoTDBPipePermissionIT#testIllegalPassword \ -DintegrationTest.forkCount=1 \ -DConfigNodeMaxHeapSize=256 \ -DDataNodeMaxHeapSize=1024 \ -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test -ntp + -pl integration-test -am -PMultiClusterIT2DualTreeManual \ + -DskipUTs \ + -ntp > ~/run-tests-$attempt.log 2>&1 if [ $? -ne 0 ]; then echo "FAILED at $attempt" From 96d3e7e9f882b9c5251327b285e1fd9940c3d887 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 24 Mar 2026 17:32:18 +0800 Subject: [PATCH 17/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index fb9a4d6b267a..fd3904f6886c 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -53,6 +53,7 @@ jobs: -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ -pl integration-test -am -PMultiClusterIT2DualTreeManual \ + -Dsurefire.failIfNoSpecifiedTests=false -DskipUTs \ -ntp > ~/run-tests-$attempt.log 2>&1 From 34dddfb63591cae04a7f660639732e43a42cef4a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 24 Mar 2026 17:35:30 +0800 Subject: [PATCH 18/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index fd3904f6886c..6d468965cb02 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -53,7 +53,7 @@ jobs: -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ -pl integration-test -am -PMultiClusterIT2DualTreeManual \ - -Dsurefire.failIfNoSpecifiedTests=false + -Dsurefire.failIfNoSpecifiedTests=false \ -DskipUTs \ -ntp > ~/run-tests-$attempt.log 2>&1 From 5eb0ab782cee6796a0d6b2aafc7afe374cfeda9d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 24 Mar 2026 17:37:55 +0800 Subject: [PATCH 19/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 6d468965cb02..f787a78182f4 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -41,6 +41,11 @@ jobs: - name: Run Test in Loop run: | attempt=1 + + mvn clean package \ + -P with-integration-tests,MultiClusterIT2DualTreeManual \ + -DskipTests -DskipUTs -ntp + while true; do echo "--- Attempt $attempt ---" From 4282a8fe554a05d6f5ad68458c11362d4e64a8bf Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 24 Mar 2026 17:38:48 +0800 Subject: [PATCH 20/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index f787a78182f4..3a748f845f2d 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -42,9 +42,9 @@ jobs: run: | attempt=1 - mvn clean package \ - -P with-integration-tests,MultiClusterIT2DualTreeManual \ - -DskipTests -DskipUTs -ntp + mvn clean package \ + -P with-integration-tests,MultiClusterIT2DualTreeManual \ + -DskipTests -DskipUTs -ntp while true; do echo "--- Attempt $attempt ---" From 70371fcf797151f84c67cfdd07e92616b127b059 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 24 Mar 2026 17:41:37 +0800 Subject: [PATCH 21/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 3a748f845f2d..03eb19e69fd9 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -49,7 +49,7 @@ jobs: while true; do echo "--- Attempt $attempt ---" - mvn clean verify \ + mvn verify \ -P with-integration-tests \ -Dtest=IoTDBPipePermissionIT#testIllegalPassword \ -DintegrationTest.forkCount=1 \ From 4af3951d624be13df23a47c2f020bc56fa873a40 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 24 Mar 2026 19:00:14 +0800 Subject: [PATCH 22/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 03eb19e69fd9..b75c13d35c29 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -42,14 +42,10 @@ jobs: run: | attempt=1 - mvn clean package \ - -P with-integration-tests,MultiClusterIT2DualTreeManual \ - -DskipTests -DskipUTs -ntp - while true; do echo "--- Attempt $attempt ---" - mvn verify \ + mvn clean verify \ -P with-integration-tests \ -Dtest=IoTDBPipePermissionIT#testIllegalPassword \ -DintegrationTest.forkCount=1 \ From 1b7c75a2f5aadf73ee3a79d5d045e824095e363e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 24 Mar 2026 19:01:02 +0800 Subject: [PATCH 23/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index b75c13d35c29..428caed29d55 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -48,7 +48,6 @@ jobs: mvn clean verify \ -P with-integration-tests \ -Dtest=IoTDBPipePermissionIT#testIllegalPassword \ - -DintegrationTest.forkCount=1 \ -DConfigNodeMaxHeapSize=256 \ -DDataNodeMaxHeapSize=1024 \ -DDataNodeMaxDirectMemorySize=768 \ From 79a9fee823930fef7806feee9be6312e45c9ed9c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 25 Mar 2026 09:56:32 +0800 Subject: [PATCH 24/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 121 ++++++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 36 deletions(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 428caed29d55..410ee577e5bd 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -1,74 +1,123 @@ -name: Pipe IT Debug (Final Stable) +name: Multi-Cluster IT on: push: - branches: [ "**" ] + branches: + - master + - "rel/*" + - "rc/*" + paths-ignore: + - "docs/**" + - "site/**" + - "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**" #queryengine pull_request: - branches: [ "**" ] + branches: + - master + - "rel/*" + - "rc/*" + - "force_ci/**" + paths-ignore: + - "docs/**" + - "site/**" + - "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**" #queryengine + # allow manually run the action: + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: - debug-specific-it: - runs-on: ubuntu-latest - name: Loop testIllegalPassword + dual-tree-manual: strategy: + fail-fast: false + max-parallel: 15 matrix: java: [17] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. cluster1: [HighPerformanceMode] cluster2: [HighPerformanceMode] - + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v5 - - name: Set up JDK + - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v5 with: distribution: corretto java-version: ${{ matrix.java }} - - - name: Cache Maven + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Cache Maven packages uses: actions/cache@v5 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - - - name: Run Test in Loop + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds run: | - attempt=1 - - while true; do - echo "--- Attempt $attempt ---" + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output - mvn clean verify \ + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ -P with-integration-tests \ + -DskipUTs \ -Dtest=IoTDBPipePermissionIT#testIllegalPassword \ - -DConfigNodeMaxHeapSize=256 \ - -DDataNodeMaxHeapSize=1024 \ - -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test -am -PMultiClusterIT2DualTreeManual \ -Dsurefire.failIfNoSpecifiedTests=false \ - -DskipUTs \ - -ntp > ~/run-tests-$attempt.log 2>&1 + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT2DualTreeManual \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" - if [ $? -ne 0 ]; then - echo "FAILED at $attempt" - exit 1 - fi - - echo "PASSED $attempt" - attempt=$((attempt + 1)) - done + if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then + echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." + fi - - name: Upload Logs + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact if: failure() uses: actions/upload-artifact@v6 with: - name: failure-logs - path: integration-test/target/cluster-logs \ No newline at end of file + name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: integration-test/target/cluster-logs + retention-days: 30 \ No newline at end of file From 829b10ac9f616a117f899aa64fd24686a3e483cd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 25 Mar 2026 10:21:10 +0800 Subject: [PATCH 25/25] Update pipe-it.yml --- .github/workflows/pipe-it.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml index 410ee577e5bd..f834c884a257 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it.yml @@ -79,7 +79,8 @@ jobs: -DskipUTs \ -Dtest=IoTDBPipePermissionIT#testIllegalPassword \ -Dsurefire.failIfNoSpecifiedTests=false \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DintegrationTest.forkCount=0 -DforkCount=0 \ + -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ -pl integration-test \ -am -PMultiClusterIT2DualTreeManual \