Merge pull request #52 from aliyun/feature/release-v1.1.9-10

Feature/release v1.1.9 10
aliyun · Jan 9, 2025 · a6be5a6 · a6be5a6
2 parents 72c47a0 + 71e2230
commit a6be5a6
Show file tree

Hide file tree

Showing 33 changed files with 283 additions and 224 deletions.
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/client/client-common/pom.xml b/client/client-common/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.aliyun.dataworks</groupId>
         <artifactId>client</artifactId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/client/client-spark-utils/pom.xml b/client/client-spark-utils/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.aliyun.dataworks</groupId>
         <artifactId>client</artifactId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/client/client-toolkits/pom.xml b/client/client-toolkits/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.aliyun.dataworks</groupId>
         <artifactId>client</artifactId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/client/migrationx/migrationx-common/pom.xml b/client/migrationx/migrationx-common/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <artifactId>migrationx</artifactId>
         <groupId>com.aliyun.dataworks</groupId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -41,10 +41,6 @@
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-core</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.hibernate.validator</groupId>
             <artifactId>hibernate-validator</artifactId>

diff --git a/client/migrationx/migrationx-domain/migrationx-domain-adf/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-adf/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>com.aliyun.dataworks</groupId>
         <artifactId>migrationx-domain</artifactId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/client/migrationx/migrationx-domain/migrationx-domain-airflow/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-airflow/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>migrationx-domain</artifactId>
         <groupId>com.aliyun.dataworks</groupId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/client/migrationx/migrationx-domain/migrationx-domain-aliyunemr/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-aliyunemr/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>migrationx-domain</artifactId>
         <groupId>com.aliyun.dataworks</groupId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/client/migrationx/migrationx-domain/migrationx-domain-azkaban/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-azkaban/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>migrationx-domain</artifactId>
         <groupId>com.aliyun.dataworks</groupId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/client/migrationx/migrationx-domain/migrationx-domain-caiyunjian/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-caiyunjian/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>migrationx-domain</artifactId>
         <groupId>com.aliyun.dataworks</groupId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/client/migrationx/migrationx-domain/migrationx-domain-core/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-core/pom.xml
@@ -21,7 +21,7 @@
     <parent>
         <groupId>com.aliyun.dataworks</groupId>
         <artifactId>migrationx-domain</artifactId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/client/migrationx/migrationx-domain/migrationx-domain-datago/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-datago/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>migrationx-domain</artifactId>
         <groupId>com.aliyun.dataworks</groupId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>migrationx-domain</artifactId>
         <groupId>com.aliyun.dataworks</groupId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -56,18 +56,9 @@
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-core</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-beans</artifactId>
-        </dependency>
         <dependency>
             <groupId>com.aliyun</groupId>
             <artifactId>dataworks_public20240518</artifactId>
-            <version>2.0.0</version>
         </dependency>
     </dependencies>
     <build>

diff --git a/...n/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/NodeSpecAdapter.java b/...n/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/NodeSpecAdapter.java
@@ -127,10 +127,19 @@ public Specification<DataWorksWorkflowSpec> getNodeSpecObject(DwNodeEntity dwNod
             } else {
                 BasicNodeSpecHandler nodeSpecHandler = (BasicNodeSpecHandler)getHandler(dwNode, context.getLocale());
                 nodeSpecHandler.setContext(context);
-                dataWorksWorkflowSpec.setFlow(toFlow(nodeSpecHandler, dwNode, context));
-                List<SpecNode> nodes = new ArrayList<>();
-                Optional.ofNullable(nodeSpecHandler.handle(dwNode)).ifPresent(nodes::add);
-                dataWorksWorkflowSpec.setNodes(nodes);
+                if (dwNode.getParentNode() == null) {
+                    dataWorksWorkflowSpec.setFlow(toFlow(nodeSpecHandler, dwNode, context));
+                    List<SpecNode> nodes = new ArrayList<>();
+                    Optional.ofNullable(nodeSpecHandler.handle(dwNode)).ifPresent(nodes::add);
+                    dataWorksWorkflowSpec.setNodes(nodes);
+                } else {
+                    // for inner node
+                    BasicNodeSpecHandler parentNodeHandler = (BasicNodeSpecHandler)getHandler(dwNode.getParentNode(), context.getLocale());
+                    dataWorksWorkflowSpec.setFlow(toFlow(parentNodeHandler, dwNode.getParentNode(), context));
+                    List<SpecNode> nodes = new ArrayList<>();
+                    Optional.ofNullable(parentNodeHandler.handle(dwNode.getParentNode(), Collections.singletonList(dwNode))).ifPresent(nodes::add);
+                    dataWorksWorkflowSpec.setNodes(nodes);
+                }
             }
         });
         return spec;

diff --git a/...va/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/entity/DwNodeEntity.java b/...va/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/entity/DwNodeEntity.java
@@ -435,6 +435,15 @@ default Long getParentId() {
         return null;
     }
 
+    /**
+     * get parent node info
+     *
+     * @return parent node
+     */
+    default DwNodeEntity getParentNode() {
+        return null;
+    }
+
     /**
      * get cu
      *

diff --git a/...aliyun/dataworks/migrationx/domain/dataworks/service/spec/entity/DwNodeEntityAdapter.java b/...aliyun/dataworks/migrationx/domain/dataworks/service/spec/entity/DwNodeEntityAdapter.java
@@ -51,11 +51,17 @@
 @Slf4j
 public class DwNodeEntityAdapter implements DwNodeEntity {
     private final DwNode dwNode;
+    private DwNode dwParentNode;
 
     public DwNodeEntityAdapter(DwNode dwNode) {
         this.dwNode = dwNode;
     }
 
+    public DwNodeEntityAdapter(DwNode dwNode, DwNode dwParentNode) {
+        this.dwNode = dwNode;
+        this.dwParentNode = dwParentNode;
+    }
+
     @Override
     public String getUuid() {
         return dwNode.getGlobalUuid();
@@ -375,6 +381,11 @@ public Long getParentId() {
         return null;
     }
 
+    @Override
+    public DwNodeEntity getParentNode() {
+        return Optional.ofNullable(dwParentNode).map(DwNodeEntityAdapter::new).orElse(null);
+    }
+
     @Override
     public String getCu() {
         return dwNode.getCu();

diff --git a/...iyun/dataworks/migrationx/domain/dataworks/service/spec/handler/BasicNodeSpecHandler.java b/...iyun/dataworks/migrationx/domain/dataworks/service/spec/handler/BasicNodeSpecHandler.java
@@ -109,7 +109,6 @@ protected boolean matchNodeType(DwNodeEntity dwNode, String nodeType) {
             .orElse(false);
     }
 
-    @SuppressWarnings("unchecked")
     @Override
     public SpecNode handle(DwNodeEntity dmNode) {
         Preconditions.checkNotNull(dmNode, "dmNode is null");

diff --git a/...un/dataworks/migrationx/domain/dataworks/service/spec/handler/DowhileNodeSpecHandler.java b/...un/dataworks/migrationx/domain/dataworks/service/spec/handler/DowhileNodeSpecHandler.java
@@ -49,13 +49,24 @@ public SpecNode handle(DwNodeEntity dmNodeBo) {
         SpecNode specNode = super.handle(dmNodeBo);
 
         if (support(dmNodeBo)) {
-            specNode.setDoWhile(buildDoWhile(dmNodeBo, context));
+            List<DwNodeEntity> innerNodes = getInnerNodes(dmNodeBo);
+            specNode.setDoWhile(buildDoWhile(dmNodeBo, innerNodes, context));
         }
         return specNode;
     }
 
-    private SpecDoWhile buildDoWhile(DwNodeEntity orcNode, SpecHandlerContext context) {
-        List<DwNodeEntity> innerNodes = getInnerNodes(orcNode);
+    @Override
+    public SpecNode handle(DwNodeEntity parentNode, List<DwNodeEntity> innerNodes) {
+        Preconditions.checkNotNull(parentNode, "node is null");
+        SpecNode specNode = super.handle(parentNode);
+
+        if (support(parentNode)) {
+            specNode.setDoWhile(buildDoWhile(parentNode, innerNodes, context));
+        }
+        return specNode;
+    }
+
+    private SpecDoWhile buildDoWhile(DwNodeEntity orcNode, List<DwNodeEntity> innerNodes, SpecHandlerContext context) {
         SpecDoWhile specDoWhile = new SpecDoWhile();
         specDoWhile.setMaxIterations(orcNode.getLoopCount());
         specDoWhile.setNodes(innerNodes.stream()

diff --git a/...un/dataworks/migrationx/domain/dataworks/service/spec/handler/ForeachNodeSpecHandler.java b/...un/dataworks/migrationx/domain/dataworks/service/spec/handler/ForeachNodeSpecHandler.java
@@ -51,13 +51,23 @@ public SpecNode handle(DwNodeEntity orcNode) {
         Preconditions.checkNotNull(orcNode, "node is null");
         SpecNode specNode = super.handle(orcNode);
         if (support(orcNode)) {
-            specNode.setForeach(buildForEach(orcNode, specNode, context));
+            List<DwNodeEntity> innerNodes = getInnerNodes(orcNode);
+            specNode.setForeach(buildForEach(orcNode, specNode, innerNodes, context));
         }
         return specNode;
     }
 
-    private SpecForEach buildForEach(DwNodeEntity orcNode, SpecNode specNode, SpecHandlerContext context) {
-        List<DwNodeEntity> innerNodes = getInnerNodes(orcNode);
+    @Override
+    public SpecNode handle(DwNodeEntity parentNode, List<DwNodeEntity> innerNodes) {
+        Preconditions.checkNotNull(parentNode, "node is null");
+        SpecNode specNode = super.handle(parentNode);
+        if (support(parentNode)) {
+            specNode.setForeach(buildForEach(parentNode, specNode, innerNodes, context));
+        }
+        return specNode;
+    }
+
+    private SpecForEach buildForEach(DwNodeEntity orcNode, SpecNode specNode, List<DwNodeEntity> innerNodes, SpecHandlerContext context) {
         SpecForEach specForEach = new SpecForEach();
         specForEach.setNodes(innerNodes.stream()
             .map(n -> getSpecAdapter().getHandler(n, context.getLocale()).handle(n))

diff --git a/...va/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/NodeSpecAdapterTest.java b/...va/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/NodeSpecAdapterTest.java
@@ -16,6 +16,7 @@
 package com.aliyun.dataworks.migrationx.domain.dataworks.service.spec;
 
 import java.util.Arrays;
+import java.util.Collections;
 
 import javax.validation.constraints.NotNull;
 
@@ -27,11 +28,12 @@
 import com.aliyun.dataworks.common.spec.domain.ref.SpecNode;
 import com.aliyun.dataworks.common.spec.domain.ref.component.SpecComponent;
 import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwNode;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwNodeIo;
+import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.NodeIo;
 import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.NodeUseType;
 import com.aliyun.dataworks.migrationx.domain.dataworks.service.spec.entity.DwNodeEntityAdapter;
 import com.aliyun.dataworks.migrationx.domain.dataworks.service.spec.handler.BasicNodeSpecHandler;
 import com.aliyun.dataworks.migrationx.domain.dataworks.service.spec.handler.ForeachNodeSpecHandler;
-
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Assert;
 import org.junit.Test;
@@ -49,7 +51,7 @@ public void testGetHandler() {
         DwNode dwNode = new DwNode();
         dwNode.setType(CodeProgramType.CONTROLLER_TRAVERSE.getName());
         @NotNull
-        BasicNodeSpecHandler handler = (BasicNodeSpecHandler) adapter.getHandler(new DwNodeEntityAdapter(dwNode), null);
+        BasicNodeSpecHandler handler = (BasicNodeSpecHandler)adapter.getHandler(new DwNodeEntityAdapter(dwNode), null);
         Assert.assertNotNull(handler);
         Assert.assertEquals(ForeachNodeSpecHandler.class, handler.getClass());
     }
@@ -61,7 +63,7 @@ public void testGetHandlerByDefault() {
         DwNode dwNode = new DwNode();
         dwNode.setType(CodeProgramType.ODPS_SQL.getName());
         @NotNull
-        BasicNodeSpecHandler handler = (BasicNodeSpecHandler) adapter.getHandler(new DwNodeEntityAdapter(dwNode), null);
+        BasicNodeSpecHandler handler = (BasicNodeSpecHandler)adapter.getHandler(new DwNodeEntityAdapter(dwNode), null);
         Assert.assertNotNull(handler);
         Assert.assertEquals(BasicNodeSpecHandler.class, handler.getClass());
     }
@@ -103,6 +105,39 @@ public void testGetNodeSpec() {
         Assert.assertEquals(3, foreach.getForeach().getNodes().size());
     }
 
+    @Test
+    public void testGetInnerNodeSpec() {
+        DwNode dwNode = new DwNode();
+        dwNode.setName("test1");
+        dwNode.setGlobalUuid("test1Uuid");
+        dwNode.setType(CodeProgramType.CONTROLLER_TRAVERSE.getName());
+        dwNode.setNodeUseType(NodeUseType.SCHEDULED);
+        DwNode sql = new DwNode();
+        sql.setName("sql");
+        sql.setGlobalUuid("sqlUuid");
+        sql.setType(CodeProgramType.ODPS_SQL.getName());
+        sql.setNodeUseType(NodeUseType.SCHEDULED);
+        NodeIo sqlInput = new DwNodeIo();
+        sqlInput.setData("autotest.start");
+        sql.setInputs(Collections.singletonList(sqlInput));
+
+        NodeSpecAdapter adapter = new NodeSpecAdapter();
+        SpecHandlerContext context = new SpecHandlerContext();
+        String spec = adapter.getNodeSpec(new DwNodeEntityAdapter(sql, dwNode), context);
+        log.info("spec: {}", spec);
+
+        Specification<DataWorksWorkflowSpec> sp = SpecUtil.parseToDomain(spec);
+        Assert.assertNotNull(sp);
+        Assert.assertNotNull(sp.getSpec());
+        Assert.assertNotNull(sp.getSpec().getNodes());
+        Assert.assertEquals(1, sp.getSpec().getNodes().size());
+        SpecNode foreach = sp.getSpec().getNodes().get(0);
+        Assert.assertNotNull(foreach.getForeach());
+        Assert.assertNotNull(foreach.getForeach().getNodes());
+        Assert.assertEquals(1, foreach.getForeach().getNodes().size());
+        Assert.assertEquals(1, foreach.getForeach().getFlow().size());
+    }
+
     @Test
     public void testComponent() {
         DwNode dwNode = new DwNode();

diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>migrationx-domain</artifactId>
         <groupId>com.aliyun.dataworks</groupId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/...a/com/aliyun/dataworks/migrationx/dolphinscheduler/v1/DolphinSchedulerApiServiceTest.java b/...a/com/aliyun/dataworks/migrationx/dolphinscheduler/v1/DolphinSchedulerApiServiceTest.java
@@ -19,7 +19,6 @@
 import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.QueryProcessDefinitionByPaginateRequest;
 import com.aliyun.migrationx.common.http.HttpClientUtil;
 
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.MockedConstruction;
@@ -34,7 +33,6 @@
  * @date 2022/10/20
  */
 @RunWith(MockitoJUnitRunner.class)
-@Ignore
 public class DolphinSchedulerApiServiceTest {
 
     @Test

diff --git a/client/migrationx/migrationx-domain/migrationx-domain-oozie/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-oozie/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>migrationx-domain</artifactId>
         <groupId>com.aliyun.dataworks</groupId>
-        <version>1.1.9-8</version>
+        <version>1.1.9-10</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>