diff --git a/client/.gitignore b/client/.gitignore deleted file mode 100644 index 60ea5a5..0000000 --- a/client/.gitignore +++ /dev/null @@ -1,169 +0,0 @@ -*.iml -*.ipr -*.iws -*.idea/* -.project -.settings/ -core/.classpath -core/.gitignore -core/.project -core/.settings/ -dest -*.egg.* -.git -build -dist -*.pyc -*~ -#* -*# -*.log* -../spec/target/classes -*.egg-info* -# .gitignore for maven -target/ -*.releaseBackup -.git/* -.SQI_PROCESS -.SQI_BETA_STATUS - -# web application files -#/web-app/WEB-INF - -# IDE support files -/.classpath -/.launch -/.project -/.settings -.idea/* -/*.launch -/*.tmproj -/ivy* -/eclipse - -# default HSQL database files for production mode -/prodDb.* - -# general HSQL database files -*Db.properties -*Db.script - -# logs -/stacktrace.log -/test/reports -/logs -*.log -*.log.* - -# plugin release file -/*.zip -/*.zip.sha1 - -# older plugin install locations -/web-app/plugins -/web-app/WEB-INF/classes - -# "temporary" build files -target/ -**/target/ -out/ -build/ - -# other -*.iws - -#.gitignore for java -*.class - -# Package Files # -*.jar -*.war -*.ear - - - -## .gitignore for eclipse - -*.pydevproject -.project -.metadata -bin/** -tmp/** -tmp/**/* -*.tmp -*.bak -*.swp -*~.nib -local.properties -.classpath -.settings/ -.loadpath - -# External tool builders -.externalToolBuilders/ - -# Locally stored "Eclipse launch configurations" -*.launch - -# CDT-specific -.cproject - -# PDT-specific -.buildpath - -## .gitignore for intellij - -*.iml -*.ipr -*.iws -.idea/ - -## .gitignore for windows - -# Windows image file caches -Thumbs.db -ehthumbs.db - -# Folder config file -Desktop.ini - -# Recycle Bin used on file shares -$RECYCLE.BIN/ - -## .gitignore for mac os x - -.DS_Store -.AppleDouble -.LSOverride - - -# Thumbnails -._* - -# Files that might appear on external disk -.Spotlight-V100 -.Trashes - -## hack for graddle wrapper -!wrapper/*.jar -!**/wrapper/*.jar - -# Packages # -# it's better to unpack these files and commit the raw source -# git has its own built in compression methods -*.7z -*.dmg -*.gz -*.iso -*.jar -*.rar -*.tar -*.zip -*.war -*.ear -*.tgz - -# NODE_MODULES -**/node_modules/ - -**/.factorypath \ No newline at end of file diff --git a/client/client-common/pom.xml b/client/client-common/pom.xml index fdcd31c..a898324 100644 --- a/client/client-common/pom.xml +++ b/client/client-common/pom.xml @@ -4,7 +4,7 @@ com.aliyun.dataworks client - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/client/client-common/src/main/conf/logback.xml b/client/client-common/src/main/conf/logback.xml index 36bba3b..3feb09d 100644 --- a/client/client-common/src/main/conf/logback.xml +++ b/client/client-common/src/main/conf/logback.xml @@ -27,7 +27,7 @@ ${LOG_FILE} ${FILE_LOG_PATTERN} - utf8 + UTF-8 @@ -64,7 +64,7 @@ ${CONSOLE_LOG_PATTERN} - utf8 + UTF-8 @@ -77,6 +77,7 @@ - + + \ No newline at end of file diff --git a/client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandAppEntrance.java b/client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandAppEntrance.java index ea1850f..6f7263b 100644 --- a/client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandAppEntrance.java +++ b/client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandAppEntrance.java @@ -96,9 +96,9 @@ private static Map> loadApps(String conf, AppType. appMeta.setName(appName); appMeta.setType(appType); log.info("register command app type: {}, name: {}, class: {}", appType, appName, appMeta.getAppClass()); - CommandAppFactory.register(appType, appName, (Class)Class.forName(appMeta.getAppClass())); - } catch (ClassNotFoundException e) { - log.info("register command app failed, appType: {}, appName: {}, class: {}, error: ", appType, appName, appMeta.getAppClass(), e); + CommandAppFactory.register(appType, appName, appMeta.getAppClass()); + } catch (Exception e) { + throw new RuntimeException(e); } })); diff --git a/client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandAppFactory.java b/client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandAppFactory.java index 52013bf..3b08683 100644 --- a/client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandAppFactory.java +++ b/client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandAppFactory.java @@ -30,10 +30,10 @@ */ @Slf4j public class CommandAppFactory { - private static final Map>> commandApps = new HashMap<>(); + private static final Map> commandApps = new HashMap<>(); - public static void register(AppType appType, String appName, Class readerAppClz) { - Map> apps = commandApps.computeIfAbsent(appType, type -> new HashMap<>()); + public static void register(AppType appType, String appName, String readerAppClz) { + Map apps = commandApps.computeIfAbsent(appType, type -> new HashMap<>()); if (apps.containsKey(appName)) { return; } @@ -42,19 +42,21 @@ public static void register(AppType appType, String appName, Class T create(AppType appType, String appName) throws InstantiationException, IllegalAccessException { + public static T create(AppType appType, String appName) throws InstantiationException, + IllegalAccessException, ClassNotFoundException { if (!commandApps.containsKey(appType)) { throw new RuntimeException("unregistered app type: " + appType); } - Map> apps = commandApps.get(appType); + Map apps = commandApps.get(appType); if (!apps.containsKey(appName)) { log.error("appName: {} not found in apps: {}", appName, apps); throw new RuntimeException("unregistered app name: " + appName); } - Class clz = apps.get(appName); - return (T)clz.newInstance(); + String clz = apps.get(appName); + Class clazz = Class.forName(clz); + return (T) clazz.newInstance(); } public static Map> getApps() { diff --git a/client/client-spark-utils/pom.xml b/client/client-spark-utils/pom.xml index 57d3907..4019036 100644 --- a/client/client-spark-utils/pom.xml +++ b/client/client-spark-utils/pom.xml @@ -4,7 +4,7 @@ com.aliyun.dataworks client - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/client/client-toolkits/pom.xml b/client/client-toolkits/pom.xml index f96d449..7f243fb 100644 --- a/client/client-toolkits/pom.xml +++ b/client/client-toolkits/pom.xml @@ -4,7 +4,7 @@ com.aliyun.dataworks client - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/client/migrationx/migrationx-common/pom.xml b/client/migrationx/migrationx-common/pom.xml index fa68e82..e1ddc07 100644 --- a/client/migrationx/migrationx-common/pom.xml +++ b/client/migrationx/migrationx-common/pom.xml @@ -22,7 +22,7 @@ migrationx com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/http/HttpClientUtil.java b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/http/HttpClientUtil.java index b799748..2065f1b 100644 --- a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/http/HttpClientUtil.java +++ b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/http/HttpClientUtil.java @@ -29,10 +29,6 @@ import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.SocketException; /** * @author 聿剑 @@ -66,7 +62,7 @@ public boolean isRetry(Exception e) { } public String executeAndGet(HttpRequestBase httpRequestBase) throws Exception { - return executeAndGet(httpRequestBase, 3, 1000); + return executeAndGet(httpRequestBase, 1, 1000); } public String executeAndGet(HttpRequestBase httpRequestBase, boolean retry) throws Exception { diff --git a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/metrics/DolphinMetricsCollector.java b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/metrics/DolphinMetricsCollector.java index cb30094..3bbe20f 100644 --- a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/metrics/DolphinMetricsCollector.java +++ b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/metrics/DolphinMetricsCollector.java @@ -53,7 +53,7 @@ public class DolphinMetricsCollector implements MetricsCollector { private String transformerType; private Date startTime; - private Consumer defaultMetricsConsumer = metrics -> metricsLogger.warn("{}", GSON.toJson(metrics)); + private Consumer defaultMetricsConsumer = metrics -> metricsLogger.info("{}", GSON.toJson(metrics)); public DolphinMetricsCollector() { startTime = new Date(); @@ -172,7 +172,7 @@ private Metrics findMetrics(Metrics tmp) { @Override public void finishCollector() { - finishCollector((summary) -> summaryLogger.warn("{}", PrettyGson.toJson(summary))); + finishCollector((summary) -> summaryLogger.info("{}", PrettyGson.toJson(summary))); } @Override diff --git a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/BeanUtils.java b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/BeanUtils.java index bd11c57..f5b5373 100644 --- a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/BeanUtils.java +++ b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/BeanUtils.java @@ -8,8 +8,13 @@ package com.aliyun.migrationx.common.utils; -import org.dozer.DozerBeanMapper; +import java.beans.PropertyDescriptor; +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.beanutils.PropertyUtils; +import org.dozer.DozerBeanMapper; /** * Desc: * @@ -33,4 +38,45 @@ public static T deepCopy(Object obj, Class clazz) { private BeanUtils() { } + + public static void copyProperties(Object src, Object dest) { + try { + org.apache.commons.beanutils.BeanUtils.copyProperties(dest, src); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static void copyProperties(Object src, Object dest, String... ignoreProperties) { + try { + copyPropertiesWithIgnore(dest, src, ignoreProperties); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static void copyPropertiesWithIgnore(Object dest, Object orig, String... ignoreProperties) + throws IllegalAccessException, InvocationTargetException { + List ignoreList = (ignoreProperties != null) ? Arrays.asList(ignoreProperties) : null; + + PropertyDescriptor[] origDescriptors = PropertyUtils.getPropertyDescriptors(orig); + for (PropertyDescriptor origDescriptor : origDescriptors) { + String name = origDescriptor.getName(); + if ("class".equals(name)) { + continue; // Ignore the "class" property + } + if (ignoreList != null && ignoreList.contains(name)) { + continue; // Ignore specified properties + } + + if (PropertyUtils.isReadable(orig, name) && PropertyUtils.isWriteable(dest, name)) { + try { + Object value = PropertyUtils.getSimpleProperty(orig, name); + org.apache.commons.beanutils.BeanUtils.setProperty(dest, name, value); + } catch (NoSuchMethodException e) { + // Ignore if the setter method does not exist in the destination bean + } + } + } + } } \ No newline at end of file diff --git a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/Config.java b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/Config.java index 0be5968..4d7d886 100644 --- a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/Config.java +++ b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/Config.java @@ -39,4 +39,9 @@ public static void init(Config config) { * '*' for all */ private List filterTasks = new ArrayList<>(); + + /** + * workflow base path + */ + private String basePath; } diff --git a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/IntlUtils.java b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/IntlUtils.java index ba8b33d..82056b9 100644 --- a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/IntlUtils.java +++ b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/IntlUtils.java @@ -15,9 +15,9 @@ package com.aliyun.migrationx.common.utils; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.Locale; @@ -33,9 +33,8 @@ import lombok.ToString; import lombok.experimental.Accessors; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; /** * @author 聿剑 @@ -44,7 +43,7 @@ public class IntlUtils { public static Map> intlResource = new HashMap<>(); - private static final String CLASS_PATH_RESOURCE = "i18n"; + public static final String CLASS_PATH_RESOURCE = "i18n"; public static IntlBo get(String key) { IntlBo intlBo = new IntlBo(); @@ -85,24 +84,34 @@ public static Map getLanguageResource() { } } - private static String readFileToStr(String fileName) { - Resource resource = new ClassPathResource(CLASS_PATH_RESOURCE + "/" + fileName); - try { - BufferedReader br = new BufferedReader(new InputStreamReader(resource.getInputStream())); - StringBuilder buffer = new StringBuilder(); - String line; - - while((line = br.readLine()) != null) { - buffer.append(line); + public static String readFileToStr(String file) throws IOException { + String path = CLASS_PATH_RESOURCE + "/" + file; + try (InputStream inputStream = IntlUtils.class.getClassLoader().getResourceAsStream(path)) { + if (inputStream != null) { + return IOUtils.toString(inputStream, StandardCharsets.UTF_8); + } else { + return null; } - - resource.getInputStream().close(); - return buffer.toString(); - } catch (IOException e) { - e.printStackTrace(); - return ""; } } + //private static String readFileToStr(String fileName) { + // Resource resource = new ClassPathResource(CLASS_PATH_RESOURCE + "/" + fileName); + // try { + // BufferedReader br = new BufferedReader(new InputStreamReader(resource.getInputStream())); + // StringBuilder buffer = new StringBuilder(); + // String line; + // + // while((line = br.readLine()) != null) { + // buffer.append(line); + // } + // + // resource.getInputStream().close(); + // return buffer.toString(); + // } catch (IOException e) { + // e.printStackTrace(); + // return ""; + // } + //} public static class IntlBo { private String value; diff --git a/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/UuidGenerators.java b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/UuidGenerators.java new file mode 100644 index 0000000..0b827c0 --- /dev/null +++ b/client/migrationx/migrationx-common/src/main/java/com/aliyun/migrationx/common/utils/UuidGenerators.java @@ -0,0 +1,23 @@ +package com.aliyun.migrationx.common.utils; + +import java.util.Objects; +import java.util.function.Function; + +import static com.aliyun.migrationx.common.utils.UuidUtils.genUuidWithoutHorizontalLine; + +public class UuidGenerators { + + private static final Function generateUuidFunc = code -> genUuidWithoutHorizontalLine(); + + public static String generateUuid(Long code) { + if (Objects.isNull(code)) { + return generateUuid(); + } + return generateUuidFunc.apply(code); + } + + public static String generateUuid() { + String uuid = genUuidWithoutHorizontalLine(); + return uuid; + } +} diff --git a/client/migrationx/migrationx-domain/migrationx-domain-adf/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-adf/pom.xml index fc58e86..0c7c3e2 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-adf/pom.xml +++ b/client/migrationx/migrationx-domain/migrationx-domain-adf/pom.xml @@ -6,7 +6,7 @@ com.aliyun.dataworks migrationx-domain - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/client/migrationx/migrationx-domain/migrationx-domain-airflow/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-airflow/pom.xml index b2fa55d..e4fecad 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-airflow/pom.xml +++ b/client/migrationx/migrationx-domain/migrationx-domain-airflow/pom.xml @@ -20,7 +20,7 @@ migrationx-domain com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml 4.0.0 diff --git a/client/migrationx/migrationx-domain/migrationx-domain-aliyunemr/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-aliyunemr/pom.xml index 2aacf79..cebb868 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-aliyunemr/pom.xml +++ b/client/migrationx/migrationx-domain/migrationx-domain-aliyunemr/pom.xml @@ -20,7 +20,7 @@ migrationx-domain com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml 4.0.0 diff --git a/client/migrationx/migrationx-domain/migrationx-domain-aliyunemr/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/aliyunemr/AliyunEmrService.java b/client/migrationx/migrationx-domain/migrationx-domain-aliyunemr/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/aliyunemr/AliyunEmrService.java index 5effc13..2689c7c 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-aliyunemr/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/aliyunemr/AliyunEmrService.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-aliyunemr/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/aliyunemr/AliyunEmrService.java @@ -28,7 +28,9 @@ import java.util.Objects; import java.util.Optional; +import com.aliyun.migrationx.common.utils.BeanUtils; import com.aliyun.migrationx.common.utils.GsonUtils; + import com.aliyuncs.DefaultAcsClient; import com.aliyuncs.emr.model.v20160408.DescribeFlowCategoryRequest; import com.aliyuncs.emr.model.v20160408.DescribeFlowCategoryResponse; @@ -56,7 +58,6 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.BeanUtils; /** * @author sam.liux diff --git a/client/migrationx/migrationx-domain/migrationx-domain-azkaban/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-azkaban/pom.xml index fdad0a3..e59fa4c 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-azkaban/pom.xml +++ b/client/migrationx/migrationx-domain/migrationx-domain-azkaban/pom.xml @@ -20,7 +20,7 @@ migrationx-domain com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml 4.0.0 diff --git a/client/migrationx/migrationx-domain/migrationx-domain-caiyunjian/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-caiyunjian/pom.xml index 7be6416..a267da2 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-caiyunjian/pom.xml +++ b/client/migrationx/migrationx-domain/migrationx-domain-caiyunjian/pom.xml @@ -20,7 +20,7 @@ migrationx-domain com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml 4.0.0 diff --git a/client/migrationx/migrationx-domain/migrationx-domain-core/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-core/pom.xml index f78bb7e..18dd19b 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-core/pom.xml +++ b/client/migrationx/migrationx-domain/migrationx-domain-core/pom.xml @@ -21,7 +21,7 @@ com.aliyun.dataworks migrationx-domain - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/client/migrationx/migrationx-domain/migrationx-domain-datago/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-datago/pom.xml index 53566b7..c80d933 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-datago/pom.xml +++ b/client/migrationx/migrationx-domain/migrationx-domain-datago/pom.xml @@ -20,7 +20,7 @@ migrationx-domain com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml 4.0.0 diff --git "a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/.tmp/test/DataStudio/\344\270\232\345\212\241\346\265\201\347\250\213/test_flow1/MaxCompute/\346\225\260\346\215\256\345\274\200\345\217\221/test1/test1.schedule.json" "b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/.tmp/test/DataStudio/\344\270\232\345\212\241\346\265\201\347\250\213/test_flow1/MaxCompute/\346\225\260\346\215\256\345\274\200\345\217\221/test1/test1.schedule.json" new file mode 100644 index 0000000..40deecf --- /dev/null +++ "b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/.tmp/test/DataStudio/\344\270\232\345\212\241\346\265\201\347\250\213/test_flow1/MaxCompute/\346\225\260\346\215\256\345\274\200\345\217\221/test1/test1.schedule.json" @@ -0,0 +1,29 @@ +{ + "version":"1.1.0", + "kind":"CycleWorkflow", + "spec":{ + "nodes":[ + { + "recurrence":"Normal", + "id":"e2f7ce7c-e5ee-49aa-a13f-2958006e257e", + "instanceMode":"T+1", + "rerunMode":"Allowed", + "script":{ + "path":"业务流程/test_flow1/MaxCompute/数据开发", + "runtime":{ + "engine":"MaxCompute", + "command":"ODPS_SQL" + }, + "id":"e2f7ce7c-e5ee-49aa-a13f-2958006e257e" + }, + "name":"test1", + "inputs":{ + + }, + "outputs":{ + + } + } + ] + } +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/AbstractDolphinSchedulerConverter.java "b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/.tmp/test/DataStudio/\344\270\232\345\212\241\346\265\201\347\250\213/test_flow1/MaxCompute/\346\225\260\346\215\256\345\274\200\345\217\221/test1/test1.sql" similarity index 100% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/AbstractDolphinSchedulerConverter.java rename to "client/migrationx/migrationx-domain/migrationx-domain-dataworks/.tmp/test/DataStudio/\344\270\232\345\212\241\346\265\201\347\250\213/test_flow1/MaxCompute/\346\225\260\346\215\256\345\274\200\345\217\221/test1/test1.sql" diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/.tmp/test/SPEC.FORMAT b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/.tmp/test/SPEC.FORMAT new file mode 100644 index 0000000..e69de29 diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/pom.xml index f9a58e3..f102511 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/pom.xml +++ b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/pom.xml @@ -20,7 +20,7 @@ migrationx-domain com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml 4.0.0 diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/File.java b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/File.java index e9bdea5..8d467b4 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/File.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/objects/entity/client/File.java @@ -87,4 +87,8 @@ public class File { private Boolean ignoreLock; private String advanceSettings; + /** + * 绝对路径，目前只有ListFiles在needAbsoluteFolderPath时会用到 + */ + private String absoluteFolderPath; } diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/NodeSpecAdapter.java b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/NodeSpecAdapter.java index 956063e..1b8fdca 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/NodeSpecAdapter.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/NodeSpecAdapter.java @@ -28,6 +28,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.alibaba.fastjson2.JSONWriter.Feature; + import com.aliyun.dataworks.common.spec.adapter.SpecAdapter; import com.aliyun.dataworks.common.spec.adapter.SpecHandlerContext; import com.aliyun.dataworks.common.spec.domain.DataWorksWorkflowSpec; @@ -55,9 +57,6 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.service.spec.entity.DwNodeEntity; import com.aliyun.dataworks.migrationx.domain.dataworks.service.spec.handler.BasicNodeSpecHandler; import com.aliyun.dataworks.migrationx.domain.dataworks.service.spec.handler.ComponentSpecHandler; - -import com.alibaba.fastjson2.JSONWriter.Feature; - import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; @@ -82,14 +81,14 @@ private void registerHandlers() { Reflections reflections = new Reflections(BasicNodeSpecHandler.class.getPackage().getName()); Set> handlerClasses = reflections.getSubTypesOf(BasicNodeSpecHandler.class); SetUtils.emptyIfNull(handlerClasses).stream() - .filter(h -> !h.equals(BasicNodeSpecHandler.class)) - .forEach(this::registerHandler); + .filter(h -> !h.equals(BasicNodeSpecHandler.class)) + .forEach(this::registerHandler); } public String getNodeSpec(DwNodeEntity dmNodeBO, SpecHandlerContext context) { context.setSpecAdapter(this); SpecWriterContext specWriterContext = new SpecWriterContext(); - SpecificationWriter writer = (SpecificationWriter) WriterFactory.getWriter(Specification.class, specWriterContext); + SpecificationWriter writer = (SpecificationWriter)WriterFactory.getWriter(Specification.class, specWriterContext); return writer.write(getNodeSpecObject(dmNodeBO, context), specWriterContext).toJSONString(Feature.PrettyFormat); } @@ -97,13 +96,17 @@ public Specification getNodeSpecObject(DwNodeEntity dwNod Preconditions.checkNotNull(dwNode, "node is null"); context.setSpecAdapter(this); Specification spec = new Specification<>(); + DataWorksWorkflowSpec dataWorksWorkflowSpec = new DataWorksWorkflowSpec(); spec.setVersion(SpecVersion.V_1_1_0.getLabel()); spec.setKind(Optional.ofNullable(dwNode.getNodeUseType()).map(useType -> { switch (useType) { case MANUAL: return SpecKind.MANUAL_NODE.getLabel(); - case MANUAL_WORKFLOW: + case MANUAL_WORKFLOW: { + dataWorksWorkflowSpec.setId(Optional.ofNullable(dwNode.getBizId()).filter(id -> id > 0).map(String::valueOf).orElse(null)); + dataWorksWorkflowSpec.setName(dwNode.getBizName()); return SpecKind.MANUAL_WORKFLOW.getLabel(); + } case COMPONENT: return SpecKind.COMPONENT.getLabel(); default: @@ -113,9 +116,6 @@ public Specification getNodeSpecObject(DwNodeEntity dwNod Map metadata = new LinkedHashMap<>(); metadata.put("owner", dwNode.getOwner()); spec.setMetadata(metadata); - DataWorksWorkflowSpec dataWorksWorkflowSpec = new DataWorksWorkflowSpec(); - dataWorksWorkflowSpec.setId(Optional.ofNullable(dwNode.getBizId()).filter(id -> id > 0).map(String::valueOf).orElse(null)); - dataWorksWorkflowSpec.setName(Optional.ofNullable(dwNode.getBizName()).orElse(null)); spec.setSpec(dataWorksWorkflowSpec); Optional.ofNullable(spec.getKind()).map(kind -> LabelEnum.getByLabel(SpecKind.class, kind)).ifPresent(kind -> { @@ -125,7 +125,7 @@ public Specification getNodeSpecObject(DwNodeEntity dwNod Optional.ofNullable(handler.handle(dwNode)).ifPresent(nodes::add); dataWorksWorkflowSpec.setComponents(nodes); } else { - BasicNodeSpecHandler nodeSpecHandler = (BasicNodeSpecHandler) getHandler(dwNode, context.getLocale()); + BasicNodeSpecHandler nodeSpecHandler = (BasicNodeSpecHandler)getHandler(dwNode, context.getLocale()); nodeSpecHandler.setContext(context); dataWorksWorkflowSpec.setFlow(toFlow(nodeSpecHandler, dwNode, context)); List nodes = new ArrayList<>(); @@ -176,7 +176,7 @@ public List toFlow(BasicNodeSpecHandler handler, DwNodeEntity dm flowDepend.setNodeId(nodeId); List inputs = handler.getNodeInputs(dmNodeBo); - flowDepend.setDepends(ListUtils.emptyIfNull(inputs).stream().map(in -> (SpecNodeOutput) in).map(dep -> { + flowDepend.setDepends(ListUtils.emptyIfNull(inputs).stream().map(in -> (SpecNodeOutput)in).map(dep -> { SpecDepend specDepend = new SpecDepend(); specDepend.setType(DependencyType.NORMAL); SpecNodeOutput art = new SpecNodeOutput(); @@ -188,21 +188,21 @@ public List toFlow(BasicNodeSpecHandler handler, DwNodeEntity dm }).collect(Collectors.toList())); if (Stream.of(DependentType.USER_DEFINE, DependentType.USER_DEFINE_AND_SELF).anyMatch( - dt -> Objects.equals(dmNodeBo.getDependentType(), dt.getValue()) && StringUtils.isNotBlank(dmNodeBo.getDependentDataNode()))) { + dt -> Objects.equals(dmNodeBo.getDependentType(), dt.getValue()) && StringUtils.isNotBlank(dmNodeBo.getDependentDataNode()))) { Optional.ofNullable(StringUtils.split(dmNodeBo.getDependentDataNode(), ",")).map(Arrays::asList).orElse(new ArrayList<>()).stream().map( - out -> { - SpecDepend specDepend = new SpecDepend(); - specDepend.setType(DependencyType.CROSS_CYCLE_OTHER_NODE); - SpecNodeOutput art = new SpecNodeOutput(); - art.setData(out); - art.setArtifactType(ArtifactType.NODE_OUTPUT); - specDepend.setOutput(art); - return specDepend; - }).forEach(sp -> flowDepend.getDepends().add(sp)); + out -> { + SpecDepend specDepend = new SpecDepend(); + specDepend.setType(DependencyType.CROSS_CYCLE_OTHER_NODE); + SpecNodeOutput art = new SpecNodeOutput(); + art.setData(out); + art.setArtifactType(ArtifactType.NODE_OUTPUT); + specDepend.setOutput(art); + return specDepend; + }).forEach(sp -> flowDepend.getDepends().add(sp)); } if (Stream.of(DependentType.SELF, DependentType.USER_DEFINE_AND_SELF, DependentType.CHILD_AND_SELF).anyMatch( - dt -> Objects.equals(dt.getValue(), dmNodeBo.getDependentType()))) { + dt -> Objects.equals(dt.getValue(), dmNodeBo.getDependentType()))) { SpecDepend specDepend = new SpecDepend(); specDepend.setType(DependencyType.CROSS_CYCLE_SELF); specDepend.setNodeId(nodeId); diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/handler/EmrNodeSpecHandler.java b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/handler/EmrNodeSpecHandler.java index 6e8c6d1..c0ba71b 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/handler/EmrNodeSpecHandler.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/service/spec/handler/EmrNodeSpecHandler.java @@ -28,9 +28,10 @@ import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; import com.aliyun.dataworks.common.spec.utils.ReflectUtils; import com.aliyun.dataworks.migrationx.domain.dataworks.service.spec.entity.DwNodeEntity; +import com.aliyun.migrationx.common.utils.BeanUtils; + import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; /** * EMR节点处理器 diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/utils/NodeUtils.java b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/utils/NodeUtils.java index 05d9fe7..056fbbf 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/utils/NodeUtils.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dataworks/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/utils/NodeUtils.java @@ -33,6 +33,7 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.client.FileNodeInputOutput; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.IoParseType; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.RerunMode; +import com.aliyun.migrationx.common.utils.BeanUtils; import com.aliyun.migrationx.common.utils.GsonUtils; import com.google.common.base.Joiner; @@ -44,7 +45,6 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.BeanUtils; /** * @author sam.liux diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/pom.xml index 14235c3..b4aacaf 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/pom.xml +++ b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/pom.xml @@ -20,7 +20,7 @@ migrationx-domain com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml 4.0.0 diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/service/DolphinSchedulerV3PackageLoader.java b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/service/DolphinSchedulerV3PackageLoader.java index 8668575..6989bdb 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/service/DolphinSchedulerV3PackageLoader.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/service/DolphinSchedulerV3PackageLoader.java @@ -21,7 +21,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -40,7 +39,6 @@ import com.google.gson.reflect.TypeToken; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.ListUtils; import org.apache.commons.io.FileUtils; /** @@ -106,9 +104,8 @@ private List readProcessMetaList(File rootDir) { .map(files -> files.stream().map(this::readProcessMetaJson) .filter(CollectionUtils::isNotEmpty) .flatMap(List::stream) - .filter(Objects::nonNull) .collect(Collectors.toList())) - .orElse(ListUtils.emptyIfNull(null)); + .orElse(Collections.emptyList()); } private List readProcessMetaJson(File jsonFile) { diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerV1Context.java b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerV1Context.java index f4c1c8c..ee9f775 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerV1Context.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/DolphinSchedulerV1Context.java @@ -22,6 +22,7 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.Project; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.UdfFunc; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.datasource.DataSource; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.entity.ResourceInfo; @@ -42,7 +43,36 @@ public class DolphinSchedulerV1Context { private List dagDatas; private Map projectCodeMap = new HashMap<>(); - private Map> subProcessCodeOutMap = new HashMap<>(); + private Map> subProcessCodeOutMap = new HashMap<>(); + + public Map> getProcessCodeTaskRelationMap() { + return processCodeTaskRelationMap; + } + + private Map> processCodeTaskRelationMap = new HashMap<>(); + + /** + * processId：SpecNode + */ + private Map subProcessCodeNodeMap = new HashMap<>(); + /** + * processId: workflow + */ + private Map subProcessCodeWorkflowMap = new HashMap<>(); + + /** + * taskDefinition.code : SpecNodeOutput.data + */ + private Map taskCodeNodeDataMap = new HashMap<>(); + private Map taskCodeNodeIdMap = new HashMap<>(); + + public Map getTaskCodeSpecNodeMap() { + return taskCodeSpecNodeMap; + } + + private Map taskCodeSpecNodeMap = new HashMap<>(); + + private Map> specNodeProcessCodeMap = new HashMap<>(); private DolphinSchedulerV1Context() { @@ -62,6 +92,17 @@ public static void initContext(List projects, List dagData context.projectCodeMap.put(Long.parseLong(project.getCode()), project); } } + + for (ProcessMeta processMeta : dagDatas) { + List taskNodes = processMeta.getProcessDefinitionJson().getTasks(); + List taskDefinitionList = context.processCodeTaskRelationMap.get(processMeta.getProcessDefinitionId()); + if (taskDefinitionList == null) { + taskDefinitionList = new ArrayList<>(); + context.processCodeTaskRelationMap.put(processMeta.getProcessDefinitionId(), taskDefinitionList); + } + taskDefinitionList.addAll(taskNodes); + } + DolphinSchedulerV1Context.context = context; } @@ -109,7 +150,7 @@ public void setUdfFuncs(List udfFuncs) { this.udfFuncs = udfFuncs; } - public void putSubProcessCodeOutMap(Long code, String out) { + public void putSubProcessCodeOutMap(Integer code, String out) { List outs = this.subProcessCodeOutMap.get(code); if (outs == null) { outs = new ArrayList<>(); @@ -121,4 +162,24 @@ public void putSubProcessCodeOutMap(Long code, String out) { public List getSubProcessCodeMap(Long code) { return this.subProcessCodeOutMap.get(code); } + + public Map getTaskCodeNodeDataMap() { + return taskCodeNodeDataMap; + } + + public Map getTaskCodeNodeIdMap() { + return taskCodeNodeIdMap; + } + + public Map getSubProcessCodeNodeMap() { + return subProcessCodeNodeMap; + } + + public Map getSubProcessCodeWorkflowMap() { + return subProcessCodeWorkflowMap; + } + + public Map> getSpecNodeProcessCodeMap() { + return specNodeProcessCodeMap; + } } diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/QueryResourceListRequest.java b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/QueryResourceListRequest.java index f9536dd..6a7a38a 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/QueryResourceListRequest.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v1/QueryResourceListRequest.java @@ -32,4 +32,5 @@ public class QueryResourceListRequest extends DolphinSchedulerRequest { private String type; private String fullName; private String tenantCode; + private int dirId; } diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v2/DolphinSchedulerV2Context.java b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v2/DolphinSchedulerV2Context.java index 78bd95f..7a1fe90 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v2/DolphinSchedulerV2Context.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v2/DolphinSchedulerV2Context.java @@ -46,6 +46,26 @@ public class DolphinSchedulerV2Context { private Map> subProcessCodeOutMap = new HashMap<>(); + /** + * processId：SpecNode + */ + private Map subProcessCodeNodeMap = new HashMap<>(); + private Map subProcessCodeWorkflowMap = new HashMap<>(); + + /** + * taskDefinition.code : SpecNodeOutput.data + */ + private Map taskCodeNodeDataMap = new HashMap<>(); + private Map taskCodeNodeIdMap = new HashMap<>(); + + public Map getTaskCodeSpecNodeMap() { + return taskCodeSpecNodeMap; + } + + private Map taskCodeSpecNodeMap = new HashMap<>(); + + private Map> specNodeProcessCodeMap = new HashMap<>(); + private DolphinSchedulerV2Context() { } @@ -190,4 +210,28 @@ public void putSubProcessCodeOutMap(Long code, String out) { public List getSubProcessCodeMap(Long code) { return this.subProcessCodeOutMap.get(code); } + + public Map getTaskCodeNodeDataMap() { + return taskCodeNodeDataMap; + } + + public Map getTaskCodeNodeIdMap() { + return taskCodeNodeIdMap; + } + + public Map getSubProcessCodeNodeMap() { + return subProcessCodeNodeMap; + } + + public Map getSubProcessCodeWorkflowMap() { + return subProcessCodeWorkflowMap; + } + + public Map> getSpecNodeProcessCodeMap() { + return specNodeProcessCodeMap; + } + + public void setSpecNodeProcessCodeMap(Map> specNodeProcessCodeMap) { + this.specNodeProcessCodeMap = specNodeProcessCodeMap; + } } diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v2/process/ResourceInfo.java b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v2/process/ResourceInfo.java index d522290..925aff6 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v2/process/ResourceInfo.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v2/process/ResourceInfo.java @@ -35,6 +35,11 @@ public class ResourceInfo { private String type; + /** + * full name of the resource that was uploaded + */ + private String resourceName; + public ResourceInfo() { // do nothing, void constructor } @@ -78,4 +83,12 @@ public String getType() { public void setType(String type) { this.type = type; } + + public String getResourceName() { + return resourceName; + } + + public void setResourceName(String resourceName) { + this.resourceName = resourceName; + } } diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinSchedulerV3Context.java b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinSchedulerV3Context.java index 0500ea6..5eef2b4 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinSchedulerV3Context.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinSchedulerV3Context.java @@ -46,6 +46,26 @@ public class DolphinSchedulerV3Context { private Map> subProcessCodeOutMap = new HashMap<>(); + /** + * processId：SpecNode + */ + private Map subProcessCodeNodeMap = new HashMap<>(); + private Map subProcessCodeWorkflowMap = new HashMap<>(); + + /** + * taskDefinition.code : SpecNodeOutput.data + */ + private Map taskCodeNodeDataMap = new HashMap<>(); + private Map taskCodeNodeIdMap = new HashMap<>(); + + public Map getTaskCodeSpecNodeMap() { + return taskCodeSpecNodeMap; + } + + private Map taskCodeSpecNodeMap = new HashMap<>(); + + private Map> specNodeProcessCodeMap = new HashMap<>(); + private DolphinSchedulerV3Context() { } @@ -191,4 +211,28 @@ public void putSubProcessCodeOutMap(Long code, String out) { public List getSubProcessCodeMap(Long code) { return this.subProcessCodeOutMap.get(code); } + + public Map getTaskCodeNodeDataMap() { + return taskCodeNodeDataMap; + } + + public Map getTaskCodeNodeIdMap() { + return taskCodeNodeIdMap; + } + + public Map getSubProcessCodeNodeMap() { + return subProcessCodeNodeMap; + } + + public Map getSubProcessCodeWorkflowMap() { + return subProcessCodeWorkflowMap; + } + + public Map> getSpecNodeProcessCodeMap() { + return specNodeProcessCodeMap; + } + + public void setSpecNodeProcessCodeMap(Map> specNodeProcessCodeMap) { + this.specNodeProcessCodeMap = specNodeProcessCodeMap; + } } diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinschedulerApiV3Service.java b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinschedulerApiV3Service.java index 69c5ee2..fbbcbc4 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinschedulerApiV3Service.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/main/java/com/aliyun/dataworks/migrationx/domain/dataworks/dolphinscheduler/v3/DolphinschedulerApiV3Service.java @@ -121,8 +121,8 @@ public Response> queryResourceList(QueryResourceListRequest req public List queryResourceListByPage(QueryResourceListRequest request, int pageNum, int pageSize) throws Exception { HttpClientUtil client = new HttpClientUtil(); - String url = String.format("resources?type=%s&fullName=%s&tenantCode=&searchVal=&pageNo=%s&pageSize=%s&id=-1", - request.getType(), request.getFullName() == null ? "" : request.getFullName(), pageNum, pageSize); + String url = String.format("resources?type=%s&fullName=%s&tenantCode=&searchVal=&pageNo=%s&pageSize=%s&id=%s", + request.getType(), request.getFullName() == null ? "" : request.getFullName(), pageNum, pageSize, request.getDirId()); HttpGet httpGet = newHttpGet(url); String responseStr = client.executeAndGet(httpGet); Response response = GsonUtils.fromJsonString(responseStr, new TypeToken>() {}.getType()); diff --git a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/test/java/com/aliyun/dataworks/migrationx/dolphinscheduler/v1/DolphinSchedulerApiServiceTest.java b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/test/java/com/aliyun/dataworks/migrationx/dolphinscheduler/v1/DolphinSchedulerApiServiceTest.java index ed24b3c..0f73698 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/test/java/com/aliyun/dataworks/migrationx/dolphinscheduler/v1/DolphinSchedulerApiServiceTest.java +++ b/client/migrationx/migrationx-domain/migrationx-domain-dolphinscheduler/src/test/java/com/aliyun/dataworks/migrationx/dolphinscheduler/v1/DolphinSchedulerApiServiceTest.java @@ -18,35 +18,35 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerApiService; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.QueryProcessDefinitionByPaginateRequest; import com.aliyun.migrationx.common.http.HttpClientUtil; -import org.junit.Before; + +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; +import org.mockito.MockedConstruction; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.when; /** * @author 聿剑 * @date 2022/10/20 */ -@RunWith(PowerMockRunner.class) -@PrepareForTest({DolphinSchedulerApiService.class, HttpClientUtil.class}) +@RunWith(MockitoJUnitRunner.class) +@Ignore public class DolphinSchedulerApiServiceTest { - @Before - public void setup() { - MockitoAnnotations.openMocks(this); - } @Test - public void testQueryProcessDefinitionByPaginate() throws Exception { - HttpClientUtil httpClient = PowerMockito.mock(HttpClientUtil.class); - PowerMockito.whenNew(HttpClientUtil.class).withAnyArguments().thenReturn(httpClient); - String response = ""; - Mockito.doReturn(response).when(httpClient).executeAndGet(Mockito.any()); - DolphinSchedulerApiService service = new DolphinSchedulerApiService("", ""); - QueryProcessDefinitionByPaginateRequest request = new QueryProcessDefinitionByPaginateRequest(); - service.queryProcessDefinitionByPaging(request); + public void testParseProjects_Normal() throws Exception { + try (MockedConstruction mockTank = mockConstruction(HttpClientUtil.class, (mock, context) -> { + when(mock.executeAndGet(any())).thenReturn(""); + })) { + DolphinSchedulerApiService service = new DolphinSchedulerApiService("", ""); + QueryProcessDefinitionByPaginateRequest request = new QueryProcessDefinitionByPaginateRequest(); + service.queryProcessDefinitionByPaging(request); + } catch (Exception e) { + throw e; + } } } diff --git a/client/migrationx/migrationx-domain/migrationx-domain-oozie/pom.xml b/client/migrationx/migrationx-domain/migrationx-domain-oozie/pom.xml index 899f308..7357e20 100644 --- a/client/migrationx/migrationx-domain/migrationx-domain-oozie/pom.xml +++ b/client/migrationx/migrationx-domain/migrationx-domain-oozie/pom.xml @@ -20,7 +20,7 @@ migrationx-domain com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml 4.0.0 diff --git a/client/migrationx/migrationx-domain/pom.xml b/client/migrationx/migrationx-domain/pom.xml index 5a4d397..a86f452 100644 --- a/client/migrationx/migrationx-domain/pom.xml +++ b/client/migrationx/migrationx-domain/pom.xml @@ -23,7 +23,7 @@ migrationx com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/client/migrationx/migrationx-reader/pom.xml b/client/migrationx/migrationx-reader/pom.xml index dc1ef22..ba16b3a 100644 --- a/client/migrationx/migrationx-reader/pom.xml +++ b/client/migrationx/migrationx-reader/pom.xml @@ -20,7 +20,7 @@ migrationx com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml 4.0.0 @@ -77,22 +77,6 @@ commons-cli commons-cli - - - org.powermock - powermock-module-junit4-rule-agent - test - - - org.powermock - powermock-module-junit4 - test - - - org.powermock - powermock-api-mockito2 - test - org.mockito mockito-inline diff --git a/client/migrationx/migrationx-reader/src/main/java/com/aliyun/dataworks/migrationx/reader/dolphinscheduler/DolphinSchedulerReader.java b/client/migrationx/migrationx-reader/src/main/java/com/aliyun/dataworks/migrationx/reader/dolphinscheduler/DolphinSchedulerReader.java index 2cca658..8242e4a 100644 --- a/client/migrationx/migrationx-reader/src/main/java/com/aliyun/dataworks/migrationx/reader/dolphinscheduler/DolphinSchedulerReader.java +++ b/client/migrationx/migrationx-reader/src/main/java/com/aliyun/dataworks/migrationx/reader/dolphinscheduler/DolphinSchedulerReader.java @@ -48,6 +48,9 @@ import com.aliyun.migrationx.common.utils.ZipUtils; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Maps; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; @@ -263,7 +266,7 @@ private void exportResourceFiles(File tmpDir) { } List resources = new ArrayList<>(); Arrays.asList("FILE", "UDF").forEach(type -> { - visitResourceByFolder(type, null, resourceDir, resources); + visitResourceByFolder(type, null, -1, resourceDir, resources); }); try { @@ -276,12 +279,12 @@ private void exportResourceFiles(File tmpDir) { } } - private void visitResourceByFolder(String type, String fullName, File resourceDir, List resources) { + private void visitResourceByFolder(String type, String fullName, int dirId, File resourceDir, List resources) { try { int pageNum = 1; int pageSize = 100; while (true) { - List data = getResource(type, fullName, pageNum, pageSize); + List data = getResource(type, fullName, dirId, pageNum, pageSize); if (data == null) { break; } @@ -296,7 +299,8 @@ private void visitResourceByFolder(String type, String fullName, File resourceDi currentDir = currentDir + File.separator + component.getFileName(); } File visitDir = new File(currentDir); - visitResourceByFolder(type, component.getFullName(), visitDir, resources); + //todo lower version has no fullName + visitResourceByFolder(type, component.getFullName(), component.getId(), visitDir, resources); } else { if (!BooleanUtils.isTrue(skipResources)) { if (!resourceDir.exists()) { @@ -317,10 +321,11 @@ private void visitResourceByFolder(String type, String fullName, File resourceDi } } - private List getResource(String type, String fullName, int pageNum, int pageSize) throws Exception { + private List getResource(String type, String fullName, int dirId, int pageNum, int pageSize) throws Exception { QueryResourceListRequest queryResourceListRequest = new QueryResourceListRequest(); queryResourceListRequest.setType(type); queryResourceListRequest.setFullName(fullName); + queryResourceListRequest.setDirId(dirId); List response = dolphinSchedulerApiService.queryResourceListByPage( queryResourceListRequest, pageNum, pageSize); return response; @@ -443,6 +448,9 @@ private void exportProcessDefinition(File projectDir, String project) throws Exc if (jsonNode.has("code") && jsonNode.get("code").asInt() > 0) { throw new RuntimeException(response); } + if (isVersion1()) { + setProcessId(processDefinitions, jsonNode); + } FileUtils.writeStringToFile( new File(processDefinitionDir, "process_definitions_page_" + p.getPageNum() + ".json"), JSONUtils.toPrettyString(jsonNode), @@ -461,6 +469,29 @@ private void exportProcessDefinition(File projectDir, String project) throws Exc }); } + private void setProcessId(List processDefinitions, JsonNode jsonNode) { + //List processMetas = GsonUtils.fromJsonString(response, new TypeToken>() {}.getType()); + Map nameIdMap = Maps.newHashMap(); + for (JsonObject process : processDefinitions) { + Integer id = null; + String name = null; + if (process.has("name")) { + name = process.get("name").getAsString(); + } + if (process.has("id")) { + id = process.get("id").getAsInt(); + } + nameIdMap.put(name, id); + } + + ArrayNode arrayNode = (ArrayNode) jsonNode; + for (int i = 0; i < arrayNode.size(); i++) { + JsonNode process = arrayNode.get(i); + Integer id = nameIdMap.get(process.get("processDefinitionName").asText()); + ((ObjectNode) process).put("processDefinitionId", id); + } + } + private int queryProcessDefinitionCount(String project) throws Exception { QueryProcessDefinitionByPaginateRequest request = new QueryProcessDefinitionByPaginateRequest(); request.setPageSize(1); diff --git a/client/migrationx/migrationx-reader/src/test/java/com/aliyun/dataworks/migrationx/reader/dolphinscheduler/DolphinSchedulerCommandAppTest.java b/client/migrationx/migrationx-reader/src/test/java/com/aliyun/dataworks/migrationx/reader/dolphinscheduler/DolphinSchedulerCommandAppTest.java deleted file mode 100644 index 64a0bbd..0000000 --- a/client/migrationx/migrationx-reader/src/test/java/com/aliyun/dataworks/migrationx/reader/dolphinscheduler/DolphinSchedulerCommandAppTest.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.aliyun.dataworks.migrationx.reader.dolphinscheduler; - -import org.junit.Ignore; -import org.junit.Test; - -@Ignore -public class DolphinSchedulerCommandAppTest { - @Test - public void testReader1x() { - DolphinSchedulerCommandApp app = new DolphinSchedulerCommandApp(); - String[] args = new String[]{ - "-e", "http://8.152.5.70:12345/", - "-t", "37fc0fd51131cdba974879d1bf9x0da78", - "-v", "1.3.5", - "-p", "test1234", - "-f", "../../temp/13" - }; - app.run(args); - } - - @Test - public void testReader2x() { - DolphinSchedulerCommandApp app = new DolphinSchedulerCommandApp(); - String[] args = new String[]{ - "-e", "http://101.200.34.168:12345", - "-t", "dbdab854de9695fb3bd2efax30f59d8d0", - "-v", "2.0.5", - "-p", "proj1", - "-f", "temp/13666515015680" - }; - app.run(args); - } - - @Test - public void testReader3x() { - DolphinSchedulerCommandApp app = new DolphinSchedulerCommandApp(); - String[] args = new String[]{ - "-e", "http://39.106.89.151:12345/", - "-t", "xx", - "-v", "3.1.5", - "-p", "code:15688846682752", - "-f", "temp/max2" - }; - app.run(args); - } - - @Test - public void testReader3x2() { - DolphinSchedulerCommandApp app = new DolphinSchedulerCommandApp(); - String[] args = new String[]{ - "-e", "http://39.106.89.151:12345/", - "-t", "xxxxx", - "-v", "3.1.5", - "-p", "p_sparks", - "-f", "temp/max2" - }; - app.run(args); - } -} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/.gitignore b/client/migrationx/migrationx-transformer/.gitignore deleted file mode 100644 index 5ff6309..0000000 --- a/client/migrationx/migrationx-transformer/.gitignore +++ /dev/null @@ -1,38 +0,0 @@ -target/ -!.mvn/wrapper/maven-wrapper.jar -!**/src/main/**/target/ -!**/src/test/**/target/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ -build/ -!**/src/main/**/build/ -!**/src/test/**/build/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/pom.xml b/client/migrationx/migrationx-transformer/pom.xml index efa6e83..237fb8a 100644 --- a/client/migrationx/migrationx-transformer/pom.xml +++ b/client/migrationx/migrationx-transformer/pom.xml @@ -20,7 +20,7 @@ migrationx com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml 4.0.0 @@ -115,21 +115,6 @@ commons-cli - - org.powermock - powermock-module-junit4-rule-agent - test - - - org.powermock - powermock-module-junit4 - test - - - org.powermock - powermock-api-mockito2 - test - org.mockito mockito-inline diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/BaseTransformerApp.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/BaseTransformerApp.java index 3047daa..0ac8703 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/BaseTransformerApp.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/BaseTransformerApp.java @@ -23,6 +23,7 @@ import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.file.LocalFileCheckPoint; import com.aliyun.dataworks.migrationx.transformer.core.transformer.Transformer; import com.aliyun.migrationx.common.context.TransformerContext; +import com.aliyun.migrationx.common.utils.JSONUtils; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -63,6 +64,7 @@ public BaseTransformerApp(Class from, Class { +public interface CheckPoint { ThreadLocal INSTANCE = ThreadLocal.withInitial(() -> new LocalFileCheckPoint()); - List doWithCheckpoint(Function> func, String projectName); + List doWithCheckpoint(Function> func, String projectName); - void doCheckpoint(WRITER writer, List workflows, String processName, String taskName); + void doCheckpoint(WRITER writer, List workflows, String processName, String taskName); - Map> loadFromCheckPoint(String projectName, String processName); + Map> loadFromCheckPoint(String projectName, String processName); static CheckPoint getInstance() { return INSTANCE.get(); diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/checkpoint/file/LocalFileCheckPoint.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/checkpoint/file/LocalFileCheckPoint.java index 81c01ab..4626ba7 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/checkpoint/file/LocalFileCheckPoint.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/checkpoint/file/LocalFileCheckPoint.java @@ -31,7 +31,6 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwNode; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwResource; -import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwWorkflow; import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.CheckPoint; import com.aliyun.migrationx.common.context.TransformerContext; @@ -42,7 +41,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class LocalFileCheckPoint implements CheckPoint { +public class LocalFileCheckPoint implements CheckPoint { /** * check point file suffix */ @@ -51,7 +50,7 @@ public class LocalFileCheckPoint implements CheckPoint { * need append line to file */ private static final boolean FILE_APPEND = true; - private final static TypeReference> REF = new TypeReference>() {}; + private final TypeReference> REF = new TypeReference>() {}; private static final PolymorphicTypeValidator ptv = BasicPolymorphicTypeValidator.builder() .allowIfSubType(DwNode.class) .allowIfSubType(DwResource.class) @@ -60,7 +59,7 @@ public class LocalFileCheckPoint implements CheckPoint { private final ObjectMapper objectMapper = new ObjectMapper().setPolymorphicTypeValidator(ptv); @Override - public List doWithCheckpoint(Function> checkpointFunc, String projectName) { + public List doWithCheckpoint(Function> checkpointFunc, String projectName) { File checkpoint = TransformerContext.getContext().getCheckpoint(); if (checkpoint == null) { log.info("checkpoint is null, skip"); @@ -71,7 +70,7 @@ public List doWithCheckpoint(Function doWithCheckpoint(Function> checkpointFunc, File target) { + public List doWithCheckpoint(Function> checkpointFunc, File target) { try (FileOutputStream out = new FileOutputStream(target, FILE_APPEND)) { try (OutputStreamWriter streamWriter = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { try (BufferedFileWriter writer = new BufferedFileWriter(streamWriter)) { @@ -84,11 +83,11 @@ public List doWithCheckpoint(Function workflows, String processName, String taskName) { + public void doCheckpoint(BufferedFileWriter writer, List workflows, String processName, String taskName) { if (writer == null) { return; } - Map> dataMap = new HashMap<>(); + Map> dataMap = new HashMap<>(); dataMap.put(taskName, workflows); String data = toJson(workflows); try { @@ -101,9 +100,9 @@ public void doCheckpoint(BufferedFileWriter writer, List workflows, } @Override - public Map> loadFromCheckPoint(String projectName, String processName) { + public Map> loadFromCheckPoint(String projectName, String processName) { File dir = TransformerContext.getContext().getLoad(); - Map> workflowMap = null; + Map> workflowMap = null; if (dir != null && dir.exists() && dir.isDirectory()) { for (File checkFile : dir.listFiles()) { if (checkFile.getName().endsWith(projectName + SUFFIX)) { @@ -121,11 +120,11 @@ private interface LineConsumer { void consume(String processName, String taskName, String line); } - private Map> loadFromCheckpoint(File checkpointFile, String qProcessName) { - final Map> workflowMap = new HashMap<>(); + private Map> loadFromCheckpoint(File checkpointFile, String qProcessName) { + final Map> workflowMap = new HashMap<>(); final LineConsumer lineConsumer = (processName, taskName, line) -> { if (qProcessName.equals(processName)) { - List taskMap = parseJson(line); + List taskMap = parseJson(line); workflowMap.putIfAbsent(taskName, taskMap); } }; @@ -172,9 +171,9 @@ private String toJson(Object o) { } } - private List parseJson(String json) { + private List parseJson(String json) { try { - List lists = objectMapper.readerFor(REF).readValue(json); + List lists = objectMapper.readerFor(REF).readValue(json); return lists; } catch (Exception e) { throw new RuntimeException(e); diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/checkpoint/memory/InMemoryCheckPoint.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/checkpoint/memory/InMemoryCheckPoint.java index c0baf48..dc6e4df 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/checkpoint/memory/InMemoryCheckPoint.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/checkpoint/memory/InMemoryCheckPoint.java @@ -19,19 +19,18 @@ import java.util.Map; import java.util.function.Function; -import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwWorkflow; import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.CheckPoint; -public class InMemoryCheckPoint implements CheckPoint { - public List doWithCheckpoint(Function> func, String projectName) { +public class InMemoryCheckPoint implements CheckPoint { + public List doWithCheckpoint(Function> func, String projectName) { return null; } - public void doCheckpoint(CacheWriter writer, List workflows, String processName, String taskName) { + public void doCheckpoint(CacheWriter writer, List workflows, String processName, String taskName) { } - public Map> loadFromCheckPoint(String projectName, String processName) { + public Map> loadFromCheckPoint(String projectName, String processName) { return null; } } diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/loader/ProjectWorkflowLoader.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/loader/ProjectWorkflowLoader.java index db757b0..0e46fbd 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/loader/ProjectWorkflowLoader.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/loader/ProjectWorkflowLoader.java @@ -15,6 +15,15 @@ package com.aliyun.dataworks.migrationx.transformer.core.loader; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + import com.aliyun.dataworks.migrationx.domain.dataworks.constants.DataWorksConstants; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwFunction; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwNode; @@ -27,7 +36,9 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.utils.ResourceUtils; import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; import com.aliyun.dataworks.migrationx.transformer.core.controller.Task; +import com.aliyun.migrationx.common.utils.BeanUtils; import com.aliyun.migrationx.common.utils.GsonUtils; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.gson.JsonObject; @@ -37,12 +48,6 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.BeanUtils; - -import java.io.File; -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; /** * @author sam.liux diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/translator/SparkSubmitTranslator.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/translator/SparkSubmitTranslator.java index 92a8217..ae42df0 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/translator/SparkSubmitTranslator.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/translator/SparkSubmitTranslator.java @@ -36,13 +36,13 @@ import com.aliyun.dataworks.migrationx.transformer.core.spark.command.SparkSubmitCommandBuilder; import com.aliyun.migrationx.common.exception.BizException; import com.aliyun.migrationx.common.exception.ErrorCode; + import com.google.common.base.Joiner; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.core.io.ClassPathResource; /** * open source spark submit task to ODPS_SPARK @@ -102,7 +102,7 @@ public boolean translate(DwWorkflow workflow, DwNode node) { } private void convertSpark(DwWorkflow workflow, DwNode node, String calcEngineType) { - SparkSubmitCommandBuilder sparkSubmitCommandBuilder = getSparkSubmitCommandBuilder(node); + SparkSubmitCommandBuilder sparkSubmitCommandBuilder = getSparkSubmitCommandBuilder(node.getCode()); if (sparkSubmitCommandBuilder == null) { return; } @@ -138,7 +138,7 @@ private void convertSpark(DwWorkflow workflow, DwNode node, String calcEngineTyp } private void convertToOdpsSpark(DwWorkflow workflow, DwNode node) { - SparkSubmitCommandBuilder sparkSubmitCommandBuilder = getSparkSubmitCommandBuilder(node); + SparkSubmitCommandBuilder sparkSubmitCommandBuilder = getSparkSubmitCommandBuilder(node.getCode()); String calcEngineType = "odps"; List appResources = handleResources(workflow, Arrays.asList(sparkSubmitCommandBuilder.getAppResource()), @@ -192,9 +192,9 @@ private void convertToOdpsSpark(DwWorkflow workflow, DwNode node) { node.setType(CodeProgramType.ODPS_SPARK.name()); } - private SparkSubmitCommandBuilder getSparkSubmitCommandBuilder(DwNode node) { + private SparkSubmitCommandBuilder getSparkSubmitCommandBuilder(String code) { List args = new ArrayList<>( - Arrays.asList(node.getCode().split(" ")) + Arrays.asList(code.split(" ")) .stream() .filter(str -> str.trim().length() > 0) .map(String::trim) @@ -208,7 +208,7 @@ private SparkSubmitCommandBuilder getSparkSubmitCommandBuilder(DwNode node) { SparkSubmitCommandBuilder sparkSubmitCommandBuilder = new SparkSubmitCommandBuilder(args); return sparkSubmitCommandBuilder; } catch (Exception e) { - LOGGER.warn("code: {}, exception: ", node.getCode(), e); + LOGGER.warn("code: {}, exception: ", code, e); } return null; } @@ -237,8 +237,7 @@ private void addPlaceholderResource(DwWorkflow workflow, String fileName, String String filePath = ResourceUtils.getPlaceholderFile(fileName); try { - ClassPathResource classPathResource = new ClassPathResource(filePath); - resource.setLocalPath(classPathResource.getFile().getAbsolutePath()); + resource.setLocalPath(new File(filePath).getAbsolutePath()); resource.setWorkflowRef(workflow); workflow.getResources().add(resource); } catch (Exception e) { diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/utils/EmrCodeUtils.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/utils/EmrCodeUtils.java index 4637289..fd68a92 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/utils/EmrCodeUtils.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/core/utils/EmrCodeUtils.java @@ -27,6 +27,7 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.Node; import com.aliyun.dataworks.migrationx.domain.dataworks.utils.NodeUtils; import com.aliyun.migrationx.common.utils.GsonUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,6 +49,17 @@ public static EmrCode asEmrCode(Node node) { return null; } + public static EmrCode asEmrCode(String type, String code) { + try { + if (NodeUtils.isEmrNode(type)) { + return GsonUtils.fromJsonString(code, EmrCode.class); + } + } catch (Exception e) { + LOGGER.error("convert to EmrCode failed code: {}, exception: ", code, e); + } + return null; + } + public static String toEmrCode(Node node) { CodeProgramType nodeType = CodeProgramType.valueOf(node.getType()); EmrCode emrCode = new EmrCode(); @@ -90,6 +102,47 @@ public static String toEmrCode(Node node) { } } + public static String toEmrCode(CodeProgramType type, String taskName, String code) { + EmrCode emrCode = new EmrCode(); + emrCode.setName(taskName); + emrCode.setDescription("DataWorks Migration"); + EmrLauncher launcher = new EmrLauncher(); + launcher.setAllocationSpec(getDefaultAllocationSpec()); + emrCode.setLauncher(launcher); + EmrProperty properties = new EmrProperty(); + properties.setArguments(Arrays.asList(code)); + properties.setTags(Arrays.asList(taskName)); + emrCode.setProperties(properties); + switch (type) { + case EMR_HIVE: + emrCode.setType(EmrJobType.HIVE_SQL); + return GsonUtils.toJsonString(emrCode); + case EMR_SHELL: + emrCode.setType(EmrJobType.SHELL); + return GsonUtils.toJsonString(emrCode); + case EMR_IMPALA: + emrCode.setType(EmrJobType.IMPALA_SQL); + return GsonUtils.toJsonString(emrCode); + case EMR_MR: + emrCode.setType(EmrJobType.MR); + return GsonUtils.toJsonString(emrCode); + case EMR_PRESTO: + emrCode.setType(EmrJobType.PRESTO_SQL); + return GsonUtils.toJsonString(emrCode); + case EMR_SPARK: + emrCode.setType(EmrJobType.SPARK); + return GsonUtils.toJsonString(emrCode); + case EMR_SPARK_SHELL: + emrCode.setType(EmrJobType.SPARK_SHELL); + return GsonUtils.toJsonString(emrCode); + case EMR_SPARK_SQL: + emrCode.setType(EmrJobType.SPARK_SQL); + return GsonUtils.toJsonString(emrCode); + default: + return code; + } + } + private static Map getDefaultAllocationSpec() { EmrAllocationSpec allocationSpec = new EmrAllocationSpec(); allocationSpec.setMemory(String.valueOf(2048)); diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/DataWorksDolphinschedulerTransformerApp.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/DataWorksDolphinschedulerTransformerApp.java index 33c1cdc..2b54997 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/DataWorksDolphinschedulerTransformerApp.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/DataWorksDolphinschedulerTransformerApp.java @@ -16,19 +16,29 @@ package com.aliyun.dataworks.migrationx.transformer.dataworks.apps; import java.io.File; +import java.nio.charset.StandardCharsets; +import com.aliyun.dataworks.common.spec.utils.JSONUtils; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.DolphinSchedulerPackage; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DataWorksPackage; import com.aliyun.dataworks.migrationx.domain.dataworks.standard.objects.Package; import com.aliyun.dataworks.migrationx.transformer.core.BaseTransformerApp; import com.aliyun.dataworks.migrationx.transformer.core.transformer.Transformer; import com.aliyun.dataworks.migrationx.transformer.dataworks.transformer.DataWorksDolphinSchedulerTransformer; +import com.aliyun.dataworks.migrationx.transformer.dataworks.transformer.DataWorksPackageFormat; +import com.aliyun.dataworks.migrationx.transformer.dataworks.transformer.DataWorksTransformerConfig; +import com.aliyun.dataworks.migrationx.transformer.dataworks.transformer.WorkflowDolphinSchedulerTransformer; import com.aliyun.migrationx.common.context.TransformerContext; +import com.aliyun.migrationx.common.exception.BizException; +import com.aliyun.migrationx.common.exception.ErrorCode; import com.aliyun.migrationx.common.metrics.enums.CollectorType; +import com.aliyun.migrationx.common.utils.Config; +import com.aliyun.migrationx.common.utils.GsonUtils; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.reflect.TypeToken; import lombok.extern.slf4j.Slf4j; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.commons.io.FileUtils; /** * @author 聿剑 @@ -49,9 +59,36 @@ public void initCollector() { @SuppressWarnings("rawtypes") @Override - protected Transformer createTransformer(File config, Package from, Package to) { + protected Transformer createTransformer(File configFile, Package from, Package to) { DolphinSchedulerPackage dolphinSchedulerPackage = (DolphinSchedulerPackage) from; DataWorksPackage dataWorksPackage = (DataWorksPackage) to; - return new DataWorksDolphinSchedulerTransformer(config, dolphinSchedulerPackage, dataWorksPackage); + DataWorksTransformerConfig config = initConfig(configFile); + log.info("dataworks transformer configFile: {} config: {}", configFile.getAbsolutePath(), config); + if (config.getFormat() != null && DataWorksPackageFormat.WORKFLOW.equals(config.getFormat())) { + return new WorkflowDolphinSchedulerTransformer(config, dolphinSchedulerPackage, dataWorksPackage); + } else { + return new DataWorksDolphinSchedulerTransformer(config, dolphinSchedulerPackage, dataWorksPackage); + } + } + + private DataWorksTransformerConfig initConfig(File configFile) { + if (!configFile.exists()) { + log.error("config file not exists: {}", configFile); + throw new RuntimeException("file not found by " + configFile.getAbsolutePath()); + } + try { + String config = FileUtils.readFileToString(configFile, StandardCharsets.UTF_8); + DataWorksTransformerConfig dataWorksTransformerConfig + = JSONUtils.parseObject(config, new TypeReference() {}); + if (dataWorksTransformerConfig == null) { + log.error("config file: {}, config class: {}", configFile, DataWorksTransformerConfig.class); + throw new BizException(ErrorCode.PARSE_CONFIG_FILE_FAILED).with(configFile); + } + Config dwConfig = GsonUtils.fromJsonString(config, new TypeToken() {}.getType()); + Config.init(dwConfig); + return dataWorksTransformerConfig; + } catch (Exception e) { + throw new RuntimeException(e); + } } } diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/V1MigrationPackageConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/V1MigrationPackageConverter.java index 145de1b..2ca3169 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/V1MigrationPackageConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/V1MigrationPackageConverter.java @@ -50,7 +50,9 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.NodeUseType; import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; import com.aliyun.dataworks.migrationx.transformer.core.utils.ZipUtils; +import com.aliyun.migrationx.common.utils.BeanUtils; import com.aliyun.migrationx.common.utils.GsonUtils; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.common.base.Joiner; @@ -63,7 +65,6 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.BeanUtils; /** * V1 Migration工具的导出包转换器，转换为dataworks-model diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/adf/AdfConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/adf/AdfConverter.java index fc847bf..629d0c1 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/adf/AdfConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/adf/AdfConverter.java @@ -1,32 +1,44 @@ package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.adf; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.time.LocalTime; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import javax.validation.constraints.NotNull; + import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; -import com.aliyun.dataworks.common.spec.domain.enums.*; +import com.aliyun.dataworks.common.spec.domain.enums.ArtifactType; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.enums.FlowType; +import com.aliyun.dataworks.common.spec.domain.enums.NodeInstanceModeType; +import com.aliyun.dataworks.common.spec.domain.enums.NodeRecurrenceType; +import com.aliyun.dataworks.common.spec.domain.enums.NodeRerunModeType; +import com.aliyun.dataworks.common.spec.domain.enums.TriggerType; import com.aliyun.dataworks.common.spec.domain.interfaces.Output; import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; import com.aliyun.dataworks.common.spec.domain.noref.SpecSubFlow; -import com.aliyun.dataworks.common.spec.domain.ref.*; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; import com.aliyun.dataworks.migrationx.domain.adf.AdfConf; import com.aliyun.dataworks.migrationx.domain.adf.AdfPackage; import com.aliyun.dataworks.migrationx.domain.adf.Pipeline; import com.aliyun.dataworks.migrationx.domain.adf.Trigger; + import com.google.common.collect.ImmutableList; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.jetbrains.annotations.NotNull; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.time.Duration; -import java.time.LocalTime; -import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; @Slf4j public class AdfConverter { diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/airflow/AirflowDagConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/airflow/AirflowDagConverter.java index 79b1a0e..60a43ed 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/airflow/AirflowDagConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/airflow/AirflowDagConverter.java @@ -44,6 +44,7 @@ import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.WorkflowConverter; import com.aliyun.migrationx.common.exception.BizException; import com.aliyun.migrationx.common.exception.ErrorCode; +import com.aliyun.migrationx.common.utils.BeanUtils; import com.aliyun.migrationx.common.utils.FileNameUtils; import com.aliyun.migrationx.common.utils.GsonUtils; @@ -56,7 +57,6 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.BeanUtils; /** * Implementation of parsing airflow dag python code to workflow definition diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DolphinSchedulerV1Converter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/DolphinSchedulerV1Converter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DolphinSchedulerV1Converter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/DolphinSchedulerV1Converter.java index 547a4c2..e0fcc6e 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DolphinSchedulerV1Converter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/DolphinSchedulerV1Converter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes; import java.util.ArrayList; import java.util.Collections; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/TaskConverterFactory.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/TaskConverterFactory.java similarity index 67% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/TaskConverterFactory.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/TaskConverterFactory.java index 4f359ea..e07ee48 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/TaskConverterFactory.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/TaskConverterFactory.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes; import java.util.Optional; import java.util.function.Supplier; @@ -23,6 +23,18 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.enums.TaskType; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.AbstractParameters; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.AbstractParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.ConditionsParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.DataxParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.DefaultParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.DependentParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.MrParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.PythonParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.ShellParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.SparkParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.SqlParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.SqoopParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.SubProcessParameterConverter; /** * @author 聿剑 diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/V1ProcessDefinitionConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/V1ProcessDefinitionConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/V1ProcessDefinitionConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/V1ProcessDefinitionConverter.java index af7d858..167d0c7 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/V1ProcessDefinitionConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/V1ProcessDefinitionConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes; import java.util.ArrayList; import java.util.Collections; @@ -46,6 +46,8 @@ import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.StoreWriter; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.filters.DolphinSchedulerConverterFilter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.AbstractParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.ProcessDefinitionConverter; import com.aliyun.migrationx.common.context.TransformerContext; import com.aliyun.migrationx.common.exception.UnSupportedTypeException; import com.aliyun.migrationx.common.metrics.DolphinMetrics; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/AbstractParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/AbstractParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/AbstractParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/AbstractParameterConverter.java index 4a63510..6942d69 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/AbstractParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/AbstractParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.io.IOException; import java.util.ArrayList; @@ -60,6 +60,7 @@ import com.aliyun.migrationx.common.context.TransformerContext; import com.aliyun.migrationx.common.metrics.DolphinMetrics; import com.aliyun.migrationx.common.utils.GsonUtils; + import com.google.common.base.Joiner; import com.google.gson.reflect.TypeToken; import lombok.extern.slf4j.Slf4j; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/ConditionsParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/ConditionsParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/ConditionsParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/ConditionsParameterConverter.java index 4d7cea6..8571e19 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/ConditionsParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/ConditionsParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.io.IOException; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DataxParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/DataxParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DataxParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/DataxParameterConverter.java index 09fae67..850e205 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DataxParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/DataxParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.util.Arrays; import java.util.HashMap; @@ -284,6 +284,6 @@ private DataSource getDataSource(int datasourceId) { return CollectionUtils.emptyIfNull(datasources).stream() .filter(s -> s.getId() == datasourceId) .findFirst() - .orElseGet(null); + .orElse(null); } } diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DefaultParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/DefaultParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DefaultParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/DefaultParameterConverter.java index deb4e93..cdbdc3d 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DefaultParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/DefaultParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.util.Arrays; import java.util.List; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DependentParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/DependentParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DependentParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/DependentParameterConverter.java index c3c9c3e..6e387ea 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/DependentParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/DependentParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/MrParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/MrParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/MrParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/MrParameterConverter.java index 19a020e..4f0b600 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/MrParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/MrParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; @@ -39,11 +39,11 @@ import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; +import com.aliyun.migrationx.common.utils.BeanUtils; import com.google.common.base.Joiner; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; /** * @author 聿剑 diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/ProcessDefinitionConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/ProcessDefinitionConverter.java similarity index 97% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/ProcessDefinitionConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/ProcessDefinitionConverter.java index e5295eb..457346b 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/ProcessDefinitionConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/ProcessDefinitionConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.text.ParseException; import java.util.List; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/PythonParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/PythonParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/PythonParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/PythonParameterConverter.java index 563a32f..e48a990 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/PythonParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/PythonParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.io.File; import java.io.IOException; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/ShellParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/ShellParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/ShellParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/ShellParameterConverter.java index 8bedce7..a234efb 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/ShellParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/ShellParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SparkParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SparkParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SparkParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SparkParameterConverter.java index 3d36965..ccd1b04 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SparkParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SparkParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; @@ -37,11 +37,11 @@ import com.aliyun.dataworks.migrationx.transformer.core.translator.TranslateUtils; import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; +import com.aliyun.migrationx.common.utils.BeanUtils; import com.google.common.base.Joiner; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; -import org.springframework.beans.BeanUtils; /** * @author 聿剑 diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SqlParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SqlParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SqlParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SqlParameterConverter.java index 28696d3..72844fe 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SqlParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SqlParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.util.Arrays; import java.util.HashMap; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SqoopParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SqoopParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SqoopParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SqoopParameterConverter.java index 82e30e8..36cfe4f 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SqoopParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SqoopParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SubProcessParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SubProcessParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SubProcessParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SubProcessParameterConverter.java index 328802e..bcd8449 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/SubProcessParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/nodes/parameters/SubProcessParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters; import java.util.Arrays; import java.util.List; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/DolphinSchedulerV1WorkflowConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/DolphinSchedulerV1WorkflowConverter.java new file mode 100644 index 0000000..c013ae1 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/DolphinSchedulerV1WorkflowConverter.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.DataWorksWorkflowSpec; +import com.aliyun.dataworks.common.spec.domain.Specification; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.enums.SpecKind; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecSubFlow; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.DolphinSchedulerPackage; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.Project; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.UdfFunc; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.datasource.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.entity.ResourceInfo; + +public class DolphinSchedulerV1WorkflowConverter { + public static final String SPEC_VERSION = "1.2.0"; + + protected List processMetaList; + protected Map nodeTypeMap = new HashMap<>(); + private DolphinSchedulerPackage + dolphinSchedulerPackage; + + private final Properties converterProperties; + + public DolphinSchedulerV1WorkflowConverter(DolphinSchedulerPackage dolphinSchedulerPackage, Properties converterProperties) { + this.dolphinSchedulerPackage = dolphinSchedulerPackage; + this.converterProperties = converterProperties; + } + + public List> convert() { + processMetaList = dolphinSchedulerPackage.getProcessDefinitions().values() + .stream().flatMap(List::stream).collect(Collectors.toList()); + + List> specifications = new ArrayList<>(); + for (ProcessMeta processMeta : processMetaList) { + List workflows = new ArrayList<>(); + //convert process to workflow + Specification specification = new Specification<>(); + specification.setKind(SpecKind.CYCLE_WORKFLOW.getLabel()); + specification.setVersion(SPEC_VERSION); + DataWorksWorkflowSpec spec = new DataWorksWorkflowSpec(); + //todo spec name + String processName = processMeta.getProcessDefinitionName(); + spec.setName(processName); + ProcessData processData = processMeta.getProcessDefinitionJson(); + V1ProcessDefinitionConverter converter = new V1ProcessDefinitionConverter(processMeta, processData.getTasks(), this.converterProperties); + SpecWorkflow workflow = converter.convert(); + workflows.add(workflow); + spec.setWorkflows(workflows); + specification.setSpec(spec); + specifications.add(specification); + } + handleSubprocess(); + handleDependents(specifications); + return specifications; + } + + /** + * subprocess + */ + private void handleSubprocess() { + Map codeWorkflowMap = DolphinSchedulerV1Context.getContext().getSubProcessCodeWorkflowMap(); + Map codeNodeMap = DolphinSchedulerV1Context.getContext().getSubProcessCodeNodeMap(); + //find subprocess + for (Map.Entry entry : codeNodeMap.entrySet()) { + //find workflow + SpecWorkflow specWorkflow = (SpecWorkflow) codeWorkflowMap.get(entry.getKey()); + SpecNodeOutput specNodeOutput; + if (specWorkflow.getOutputs().isEmpty()) { + specNodeOutput = new SpecNodeOutput(); + specNodeOutput.setData(specWorkflow.getId()); + specWorkflow.getOutputs().add(specNodeOutput); + } else { + specNodeOutput = (SpecNodeOutput) specWorkflow.getOutputs().get(0); + } + + SpecNode subprocess = (SpecNode) entry.getValue(); + SpecSubFlow subflow = new SpecSubFlow(); + subflow.setOutput(specNodeOutput.getData()); + subprocess.setSubflow(subflow); + } + } + + /** + * dependent process + * + * @param specifications + */ + private void handleDependents(List> specifications) { + Map nodeIdWorkflowMap = new HashMap<>(); + for (Specification specification : specifications) { + specification.getSpec().getWorkflows().forEach(workflow -> { + workflow.getNodes().forEach(node -> { + nodeIdWorkflowMap.put(node.getId(), workflow); + }); + }); + } + Map taskCodeSpecNodeMap = DolphinSchedulerV1Context.getContext().getTaskCodeSpecNodeMap(); + Map> depMap = DolphinSchedulerV1Context.getContext().getSpecNodeProcessCodeMap(); + + for (Map.Entry> entry : depMap.entrySet()) { + SpecNode specNode = (SpecNode) entry.getKey(); + SpecWorkflow workflow = nodeIdWorkflowMap.get(specNode.getId()); + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + + List dependents = entry.getValue(); + List depends = new ArrayList<>(); + for (String id : dependents) { + SpecNode depNode = (SpecNode) taskCodeSpecNodeMap.get(id); + SpecNodeOutput specNodeOutput = new SpecNodeOutput(); + specNodeOutput.setData(depNode.getId()); + SpecDepend specDepend = new SpecDepend(null, DependencyType.NORMAL, specNodeOutput); + depends.add(specDepend); + } + specFlowDepend.setDepends(depends); + + specFlowDepend.setNodeId(specNode); + workflow.getDependencies().add(specFlowDepend); + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/ParamListConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/ParamListConverter.java new file mode 100644 index 0000000..4079700 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/ParamListConverter.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.enums.VariableScopeType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.entity.Property; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.enums.Direct; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import org.apache.commons.collections4.ListUtils; + +public class ParamListConverter { + + public static final String SYSTEM_VARIABLE_TAG = "$"; + + private final List paramList; + + private final TaskNode taskDefinition; + + public ParamListConverter(List paramList) { + this(paramList, null); + } + + public ParamListConverter(List paramList, TaskNode taskDefinition) { + super(); + this.paramList = paramList; + this.taskDefinition = taskDefinition; + } + + public List convert() { + return ListUtils.emptyIfNull(paramList).stream().map(p -> { + // don't convert global out param + if (Objects.isNull(taskDefinition) && Direct.OUT.equals(p.getDirect())) { + return null; + } + + SpecVariable specVariable = new SpecVariable(); + + specVariable.setId(UuidGenerators.generateUuid()); + specVariable.setName(p.getProp()); + specVariable.setValue(p.getValue()); + specVariable.setDescription(p.getType().name()); + if (Direct.IN.equals(p.getDirect())) { + if (specVariable.getValue().startsWith(SYSTEM_VARIABLE_TAG)) { + specVariable.setType(VariableType.SYSTEM); + } else { + specVariable.setType(VariableType.CONSTANT); + } + specVariable.setScope(Objects.isNull(taskDefinition) ? VariableScopeType.FLOW : VariableScopeType.NODE_PARAMETER); + } else { + specVariable.setType(VariableType.NODE_OUTPUT); + specVariable.setScope(VariableScopeType.NODE_CONTEXT); + } + return specVariable; + } + ).filter(Objects::nonNull).collect(Collectors.toList()); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/SpecFlowDependConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/SpecFlowDependConverter.java new file mode 100644 index 0000000..1a48b51 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/SpecFlowDependConverter.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.DataWorksWorkflowSpec; +import com.aliyun.dataworks.common.spec.domain.enums.ArtifactType; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.domain.ref.SpecArtifact; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.ProcessTaskRelation; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; + +@Slf4j +public class SpecFlowDependConverter { + + private static final Map> ARTIFACT_TYPE_CLASS_MAP = new EnumMap<>(ArtifactType.class); + + private final DataWorksWorkflowSpec spec; + + private final SpecWorkflow specWorkflow; + + private final List processTaskRelationList; + + static { + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.TABLE, SpecTable.class); + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.VARIABLE, SpecVariable.class); + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.NODE_OUTPUT, SpecNodeOutput.class); + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.FILE, SpecArtifact.class); + } + + public SpecFlowDependConverter(DataWorksWorkflowSpec spec, SpecWorkflow specWorkflow, List processTaskRelationList) { + this.spec = spec; + this.specWorkflow = specWorkflow; + this.processTaskRelationList = processTaskRelationList; + } + + public List convert() { + if (Objects.nonNull(specWorkflow)) { + specWorkflow.setDependencies(convertTaskRelationList(processTaskRelationList)); + return specWorkflow.getDependencies(); + } + spec.setFlow(convertTaskRelationList(processTaskRelationList)); + return spec.getFlow(); + } + + private List convertTaskRelationList(List taskRelationList) { + List flow = Optional.ofNullable(specWorkflow).map(SpecWorkflow::getDependencies).orElse(Optional.ofNullable(spec) + .map(DataWorksWorkflowSpec::getFlow).orElse(new ArrayList<>())); + Map> nodeIdDependMap = flow.stream().collect( + Collectors.toMap(o -> o.getNodeId().getId(), SpecFlowDepend::getDepends)); + + Map taskCodeNodeDataMap = DolphinSchedulerV2Context.getContext().getTaskCodeNodeDataMap(); + Map taskCodeNodeIdMap = DolphinSchedulerV2Context.getContext().getTaskCodeNodeIdMap(); + for (ProcessTaskRelation processTaskRelation : ListUtils.emptyIfNull(taskRelationList)) { + long preTaskCode = processTaskRelation.getPreTaskCode(); + if (preTaskCode == 0L) { + continue; + } + long postTaskCode = processTaskRelation.getPostTaskCode(); + String nodeId = taskCodeNodeIdMap.get(postTaskCode); + SpecNode currentNode; + if (nodeId == null) { + log.warn("can not find nodeId {}", postTaskCode); + continue; + } else { + currentNode = specWorkflow.getNodes().stream().filter(node -> nodeId.equals(node.getId())) + .findAny().orElse(null); + } + final SpecNode finalCurrentNode = currentNode; + + String data = taskCodeNodeDataMap.get(preTaskCode); + if (data != null) { + specWorkflow.getNodes().stream() + .filter(node -> { + return CollectionUtils.emptyIfNull(node.getOutputs()) + .stream() + .map(output -> (SpecNodeOutput) output) + .anyMatch(output -> data.equals(output.getData())); + }) + .findAny() + .ifPresent(node -> { + SpecNodeOutput specNodeOutput = (SpecNodeOutput) node.getOutputs().get(0); + SpecDepend specDepend = new SpecDepend(null, DependencyType.NORMAL, specNodeOutput); + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + specFlowDepend.setDepends(Arrays.asList(specDepend)); + specFlowDepend.setNodeId(finalCurrentNode); + //flow.add(specFlowDepend); + specWorkflow.getDependencies().add(specFlowDepend); + }); + } + } + return flow; + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/TaskConverterFactoryV1.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/TaskConverterFactoryV1.java new file mode 100644 index 0000000..fb51666 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/TaskConverterFactoryV1.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow; + +import java.util.Properties; + +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.enums.TaskType; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.AbstractParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.CustomParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.DataxParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.DependentParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.MrParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.ProcedureParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.PythonParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.ShellParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.SparkParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.SqlParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.SqoopParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.SubProcessParameterConverter; +import com.aliyun.migrationx.common.utils.Config; + +public class TaskConverterFactoryV1 { + public static AbstractParameterConverter create( + Properties properties, SpecWorkflow specWorkflow, + ProcessMeta processMeta, TaskNode taskDefinition) throws Throwable { + if (Config.INSTANCE.getTempTaskTypes().contains(taskDefinition.getType())) { + return new CustomParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + } + TaskType taskType = taskDefinition.getType(); + if (taskType == null) { + throw new RuntimeException("task type is null"); + } + + switch (taskType) { + case SHELL: + return new ShellParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case PYTHON: + return new PythonParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SQL: + return new SqlParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case MR: + return new MrParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SPARK: + return new SparkParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SUB_PROCESS: + return new SubProcessParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case DEPENDENT: + return new DependentParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SQOOP: + return new SqoopParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case DATAX: + return new DataxParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case PROCEDURE: + return new ProcedureParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + default: + throw new RuntimeException("unsupported task type: " + taskType); + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/TriggerConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/TriggerConverter.java new file mode 100644 index 0000000..5601c41 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/TriggerConverter.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow; + +import java.text.ParseException; +import java.util.Objects; + +import com.aliyun.dataworks.common.spec.domain.enums.NodeRecurrenceType; +import com.aliyun.dataworks.common.spec.domain.enums.TriggerType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.Schedule; +import com.aliyun.dataworks.migrationx.domain.dataworks.utils.CronExpressUtil; +import com.aliyun.migrationx.common.utils.DateUtils; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class TriggerConverter { + + private final Schedule schedule; + + private final SpecTrigger currTrigger; + + private final TaskNode taskDefinition; + + private SpecTrigger convertRes; + + public TriggerConverter(Schedule schedule) { + super(); + this.schedule = schedule; + this.currTrigger = null; + this.taskDefinition = null; + this.convertRes = new SpecTrigger(); + } + + public TriggerConverter(SpecTrigger trigger, TaskNode taskDefinition) { + super(); + this.schedule = null; + this.currTrigger = trigger; + this.taskDefinition = taskDefinition; + this.convertRes = new SpecTrigger(); + } + + public SpecTrigger convert() { + if (Objects.nonNull(schedule)) { + convertBySchedule(); + } else if (Objects.nonNull(currTrigger) && Objects.nonNull(taskDefinition)) { + convertByTaskDefinition(); + } else { + return null; + } + return convertRes; + } + + /** + * convert workflow trigger + */ + private void convertBySchedule() { + if (Objects.isNull(schedule)) { + convertRes.setType(TriggerType.MANUAL); + } else { + convertRes.setId(UuidGenerators.generateUuid(Long.valueOf(schedule.getId()))); + convertRes.setType(TriggerType.SCHEDULER); + convertRes.setStartTime(DateUtils.convertDateToString(schedule.getStartTime())); + convertRes.setEndTime(DateUtils.convertDateToString(schedule.getEndTime())); + convertRes.setCron(dolphinCron2SpecCron(schedule.getCrontab())); + convertRes.setTimezone(schedule.getTimezoneId()); + convertRes.setDelaySeconds(0); + } + } + + /** + * convert task trigger, especially for delay seconds. + * Because node will use same trigger with parent workflow except delay seconds. + */ + private void convertByTaskDefinition() { + if (currTrigger == null || taskDefinition == null) { + throw new RuntimeException("currTrigger or taskDefinition null"); + } + //convertRes = BeanUtils.deepCopy(currTrigger, SpecTrigger.class); + convertRes = currTrigger; + convertRes.setRecurrence("NORMAL".equals(taskDefinition.getRunFlag()) ? NodeRecurrenceType.NORMAL : NodeRecurrenceType.PAUSE); + convertRes.setId(UuidGenerators.generateUuid()); + } + + private String dolphinCron2SpecCron(String dolphinCron) { + try { + return CronExpressUtil.quartzCronExpressionToDwCronExpress(dolphinCron); + } catch (ParseException e) { + log.warn("dolphin cron parse error: {}", dolphinCron); + return dolphinCron; + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/V1ProcessDefinitionConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/V1ProcessDefinitionConverter.java new file mode 100644 index 0000000..bf9ba74 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/V1ProcessDefinitionConverter.java @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.ControllerBranchCode.Branch; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.noref.SpecBranches; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.AbstractParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.utils.CronExpressUtil; +import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.CheckPoint; +import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.StoreWriter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.filters.DolphinSchedulerConverterFilter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters.AbstractParameterConverter; +import com.aliyun.migrationx.common.context.TransformerContext; +import com.aliyun.migrationx.common.exception.UnSupportedTypeException; +import com.aliyun.migrationx.common.metrics.DolphinMetrics; +import com.aliyun.migrationx.common.utils.Config; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import com.google.common.base.Joiner; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class V1ProcessDefinitionConverter { + private List specNodes = new ArrayList<>(); + + private static final SpecScriptRuntime WORKFLOW_RUNTIME = new SpecScriptRuntime(); + + private static final SpecScriptRuntime MANUAL_WORKFLOW_RUNTIME = new SpecScriptRuntime(); + + static { + WORKFLOW_RUNTIME.setEngine(CodeProgramType.VIRTUAL_WORKFLOW.getCalcEngineType().getLabel()); + WORKFLOW_RUNTIME.setCommand("WORKFLOW"); + + MANUAL_WORKFLOW_RUNTIME.setEngine(CodeProgramType.VIRTUAL_WORKFLOW.getCalcEngineType().getLabel()); + MANUAL_WORKFLOW_RUNTIME.setCommand("MANUAL_WORKFLOW"); + } + + private ProcessMeta processDefinition; + private Properties converterProperties; + private List taskDefinitionList; + + private CheckPoint checkPoint; + private SpecWorkflow specWorkflow; + + private DolphinSchedulerConverterFilter filter; + + public V1ProcessDefinitionConverter(ProcessMeta processMeta, List taskDefinitionList, Properties converterProperties) { + this.processDefinition = processMeta; + this.converterProperties = converterProperties; + this.taskDefinitionList = taskDefinitionList; + this.filter = new DolphinSchedulerConverterFilter(); + checkPoint = CheckPoint.getInstance(); + } + + public static String toWorkflowName(ProcessMeta processMeta) { + return com.aliyun.dataworks.migrationx.domain.dataworks.utils.StringUtils.toValidName(Joiner.on("_").join( + processMeta.getProjectName(), processMeta.getProcessDefinitionName())); + } + + private SpecWorkflow initWorkflow() { + SpecWorkflow specWorkflow = new SpecWorkflow(); + specWorkflow.setDependencies(new ArrayList<>()); + specWorkflow.setNodes(new ArrayList<>()); + specWorkflow.setInputs(new ArrayList<>()); + specWorkflow.setOutputs(new ArrayList<>()); + return specWorkflow; + } + + public SpecWorkflow convert() { + specWorkflow = convertProcess(processDefinition); + DolphinSchedulerV1Context.getContext().getSubProcessCodeWorkflowMap().put(processDefinition.getProcessDefinitionId(), specWorkflow); + convertTasks(); + convertTrigger(specWorkflow); + //convertTaskRelations(specWorkflow); + handleBranch(specWorkflow); + return specWorkflow; + } + + protected SpecWorkflow convertProcess(ProcessMeta processDefinition) { + log.info("convert workflow,processDefinition: {}", processDefinition.getProcessDefinitionName()); + + SpecWorkflow specWorkflow = initWorkflow(); + specWorkflow.setId(UuidGenerators.generateUuid()); + specWorkflow.setName(toWorkflowName(processDefinition)); + specWorkflow.setDescription(processDefinition.getProcessDefinitionDescription()); + + ProcessData processData = processDefinition.getProcessDefinitionJson(); + + List specVariableList = new ParamListConverter(processData.getGlobalParams()).convert(); + log.info("convert workflow,global params: {}", specVariableList); + + SpecScript script = new SpecScript(); + script.setParameters(specVariableList); + script.setRuntime(WORKFLOW_RUNTIME); + + //todo + script.setPath(getScriptPath(specWorkflow.getName())); + specWorkflow.setScript(script); + + specWorkflow.getOutputs().add(buildDefaultOutput(specWorkflow)); + return specWorkflow; + } + + protected String getScriptPath(String name) { + String defaultPath = StringUtils.defaultString(Config.INSTANCE.getBasePath(), StringUtils.EMPTY); + return FilenameUtils.concat(defaultPath, name); + } + + protected List convertTaskDefinitions(SpecWorkflow specWorkflow) { + return convertTasks(); + } + + protected void convertTrigger(SpecWorkflow specWorkflow) { + processDefinition.getScheduleCrontab(); + String crontab = processDefinition.getScheduleCrontab(); + if (StringUtils.isNotBlank(crontab)) { + SpecTrigger trigger = new SpecTrigger(); + trigger.setCron(convertCrontab(crontab)); + specWorkflow.setTrigger(trigger); + log.info("convert workflow,crontab: {}", crontab); + } + } + + protected String convertCrontab(String scheduleCrontab) { + try { + return CronExpressUtil.quartzCronExpressionToDwCronExpress(scheduleCrontab); + } catch (ParseException e) { + log.error("convert quartz cron expression error: ", e); + } + + return "day"; + } + + /** + * when handling task, spec node maybe not exists, post handle branch + * change branch output data (task code) to specNode id + */ + private void handleBranch(SpecWorkflow specWorkflow) { + Map taskCodeSpecNodeMap = DolphinSchedulerV1Context.getContext().getTaskCodeSpecNodeMap(); + for (SpecNode specNode : specWorkflow.getNodes()) { + if (specNode.getBranch() != null && specNode.getBranch().getBranches() != null) { + List specBranches = specNode.getBranch().getBranches(); + List branchList = new ArrayList<>(); + for (SpecBranches specBranch : specBranches) { + SpecNodeOutput specNodeOutput = specBranch.getOutput(); + if (specNodeOutput != null && specNodeOutput.getData() != null) { + String data = specNodeOutput.getData(); + //current data is task code, convert to specNode id + Long taskCode = Long.parseLong(data); + SpecNode branchSpecNode = (SpecNode) taskCodeSpecNodeMap.get(taskCode); + specNodeOutput.setData(branchSpecNode.getId()); + Branch branch = new Branch(); + branch.setCondition(specBranch.getWhen()); + branch.setNodeoutput(branchSpecNode.getId()); + branchList.add(branch); + } + } + if (branchList.size() > 0) { + String content = com.aliyun.dataworks.common.spec.utils.GsonUtils.toJsonString(branchList); + specNode.getScript().setContent(content); + } + } + } + } + + + private SpecNodeOutput buildDefaultOutput(SpecWorkflow specWorkflow) { + SpecNodeOutput specNodeOutput = new SpecNodeOutput(); + specNodeOutput.setIsDefault(true); + specNodeOutput.setData(specWorkflow.getId()); + specNodeOutput.setRefTableName(specWorkflow.getName()); + return specNodeOutput; + } + + public List convertTasks() { + String projectName = processDefinition.getProjectName(); + String processName = processDefinition.getProcessDefinitionName(); + Map> loadedTasks = checkPoint.loadFromCheckPoint(projectName, processName); + SpecWorkflow workflow = new SpecWorkflow(); + + final Function> processFunc = (StoreWriter writer) -> + Optional.ofNullable(processDefinition.getProcessDefinitionJson()) + .map(pd -> toSpecNodes(pd, loadedTasks, writer)) + .orElse(Collections.emptyList()); + + List specNodes = checkPoint.doWithCheckpoint(processFunc, projectName); + + //processSubProcessDefinitionDepends(); + return specNodes; + } + + private List toSpecNodes(ProcessData pd, Map> loadedTasks, StoreWriter writer) { + return ListUtils.emptyIfNull(pd.getTasks()) + .stream() + .filter(s -> { + if (Thread.currentThread().isInterrupted()) { + throw new RuntimeException(new InterruptedException()); + } + return true; + }) + .filter(task -> filter.filter(processDefinition.getProjectName(), + processDefinition.getProcessDefinitionName(), task.getName())) + .filter(taskNode -> !inSkippedList(taskNode)) + .map(taskNode -> { + List specNodes = convertTaskToSpecNodeWithLoadedTask(taskNode, loadedTasks); + checkPoint.doCheckpoint(writer, specNodes, processDefinition.getProcessDefinitionName(), taskNode.getName()); + return specNodes; + }) + .collect(Collectors.toList()) + .stream().flatMap(List::stream) + .collect(Collectors.toList()); + } + + private List convertTaskToSpecNodeWithLoadedTask(TaskNode taskNode, Map> loadedTasks) { + List workflows = loadedTasks.get(taskNode.getName()); + if (workflows != null) { + markSuccessProcess(workflows, taskNode); + log.info("loaded task {} from checkpoint", taskNode.getName()); + return workflows; + } + return converter(taskNode); + } + + private List converter(TaskNode taskNode) { + AbstractParameterConverter converter; + try { + converter = TaskConverterFactoryV1.create(converterProperties, this.specWorkflow, processDefinition, taskNode); + SpecNode specNode = converter.convert(); + if (specNode != null) { + DolphinSchedulerV1Context.getContext().getTaskCodeSpecNodeMap().put(taskNode.getId(), specNode); + return Arrays.asList(specNode); + } else { + return Collections.emptyList(); + } + } catch (UnSupportedTypeException e) { + markFailedProcess(taskNode, e.getMessage()); + if (Config.INSTANCE.isSkipUnSupportType()) { + List list = Collections.emptyList(); + return list; + } else { + throw e; + } + } catch (Throwable e) { + log.error("task converter error: ", e); + throw new RuntimeException(e); + } + } + + protected void markSuccessProcess(List specNodes, TaskNode taskNode) { + for (SpecNode node : specNodes) { + DolphinMetrics metrics = DolphinMetrics.builder() + .projectName(processDefinition.getProjectName()) + .processName(processDefinition.getProcessDefinitionName()) + .taskName(taskNode.getName()) + .taskType(taskNode.getType().name()) + .build(); + metrics.setWorkflowName(this.specWorkflow.getName()); + metrics.setDwName(node.getName()); + metrics.setDwType(taskNode.getType().name()); + TransformerContext.getCollector().markSuccessMiddleProcess(metrics); + } + } + + private boolean inSkippedList(TaskNode taskNode) { + if (Config.INSTANCE.getSkipTypes().contains(taskNode.getType()) + || Config.INSTANCE.getSkipTaskCodes().contains(taskNode.getName())) { + log.warn("task name {} in skipped list", taskNode.getName()); + markSkippedProcess(taskNode); + return true; + } else { + return false; + } + } + + protected void markSkippedProcess(TaskNode taskNode) { + DolphinMetrics metrics = DolphinMetrics.builder() + .projectName(processDefinition.getProjectName()) + .processName(processDefinition.getProcessDefinitionName()) + .taskName(taskNode.getName()) + .taskType(taskNode.getType().name()) + .timestamp(System.currentTimeMillis()) + .build(); + TransformerContext.getCollector().markSkippedProcess(metrics); + } + + protected void markFailedProcess(TaskNode taskNode, String errorMsg) { + DolphinMetrics metrics = DolphinMetrics.builder() + .projectName(processDefinition.getProjectName()) + .processName(processDefinition.getProcessDefinitionName()) + .taskName(taskNode.getName()) + .taskType(taskNode.getType().name()) + .timestamp(System.currentTimeMillis()) + .build(); + metrics.setErrorMsg(errorMsg); + TransformerContext.getCollector().markSkippedProcess(metrics); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/AbstractParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/AbstractParameterConverter.java new file mode 100644 index 0000000..a5fba90 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/AbstractParameterConverter.java @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.io.File; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.dw.types.LanguageEnum; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.enums.NodeRecurrenceType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableScopeType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.interfaces.Output; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.domain.ref.SpecFileResource; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScheduleStrategy; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.enums.TaskType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.AbstractParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.conditions.ConditionsParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.datax.DataxParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.dependent.DependentParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.flink.FlinkParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.http.HttpParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.mr.MapReduceParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.procedure.ProcedureParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.python.PythonParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.shell.ShellParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.spark.SparkParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.sql.SqlParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.sqoop.SqoopParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.subprocess.SubProcessParameters; +import com.aliyun.dataworks.migrationx.transformer.core.utils.SpecFileResourceTypeUtils; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.utils.ConverterTypeUtils; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.ParamListConverter; +import com.aliyun.migrationx.common.exception.BizException; +import com.aliyun.migrationx.common.exception.ErrorCode; +import com.aliyun.migrationx.common.utils.BeanUtils; +import com.aliyun.migrationx.common.utils.Config; +import com.aliyun.migrationx.common.utils.GsonUtils; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import com.google.gson.reflect.TypeToken; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public abstract class AbstractParameterConverter { + + protected static final String RESOURCE_REFERENCE_FORMAT = "%s@resource_reference{\"%s\"}"; + protected static final String RESOURCE_REFERENCE_PREFIX = "##"; + + protected final TaskNode taskDefinition; + protected final ProcessMeta processDefinition; + protected final Properties properties; + + protected static Map> taskTypeClassMap; + + protected T parameter; + + protected final SpecWorkflow specWorkflow; + + static { + taskTypeClassMap = new EnumMap<>(TaskType.class); + taskTypeClassMap.put(TaskType.SQL, SqlParameters.class); + taskTypeClassMap.put(TaskType.DEPENDENT, DependentParameters.class); + taskTypeClassMap.put(TaskType.FLINK, FlinkParameters.class); + taskTypeClassMap.put(TaskType.SPARK, SparkParameters.class); + taskTypeClassMap.put(TaskType.DATAX, DataxParameters.class); + taskTypeClassMap.put(TaskType.SHELL, ShellParameters.class); + taskTypeClassMap.put(TaskType.HTTP, HttpParameters.class); + taskTypeClassMap.put(TaskType.PROCEDURE, ProcedureParameters.class); + taskTypeClassMap.put(TaskType.CONDITIONS, ConditionsParameters.class); + taskTypeClassMap.put(TaskType.SQOOP, SqoopParameters.class); + taskTypeClassMap.put(TaskType.SUB_PROCESS, SubProcessParameters.class); + taskTypeClassMap.put(TaskType.PYTHON, PythonParameters.class); + taskTypeClassMap.put(TaskType.MR, MapReduceParameters.class); + } + + protected AbstractParameterConverter(Properties properties, + SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(); + this.properties = properties; + //this.spec = spec; + this.specWorkflow = specWorkflow; + this.processDefinition = processMeta; + this.taskDefinition = taskDefinition; + + TaskType taskType = taskDefinition.getType(); + try { + parameter = GsonUtils.fromJsonString( + taskDefinition.getParams(), TypeToken.get(taskTypeClassMap.get(taskType)).getType()); + } catch (Exception ex) { + log.error("parse task {}, {}, parameter {} error: ", taskType, taskTypeClassMap.get(taskType), ex); + } + } + + /** + * Each node translates the specific logic of the parameters + */ + protected abstract void convertParameter(SpecNode specNode); + + public SpecNode convert() { + SpecNode specNode = newSpecNode(taskDefinition); + + convertParameter(specNode); + this.specWorkflow.getNodes().add(specNode); + + // hint: the node returned may not be the final result of the conversion + return specNode; + } + + private SpecNode initSpecNode() { + SpecNode specNode = new SpecNode(); + specNode.setInputs(new ArrayList<>()); + specNode.setOutputs(new ArrayList<>()); + specNode.setFileResources(new ArrayList<>()); + specNode.setFunctions(new ArrayList<>()); + return specNode; + } + + /** + * common new SpecNode method, almost all nodes use this method + * + * @param taskDefinition taskDefinition + * @return SpecNode + */ + protected SpecNode newSpecNode(TaskNode taskDefinition) { + SpecNode specNode = initSpecNode(); + specNode.setId(UuidGenerators.generateUuid()); + specNode.setName(taskDefinition.getName()); + specNode.setDescription(taskDefinition.getDesc()); + specNode.setRerunTimes(taskDefinition.getMaxRetryTimes()); + // Unit conversion, minutes to milliseconds + specNode.setRerunInterval((int) Duration.ofMinutes(taskDefinition.getRetryInterval()).toMillis()); + + try { + Integer timeout = Integer.parseInt(taskDefinition.getTimeout()); + specNode.setTimeout(timeout); + } catch (Exception e) { + log.warn("parse task {}, timeout {} error: ", taskDefinition.getId(), taskDefinition.getTimeout()); + } + + resetNodeStrategy(specNode); + + SpecNodeOutput defaultOutput = buildDefaultNodeOutput(specNode); + DolphinSchedulerV1Context.getContext().getTaskCodeNodeDataMap().put(taskDefinition.getId(), defaultOutput.getData()); + DolphinSchedulerV1Context.getContext().getTaskCodeNodeIdMap().put(taskDefinition.getId(), specNode.getId()); + //specNode.getOutputs().add(BeanUtils.deepCopy(defaultOutput, SpecNodeOutput.class)); + + specNode.getOutputs().add(defaultOutput); + + specNode.setTrigger(convertByTaskDefinition()); + + return specNode; + } + + private SpecTrigger convertByTaskDefinition() { + SpecTrigger specTrigger = new SpecTrigger(); + specTrigger.setRecurrence("NORMAL".equals(taskDefinition.getRunFlag()) ? NodeRecurrenceType.NORMAL : NodeRecurrenceType.PAUSE); + specTrigger.setId(UuidGenerators.generateUuid()); + return specTrigger; + } + + protected SpecNodeOutput buildDefaultNodeOutput(SpecNode specNode) { + SpecNodeOutput specNodeOutput = new SpecNodeOutput(); + specNodeOutput.setIsDefault(true); + //specNodeOutput.setId(generateUuid()); + specNodeOutput.setData(specNode.getId()); + specNodeOutput.setRefTableName(specNode.getName()); + return specNodeOutput; + } + + protected SpecNodeOutput getDefaultOutput(SpecNode specNode) { + return getDefaultOutput(Optional.ofNullable(specNode).map(SpecNode::getOutputs).orElse(null)); + } + + protected SpecNodeOutput getDefaultOutput(SpecWorkflow specWorkflow, boolean throwException) { + return getDefaultOutput(Optional.ofNullable(specWorkflow).map(SpecWorkflow::getOutputs).orElse(null), throwException); + } + + protected SpecNodeOutput getDefaultOutput(SpecNode specNode, boolean throwException) { + return getDefaultOutput(Optional.ofNullable(specNode).map(SpecNode::getOutputs).orElse(null), throwException); + } + + protected SpecNodeOutput getDefaultOutput(List outputList) { + return getDefaultOutput(outputList, false); + } + + protected SpecNodeOutput getDefaultOutput(List outputList, boolean throwException) { + Optional first = ListUtils.emptyIfNull(outputList).stream().filter( + output -> output instanceof SpecNodeOutput && ((SpecNodeOutput) output).getIsDefault()) + .findFirst(); + if (throwException && !first.isPresent()) { + throw new BizException(ErrorCode.PARAMETER_NOT_SET, "defaultOutput"); + } + return (SpecNodeOutput) first.orElse(null); + } + + protected List getContextOutputs(SpecNode specNode) { + return specNode.getOutputs().stream() + .filter(v -> v instanceof SpecVariable && VariableScopeType.NODE_CONTEXT.equals(((SpecVariable) v).getScope()) + && VariableType.NODE_OUTPUT.equals(((SpecVariable) v).getType())) + .map(v -> { + SpecVariable variable = BeanUtils.deepCopy(v, SpecVariable.class); + variable.setNode(new SpecDepend(specNode, DependencyType.NORMAL, null)); + return variable; + }).collect(Collectors.toList()); + } + + protected List convertSpecNodeParam(SpecNode specNode) { + ParamListConverter paramListConverter = new ParamListConverter(this.parameter.getLocalParams(), taskDefinition); + List specVariableList = paramListConverter.convert(); + for (SpecVariable specVariable : specVariableList) { + // all outputs are context output, all inputs are all script inputs + if (VariableType.NODE_OUTPUT.equals(specVariable.getType())) { + SpecDepend nodeDepend = new SpecDepend(); + nodeDepend.setNodeId(specNode); + nodeDepend.setOutput(getDefaultOutput(specNode)); + specVariable.setNode(nodeDepend); + specNode.getOutputs().add(specVariable); + } + } + return specVariableList; + } + + /** + * convert resource with fileResources info in workflow, if not exists in fileResources, create and add in fileResources + * + * @param specNode node need to convert + */ + protected void convertFileResourceList(SpecNode specNode) { + ListUtils.emptyIfNull(parameter.getResourceFilesList()).forEach(resourceInfo -> { + SpecFileResource specFileResource = new SpecFileResource(); + specFileResource.setRuntimeResource(specNode.getRuntimeResource()); + specFileResource.setName(getFileNameByPath(resourceInfo.getResourceName())); + specFileResource.setType(SpecFileResourceTypeUtils.getResourceTypeBySuffix(specFileResource.getName())); + checkFileSameName(specFileResource.getName(), resourceInfo.getResourceName()); + specNode.getFileResources().add(specFileResource); + }); + } + + private void checkFileSameName(String fileName, String fullName) { + //String fullNameIn = context.getFileNameMap().get(fileName); + String fullNameIn = null; + if (Objects.nonNull(fullNameIn) && !fullNameIn.equals(fullName)) { + log.warn("存在同名资源冲突风险, {} 和 {} 导入后会同名", fullNameIn, fullName); + } else { + //context.getFileNameMap().put(fileName, fullName); + } + } + + protected String buildFileResourceReference(SpecNode specNode, String prefix) { + StringBuilder stringBuilder = new StringBuilder(); + Optional.ofNullable(specNode).map(SpecNode::getFileResources) + .ifPresent(fileResources -> + fileResources.forEach(fileResource -> + stringBuilder.append(String.format(RESOURCE_REFERENCE_FORMAT, prefix, fileResource.getName())).append("\n"))); + return stringBuilder.append("\n").toString(); + } + + /** + * add relation before join node. if the node depend on a whole workflow, need depend on workflow output + * + * @param postNode post join node + * @param preNodeList pre node list + * @param preNodeOutputList pre workflow output list + */ + protected void addRelation(SpecNode postNode, List preNodeList, List preNodeOutputList) { + SpecFlowDepend specFlowDepend = newSpecFlowDepend(); + specFlowDepend.setNodeId(postNode); + ListUtils.emptyIfNull(preNodeList).forEach(preNode -> { + SpecNodeOutput preNodeOutput = getDefaultOutput(preNode); + postNode.getInputs().add(preNodeOutput); + postNode.getInputs().addAll(getContextOutputs(preNode)); + specFlowDepend.getDepends().add(new SpecDepend(preNode, DependencyType.NORMAL, preNodeOutput)); + }); + + ListUtils.emptyIfNull(preNodeOutputList).forEach(preNodeOutput -> { + postNode.getInputs().add(preNodeOutput); + SpecDepend specDepend = new SpecDepend(); + specDepend.setType(DependencyType.NORMAL); + specDepend.setOutput(preNodeOutput); + specFlowDepend.getDepends().add(specDepend); + }); + getWorkflowDependencyList().add(specFlowDepend); + } + + protected SpecFlowDepend newSpecFlowDepend() { + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + specFlowDepend.setDepends(new ArrayList<>()); + return specFlowDepend; + } + + protected void addRelation(SpecNode postNode, List preNodeList) { + addRelation(postNode, preNodeList, null); + } + + protected String getFileNameByPath(String path) { + File file = new File(path); + return file.getName(); + } + + protected String getScriptPath(SpecNode specNode) { + if (Objects.isNull(specNode)) { + return StringUtils.EMPTY; + } + String defaultPath = StringUtils.defaultString(Config.INSTANCE.getBasePath(), StringUtils.EMPTY); + String workFlowPath = Optional.ofNullable(specWorkflow) + .map(SpecWorkflow::getName) + .orElse(StringUtils.EMPTY); + return FilenameUtils.concat(FilenameUtils.concat(defaultPath, workFlowPath), specNode.getName()); + } + + protected List getWorkflowDependencyList() { + if (Objects.nonNull(specWorkflow)) { + return specWorkflow.getDependencies(); + } else { + return Collections.emptyList(); + } + } + + protected SpecTrigger getWorkflowTrigger() { + if (Objects.nonNull(specWorkflow)) { + return specWorkflow.getTrigger(); + } + // may be manual flow + return null; + } + + /** + * get workflow, but it will be null in spec version < 1.2.0 + * + * @return SpecWorkflow if it is not null + */ + protected SpecWorkflow getWorkFlow() { + return specWorkflow; + } + + protected void resetNodeStrategy(SpecNode specNode) { + if (specNode.getStrategy() == null) { + specNode.setStrategy(new SpecScheduleStrategy()); + } + SpecScheduleStrategy strategy = specNode.getStrategy(); + strategy.setPriority(specNode.getPriority()); + strategy.setTimeout(specNode.getTimeout()); + strategy.setRerunInterval(specNode.getRerunInterval()); + strategy.setRerunTimes(specNode.getRerunTimes()); + strategy.setIgnoreBranchConditionSkip(specNode.getIgnoreBranchConditionSkip()); + strategy.setInstanceMode(specNode.getInstanceMode()); + strategy.setRerunMode(specNode.getRerunMode()); + + Optional.ofNullable(getWorkFlow()) + .map(SpecWorkflow::getStrategy) + .map(SpecScheduleStrategy::getFailureStrategy) + .ifPresent(strategy::setFailureStrategy); + } + + protected String codeToLanguageIdentifier(CodeProgramType nodeType) { + LanguageEnum languageEnum = codeToLanguage(nodeType); + if (languageEnum == null) { + log.warn("can not find language by {}", nodeType); + return null; + } + return languageEnum.getIdentifier(); + } + + protected LanguageEnum codeToLanguage(CodeProgramType nodeType) { + switch (nodeType) { + case SHELL: + case DIDE_SHELL: + case CDH_SHELL: + case EMR_SPARK_SHELL: + case CDH_SPARK_SHELL: + case EMR_SHELL: + case EMR_HIVE_CLI: + case PERL: + return LanguageEnum.SHELL_SCRIPT; + case EMR_SPARK_SQL: + return LanguageEnum.SPARK_SQL; + case CDH_HIVE: + case HIVE: + case EMR_HIVE: + return LanguageEnum.HIVE_SQL; + case EMR_IMPALA: + case CDH_IMPALA: + return LanguageEnum.IMPALA_SQL; + case CLICK_SQL: + return LanguageEnum.CLICKHOUSE_SQL; + case ODPS_SQL: + case ODPS_PERL: + return LanguageEnum.ODPS_SQL; + case ODPS_SCRIPT: + return LanguageEnum.ODPS_SCRIPT; + case EMR_PRESTO: + case CDH_PRESTO: + return LanguageEnum.PRESTO_SQL; + case PYODPS: + return LanguageEnum.PYTHON2; + case PYODPS3: + return LanguageEnum.PYTHON3; + case DATAX2: + case DATAX: + case RI: + case DI: + return LanguageEnum.JSON; + case HOLOGRES_SQL: + return LanguageEnum.HOLOGRES_SQL; + default: + return null; + } + } + + protected String getConverterType(String convertType, String defaultConvertType) { + String projectName = processDefinition.getProjectName(); + String processName = processDefinition.getProcessDefinitionName(); + String taskName = taskDefinition.getName(); + return ConverterTypeUtils.getConverterType(convertType, projectName, processName, taskName, defaultConvertType); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/CustomParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/CustomParameterConverter.java new file mode 100644 index 0000000..451fed0 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/CustomParameterConverter.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.AbstractParameters; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import org.apache.commons.collections4.ListUtils; + +public class CustomParameterConverter extends AbstractParameterConverter { + public CustomParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + + SpecScript script = new SpecScript(); + CodeProgramType codeProgramType = CodeProgramType.CUSTOM; + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + script.setContent(JSONUtils.toJsonString(taskDefinition)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/DataxParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/DataxParameterConverter.java new file mode 100644 index 0000000..2bf751c --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/DataxParameterConverter.java @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.datasource.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.entity.Property; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.enums.DbType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.enums.Flag; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.datax.DataxParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.datax.DataxUtils; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.utils.ParameterUtils; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.entity.Connection; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.entity.Parameter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.entity.Step; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; +import com.alibaba.druid.sql.ast.statement.SQLSelect; +import com.alibaba.druid.sql.ast.statement.SQLSelectItem; +import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; +import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; +import com.alibaba.druid.sql.parser.SQLStatementParser; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.tuple.Pair; + +@Slf4j +public class DataxParameterConverter extends AbstractParameterConverter { + /** + * select all + */ + private static final String SELECT_ALL_CHARACTER = "*"; + + private static final int DATAX_CHANNEL_COUNT = 1; + + public DataxParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + String type = properties.getProperty(Constants.CONVERTER_TARGET_DATAX_NODE_TYPE_AS, CodeProgramType.DI.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String json = buildDataxJsonFile(new HashMap<>()); + script.setContent(json); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + private String buildDataxJsonFile(Map paramsMap) { + String json; + + if (parameter.getCustomConfig() == Flag.YES.ordinal()) { + json = parameter.getJson().replaceAll("\\r\\n", System.lineSeparator()); + } else { + ObjectNode job = JSONUtils.createObjectNode(); + job.put("transform", false); + job.put("type", "job"); + job.put("version", "2.0"); + ArrayNode steps = buildDataxJobSteps(); + job.putArray("steps").addAll(steps); + job.set("setting", buildDataxJobSettingJson()); + json = job.toString(); + } + + // replace placeholder + json = ParameterUtils.convertParameterPlaceholders(json, ParameterUtils.convert(paramsMap)); + + log.debug("datax job json : {}", json); + return json; + } + + private ArrayNode buildDataxJobSteps() { + DataSource source = getDataSource(parameter.getDataSource()); + DataSource target = getDataSource(parameter.getDataTarget()); + Step reader = new Step(); + Step writer = new Step(); + reader.setName("Reader"); + reader.setCategory("reader"); + reader.setStepType(source.getType().name().toLowerCase()); + + writer.setName("Writer"); + writer.setCategory("writer"); + writer.setStepType(target.getType().name().toLowerCase()); + + Parameter readParameter = new Parameter(); + readParameter.setEncoding("UTF-8"); + readParameter.setEnvType(1); + readParameter.setUseSpecialSecret(false); + + Parameter writeParameter = new Parameter(); + writeParameter.setEncoding("UTF-8"); + writeParameter.setEnvType(1); + writeParameter.setUseSpecialSecret(false); + + Pair pair = tryGrammaticalAnalysisSqlColumnNames(source.getType(), parameter.getSql()); + String[] srcColumnNames = pair.getRight(); + if (srcColumnNames != null && srcColumnNames.length > 0) { + readParameter.setColumn(Arrays.asList(srcColumnNames)); + String[] tgtColumnNames = DataxUtils.convertKeywordsColumns(target.getType(), srcColumnNames); + if (tgtColumnNames != null && tgtColumnNames.length > 0) { + writeParameter.setColumn(Arrays.asList(srcColumnNames)); + } + } + Connection sourceConnection = new Connection(); + sourceConnection.setDatasource(source.getName()); + sourceConnection.setTable(Arrays.asList(pair.getLeft())); + readParameter.setConnection(Arrays.asList(sourceConnection)); + Connection targetConnection = new Connection(); + targetConnection.setDatasource(target.getName()); + targetConnection.setTable(Arrays.asList(parameter.getTargetTable())); + writeParameter.setConnection(Arrays.asList(targetConnection)); + reader.setParameter(readParameter); + writer.setParameter(writeParameter); + + ObjectNode steps = JSONUtils.createObjectNode(); + ArrayNode tableArr = steps.putArray("steps"); + JsonNode readerNode = JSONUtils.toJsonNode(reader); + tableArr.add(readerNode); + JsonNode writerNode = JSONUtils.toJsonNode(writer); + tableArr.add(writerNode); + return tableArr; + } + + /** + * build datax setting config + * + * @return datax setting config JSONObject + */ + private ObjectNode buildDataxJobSettingJson() { + + ObjectNode speed = JSONUtils.createObjectNode(); + + speed.put("channel", DATAX_CHANNEL_COUNT); + + if (parameter.getJobSpeedByte() > 0) { + speed.put("byte", parameter.getJobSpeedByte()); + } + + if (parameter.getJobSpeedRecord() > 0) { + speed.put("record", parameter.getJobSpeedRecord()); + } + + ObjectNode errorLimit = JSONUtils.createObjectNode(); + errorLimit.put("record", 0); + errorLimit.put("percentage", 0); + + ObjectNode setting = JSONUtils.createObjectNode(); + setting.set("speed", speed); + setting.set("errorLimit", errorLimit); + + return setting; + } + + /** + * try grammatical parsing column + * + * @param dbType database type + * @param sql sql for data synchronization + * @return column name array + * @throws RuntimeException if error throws RuntimeException + */ + private Pair tryGrammaticalAnalysisSqlColumnNames(DbType dbType, String sql) { + String[] columnNames; + String sourceTable = null; + try { + SQLStatementParser parser = DataxUtils.getSqlStatementParser(dbType, sql); + if (parser == null) { + log.warn("database driver [{}] is not support grammatical analysis sql", dbType); + return Pair.of(null, new String[0]); + } + + SQLStatement sqlStatement = parser.parseStatement(); + SQLSelectStatement sqlSelectStatement = (SQLSelectStatement) sqlStatement; + SQLSelect sqlSelect = sqlSelectStatement.getSelect(); + + List selectItemList = null; + + if (sqlSelect.getQuery() instanceof SQLSelectQueryBlock) { + SQLSelectQueryBlock block = (SQLSelectQueryBlock) sqlSelect.getQuery(); + selectItemList = block.getSelectList(); + if (block.getFrom() instanceof SQLExprTableSource) { + SQLExprTableSource expr = (SQLExprTableSource) block.getFrom(); + if (expr.getExpr() instanceof SQLIdentifierExpr) { + sourceTable = ((SQLIdentifierExpr) expr.getExpr()).getName(); + } + } + } else if (sqlSelect.getQuery() instanceof SQLUnionQuery) { + SQLUnionQuery unionQuery = (SQLUnionQuery) sqlSelect.getQuery(); + SQLSelectQueryBlock block = (SQLSelectQueryBlock) unionQuery.getRight(); + selectItemList = block.getSelectList(); + if (block.getFrom() instanceof SQLExprTableSource) { + SQLExprTableSource expr = (SQLExprTableSource) block.getFrom(); + if (expr.getExpr() instanceof SQLIdentifierExpr) { + sourceTable = ((SQLIdentifierExpr) expr.getExpr()).getName(); + } + } + } + + if (selectItemList == null) { + throw new RuntimeException(String.format("select query type [%s] is not support", sqlSelect.getQuery().toString())); + } + + columnNames = new String[selectItemList.size()]; + for (int i = 0; i < selectItemList.size(); i++) { + SQLSelectItem item = selectItemList.get(i); + + String columnName = null; + + if (item.getAlias() != null) { + columnName = item.getAlias(); + } else if (item.getExpr() != null) { + if (item.getExpr() instanceof SQLPropertyExpr) { + SQLPropertyExpr expr = (SQLPropertyExpr) item.getExpr(); + columnName = expr.getName(); + } else if (item.getExpr() instanceof SQLIdentifierExpr) { + SQLIdentifierExpr expr = (SQLIdentifierExpr) item.getExpr(); + columnName = expr.getName(); + } + } else { + throw new RuntimeException( + String.format("grammatical analysis sql column [ %s ] failed", item)); + } + + if (SELECT_ALL_CHARACTER.equals(item.toString())) { + log.info("sql contains *, grammatical analysis failed"); + return Pair.of(sourceTable, new String[]{"*"}); + } + + if (columnName == null) { + throw new RuntimeException( + String.format("grammatical analysis sql column [ %s ] failed", item)); + } + + columnNames[i] = columnName; + } + } catch (Exception e) { + log.warn(e.getMessage(), e); + return Pair.of(null, new String[0]); + } + + return Pair.of(sourceTable, columnNames); + } + + private DataSource getDataSource(int datasourceId) { + List datasources = DolphinSchedulerV1Context.getContext().getDataSources(); + return CollectionUtils.emptyIfNull(datasources).stream() + .filter(s -> s.getId() == datasourceId) + .findFirst() + .orElse(null); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/DependentParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/DependentParameterConverter.java new file mode 100644 index 0000000..4ecd422 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/DependentParameterConverter.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.DependentTaskModel; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.dependent.DependentParameters; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class DependentParameterConverter extends AbstractParameterConverter { + + public DependentParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + CodeProgramType codeProgramType = CodeProgramType.VIRTUAL; + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + DolphinSchedulerV1Context context = DolphinSchedulerV1Context.getContext(); + List deps = convertDeps(); + context.getSpecNodeProcessCodeMap().put(specNode, deps); + } + + public List convertDeps() { + log.info("params : {}", taskDefinition.getParams()); + DependentParameters dependentParameters = taskDefinition.getDependence(); + + if (dependentParameters == null || dependentParameters.getDependTaskList() == null || dependentParameters.getDependTaskList().isEmpty()) { + log.warn("no dependence param {}", taskDefinition.getParams()); + return Collections.emptyList(); + } + DolphinSchedulerV1Context context = DolphinSchedulerV1Context.getContext(); + + // 本节点的条件依赖 + List dependencies = dependentParameters.getDependTaskList(); + List taskIds = new ArrayList<>(); + ListUtils.emptyIfNull(dependencies).forEach(dependModel -> + ListUtils.emptyIfNull(dependModel.getDependItemList()).forEach(depItem -> { + String tasks = depItem.getDepTasks(); + if (StringUtils.equalsIgnoreCase("all", tasks)) { + //all leaf task of process definition + //1. get all relation of process + // preTaskCode -> postTaskCode + //Map> relations = findRelations(depItem); + List taskDefinitions = context.getProcessCodeTaskRelationMap().get(depItem.getDefinitionId()); + for (Object task : CollectionUtils.emptyIfNull(taskDefinitions)) { + TaskNode taskNode = (TaskNode) task; + //2. find all taskCode not in preTaskCode (not as a pre dependent, leaf task) + if (!taskIds.contains(taskNode.getId())) { + taskIds.add(taskNode.getId()); + } + } + } else { + if (!taskIds.contains(depItem.getDepTasks())) { + taskIds.add(depItem.getDepTasks()); + } + } + })); + return taskIds; + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/MrParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/MrParameterConverter.java new file mode 100644 index 0000000..c821a66 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/MrParameterConverter.java @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.EmrAllocationSpec; +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.EmrCode; +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.EmrLauncher; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.utils.MapReduceArgsUtils; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.entity.ResourceInfo; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.mr.MapReduceParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.utils.DataStudioCodeUtils; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; +import com.aliyun.migrationx.common.utils.BeanUtils; + +import com.google.common.base.Joiner; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class MrParameterConverter extends AbstractParameterConverter { + public static final String MR_YARN_QUEUE = "mapreduce.job.queuename"; + + public MrParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + + SpecScript script = new SpecScript(); + String type = getConverterType(); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + //todo + //String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + String code = convertCode(codeProgramType); + //script.setContent(resourceReference + parameter.getRawScript()); + script.setContent(code); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String convertCode(CodeProgramType codeProgramType) { + DolphinSchedulerV1Context context = DolphinSchedulerV1Context.getContext(); + List resourceInfos = context.getResources(); + + Optional.ofNullable(parameter).map(MapReduceParameters::getMainJar) + .flatMap(mainJar -> ListUtils.emptyIfNull(resourceInfos) + .stream().filter(res -> Objects.equals(res.getId(), mainJar.getId())) + .findFirst()).ifPresent(res -> parameter.setMainJar(res)); + + ListUtils.emptyIfNull(Optional.ofNullable(parameter).map(MapReduceParameters::getResourceFilesList) + .orElse(ListUtils.emptyIfNull(null))) + .forEach(res -> ListUtils.emptyIfNull(resourceInfos).stream() + .filter(res1 -> Objects.equals(res1.getId(), res.getId())) + .forEach(res1 -> BeanUtils.copyProperties(res1, res))); + + String type = codeProgramType.getName(); + + List resources = ListUtils.emptyIfNull(parameter.getResourceFilesList()).stream() + .filter(Objects::nonNull) + .map(ResourceInfo::getName).distinct().collect(Collectors.toList()); + + List codeLines = new ArrayList<>(); + codeLines.add(DataStudioCodeUtils.addResourceReference(CodeProgramType.valueOf(type), "", resources)); + + // convert to EMR_MR + if (StringUtils.equalsIgnoreCase(CodeProgramType.EMR_MR.name(), type)) { + String command = Joiner.on(" ").join(MapReduceArgsUtils.buildArgs(parameter).stream() + .map(String::valueOf).collect(Collectors.toList())); + codeLines.add(command); + + String code = Joiner.on("\n").join(codeLines); + code = EmrCodeUtils.toEmrCode(codeProgramType, taskDefinition.getName(), code); + EmrCode emrCode = EmrCodeUtils.asEmrCode(type, code); + Optional.ofNullable(emrCode).map(EmrCode::getLauncher) + .map(EmrLauncher::getAllocationSpec) + .map(EmrAllocationSpec::of) + .ifPresent(spec -> { + spec.setQueue(parameter.getQueue()); + emrCode.getLauncher().setAllocationSpec(spec.toMap()); + }); + return emrCode.getContent(); + } + + // convert to ODPS_MR + if (StringUtils.equalsIgnoreCase(CodeProgramType.ODPS_MR.name(), type)) { + String command = Joiner.on(" ").join( + "jar", "-resources", + Optional.ofNullable(parameter.getMainJar().getName()).orElse(""), + "-classpath", + Joiner.on(",").join(resources), + Optional.ofNullable(parameter.getMainClass()).orElse(""), + Optional.ofNullable(parameter.getMainArgs()).orElse(""), + Optional.ofNullable(parameter.getOthers()).orElse("") + ); + codeLines.add(command); + + String code = Joiner.on("\n").join(codeLines); + code = EmrCodeUtils.toEmrCode(codeProgramType, taskDefinition.getName(), code); + EmrCode emrCode = EmrCodeUtils.asEmrCode(type, code); + Optional.ofNullable(emrCode).map(EmrCode::getLauncher) + .map(EmrLauncher::getAllocationSpec) + .map(EmrAllocationSpec::of) + .ifPresent(spec -> { + spec.setQueue(parameter.getQueue()); + emrCode.getLauncher().setAllocationSpec(spec.toMap()); + }); + return emrCode.getContent(); + } + + throw new RuntimeException("Unsupported code type: " + type); + } + + private String getConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_MR_NODE_TYPE_AS); + String defaultConvertType = CodeProgramType.EMR_MR.name(); + return getConverterType(convertType, defaultConvertType); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/ProcedureParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/ProcedureParameterConverter.java new file mode 100644 index 0000000..6ee22b2 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/ProcedureParameterConverter.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.datasource.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.enums.DbType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.procedure.ProcedureParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.migrationx.common.utils.GsonUtils; + +import com.google.gson.reflect.TypeToken; +import org.apache.commons.collections4.ListUtils; + +public class ProcedureParameterConverter extends AbstractParameterConverter { + + public ProcedureParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_SHELL_NODE_TYPE_AS, CodeProgramType.DIDE_SHELL.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(convertType); + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + //todo + //String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + //script.setContent(resourceReference + parameter.getRawScript()); + script.setContent(getCode()); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String getCode() { + String sqlNodeMapStr = properties.getProperty( + Constants.CONVERTER_TARGET_SQL_NODE_TYPE_MAP, "{}"); + Map sqlTypeNodeTypeMapping = GsonUtils.fromJsonString(sqlNodeMapStr, + new TypeToken>() {}.getType()); + sqlTypeNodeTypeMapping = Optional.ofNullable(sqlTypeNodeTypeMapping).orElse(new HashMap<>(1)); + + String defaultNodeTypeIfNotSupport = getConverterType(); + + DbType codeProgramType = sqlTypeNodeTypeMapping.get(parameter.getType()); + + //add ref datasource + if (parameter.getDatasource() > 0) { + List datasources = DolphinSchedulerV1Context.getContext().getDataSources(); + + //todo + //CollectionUtils.emptyIfNull(datasources).stream() + // .filter(s -> s.getId() == parameter.getDatasource()) + // .findFirst() + // .ifPresent(s -> dwNode.setConnection(s.getName())); + } + String code = parameter.getMethod(); + return code; + } + + private String getConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_UNSUPPORTED_NODE_TYPE_AS); + return getConverterType(convertType, CodeProgramType.VIRTUAL.name()); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/PythonParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/PythonParameterConverter.java new file mode 100644 index 0000000..46a843c --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/PythonParameterConverter.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.python.PythonParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; + +import org.apache.commons.collections4.ListUtils; + +public class PythonParameterConverter extends AbstractParameterConverter { + + public PythonParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_PYTHON_NODE_TYPE_AS, CodeProgramType.PYTHON.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(convertType); + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + + String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + String pathImportCode = buildPathImportCode(specNode); + script.setContent(resourceReference + pathImportCode + parameter.getRawScript()); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + private String buildPathImportCode(SpecNode specNode) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("import os\n").append("import sys\n\n"); + Optional.ofNullable(specNode).map(SpecNode::getFileResources).ifPresent(fileResources -> + fileResources.forEach(fileResource -> { + String fileName = fileResource.getName(); + stringBuilder.append(String.format("sys.path.append(os.path.dirname(os.path.abspath('%s')))%n", fileName)); + })); + stringBuilder.append("\n"); + return stringBuilder.toString(); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/ShellParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/ShellParameterConverter.java new file mode 100644 index 0000000..98f4b28 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/ShellParameterConverter.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.shell.ShellParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; + +import org.apache.commons.collections4.ListUtils; + +public class ShellParameterConverter extends AbstractParameterConverter { + + public ShellParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_SHELL_NODE_TYPE_AS, CodeProgramType.DIDE_SHELL.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(convertType); + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + script.setContent(resourceReference + parameter.getRawScript()); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SparkParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SparkParameterConverter.java new file mode 100644 index 0000000..b55b67a --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SparkParameterConverter.java @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.OdpsSparkCode; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.utils.ArgsUtils; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.entity.ResourceInfo; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.enums.ProgramType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.spark.SparkParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.spark.SparkConstants; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; +import com.aliyun.migrationx.common.utils.BeanUtils; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class SparkParameterConverter extends AbstractParameterConverter { + + public SparkParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + /** + * Each node translates the specific logic of the parameters + */ + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + + String type = getConverterType(); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + //String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + //script.setContent(resourceReference + parameter.getRawScript()); + script.setContent(convertCode(codeProgramType)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String convertCode(CodeProgramType codeProgramType) { + DolphinSchedulerV1Context context = DolphinSchedulerV1Context.getContext(); + List resourceInfos = context.getResources(); + + Optional.ofNullable(parameter).map(SparkParameters::getMainJar) + .flatMap(mainJar -> ListUtils.emptyIfNull(resourceInfos) + .stream().filter(res -> Objects.equals(res.getId(), mainJar.getId())) + .findFirst()).ifPresent(res -> parameter.setMainJar(res)); + + ListUtils.emptyIfNull(Optional.ofNullable(parameter).map(SparkParameters::getResourceFilesList) + .orElse(ListUtils.emptyIfNull(null))) + .forEach(res -> ListUtils.emptyIfNull(resourceInfos).stream() + .filter(res1 -> Objects.equals(res1.getId(), res.getId())) + .forEach(res1 -> { + BeanUtils.copyProperties(res1, res); + res.setRes(res1.getName()); + res.setName(res1.getName()); + })); + + if (CodeProgramType.EMR_SPARK.equals(codeProgramType)) { + List cmd = populateSparkOptions(parameter); + String code = String.join(" ", cmd); + return EmrCodeUtils.toEmrCode(codeProgramType, taskDefinition.getName(), code); + } else if (CodeProgramType.ODPS_SPARK.equals(codeProgramType)) { + OdpsSparkCode odpsSparkCode = populateSparkOdpsCode(); + return odpsSparkCode.toString(); + } + throw new IllegalArgumentException("Unsupported code program type: " + codeProgramType); + } + + private List populateSparkOptions(SparkParameters sparkParameters) { + List args = new ArrayList<>(); + + ProgramType programType = sparkParameters.getProgramType(); + ResourceInfo mainJar = sparkParameters.getMainJar(); + if (programType != ProgramType.SQL) { + String resource = mainJar.getName(); + if (resource != null) { + String[] resources = resource.split("/"); + if (resources.length > 0) { + resource = resources[resources.length - 1]; + } + } else { + DolphinSchedulerV1Context context = DolphinSchedulerV1Context.getContext(); + resource = CollectionUtils.emptyIfNull(context.getResources()) + .stream() + .filter(r -> r.getId() == mainJar.getId()) + .findAny() + .map(r -> r.getName()) + .orElse(null); + mainJar.setName(resource); + } + String dwResource = "##@resource_reference{\"" + resource + "\"} \n"; + args.add(dwResource + SparkConstants.SPARK_SUBMIT_COMMAND); + } else { + args.add(SparkConstants.SPARK_SUBMIT_COMMAND); + } + + String deployMode = StringUtils.isNotEmpty(sparkParameters.getDeployMode()) ? sparkParameters.getDeployMode() + : SparkConstants.DEPLOY_MODE_LOCAL; + + if (!SparkConstants.DEPLOY_MODE_LOCAL.equals(deployMode)) { + args.add(SparkConstants.MASTER); + String masterUrl = SparkConstants.SPARK_ON_YARN; + args.add(masterUrl); + } + args.add(SparkConstants.DEPLOY_MODE); + args.add(deployMode); + + String mainClass = sparkParameters.getMainClass(); + if (programType != ProgramType.PYTHON && programType != ProgramType.SQL && StringUtils.isNotEmpty(mainClass)) { + args.add(SparkConstants.MAIN_CLASS); + args.add(mainClass); + } + + populateSparkResourceDefinitions(args, sparkParameters); + + String appName = sparkParameters.getAppName(); + if (StringUtils.isNotEmpty(appName)) { + args.add(SparkConstants.SPARK_NAME); + args.add(ArgsUtils.escape(appName)); + } + + String others = sparkParameters.getOthers(); + + // --conf --files --jars --packages + if (StringUtils.isNotEmpty(others)) { + args.add(others); + } + + //jar + if (programType != ProgramType.SQL && mainJar != null) { + args.add(mainJar.getName()); + } + + String mainArgs = sparkParameters.getMainArgs(); + if (programType != ProgramType.SQL && StringUtils.isNotEmpty(mainArgs)) { + args.add(mainArgs); + } + + return args; + } + + private OdpsSparkCode populateSparkOdpsCode() { + OdpsSparkCode odpsSparkCode = new OdpsSparkCode(); + odpsSparkCode.setResourceReferences(new ArrayList<>()); + odpsSparkCode.setSparkJson(new OdpsSparkCode.CodeJson()); + + ResourceInfo mainJar = parameter.getMainJar(); + String resource = mainJar.getName(); + if (StringUtils.isEmpty(resource)) { + resource = getResourceName(mainJar.getId()); + } + + if (resource != null) { + String[] resources = resource.split("/"); + if (resources.length > 0) { + resource = resources[resources.length - 1]; + } + mainJar.setName(resource); + //String dwResource = "##@resource_reference{\"" + resource + "\"} \n"; + odpsSparkCode.getResourceReferences().add(resource); + odpsSparkCode.getSparkJson().setMainJar(resource); + } + String mainClass = parameter.getMainClass(); + odpsSparkCode.getSparkJson().setMainClass(mainClass); + odpsSparkCode.getSparkJson().setVersion("2.x"); + odpsSparkCode.getSparkJson().setLanguage("java"); + + String mainArgs = parameter.getMainArgs(); + odpsSparkCode.getSparkJson().setArgs(mainArgs); + List confs = new ArrayList<>(); + populateOdpsSparkResourceDefinitions(confs, parameter); + odpsSparkCode.getSparkJson().setConfigs(confs); + return odpsSparkCode; + } + + private void populateOdpsSparkResourceDefinitions(List args, SparkParameters sparkParameters) { + int driverCores = sparkParameters.getDriverCores(); + if (driverCores > 0) { + args.add(String.format("spark.driver.cores=%d", driverCores)); + } + + String driverMemory = sparkParameters.getDriverMemory(); + if (StringUtils.isNotEmpty(driverMemory)) { + args.add(String.format("spark.driver.memory=%s", driverMemory)); + } + + int numExecutors = sparkParameters.getNumExecutors(); + if (numExecutors > 0) { + args.add(String.format("spark.executor.instances=%d", numExecutors)); + } + + int executorCores = sparkParameters.getExecutorCores(); + if (executorCores > 0) { + args.add(String.format("spark.executor.cores=%d", executorCores)); + } + + String executorMemory = sparkParameters.getExecutorMemory(); + if (StringUtils.isNotEmpty(executorMemory)) { + args.add(String.format("spark.executor.memory=%s", executorMemory)); + } + } + + private void populateSparkResourceDefinitions(List args, SparkParameters sparkParameters) { + int driverCores = sparkParameters.getDriverCores(); + if (driverCores > 0) { + args.add(String.format(SparkConstants.DRIVER_CORES, driverCores)); + } + + String driverMemory = sparkParameters.getDriverMemory(); + if (StringUtils.isNotEmpty(driverMemory)) { + args.add(String.format(SparkConstants.DRIVER_MEMORY, driverMemory)); + } + + int numExecutors = sparkParameters.getNumExecutors(); + if (numExecutors > 0) { + args.add(String.format(SparkConstants.NUM_EXECUTORS, numExecutors)); + } + + int executorCores = sparkParameters.getExecutorCores(); + if (executorCores > 0) { + args.add(String.format(SparkConstants.EXECUTOR_CORES, executorCores)); + } + + String executorMemory = sparkParameters.getExecutorMemory(); + if (StringUtils.isNotEmpty(executorMemory)) { + args.add(String.format(SparkConstants.EXECUTOR_MEMORY, executorMemory)); + } + } + + private String getResourceName(Integer id) { + if (id == null) { + return null; + } + DolphinSchedulerV1Context context = DolphinSchedulerV1Context.getContext(); + return CollectionUtils.emptyIfNull(context.getResources()) + .stream() + .filter(r -> r.getId() == id) + .findAny() + .map(r -> r.getName()) + .orElse(null); + } + + private String getConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_SPARK_SUBMIT_TYPE_AS); + String defaultConvertType = CodeProgramType.EMR_SPARK_SHELL.name(); + return getConverterType(convertType, defaultConvertType); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SqlParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SqlParameterConverter.java new file mode 100644 index 0000000..dfbc0ee --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SqlParameterConverter.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.datasource.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.enums.DbType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.sql.SqlParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; +import com.aliyun.migrationx.common.utils.GsonUtils; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import com.google.gson.reflect.TypeToken; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang.StringUtils; + +@Slf4j +public class SqlParameterConverter extends AbstractParameterConverter { + + public SqlParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + String sqlNodeMapStr = properties.getProperty( + Constants.CONVERTER_TARGET_SQL_NODE_TYPE_MAP, "{}"); + Map sqlTypeNodeTypeMapping = GsonUtils.fromJsonString(sqlNodeMapStr, + new TypeToken>() {}.getType()); + + //String type = getSQLConverterType(); + String type = Optional.ofNullable(sqlTypeNodeTypeMapping) + .map(s -> s.get(parameter.getType())) + .orElseGet(() -> { + if (DbType.HIVE.equals(parameter.getType())) { + return CodeProgramType.EMR_HIVE.name(); + } else if (DbType.SPARK.equals(parameter.getType())) { + return CodeProgramType.EMR_SPARK.name(); + } else if (parameter.getType() != null) { + return parameter.getType().name(); + } else { + String defaultNodeTypeIfNotSupport = getSQLConverterType(); + log.warn("using default node Type {} for node {}", defaultNodeTypeIfNotSupport, taskDefinition.getName()); + return defaultNodeTypeIfNotSupport; + } + }); + + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String content = parameter.getSql(); + + if (CodeProgramType.EMR_HIVE.name().equals(codeProgramType) || CodeProgramType.EMR_SPARK.name().equals(codeProgramType)) { + content = EmrCodeUtils.toEmrCode(codeProgramType, taskDefinition.getName(), content); + } + + script.setContent(content); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + private String getConnectionName(String codeProgramType) { + String mappingJson = properties.getProperty(Constants.WORKFLOW_CONVERTER_CONNECTION_MAPPING); + if (StringUtils.isNotEmpty(mappingJson)) { + Map connectionMapping = JSONUtils.parseObject(mappingJson, Map.class); + if (connectionMapping == null) { + log.error("parse connection mapping with {} error", mappingJson); + } else { + String connectionName = connectionMapping.get(codeProgramType); + log.info("Got connectionName {} by {}", connectionName, codeProgramType); + return connectionName; + } + } + + if (!CodeProgramType.EMR_HIVE.name().equals(codeProgramType) && !CodeProgramType.EMR_SPARK.name().equals(codeProgramType)) { + //add ref datasource + List datasources = DolphinSchedulerV1Context.getContext().getDataSources(); + if (parameter.getDatasource() > 0) { + return CollectionUtils.emptyIfNull(datasources).stream() + .filter(s -> s.getId() == parameter.getDatasource()) + .findFirst() + .map(s -> s.getName()) + .orElse(null); + } + } + return null; + } + + private String getSQLConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_COMMAND_SQL_TYPE_AS); + return getConverterType(convertType, CodeProgramType.SQL_COMPONENT.name()); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SqoopParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SqoopParameterConverter.java new file mode 100644 index 0000000..6250bdb --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SqoopParameterConverter.java @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.entity.ResourceInfo; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.sqoop.SqoopParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.sqoop.sources.SourceHdfsParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.sqoop.sources.SourceHiveParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.sqoop.sources.SourceMysqlParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.sqoop.targets.TargetHdfsParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.sqoop.targets.TargetHiveParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.sqoop.targets.TargetMysqlParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.CodeModeType; +import com.aliyun.dataworks.migrationx.transformer.core.sqoop.DIConfigTemplate; +import com.aliyun.dataworks.migrationx.transformer.core.sqoop.DIJsonProcessor; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import com.google.common.base.Joiner; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +public class SqoopParameterConverter extends AbstractParameterConverter { + + public SqoopParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + CodeProgramType codeProgramType = CodeProgramType.DI; + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String content = convertCode(); + script.setContent(content); + //todo + //String codeMode = CodeModeType.CODE.getValue(); + + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String convertCode() { + DIJsonProcessor diProcessor = DIJsonProcessor.from(DIConfigTemplate.DI_CODE_TEMPLATE); + DIJsonProcessor readerJp = diProcessor.getConfiguration("steps[0]"); + DIJsonProcessor writerJp = diProcessor.getConfiguration("steps[1]"); + processSqoopSource(readerJp); + processSqoopTarget(writerJp); + diProcessor.set("extend.mode", CodeModeType.CODE.getValue()); + return diProcessor.toJSON(); + } + + private void processSqoopTarget(DIJsonProcessor writerJp) { + DolphinSchedulerV1Context context = DolphinSchedulerV1Context.getContext(); + List resourceInfos = context.getResources(); + + writerJp.set("stepType", StringUtils.lowerCase(parameter.getTargetType())); + if (StringUtils.equalsIgnoreCase(parameter.getTargetType(), "mysql")) { + TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(parameter.getTargetParams(), + TargetMysqlParameter.class); + Optional.ofNullable(targetMysqlParameter).ifPresent(p -> { + ListUtils.emptyIfNull(resourceInfos).stream() + .filter(ds -> Objects.equals(ds.getId(), targetMysqlParameter.getTargetDatasource())) + .findFirst().ifPresent(ds -> writerJp.set("parameter.datasource", ds.getName())); + writerJp.set("parameter.table", targetMysqlParameter.getTargetTable()); + writerJp.set("parameter.preSql", Optional.ofNullable(StringUtils.split( + targetMysqlParameter.getPreQuery(), ",")).orElse(new String[]{})); + writerJp.set("parameter.column", StringUtils.isBlank(targetMysqlParameter.getTargetColumns()) ? + new String[]{"*"} : StringUtils.split(targetMysqlParameter.getTargetColumns(), ",")); + if (StringUtils.equalsIgnoreCase(targetMysqlParameter.getTargetUpdateMode(), "updateonly")) { + writerJp.set("parameter.writeMode", "update"); + } else { + writerJp.set("parameter.writeMode", "replace"); + } + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getTargetType(), "hive")) { + TargetHiveParameter targetHiveParameter = JSONUtils.parseObject(parameter.getTargetParams(), + TargetHiveParameter.class); + Optional.ofNullable(targetHiveParameter).ifPresent(p -> { + ListUtils.emptyIfNull(resourceInfos).stream() + .filter(ds -> StringUtils.equals(java.lang.String.valueOf(ds.getId()), targetHiveParameter.getHiveDatabase())) + .findFirst().ifPresent(ds -> writerJp.set("parameter.datasource", ds.getName())); + writerJp.set("parameter.table", targetHiveParameter.getHiveTable()); + String[] keys = Optional.ofNullable(StringUtils.split( + targetHiveParameter.getHivePartitionKey(), ",")).orElse(new String[]{}); + String[] values = Optional.ofNullable(StringUtils.split( + targetHiveParameter.getHivePartitionValue(), ",")).orElse(new String[]{}); + List partitions = new ArrayList<>(); + for (int i = 0; i < Math.min(keys.length, values.length); i++) { + partitions.add(Joiner.on("=").join(keys[i], values[i])); + } + writerJp.set("parameter.partition", Joiner.on(",").join(partitions)); + writerJp.set("parameter.hdfsUsername", "hdfs"); + writerJp.set("parameter.writeMode", "append"); + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getTargetType(), "hdfs")) { + TargetHdfsParameter targetHdfsParameter = JSONUtils.parseObject(parameter.getSourceParams(), + TargetHdfsParameter.class); + Optional.ofNullable(targetHdfsParameter).ifPresent(p -> { + writerJp.set("parameter.path", p.getTargetPath()); + writerJp.set("parameter.compress", targetHdfsParameter.getCompressionCodec()); + writerJp.set("parameter.datasource", "hdfs"); + writerJp.set("parameter.fileType", targetHdfsParameter.getFileType()); + if (StringUtils.equalsIgnoreCase("parquet", targetHdfsParameter.getFileType())) { + writerJp.set("parameter.writeMode", "noConflict"); + } else { + writerJp.set("parameter.writeMode", "append"); + } + }); + } + } + + private void processSqoopSource(DIJsonProcessor readerJp) { + DolphinSchedulerV1Context context = DolphinSchedulerV1Context.getContext(); + List resourceInfos = context.getResources(); + + readerJp.set("stepType", StringUtils.lowerCase(parameter.getSourceType())); + if (StringUtils.equalsIgnoreCase(parameter.getSourceType(), "mysql")) { + SourceMysqlParameter mysqlParameter = JSONUtils.parseObject(parameter.getSourceParams(), + SourceMysqlParameter.class); + Optional.ofNullable(mysqlParameter).ifPresent(p -> { + ListUtils.emptyIfNull(resourceInfos).stream() + .filter(ds -> Objects.equals(ds.getId(), mysqlParameter.getSrcDatasource())) + .findFirst().ifPresent(ds -> readerJp.set("parameter.datasource", ds.getName())); + readerJp.set("parameter.table", mysqlParameter.getSrcTable()); + readerJp.set("parameter.where", StringUtils.substringAfter( + StringUtils.lowerCase(mysqlParameter.getSrcQuerySql()), "where")); + readerJp.set("parameter.column", StringUtils.isBlank(mysqlParameter.getSrcColumns()) ? + new String[]{"*"} : StringUtils.split(mysqlParameter.getSrcColumns(), ",")); + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getSourceType(), "hive")) { + SourceHiveParameter sourceHiveParameter = JSONUtils.parseObject(parameter.getSourceParams(), + SourceHiveParameter.class); + Optional.ofNullable(sourceHiveParameter).ifPresent(p -> { + ListUtils.emptyIfNull(resourceInfos).stream() + .filter(ds -> StringUtils.equals(String.valueOf(ds.getId()), sourceHiveParameter.getHiveDatabase())) + .findFirst().ifPresent(ds -> readerJp.set("parameter.datasource", ds.getName())); + readerJp.set("parameter.table", sourceHiveParameter.getHiveTable()); + String[] keys = Optional.ofNullable(StringUtils.split( + sourceHiveParameter.getHivePartitionKey(), ",")).orElse(new String[]{}); + String[] values = Optional.ofNullable(StringUtils.split( + sourceHiveParameter.getHivePartitionValue(), ",")).orElse(new String[]{}); + List partitions = new ArrayList<>(); + for (int i = 0; i < Math.min(keys.length, values.length); i++) { + partitions.add(Joiner.on("=").join(keys[i], values[i])); + } + readerJp.set("parameter.partition", Joiner.on(",").join(partitions)); + readerJp.set("parameter.readMode", "hdfs"); + readerJp.set("parameter.hdfsUsername", "hdfs"); + List> columns = Arrays.stream(keys).map(key -> { + Map column = new HashMap<>(); + column.put("type", "string"); + column.put("value", key); + return column; + }).collect(Collectors.toList()); + readerJp.set("parameter.hivePartitionColumn", columns); + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getSourceType(), "hdfs")) { + SourceHdfsParameter sourceHdfsParameter = JSONUtils.parseObject(parameter.getSourceParams(), + SourceHdfsParameter.class); + Optional.ofNullable(sourceHdfsParameter).ifPresent( + p -> readerJp.set("parameter.path", p.getExportDir())); + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SubProcessParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SubProcessParameterConverter.java new file mode 100644 index 0000000..965be05 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v1/workflow/parameters/SubProcessParameterConverter.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.parameters; + +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.DolphinSchedulerV1Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.ProcessMeta; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.TaskNode; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v1.v139.task.subprocess.SubProcessParameters; + +import org.apache.commons.collections4.ListUtils; + +public class SubProcessParameterConverter extends AbstractParameterConverter { + + public SubProcessParameterConverter(Properties properties, SpecWorkflow specWorkflow, ProcessMeta processMeta, TaskNode taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + CodeProgramType codeProgramType = CodeProgramType.SUB_PROCESS; + SpecScript script = new SpecScript(); + SpecScriptRuntime runtime = new SpecScriptRuntime(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + script.setPath(getScriptPath(specNode)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + + //processDefinitionId + Integer depProcessId = this.parameter.getProcessDefinitionId(); + DolphinSchedulerV1Context.getContext().getSubProcessCodeNodeMap().put(depProcessId, specNode); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/DolphinSchedulerV2Converter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/DolphinSchedulerV2Converter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/DolphinSchedulerV2Converter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/DolphinSchedulerV2Converter.java index 9ba98a5..830b6df 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/DolphinSchedulerV2Converter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/DolphinSchedulerV2Converter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes; import java.io.File; import java.io.IOException; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/TaskConverterFactoryV2.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/TaskConverterFactoryV2.java similarity index 59% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/TaskConverterFactoryV2.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/TaskConverterFactoryV2.java index b31e94e..a7ed50b 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/TaskConverterFactoryV2.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/TaskConverterFactoryV2.java @@ -13,12 +13,28 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.TaskType; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.AbstractParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.ConditionsParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.CustomParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.DataxParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.DependentParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.HttpParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.MrParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.ProcedureParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.PythonParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.ShellParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.SparkParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.SqlParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.SqoopParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.SubProcessParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.SwitchParameterConverter; +import com.aliyun.migrationx.common.exception.UnSupportedTypeException; import com.aliyun.migrationx.common.utils.Config; public class TaskConverterFactoryV2 { @@ -57,7 +73,7 @@ public static AbstractParameterConverter create( case SQOOP: return new SqoopParameterConverter(processMeta, taskDefinition, converterContext); default: - throw new RuntimeException("unsupported converter task type: " + taskType); + throw new UnSupportedTypeException(taskDefinition.getTaskType()); } } } diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/V2ProcessDefinitionConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/V2ProcessDefinitionConverter.java similarity index 97% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/V2ProcessDefinitionConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/V2ProcessDefinitionConverter.java index d02e66f..d56f117 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/V2ProcessDefinitionConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/V2ProcessDefinitionConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes; import java.util.ArrayList; import java.util.Collections; @@ -46,7 +46,8 @@ import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.StoreWriter; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.filters.DolphinSchedulerConverterFilter; -import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.ProcessDefinitionConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.ProcessDefinitionConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters.AbstractParameterConverter; import com.aliyun.migrationx.common.context.TransformerContext; import com.aliyun.migrationx.common.exception.UnSupportedTypeException; import com.aliyun.migrationx.common.metrics.DolphinMetrics; @@ -201,6 +202,7 @@ private List convertTaskToWorkFlow(TaskDefinition taskDefinition) { } catch (UnSupportedTypeException e) { markFailedProcess(taskDefinition, e.getMessage()); if (Config.INSTANCE.isSkipUnSupportType()) { + log.warn("task {} with type {} unsupported, skip", taskDefinition.getTaskType(), taskDefinition.getName()); return Collections.emptyList(); } else { throw e; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/AbstractParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/AbstractParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/AbstractParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/AbstractParameterConverter.java index 2a01feb..260b382 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/AbstractParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/AbstractParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.io.IOException; import java.util.ArrayList; @@ -64,7 +64,7 @@ import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.AbstractBaseConverter; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.utils.ConverterTypeUtils; -import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.ConditionsParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.ConditionsParameterConverter; import com.aliyun.migrationx.common.context.TransformerContext; import com.aliyun.migrationx.common.metrics.DolphinMetrics; import com.aliyun.migrationx.common.utils.GsonUtils; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/ConditionsParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/ConditionsParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/ConditionsParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/ConditionsParameterConverter.java index 7ddde85..49b0378 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/ConditionsParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/ConditionsParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.io.IOException; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/CustomParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/CustomParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/CustomParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/CustomParameterConverter.java index 4ee2839..5faa098 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/CustomParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/CustomParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.io.IOException; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/DataxParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/DataxParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/DataxParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/DataxParameterConverter.java index f19a677..19859a8 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/DataxParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/DataxParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.util.Arrays; import java.util.HashMap; @@ -284,6 +284,6 @@ private DataSource getDataSource(int datasourceId) { return CollectionUtils.emptyIfNull(datasources).stream() .filter(s -> s.getId() == datasourceId) .findFirst() - .orElseGet(null); + .orElse(null); } } diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/DependentParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/DependentParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/DependentParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/DependentParameterConverter.java index 90a3eaf..dfe1a7e 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/DependentParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/DependentParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/HttpParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/HttpParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/HttpParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/HttpParameterConverter.java index b219b90..540ffc2 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/HttpParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/HttpParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.io.IOException; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/MrParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/MrParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/MrParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/MrParameterConverter.java index ea4a1e2..685cedc 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/MrParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/MrParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; @@ -43,13 +43,13 @@ import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; +import com.aliyun.migrationx.common.utils.BeanUtils; import com.google.common.base.Joiner; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; import static com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.mr.MapReduceTaskConstants.MR_NAME; import static com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.mr.MapReduceTaskConstants.MR_QUEUE; @@ -99,7 +99,7 @@ public List convertParameter() { .filter(r -> r.getId() == mainJar.getId()) .findAny() .map(r -> r.getName()) - .orElseGet(() -> ""); + .orElse(""); resources.add(resourceName); codeLines.add(DataStudioCodeUtils.addResourceReference(CodeProgramType.valueOf(dwNode.getType()), "", resources)); } @@ -158,7 +158,7 @@ public static List buildArgs(MapReduceParameters param) { .filter(r -> r.getId() == mainJar.getId()) .findAny() .map(r -> r.getName()) - .orElseGet(() -> ""); + .orElse(""); String resource = DataStudioCodeUtils.addResourceReference(CodeProgramType.EMR_MR, "", Arrays.asList(resourceName)); args.add(resource + resourceName); } diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/ProcedureParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/ProcedureParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/ProcedureParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/ProcedureParameterConverter.java index f54f9a3..a381526 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/ProcedureParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/ProcedureParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.util.Arrays; import java.util.HashMap; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/PythonParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/PythonParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/PythonParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/PythonParameterConverter.java index 8769062..d3f7956 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/PythonParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/PythonParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.io.File; import java.io.IOException; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/ShellParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/ShellParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/ShellParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/ShellParameterConverter.java index d49395f..133b673 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/ShellParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/ShellParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SparkParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SparkParameterConverter.java similarity index 95% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SparkParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SparkParameterConverter.java index 685e542..7125ac0 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SparkParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SparkParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.io.IOException; import java.util.ArrayList; @@ -86,7 +86,7 @@ private List populateSparkOptions(SparkParameters sparkParameters) { .filter(r -> r.getId() == mainJar.getId()) .findAny() .map(r -> r.getName()) - .orElseGet(null); + .orElse(null); mainJar.setName(resource); } String dwResource = "##@resource_reference{\"" + resource + "\"} \n"; @@ -142,7 +142,7 @@ private List populateSparkOptions(SparkParameters sparkParameters) { private OdpsSparkCode populateSparkOdpsCode() { OdpsSparkCode odpsSparkCode = new OdpsSparkCode(); - odpsSparkCode.setResourceReferences(odpsSparkCode.getResourceReferences()); + odpsSparkCode.setResourceReferences(new ArrayList<>()); odpsSparkCode.setSparkJson(new OdpsSparkCode.CodeJson()); ResourceInfo mainJar = parameter.getMainJar(); @@ -237,8 +237,17 @@ private String getResourceName(Integer id) { .stream() .filter(r -> r.getId() == id) .findAny() - .map(r -> r.getName()) - .orElseGet(null); + .map(r -> { + String name = r.getName(); + if (StringUtils.isEmpty(name)) { + name = r.getResourceName(); + } + if (StringUtils.isEmpty(name)) { + name = r.getFullName(); + } + return name; + }) + .orElse(null); } private String getSparkConverterType() { diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SqlParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SqlParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SqlParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SqlParameterConverter.java index 26b1d04..8ae9ce2 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SqlParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SqlParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.util.Arrays; import java.util.List; @@ -110,7 +110,7 @@ private String getConnectionName(String codeProgramType) { .filter(s -> s.getId() == parameter.getDatasource()) .findFirst() .map(s -> s.getName()) - .orElseGet(null); + .orElse(null); } } return null; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SqoopParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SqoopParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SqoopParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SqoopParameterConverter.java index fa48a38..a6055ec 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SqoopParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SqoopParameterConverter.java @@ -1,4 +1,4 @@ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SubProcessParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SubProcessParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SubProcessParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SubProcessParameterConverter.java index d3d48ff..0eea9ad 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SubProcessParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SubProcessParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SwitchParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SwitchParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SwitchParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SwitchParameterConverter.java index 6baabeb..ca6a163 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/SwitchParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/nodes/parameters/SwitchParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.parameters; import java.io.IOException; import java.util.ArrayList; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/DolphinSchedulerV2WorkflowConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/DolphinSchedulerV2WorkflowConverter.java new file mode 100644 index 0000000..a9fabfa --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/DolphinSchedulerV2WorkflowConverter.java @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.DataWorksWorkflowSpec; +import com.aliyun.dataworks.common.spec.domain.Specification; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.enums.SpecKind; +import com.aliyun.dataworks.common.spec.domain.interfaces.Output; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecSubFlow; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.DolphinSchedulerPackage; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.Project; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.entity.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.entity.ResourceComponent; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.entity.UdfFunc; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +@Slf4j +public class DolphinSchedulerV2WorkflowConverter { + public static final String SPEC_VERSION = "1.2.0"; + + private DolphinSchedulerPackage dolphinSchedulerPackage; + + private final List dagDataList; + private final Properties converterProperties; + + public DolphinSchedulerV2WorkflowConverter(DolphinSchedulerPackage dolphinSchedulerPackage, + Properties converterProperties) { + this.dolphinSchedulerPackage = dolphinSchedulerPackage; + this.converterProperties = converterProperties; + this.dagDataList = dolphinSchedulerPackage.getProcessDefinitions().values().stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + if (dagDataList.isEmpty()) { + throw new RuntimeException("process list empty"); + } + } + + public List> convert() { + List> specifications = new ArrayList<>(); + for (DagData dagData : dagDataList) { + List workflows = new ArrayList<>(); + //convert process to workflow + Specification specification = new Specification<>(); + specification.setKind(SpecKind.CYCLE_WORKFLOW.getLabel()); + specification.setVersion(SPEC_VERSION); + DataWorksWorkflowSpec spec = new DataWorksWorkflowSpec(); + String processName = dagData.getProcessDefinition().getName(); + spec.setName(processName); + V2ProcessDefinitionConverter converter = new V2ProcessDefinitionConverter(dagData, this.converterProperties); + SpecWorkflow workflow = converter.convert(); + workflows.add(workflow); + spec.setWorkflows(workflows); + specification.setSpec(spec); + specifications.add(specification); + } + handleSubprocess(); + handleDependents(specifications); + return specifications; + } + + /** + * subprocess + */ + private void handleSubprocess() { + Map codeWorkflowMap = DolphinSchedulerV2Context.getContext().getSubProcessCodeWorkflowMap(); + Map codeNodeMap = DolphinSchedulerV2Context.getContext().getSubProcessCodeNodeMap(); + //find subprocess + for (Map.Entry entry : codeNodeMap.entrySet()) { + //find workflow + SpecWorkflow specWorkflow = (SpecWorkflow) codeWorkflowMap.get(entry.getKey()); + SpecNodeOutput specNodeOutput; + if (specWorkflow.getOutputs().isEmpty()) { + specNodeOutput = new SpecNodeOutput(); + specNodeOutput.setData(specWorkflow.getId()); + specWorkflow.getOutputs().add(specNodeOutput); + } else { + specNodeOutput = (SpecNodeOutput) specWorkflow.getOutputs().get(0); + } + + SpecNode subprocess = (SpecNode) entry.getValue(); + SpecSubFlow subflow = new SpecSubFlow(); + subflow.setOutput(specNodeOutput.getData()); + subprocess.setSubflow(subflow); + } + } + + /** + * dependent process + * + * @param specifications + */ + private void handleDependents(List> specifications) { + Map nodeIdWorkflowMap = new HashMap<>(); + for (Specification specification : specifications) { + specification.getSpec().getWorkflows().forEach(workflow -> { + workflow.getNodes().forEach(node -> { + nodeIdWorkflowMap.put(node.getId(), workflow); + }); + }); + } + Map taskCodeSpecNodeMap = DolphinSchedulerV2Context.getContext().getTaskCodeSpecNodeMap(); + Map> depMap = DolphinSchedulerV2Context.getContext().getSpecNodeProcessCodeMap(); + + for (Map.Entry> entry : depMap.entrySet()) { + SpecNode specNode = (SpecNode) entry.getKey(); + SpecWorkflow workflow = nodeIdWorkflowMap.get(specNode.getId()); + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + + List dependents = entry.getValue(); + List depends = new ArrayList<>(); + for (Long code : dependents) { + SpecNode depNode = (SpecNode) taskCodeSpecNodeMap.get(code); + if (depNode == null) { + log.warn("can not find spec node: {}", code); + continue; + } + SpecNodeOutput specNodeOutput = new SpecNodeOutput(); + List outputs = depNode.getOutputs(); + String data = depNode.getId(); + if (CollectionUtils.isNotEmpty(outputs)) { + data = ((SpecNodeOutput) outputs.get(0)).getData(); + } + specNodeOutput.setData(data); + SpecDepend specDepend = new SpecDepend(null, DependencyType.NORMAL, specNodeOutput); + depends.add(specDepend); + } + specFlowDepend.setDepends(depends); + + specFlowDepend.setNodeId(specNode); + workflow.getDependencies().add(specFlowDepend); + } + mergeDeps(nodeIdWorkflowMap); + } + + private void mergeDeps(Map nodeIdWorkflowMap) { + nodeIdWorkflowMap.values().stream().forEach(workflow -> { + List depends = workflow.getDependencies(); + + if (CollectionUtils.isNotEmpty(depends)) { + Map> map = new HashMap<>(); + depends.stream().forEach(dep -> { + List specNodes = map.get(dep.getNodeId()); + if (CollectionUtils.isNotEmpty(specNodes)) { + specNodes.addAll(dep.getDepends()); + } else { + specNodes = new ArrayList<>(); + specNodes.addAll(dep.getDepends()); + map.put(dep.getNodeId(), specNodes); + } + }); + List specFlowDepends = new ArrayList<>(); + for (Map.Entry> entry : map.entrySet()) { + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + specFlowDepend.setNodeId(entry.getKey()); + specFlowDepend.setDepends(entry.getValue()); + specFlowDepends.add(specFlowDepend); + } + workflow.setDependencies(specFlowDepends); + } + }); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/ParamListConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/ParamListConverter.java new file mode 100644 index 0000000..c9e9679 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/ParamListConverter.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.enums.VariableScopeType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.Direct; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.process.Property; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import org.apache.commons.collections4.ListUtils; + +public class ParamListConverter { + + public static final String SYSTEM_VARIABLE_TAG = "$"; + + private final List paramList; + + private final TaskDefinition taskDefinition; + + public ParamListConverter(List paramList) { + this(paramList, null); + } + + public ParamListConverter(List paramList, TaskDefinition taskDefinition) { + super(); + this.paramList = paramList; + this.taskDefinition = taskDefinition; + } + + public List convert() { + return ListUtils.emptyIfNull(paramList).stream().map(p -> { + // don't convert global out param + if (Objects.isNull(taskDefinition) && Direct.OUT.equals(p.getDirect())) { + return null; + } + + SpecVariable specVariable = new SpecVariable(); + + specVariable.setId(UuidGenerators.generateUuid()); + specVariable.setName(p.getProp()); + specVariable.setValue(p.getValue()); + specVariable.setDescription(p.getType().name()); + if (Direct.IN.equals(p.getDirect())) { + if (specVariable.getValue().startsWith(SYSTEM_VARIABLE_TAG)) { + specVariable.setType(VariableType.SYSTEM); + } else { + specVariable.setType(VariableType.CONSTANT); + } + specVariable.setScope(Objects.isNull(taskDefinition) ? VariableScopeType.FLOW : VariableScopeType.NODE_PARAMETER); + } else { + specVariable.setType(VariableType.NODE_OUTPUT); + specVariable.setScope(VariableScopeType.NODE_CONTEXT); + } + return specVariable; + } + ).filter(Objects::nonNull).collect(Collectors.toList()); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/SpecFlowDependConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/SpecFlowDependConverter.java new file mode 100644 index 0000000..c30bb0a --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/SpecFlowDependConverter.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.DataWorksWorkflowSpec; +import com.aliyun.dataworks.common.spec.domain.enums.ArtifactType; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.domain.ref.SpecArtifact; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.ProcessTaskRelation; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; + +@Slf4j +public class SpecFlowDependConverter { + + private static final Map> ARTIFACT_TYPE_CLASS_MAP = new EnumMap<>(ArtifactType.class); + + private final DataWorksWorkflowSpec spec; + + private final SpecWorkflow specWorkflow; + + private final List processTaskRelationList; + + static { + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.TABLE, SpecTable.class); + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.VARIABLE, SpecVariable.class); + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.NODE_OUTPUT, SpecNodeOutput.class); + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.FILE, SpecArtifact.class); + } + + public SpecFlowDependConverter(DataWorksWorkflowSpec spec, SpecWorkflow specWorkflow, List processTaskRelationList) { + this.spec = spec; + this.specWorkflow = specWorkflow; + this.processTaskRelationList = processTaskRelationList; + } + + public List convert() { + if (Objects.nonNull(specWorkflow)) { + specWorkflow.setDependencies(convertTaskRelationList(processTaskRelationList)); + return specWorkflow.getDependencies(); + } + spec.setFlow(convertTaskRelationList(processTaskRelationList)); + return spec.getFlow(); + } + + private List convertTaskRelationList(List taskRelationList) { + List flow = Optional.ofNullable(specWorkflow).map(SpecWorkflow::getDependencies).orElse(Optional.ofNullable(spec) + .map(DataWorksWorkflowSpec::getFlow).orElse(new ArrayList<>())); + Map> nodeIdDependMap = flow.stream().collect( + Collectors.toMap(o -> o.getNodeId().getId(), SpecFlowDepend::getDepends)); + + Map taskCodeNodeDataMap = DolphinSchedulerV2Context.getContext().getTaskCodeNodeDataMap(); + Map taskCodeNodeIdMap = DolphinSchedulerV2Context.getContext().getTaskCodeNodeIdMap(); + for (ProcessTaskRelation processTaskRelation : ListUtils.emptyIfNull(taskRelationList)) { + long preTaskCode = processTaskRelation.getPreTaskCode(); + if (preTaskCode == 0L) { + continue; + } + long postTaskCode = processTaskRelation.getPostTaskCode(); + String nodeId = taskCodeNodeIdMap.get(postTaskCode); + SpecNode currentNode; + if (nodeId == null) { + log.warn("can not find nodeId {}", postTaskCode); + continue; + } else { + currentNode = specWorkflow.getNodes().stream().filter(node -> nodeId.equals(node.getId())) + .findAny().orElse(null); + } + final SpecNode finalCurrentNode = currentNode; + + String data = taskCodeNodeDataMap.get(preTaskCode); + if (data != null) { + specWorkflow.getNodes().stream() + .filter(node -> { + return CollectionUtils.emptyIfNull(node.getOutputs()) + .stream() + .map(output -> (SpecNodeOutput) output) + .anyMatch(output -> data.equals(output.getData())); + }) + .findAny() + .ifPresent(node -> { + SpecNodeOutput specNodeOutput = (SpecNodeOutput) node.getOutputs().get(0); + SpecDepend specDepend = new SpecDepend(null, DependencyType.NORMAL, specNodeOutput); + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + specFlowDepend.setDepends(Arrays.asList(specDepend)); + specFlowDepend.setNodeId(finalCurrentNode); + //flow.add(specFlowDepend); + specWorkflow.getDependencies().add(specFlowDepend); + }); + } + } + return flow; + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/TaskConverterFactoryV2.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/TaskConverterFactoryV2.java new file mode 100644 index 0000000..37344a2 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/TaskConverterFactoryV2.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow; + +import java.util.Properties; + +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.TaskType; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.AbstractParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.CustomParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.DataxParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.DependentParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.MrParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.ProcedureParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.PythonParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.ShellParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.SparkParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.SqlParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.SqoopParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.SubProcessParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.SwitchParameterConverter; +import com.aliyun.migrationx.common.exception.UnSupportedTypeException; +import com.aliyun.migrationx.common.utils.Config; + +public class TaskConverterFactoryV2 { + public static AbstractParameterConverter create( + Properties properties, SpecWorkflow specWorkflow, + DagData processMeta, TaskDefinition taskDefinition) throws Throwable { + if (Config.INSTANCE.getTempTaskTypes().contains(taskDefinition.getTaskType())) { + return new CustomParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + } + TaskType taskType = TaskType.of(taskDefinition.getTaskType()); + if (taskType == null) { + throw new UnSupportedTypeException(taskDefinition.getTaskType()); + } + + switch (taskType) { + case SHELL: + return new ShellParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case PYTHON: + return new PythonParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SQL: + return new SqlParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case MR: + return new MrParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SPARK: + return new SparkParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SUB_PROCESS: + return new SubProcessParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case DEPENDENT: + return new DependentParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SWITCH: + return new SwitchParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SQOOP: + return new SqoopParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case DATAX: + return new DataxParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case PROCEDURE: + return new ProcedureParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + default: + throw new UnSupportedTypeException(taskDefinition.getTaskType()); + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/TriggerConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/TriggerConverter.java new file mode 100644 index 0000000..46b3f96 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/TriggerConverter.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow; + +import java.text.ParseException; +import java.time.Duration; +import java.util.Objects; + +import com.aliyun.dataworks.common.spec.domain.enums.NodeRecurrenceType; +import com.aliyun.dataworks.common.spec.domain.enums.TriggerType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.Schedule; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.Flag; +import com.aliyun.dataworks.migrationx.domain.dataworks.utils.CronExpressUtil; +import com.aliyun.migrationx.common.utils.DateUtils; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class TriggerConverter { + + private final Schedule schedule; + + private final SpecTrigger currTrigger; + + private final TaskDefinition taskDefinition; + + private SpecTrigger convertRes; + + public TriggerConverter(Schedule schedule) { + super(); + this.schedule = schedule; + this.currTrigger = null; + this.taskDefinition = null; + this.convertRes = new SpecTrigger(); + } + + public TriggerConverter(SpecTrigger trigger, TaskDefinition taskDefinition) { + super(); + this.schedule = null; + this.currTrigger = trigger; + this.taskDefinition = taskDefinition; + this.convertRes = new SpecTrigger(); + } + + public SpecTrigger convert() { + if (Objects.nonNull(schedule)) { + convertBySchedule(); + } else if (Objects.nonNull(currTrigger) && Objects.nonNull(taskDefinition)) { + convertByTaskDefinition(); + } else { + return null; + } + return convertRes; + } + + /** + * convert workflow trigger + */ + private void convertBySchedule() { + if (Objects.isNull(schedule)) { + convertRes.setType(TriggerType.MANUAL); + } else { + convertRes.setId(UuidGenerators.generateUuid(Long.valueOf(schedule.getId()))); + convertRes.setType(TriggerType.SCHEDULER); + convertRes.setStartTime(DateUtils.convertDateToString(schedule.getStartTime())); + convertRes.setEndTime(DateUtils.convertDateToString(schedule.getEndTime())); + convertRes.setCron(dolphinCron2SpecCron(schedule.getCrontab())); + convertRes.setTimezone(schedule.getTimezoneId()); + convertRes.setDelaySeconds(0); + } + } + + /** + * convert task trigger, especially for delay seconds. + * Because node will use same trigger with parent workflow except delay seconds. + */ + private void convertByTaskDefinition() { + if (currTrigger == null || taskDefinition == null) { + throw new RuntimeException("currTrigger or taskDefinition null"); + } + //convertRes = BeanUtils.deepCopy(currTrigger, SpecTrigger.class); + convertRes = currTrigger; + convertRes.setDelaySeconds((int) Duration.ofMinutes(taskDefinition.getDelayTime()).getSeconds()); + convertRes.setRecurrence(Flag.YES.equals(taskDefinition.getFlag()) ? NodeRecurrenceType.NORMAL : NodeRecurrenceType.PAUSE); + convertRes.setId(UuidGenerators.generateUuid()); + } + + private String dolphinCron2SpecCron(String dolphinCron) { + try { + return CronExpressUtil.quartzCronExpressionToDwCronExpress(dolphinCron); + } catch (ParseException e) { + log.warn("dolphin cron parse error: {}", dolphinCron); + return dolphinCron; + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/V2ProcessDefinitionConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/V2ProcessDefinitionConverter.java new file mode 100644 index 0000000..b2909ea --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/V2ProcessDefinitionConverter.java @@ -0,0 +1,367 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.ControllerBranchCode.Branch; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.FailureStrategy; +import com.aliyun.dataworks.common.spec.domain.enums.NodeInstanceModeType; +import com.aliyun.dataworks.common.spec.domain.enums.NodeRerunModeType; +import com.aliyun.dataworks.common.spec.domain.noref.SpecBranches; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScheduleStrategy; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.ProcessDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.ProcessTaskRelation; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.Schedule; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.Priority; +import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.CheckPoint; +import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.StoreWriter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.filters.DolphinSchedulerConverterFilter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters.AbstractParameterConverter; +import com.aliyun.migrationx.common.context.TransformerContext; +import com.aliyun.migrationx.common.exception.UnSupportedTypeException; +import com.aliyun.migrationx.common.metrics.DolphinMetrics; +import com.aliyun.migrationx.common.utils.Config; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import com.google.common.base.Joiner; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class V2ProcessDefinitionConverter { + private List specNodes = new ArrayList<>(); + + private static final SpecScriptRuntime WORKFLOW_RUNTIME = new SpecScriptRuntime(); + + private static final SpecScriptRuntime MANUAL_WORKFLOW_RUNTIME = new SpecScriptRuntime(); + + static { + WORKFLOW_RUNTIME.setEngine(CodeProgramType.VIRTUAL_WORKFLOW.getCalcEngineType().getLabel()); + WORKFLOW_RUNTIME.setCommand("WORKFLOW"); + + MANUAL_WORKFLOW_RUNTIME.setEngine(CodeProgramType.VIRTUAL_WORKFLOW.getCalcEngineType().getLabel()); + MANUAL_WORKFLOW_RUNTIME.setCommand("MANUAL_WORKFLOW"); + } + + private ProcessDefinition processDefinition; + private DagData dagData; + private Properties converterProperties; + private List taskDefinitionList; + + private DolphinSchedulerConverterFilter filter; + + public V2ProcessDefinitionConverter(DagData dagData, Properties converterProperties) { + this.dagData = dagData; + this.converterProperties = converterProperties; + this.processDefinition = dagData.getProcessDefinition(); + this.taskDefinitionList = dagData.getTaskDefinitionList(); + this.filter = new DolphinSchedulerConverterFilter(); + } + + public static String toWorkflowName(ProcessDefinition processDefinition) { + if (processDefinition.getProjectName() == null) { + return processDefinition.getName(); + } + return com.aliyun.dataworks.migrationx.domain.dataworks.utils.StringUtils.toValidName(Joiner.on("_").join( + processDefinition.getProjectName(), processDefinition.getName())); + } + + private SpecWorkflow initWorkflow() { + SpecWorkflow specWorkflow = new SpecWorkflow(); + specWorkflow.setDependencies(new ArrayList<>()); + specWorkflow.setNodes(new ArrayList<>()); + specWorkflow.setInputs(new ArrayList<>()); + specWorkflow.setOutputs(new ArrayList<>()); + return specWorkflow; + } + + public SpecWorkflow convert() { + SpecWorkflow specWorkflow = convertProcess(processDefinition); + DolphinSchedulerV2Context.getContext().getSubProcessCodeWorkflowMap().put(processDefinition.getCode(), specWorkflow); + convertTaskDefinitions(specWorkflow); + convertTrigger(specWorkflow); + convertTaskRelations(specWorkflow); + handleBranch(specWorkflow); + return specWorkflow; + } + + protected SpecWorkflow convertProcess(ProcessDefinition processDefinition) { + log.info("convert workflow,processDefinition: {}", processDefinition.getName()); + + SpecWorkflow specWorkflow = initWorkflow(); + specWorkflow.setId(UuidGenerators.generateUuid(processDefinition.getCode())); + specWorkflow.setName(toWorkflowName(processDefinition)); + specWorkflow.setDescription(processDefinition.getDescription()); + + List specVariableList = new ParamListConverter(processDefinition.getGlobalParamList()).convert(); + log.info("convert workflow,global params: {}", specVariableList); + + SpecScript script = new SpecScript(); + script.setParameters(specVariableList); + script.setRuntime(WORKFLOW_RUNTIME); + script.setPath(getScriptPath(specWorkflow.getName())); + specWorkflow.setScript(script); + + specWorkflow.getOutputs().add(buildDefaultOutput(specWorkflow)); + return specWorkflow; + } + + protected String getScriptPath(String name) { + String defaultPath = StringUtils.defaultString(Config.INSTANCE.getBasePath(), StringUtils.EMPTY); + return FilenameUtils.concat(defaultPath, name); + } + + protected List convertTaskDefinitions(SpecWorkflow specWorkflow) { + return convertTasks(specWorkflow); + } + + protected void convertTrigger(SpecWorkflow specWorkflow) { + Schedule schedule = dagData.getSchedule(); + if (Objects.nonNull(schedule)) { + SpecTrigger trigger = new TriggerConverter(schedule).convert(); + specWorkflow.setTrigger(trigger); + specWorkflow.setStrategy(buildSpecScheduleStrategy(processDefinition, schedule)); + log.info("convert workflow,schedule: {}", schedule); + } + } + + /** + * when handling task, spec node maybe not exists, post handle branch + * change branch output data (task code) to specNode id + */ + private void handleBranch(SpecWorkflow specWorkflow) { + Map taskCodeSpecNodeMap = DolphinSchedulerV2Context.getContext().getTaskCodeSpecNodeMap(); + for (SpecNode specNode : specWorkflow.getNodes()) { + if (specNode.getBranch() != null && specNode.getBranch().getBranches() != null) { + List specBranches = specNode.getBranch().getBranches(); + List branchList = new ArrayList<>(); + for (SpecBranches specBranch : specBranches) { + SpecNodeOutput specNodeOutput = specBranch.getOutput(); + if (specNodeOutput != null && specNodeOutput.getData() != null) { + String data = specNodeOutput.getData(); + //current data is task code, convert to specNode id + Long taskCode = Long.parseLong(data); + SpecNode branchSpecNode = (SpecNode) taskCodeSpecNodeMap.get(taskCode); + specNodeOutput.setData(branchSpecNode.getId()); + Branch branch = new Branch(); + branch.setCondition(specBranch.getWhen()); + branch.setNodeoutput(branchSpecNode.getId()); + branchList.add(branch); + } + } + if (branchList.size() > 0) { + String content = com.aliyun.dataworks.common.spec.utils.GsonUtils.toJsonString(branchList); + specNode.getScript().setContent(content); + } + } + } + } + + protected void convertTaskRelations(SpecWorkflow specWorkflow) { + List processTaskRelationList = dagData.getProcessTaskRelationList(); + log.info("convert workflow,processTaskRelationList: {}", processTaskRelationList); + new SpecFlowDependConverter(null, specWorkflow, processTaskRelationList).convert(); + } + + private SpecScheduleStrategy buildSpecScheduleStrategy(ProcessDefinition processDefinition, Schedule schedule) { + SpecScheduleStrategy strategy = new SpecScheduleStrategy(); + strategy.setPriority(convertPriority(schedule.getProcessInstancePriority())); + strategy.setTimeout(processDefinition.getTimeout()); + strategy.setInstanceMode(NodeInstanceModeType.T_PLUS_1); + strategy.setRerunMode(NodeRerunModeType.ALL_ALLOWED); + strategy.setRerunTimes(0); + strategy.setRerunInterval(0); + strategy.setIgnoreBranchConditionSkip(false); + strategy.setFailureStrategy( + com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.FailureStrategy.CONTINUE.equals(schedule.getFailureStrategy()) + ? FailureStrategy.CONTINUE + : FailureStrategy.BREAK); + return strategy; + } + + protected Integer convertPriority(Priority priority) { + return Priority.LOWEST.getCode() - priority.getCode(); + } + + private SpecNodeOutput buildDefaultOutput(SpecWorkflow specWorkflow) { + SpecNodeOutput specNodeOutput = new SpecNodeOutput(); + specNodeOutput.setIsDefault(true); + //String data = String.valueOf(processDefinition.getCode()); + String data = String.format("%s.%s", processDefinition.getProjectName(), processDefinition.getName()); + specNodeOutput.setData(data); + specNodeOutput.setRefTableName(specWorkflow.getName()); + return specNodeOutput; + } + + public List convertTasks(SpecWorkflow specWorkflow) { + CheckPoint checkPoint = CheckPoint.getInstance(); + String projectName = processDefinition.getProjectName(); + String processName = processDefinition.getName(); + + Map> loadedTasks = checkPoint.loadFromCheckPoint(projectName, processName); + + final Function> processFunc = (StoreWriter writer) -> + ListUtils.emptyIfNull(taskDefinitionList) + .stream() + .filter(s -> { + if (Thread.currentThread().isInterrupted()) { + throw new RuntimeException(new InterruptedException()); + } + return true; + }) + .filter(task -> { + boolean passed = filter.filter(projectName, processName, task.getName()); + if (!passed) { + log.warn("task filtered {}", task.getName()); + } + return passed; + }) + .map(task -> { + List specNodes = convertTaskToWorkflowWithLoadedTask(specWorkflow, task, loadedTasks); + checkPoint.doCheckpoint(writer, specNodes, processName, task.getName()); + return specNodes; + }) + .flatMap(List::stream) + .collect(Collectors.toList()); + + specNodes = checkPoint.doWithCheckpoint(processFunc, projectName); + + log.info("successfully converted {}, size {}", processDefinition.getName(), specNodes.size()); + return specNodes; + } + + private List convertTaskToWorkflowWithLoadedTask(SpecWorkflow specWorkflow, TaskDefinition taskDefinition, Map> loadedTasks) { + List specNodes = loadedTasks.get(taskDefinition.getName()); + if (specNodes != null) { + markSuccessProcess(specWorkflow, taskDefinition); + log.info("loaded task {} from checkpoint", taskDefinition.getName()); + return specNodes; + } + return convertTaskDefinition(specWorkflow, taskDefinition); + } + + private List convertTaskDefinition(SpecWorkflow specWorkflow, TaskDefinition taskDefinition) { + if (inSkippedList(taskDefinition)) { + log.warn("task {} in skipped list", taskDefinition.getName()); + return Collections.emptyList(); + } + + try { + AbstractParameterConverter converter = TaskConverterFactoryV2.create(converterProperties, specWorkflow, dagData, taskDefinition); + SpecNode specNode = converter.convert(); + if (specNode != null) { + DolphinSchedulerV2Context.getContext().getTaskCodeSpecNodeMap().put(taskDefinition.getCode(), specNode); + return Arrays.asList(specNode); + } else { + return Collections.emptyList(); + } + } catch (UnSupportedTypeException e) { + markFailedProcess(taskDefinition, e.getMessage()); + if (Config.INSTANCE.isSkipUnSupportType()) { + log.warn("task {} with type {} unsupported, skip", taskDefinition.getTaskType(), taskDefinition.getName()); + return Collections.emptyList(); + } else { + throw e; + } + } catch (Throwable e) { + log.error("task converter error, taskName {} ", taskDefinition.getName(), e); + if (Config.INSTANCE.isTransformContinueWithError()) { + return Collections.emptyList(); + } else { + throw new RuntimeException(e); + } + } + } + + protected void markSuccessProcess(SpecWorkflow workflow, TaskDefinition taskDefinition) { + for (SpecNode node : workflow.getNodes()) { + DolphinMetrics metrics = DolphinMetrics.builder() + .projectName(taskDefinition.getProjectName()) + .projectCode(taskDefinition.getProjectCode()) + .processName(processDefinition.getName()) + .processCode(processDefinition.getCode()) + .taskName(taskDefinition.getName()) + .taskCode(taskDefinition.getCode()) + .taskType(taskDefinition.getTaskType()) + .build(); + metrics.setWorkflowName(workflow.getName()); + metrics.setDwName(node.getName()); + String type = node.getScript().getRuntime().getCommand(); + metrics.setDwType(type); + TransformerContext.getCollector().markSuccessMiddleProcess(metrics); + } + } + + private boolean inSkippedList(TaskDefinition taskDefinition) { + if (Config.INSTANCE.getSkipTypes().contains(taskDefinition.getTaskType()) + || Config.INSTANCE.getSkipTaskCodes().contains(String.valueOf(taskDefinition.getCode()))) { + log.warn("task name {} code {} in skipped list", taskDefinition.getName(), taskDefinition.getCode()); + markSkippedProcess(taskDefinition); + return true; + } else { + return false; + } + } + + protected void markFailedProcess(TaskDefinition taskDefinition, String errorMsg) { + DolphinMetrics metrics = DolphinMetrics.builder() + .projectName(taskDefinition.getProjectName()) + .projectCode(taskDefinition.getProjectCode()) + .processName(processDefinition.getName()) + .processCode(processDefinition.getCode()) + .taskName(taskDefinition.getName()) + .taskCode(taskDefinition.getCode()) + .taskType(taskDefinition.getTaskType()) + .build(); + metrics.setErrorMsg(errorMsg); + TransformerContext.getCollector().markFailedMiddleProcess(metrics); + } + + protected void markSkippedProcess(TaskDefinition taskDefinition) { + DolphinMetrics metrics = DolphinMetrics.builder() + .projectName(taskDefinition.getProjectName()) + .projectCode(taskDefinition.getProjectCode()) + .processName(processDefinition.getName()) + .processCode(processDefinition.getCode()) + .taskName(taskDefinition.getName()) + .taskCode(taskDefinition.getCode()) + .taskType(taskDefinition.getTaskType()) + .build(); + TransformerContext.getCollector().markSkippedProcess(metrics); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/AbstractParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/AbstractParameterConverter.java new file mode 100644 index 0000000..708f7c5 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/AbstractParameterConverter.java @@ -0,0 +1,457 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.io.File; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.dw.types.LanguageEnum; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.enums.NodeRecurrenceType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableScopeType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.interfaces.Output; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.domain.ref.SpecFileResource; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScheduleStrategy; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.ProcessDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.Priority; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.TaskType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.AbstractParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.conditions.ConditionsParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.datax.DataxParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.dependent.DependentParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.flink.FlinkParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.http.HttpParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.mr.MapReduceParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.procedure.ProcedureParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.python.PythonParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.shell.ShellParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.spark.SparkParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.sql.SqlParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.sqoop.SqoopParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.subprocess.SubProcessParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.switchtask.SwitchParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.Flag; +import com.aliyun.dataworks.migrationx.transformer.core.utils.SpecFileResourceTypeUtils; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.utils.ConverterTypeUtils; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.ParamListConverter; +import com.aliyun.migrationx.common.exception.BizException; +import com.aliyun.migrationx.common.exception.ErrorCode; +import com.aliyun.migrationx.common.utils.BeanUtils; +import com.aliyun.migrationx.common.utils.Config; +import com.aliyun.migrationx.common.utils.GsonUtils; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import com.google.gson.reflect.TypeToken; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public abstract class AbstractParameterConverter { + + protected static final String RESOURCE_REFERENCE_FORMAT = "%s@resource_reference{\"%s\"}"; + protected static final String RESOURCE_REFERENCE_PREFIX = "##"; + + protected final TaskDefinition taskDefinition; + protected final DagData processMeta; + protected final ProcessDefinition processDefinition; + + protected final Properties properties; + + protected static Map> taskTypeClassMap; + + protected T parameter; + + /** + * The purpose of setting this field private is to mask the differences in how subclasses perceive spec and workflow + */ + protected final SpecWorkflow specWorkflow; + + static { + taskTypeClassMap = new EnumMap<>(TaskType.class); + taskTypeClassMap.put(TaskType.SQL, SqlParameters.class); + taskTypeClassMap.put(TaskType.DEPENDENT, DependentParameters.class); + taskTypeClassMap.put(TaskType.FLINK, FlinkParameters.class); + taskTypeClassMap.put(TaskType.SPARK, SparkParameters.class); + taskTypeClassMap.put(TaskType.DATAX, DataxParameters.class); + taskTypeClassMap.put(TaskType.SHELL, ShellParameters.class); + taskTypeClassMap.put(TaskType.HTTP, HttpParameters.class); + taskTypeClassMap.put(TaskType.PROCEDURE, ProcedureParameters.class); + taskTypeClassMap.put(TaskType.CONDITIONS, ConditionsParameters.class); + taskTypeClassMap.put(TaskType.SQOOP, SqoopParameters.class); + taskTypeClassMap.put(TaskType.SUB_PROCESS, SubProcessParameters.class); + taskTypeClassMap.put(TaskType.PYTHON, PythonParameters.class); + taskTypeClassMap.put(TaskType.MR, MapReduceParameters.class); + taskTypeClassMap.put(TaskType.SWITCH, SwitchParameters.class); + } + + protected AbstractParameterConverter(Properties properties, + SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(); + this.properties = properties; + //this.spec = spec; + this.specWorkflow = specWorkflow; + this.processMeta = processMeta; + this.processDefinition = processMeta.getProcessDefinition(); + this.taskDefinition = taskDefinition; + + TaskType taskType = TaskType.valueOf(taskDefinition.getTaskType()); + try { + this.parameter = GsonUtils.fromJsonString( + taskDefinition.getTaskParams(), TypeToken.get(taskTypeClassMap.get(taskType)).getType()); + } catch (Exception ex) { + log.error("parse task {}, parameter {} error: ", taskType, taskTypeClassMap.get(taskType), ex); + } + } + + /** + * Each node translates the specific logic of the parameters + */ + protected abstract void convertParameter(SpecNode specNode); + + public SpecNode convert() { + SpecNode specNode = newSpecNode(taskDefinition); + + convertParameter(specNode); + this.specWorkflow.getNodes().add(specNode); + + // hint: the node returned may not be the final result of the conversion + return specNode; + } + + private SpecNode initSpecNode() { + SpecNode specNode = new SpecNode(); + specNode.setInputs(new ArrayList<>()); + specNode.setOutputs(new ArrayList<>()); + specNode.setFileResources(new ArrayList<>()); + specNode.setFunctions(new ArrayList<>()); + return specNode; + } + + /** + * common new SpecNode method, almost all nodes use this method + * + * @param taskDefinition taskDefinition + * @return SpecNode + */ + protected SpecNode newSpecNode(TaskDefinition taskDefinition) { + SpecNode specNode = initSpecNode(); + specNode.setId(UuidGenerators.generateUuid(taskDefinition.getCode())); + specNode.setName(taskDefinition.getName()); + specNode.setDescription(taskDefinition.getDescription()); + specNode.setRerunTimes(taskDefinition.getFailRetryTimes()); + // Unit conversion, minutes to milliseconds + specNode.setRerunInterval((int) Duration.ofMinutes(taskDefinition.getFailRetryInterval()).toMillis()); + specNode.setTimeout(taskDefinition.getTimeout()); + specNode.setPriority(convertPriority(taskDefinition.getTaskPriority())); + resetNodeStrategy(specNode); + + SpecNodeOutput defaultOutput = buildDefaultNodeOutput(specNode); + DolphinSchedulerV2Context.getContext().getTaskCodeNodeDataMap().put(taskDefinition.getCode(), defaultOutput.getData()); + DolphinSchedulerV2Context.getContext().getTaskCodeNodeIdMap().put(taskDefinition.getCode(), specNode.getId()); + //specNode.getOutputs().add(BeanUtils.deepCopy(defaultOutput, SpecNodeOutput.class)); + + specNode.getOutputs().add(defaultOutput); + + specNode.setTrigger(convertByTaskDefinition()); + + return specNode; + } + + private SpecTrigger convertByTaskDefinition() { + SpecTrigger specTrigger = new SpecTrigger(); + specTrigger.setDelaySeconds((int) Duration.ofMinutes(taskDefinition.getDelayTime()).getSeconds()); + specTrigger.setRecurrence(Flag.YES.equals(taskDefinition.getFlag()) ? NodeRecurrenceType.NORMAL : NodeRecurrenceType.PAUSE); + specTrigger.setId(UuidGenerators.generateUuid()); + return specTrigger; + } + + protected Integer convertPriority(Priority priority) { + return Priority.LOWEST.getCode() - priority.getCode(); + } + + protected SpecNodeOutput buildDefaultNodeOutput(SpecNode specNode) { + SpecNodeOutput specNodeOutput = new SpecNodeOutput(); + specNodeOutput.setIsDefault(true); + //String data = String.format("%s.%s", processDefinition.getCode(), taskDefinition.getCode()); + String data = String.format("%s.%s.%s", processDefinition.getProjectName(), processDefinition.getName(), taskDefinition.getName()); + specNodeOutput.setData(data); + specNodeOutput.setRefTableName(specNode.getName()); + return specNodeOutput; + } + + protected SpecNodeOutput getDefaultOutput(SpecNode specNode) { + return getDefaultOutput(Optional.ofNullable(specNode).map(SpecNode::getOutputs).orElse(null)); + } + + protected SpecNodeOutput getDefaultOutput(SpecWorkflow specWorkflow, boolean throwException) { + return getDefaultOutput(Optional.ofNullable(specWorkflow).map(SpecWorkflow::getOutputs).orElse(null), throwException); + } + + protected SpecNodeOutput getDefaultOutput(SpecNode specNode, boolean throwException) { + return getDefaultOutput(Optional.ofNullable(specNode).map(SpecNode::getOutputs).orElse(null), throwException); + } + + protected SpecNodeOutput getDefaultOutput(List outputList) { + return getDefaultOutput(outputList, false); + } + + protected SpecNodeOutput getDefaultOutput(List outputList, boolean throwException) { + Optional first = ListUtils.emptyIfNull(outputList).stream().filter( + output -> output instanceof SpecNodeOutput && ((SpecNodeOutput) output).getIsDefault()) + .findFirst(); + if (throwException && !first.isPresent()) { + throw new BizException(ErrorCode.PARAMETER_NOT_SET, "defaultOutput"); + } + return (SpecNodeOutput) first.orElse(null); + } + + protected List getContextOutputs(SpecNode specNode) { + return specNode.getOutputs().stream() + .filter(v -> v instanceof SpecVariable && VariableScopeType.NODE_CONTEXT.equals(((SpecVariable) v).getScope()) + && VariableType.NODE_OUTPUT.equals(((SpecVariable) v).getType())) + .map(v -> { + SpecVariable variable = BeanUtils.deepCopy(v, SpecVariable.class); + variable.setNode(new SpecDepend(specNode, DependencyType.NORMAL, null)); + return variable; + }).collect(Collectors.toList()); + } + + protected List convertSpecNodeParam(SpecNode specNode) { + ParamListConverter paramListConverter = new ParamListConverter(taskDefinition.getTaskParamList(), taskDefinition); + List specVariableList = paramListConverter.convert(); + for (SpecVariable specVariable : specVariableList) { + // all outputs are context output, all inputs are all script inputs + if (VariableType.NODE_OUTPUT.equals(specVariable.getType())) { + SpecDepend nodeDepend = new SpecDepend(); + nodeDepend.setNodeId(specNode); + nodeDepend.setOutput(getDefaultOutput(specNode)); + specVariable.setNode(nodeDepend); + specNode.getOutputs().add(specVariable); + } + } + return specVariableList; + } + + /** + * convert resource with fileResources info in workflow, if not exists in fileResources, create and add in fileResources + * + * @param specNode node need to convert + */ + protected void convertFileResourceList(SpecNode specNode) { + ListUtils.emptyIfNull(parameter.getResourceFilesList()).forEach(resourceInfo -> { + SpecFileResource specFileResource = new SpecFileResource(); + specFileResource.setRuntimeResource(specNode.getRuntimeResource()); + specFileResource.setName(getFileNameByPath(resourceInfo.getResourceName())); + specFileResource.setType(SpecFileResourceTypeUtils.getResourceTypeBySuffix(specFileResource.getName())); + checkFileSameName(specFileResource.getName(), resourceInfo.getResourceName()); + specNode.getFileResources().add(specFileResource); + }); + } + + private void checkFileSameName(String fileName, String fullName) { + //String fullNameIn = context.getFileNameMap().get(fileName); + String fullNameIn = null; + if (Objects.nonNull(fullNameIn) && !fullNameIn.equals(fullName)) { + log.warn("存在同名资源冲突风险, {} 和 {} 导入后会同名", fullNameIn, fullName); + } else { + //context.getFileNameMap().put(fileName, fullName); + } + } + + protected String buildFileResourceReference(SpecNode specNode, String prefix) { + StringBuilder stringBuilder = new StringBuilder(); + Optional.ofNullable(specNode).map(SpecNode::getFileResources) + .ifPresent(fileResources -> + fileResources.forEach(fileResource -> + stringBuilder.append(String.format(RESOURCE_REFERENCE_FORMAT, prefix, fileResource.getName())).append("\n"))); + return stringBuilder.append("\n").toString(); + } + + /** + * add relation before join node. if the node depend on a whole workflow, need depend on workflow output + * + * @param postNode post join node + * @param preNodeList pre node list + * @param preNodeOutputList pre workflow output list + */ + protected void addRelation(SpecNode postNode, List preNodeList, List preNodeOutputList) { + SpecFlowDepend specFlowDepend = newSpecFlowDepend(); + specFlowDepend.setNodeId(postNode); + ListUtils.emptyIfNull(preNodeList).forEach(preNode -> { + SpecNodeOutput preNodeOutput = getDefaultOutput(preNode); + postNode.getInputs().add(preNodeOutput); + postNode.getInputs().addAll(getContextOutputs(preNode)); + specFlowDepend.getDepends().add(new SpecDepend(preNode, DependencyType.NORMAL, preNodeOutput)); + }); + + ListUtils.emptyIfNull(preNodeOutputList).forEach(preNodeOutput -> { + postNode.getInputs().add(preNodeOutput); + SpecDepend specDepend = new SpecDepend(); + specDepend.setType(DependencyType.NORMAL); + specDepend.setOutput(preNodeOutput); + specFlowDepend.getDepends().add(specDepend); + }); + getWorkflowDependencyList().add(specFlowDepend); + } + + protected SpecFlowDepend newSpecFlowDepend() { + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + specFlowDepend.setDepends(new ArrayList<>()); + return specFlowDepend; + } + + protected void addRelation(SpecNode postNode, List preNodeList) { + addRelation(postNode, preNodeList, null); + } + + protected String getFileNameByPath(String path) { + File file = new File(path); + return file.getName(); + } + + protected String getScriptPath(SpecNode specNode) { + if (Objects.isNull(specNode)) { + return StringUtils.EMPTY; + } + String defaultPath = StringUtils.defaultString(Config.INSTANCE.getBasePath(), StringUtils.EMPTY); + String workFlowPath = Optional.ofNullable(specWorkflow) + .map(SpecWorkflow::getName) + .orElse(StringUtils.EMPTY); + return FilenameUtils.concat(FilenameUtils.concat(defaultPath, workFlowPath), specNode.getName()); + } + + protected List getWorkflowDependencyList() { + if (Objects.nonNull(specWorkflow)) { + return specWorkflow.getDependencies(); + } else { + return Collections.emptyList(); + } + } + + protected SpecTrigger getWorkflowTrigger() { + if (Objects.nonNull(specWorkflow)) { + return specWorkflow.getTrigger(); + } + // may be manual flow + return null; + } + + /** + * get workflow, but it will be null in spec version < 1.2.0 + * + * @return SpecWorkflow if it is not null + */ + protected SpecWorkflow getWorkFlow() { + return specWorkflow; + } + + protected void resetNodeStrategy(SpecNode specNode) { + if (specNode.getStrategy() == null) { + specNode.setStrategy(new SpecScheduleStrategy()); + } + SpecScheduleStrategy strategy = specNode.getStrategy(); + strategy.setPriority(specNode.getPriority()); + strategy.setTimeout(specNode.getTimeout()); + strategy.setRerunInterval(specNode.getRerunInterval()); + strategy.setRerunTimes(specNode.getRerunTimes()); + strategy.setIgnoreBranchConditionSkip(specNode.getIgnoreBranchConditionSkip()); + strategy.setInstanceMode(specNode.getInstanceMode()); + strategy.setRerunMode(specNode.getRerunMode()); + + Optional.ofNullable(getWorkFlow()) + .map(SpecWorkflow::getStrategy) + .map(SpecScheduleStrategy::getFailureStrategy) + .ifPresent(strategy::setFailureStrategy); + } + + protected String codeToLanguageIdentifier(CodeProgramType nodeType) { + LanguageEnum languageEnum = codeToLanguage(nodeType); + if (languageEnum == null) { + log.warn("can not find language by {}", nodeType); + return null; + } + return languageEnum.getIdentifier(); + } + + protected LanguageEnum codeToLanguage(CodeProgramType nodeType) { + switch (nodeType) { + case SHELL: + case DIDE_SHELL: + case CDH_SHELL: + case EMR_SPARK_SHELL: + case CDH_SPARK_SHELL: + case EMR_SHELL: + case EMR_HIVE_CLI: + case PERL: + return LanguageEnum.SHELL_SCRIPT; + case EMR_SPARK_SQL: + return LanguageEnum.SPARK_SQL; + case CDH_HIVE: + case HIVE: + case EMR_HIVE: + return LanguageEnum.HIVE_SQL; + case EMR_IMPALA: + case CDH_IMPALA: + return LanguageEnum.IMPALA_SQL; + case CLICK_SQL: + return LanguageEnum.CLICKHOUSE_SQL; + case ODPS_SQL: + case ODPS_PERL: + return LanguageEnum.ODPS_SQL; + case ODPS_SCRIPT: + return LanguageEnum.ODPS_SCRIPT; + case EMR_PRESTO: + case CDH_PRESTO: + return LanguageEnum.PRESTO_SQL; + case PYODPS: + return LanguageEnum.PYTHON2; + case PYODPS3: + return LanguageEnum.PYTHON3; + case DATAX2: + case DATAX: + case RI: + case DI: + return LanguageEnum.JSON; + case HOLOGRES_SQL: + return LanguageEnum.HOLOGRES_SQL; + default: + return null; + } + } + + protected String getConverterType(String convertType, String defaultConvertType) { + String projectName = processDefinition.getProjectName(); + String processName = processDefinition.getName(); + String taskName = taskDefinition.getName(); + return ConverterTypeUtils.getConverterType(convertType, projectName, processName, taskName, defaultConvertType); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/CustomParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/CustomParameterConverter.java new file mode 100644 index 0000000..6dcef2c --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/CustomParameterConverter.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.AbstractParameters; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import org.apache.commons.collections4.ListUtils; + +public class CustomParameterConverter extends AbstractParameterConverter { + public CustomParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + + SpecScript script = new SpecScript(); + CodeProgramType codeProgramType = CodeProgramType.CUSTOM; + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + script.setContent(JSONUtils.toJsonString(taskDefinition)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/DataxParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/DataxParameterConverter.java new file mode 100644 index 0000000..adbbf0f --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/DataxParameterConverter.java @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.entity.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.DbType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.Flag; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.process.Property; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.datax.DataxParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.datax.DataxUtils; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.utils.ParameterUtils; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.entity.Connection; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.entity.Parameter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.entity.Step; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; +import com.alibaba.druid.sql.ast.statement.SQLSelect; +import com.alibaba.druid.sql.ast.statement.SQLSelectItem; +import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; +import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; +import com.alibaba.druid.sql.parser.SQLStatementParser; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.tuple.Pair; + +@Slf4j +public class DataxParameterConverter extends AbstractParameterConverter { + /** + * select all + */ + private static final String SELECT_ALL_CHARACTER = "*"; + + private static final int DATAX_CHANNEL_COUNT = 1; + + public DataxParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + String type = properties.getProperty(Constants.CONVERTER_TARGET_DATAX_NODE_TYPE_AS, CodeProgramType.DI.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String json = buildDataxJsonFile(new HashMap<>()); + script.setContent(json); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + private String buildDataxJsonFile(Map paramsMap) { + String json; + + if (parameter.getCustomConfig() == Flag.YES.ordinal()) { + json = parameter.getJson().replaceAll("\\r\\n", System.lineSeparator()); + } else { + ObjectNode job = JSONUtils.createObjectNode(); + job.put("transform", false); + job.put("type", "job"); + job.put("version", "2.0"); + ArrayNode steps = buildDataxJobSteps(); + job.putArray("steps").addAll(steps); + job.set("setting", buildDataxJobSettingJson()); + json = job.toString(); + } + + // replace placeholder + json = ParameterUtils.convertParameterPlaceholders(json, ParameterUtils.convert(paramsMap)); + + log.debug("datax job json : {}", json); + return json; + } + + private ArrayNode buildDataxJobSteps() { + DataSource source = getDataSource(parameter.getDataSource()); + DataSource target = getDataSource(parameter.getDataTarget()); + Step reader = new Step(); + Step writer = new Step(); + reader.setName("Reader"); + reader.setCategory("reader"); + reader.setStepType(source.getType().name().toLowerCase()); + + writer.setName("Writer"); + writer.setCategory("writer"); + writer.setStepType(target.getType().name().toLowerCase()); + + Parameter readParameter = new Parameter(); + readParameter.setEncoding("UTF-8"); + readParameter.setEnvType(1); + readParameter.setUseSpecialSecret(false); + + Parameter writeParameter = new Parameter(); + writeParameter.setEncoding("UTF-8"); + writeParameter.setEnvType(1); + writeParameter.setUseSpecialSecret(false); + + Pair pair = tryGrammaticalAnalysisSqlColumnNames(source.getType(), parameter.getSql()); + String[] srcColumnNames = pair.getRight(); + if (srcColumnNames != null && srcColumnNames.length > 0) { + readParameter.setColumn(Arrays.asList(srcColumnNames)); + String[] tgtColumnNames = DataxUtils.convertKeywordsColumns(target.getType(), srcColumnNames); + if (tgtColumnNames != null && tgtColumnNames.length > 0) { + writeParameter.setColumn(Arrays.asList(srcColumnNames)); + } + } + Connection sourceConnection = new Connection(); + sourceConnection.setDatasource(source.getName()); + sourceConnection.setTable(Arrays.asList(pair.getLeft())); + readParameter.setConnection(Arrays.asList(sourceConnection)); + Connection targetConnection = new Connection(); + targetConnection.setDatasource(target.getName()); + targetConnection.setTable(Arrays.asList(parameter.getTargetTable())); + writeParameter.setConnection(Arrays.asList(targetConnection)); + reader.setParameter(readParameter); + writer.setParameter(writeParameter); + + ObjectNode steps = JSONUtils.createObjectNode(); + ArrayNode tableArr = steps.putArray("steps"); + JsonNode readerNode = JSONUtils.toJsonNode(reader); + tableArr.add(readerNode); + JsonNode writerNode = JSONUtils.toJsonNode(writer); + tableArr.add(writerNode); + return tableArr; + } + + /** + * build datax setting config + * + * @return datax setting config JSONObject + */ + private ObjectNode buildDataxJobSettingJson() { + + ObjectNode speed = JSONUtils.createObjectNode(); + + speed.put("channel", DATAX_CHANNEL_COUNT); + + if (parameter.getJobSpeedByte() > 0) { + speed.put("byte", parameter.getJobSpeedByte()); + } + + if (parameter.getJobSpeedRecord() > 0) { + speed.put("record", parameter.getJobSpeedRecord()); + } + + ObjectNode errorLimit = JSONUtils.createObjectNode(); + errorLimit.put("record", 0); + errorLimit.put("percentage", 0); + + ObjectNode setting = JSONUtils.createObjectNode(); + setting.set("speed", speed); + setting.set("errorLimit", errorLimit); + + return setting; + } + + /** + * try grammatical parsing column + * + * @param dbType database type + * @param sql sql for data synchronization + * @return column name array + * @throws RuntimeException if error throws RuntimeException + */ + private Pair tryGrammaticalAnalysisSqlColumnNames(DbType dbType, String sql) { + String[] columnNames; + String sourceTable = null; + try { + SQLStatementParser parser = DataxUtils.getSqlStatementParser(dbType, sql); + if (parser == null) { + log.warn("database driver [{}] is not support grammatical analysis sql", dbType); + return Pair.of(null, new String[0]); + } + + SQLStatement sqlStatement = parser.parseStatement(); + SQLSelectStatement sqlSelectStatement = (SQLSelectStatement) sqlStatement; + SQLSelect sqlSelect = sqlSelectStatement.getSelect(); + + List selectItemList = null; + + if (sqlSelect.getQuery() instanceof SQLSelectQueryBlock) { + SQLSelectQueryBlock block = (SQLSelectQueryBlock) sqlSelect.getQuery(); + selectItemList = block.getSelectList(); + if (block.getFrom() instanceof SQLExprTableSource) { + SQLExprTableSource expr = (SQLExprTableSource) block.getFrom(); + if (expr.getExpr() instanceof SQLIdentifierExpr) { + sourceTable = ((SQLIdentifierExpr) expr.getExpr()).getName(); + } + } + } else if (sqlSelect.getQuery() instanceof SQLUnionQuery) { + SQLUnionQuery unionQuery = (SQLUnionQuery) sqlSelect.getQuery(); + SQLSelectQueryBlock block = (SQLSelectQueryBlock) unionQuery.getRight(); + selectItemList = block.getSelectList(); + if (block.getFrom() instanceof SQLExprTableSource) { + SQLExprTableSource expr = (SQLExprTableSource) block.getFrom(); + if (expr.getExpr() instanceof SQLIdentifierExpr) { + sourceTable = ((SQLIdentifierExpr) expr.getExpr()).getName(); + } + } + } + + if (selectItemList == null) { + throw new RuntimeException(String.format("select query type [%s] is not support", sqlSelect.getQuery().toString())); + } + + columnNames = new String[selectItemList.size()]; + for (int i = 0; i < selectItemList.size(); i++) { + SQLSelectItem item = selectItemList.get(i); + + String columnName = null; + + if (item.getAlias() != null) { + columnName = item.getAlias(); + } else if (item.getExpr() != null) { + if (item.getExpr() instanceof SQLPropertyExpr) { + SQLPropertyExpr expr = (SQLPropertyExpr) item.getExpr(); + columnName = expr.getName(); + } else if (item.getExpr() instanceof SQLIdentifierExpr) { + SQLIdentifierExpr expr = (SQLIdentifierExpr) item.getExpr(); + columnName = expr.getName(); + } + } else { + throw new RuntimeException( + String.format("grammatical analysis sql column [ %s ] failed", item)); + } + + if (SELECT_ALL_CHARACTER.equals(item.toString())) { + log.info("sql contains *, grammatical analysis failed"); + return Pair.of(sourceTable, new String[]{"*"}); + } + + if (columnName == null) { + throw new RuntimeException( + String.format("grammatical analysis sql column [ %s ] failed", item)); + } + + columnNames[i] = columnName; + } + } catch (Exception e) { + log.warn(e.getMessage(), e); + return Pair.of(null, new String[0]); + } + + return Pair.of(sourceTable, columnNames); + } + + private DataSource getDataSource(int datasourceId) { + List datasources = DolphinSchedulerV2Context.getContext().getDataSources(); + return CollectionUtils.emptyIfNull(datasources).stream() + .filter(s -> s.getId() == datasourceId) + .findFirst() + .orElse(null); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/DependentParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/DependentParameterConverter.java new file mode 100644 index 0000000..d2d7546 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/DependentParameterConverter.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.model.DependentTaskModel; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.dependent.DependentParameters; +import com.aliyun.migrationx.common.utils.GsonUtils; + +import com.google.gson.JsonObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; + +@Slf4j +public class DependentParameterConverter extends AbstractParameterConverter { + + public DependentParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + CodeProgramType codeProgramType = CodeProgramType.VIRTUAL; + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + DolphinSchedulerV2Context context = DolphinSchedulerV2Context.getContext(); + List deps = convertDeps(); + context.getSpecNodeProcessCodeMap().put(specNode, deps); + } + + public List convertDeps() { + log.info("params : {}", taskDefinition.getTaskParams()); + JsonObject param = GsonUtils.fromJsonString(taskDefinition.getTaskParams(), JsonObject.class); + DependentParameters dependentParameters = null; + if (param.get("dependence") != null) { + dependentParameters = GsonUtils.fromJson(param.getAsJsonObject("dependence"), DependentParameters.class); + } + if (dependentParameters == null || dependentParameters.getDependTaskList() == null || dependentParameters.getDependTaskList().isEmpty()) { + log.warn("no dependence param {}", taskDefinition.getTaskParams()); + return Collections.emptyList(); + } + DolphinSchedulerV2Context context = DolphinSchedulerV2Context.getContext(); + + // 本节点的条件依赖 + List dependencies = dependentParameters.getDependTaskList(); + List codes = new ArrayList<>(); + ListUtils.emptyIfNull(dependencies).forEach(dependModel -> + ListUtils.emptyIfNull(dependModel.getDependItemList()).forEach(depItem -> { + long depCode = depItem.getDepTaskCode(); + if (depCode == 0L) { + //all leaf task of process definition + //1. get all relation of process + // preTaskCode -> postTaskCode + //Map> relations = findRelations(depItem); + + List taskDefinitions = context.getProcessCodeTaskRelationMap().get(depItem.getDefinitionCode()); + for (TaskDefinition task : CollectionUtils.emptyIfNull(taskDefinitions)) { + //2. find all taskCode not in preTaskCode (not as a pre dependent, leaf task) + if (!codes.contains(task.getCode())) { + codes.add(task.getCode()); + } + } + } else { + if (!codes.contains(depCode)) { + codes.add(depCode); + } + } + })); + return codes; + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/MrParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/MrParameterConverter.java new file mode 100644 index 0000000..8155278 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/MrParameterConverter.java @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.EmrAllocationSpec; +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.EmrCode; +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.EmrLauncher; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.utils.ArgsUtils; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.ProgramType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.process.ResourceInfo; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.mr.MapReduceParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.utils.ParameterUtils; +import com.aliyun.dataworks.migrationx.domain.dataworks.utils.DataStudioCodeUtils; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; +import com.aliyun.migrationx.common.utils.BeanUtils; + +import com.google.common.base.Joiner; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +import static com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.mr.MapReduceTaskConstants.MR_NAME; +import static com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.mr.MapReduceTaskConstants.MR_QUEUE; +import static com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.utils.TaskConstants.D; + +@Slf4j +public class MrParameterConverter extends AbstractParameterConverter { + public static final String MR_YARN_QUEUE = "mapreduce.job.queuename"; + + public MrParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + + SpecScript script = new SpecScript(); + String type = getConverterType(); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + //todo + //String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + String code = convertCode(codeProgramType, taskDefinition.getName()); + //script.setContent(resourceReference + parameter.getRawScript()); + script.setContent(code); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String convertCode(CodeProgramType codeProgramType, String taskName) { + List resourceInfos = DolphinSchedulerV2Context.getContext().getResources(); + + Optional.ofNullable(parameter).map(MapReduceParameters::getMainJar) + .flatMap(mainJar -> ListUtils.emptyIfNull(resourceInfos) + .stream().filter(res -> Objects.equals(res.getId(), mainJar.getId())) + .findFirst()).ifPresent(res -> parameter.setMainJar(res)); + + ListUtils.emptyIfNull(Optional.ofNullable(parameter).map(MapReduceParameters::getResourceFilesList) + .orElse(ListUtils.emptyIfNull(null))) + .forEach(res -> ListUtils.emptyIfNull(resourceInfos).stream() + .filter(res1 -> Objects.equals(res1.getId(), res.getId())) + .forEach(res1 -> BeanUtils.copyProperties(res1, res))); + + // convert to EMR_MR + if (StringUtils.equalsIgnoreCase(CodeProgramType.EMR_MR.name(), codeProgramType.getName())) { + String cmd = buildCommand(parameter); + return EmrCodeUtils.toEmrCode(codeProgramType, taskName, cmd); + } else if (StringUtils.equalsIgnoreCase(CodeProgramType.ODPS_MR.name(), codeProgramType.getName())) { + ResourceInfo mainJar = parameter.getMainJar(); + List codeLines = new ArrayList<>(); + List resources = new ArrayList<>(); + if (mainJar != null) { + DolphinSchedulerV2Context context = DolphinSchedulerV2Context.getContext(); + String resourceName = CollectionUtils.emptyIfNull(context.getResources()) + .stream() + .filter(r -> r.getId() == mainJar.getId()) + .findAny() + .map(r -> r.getName()) + .orElse(""); + resources.add(resourceName); + codeLines.add(DataStudioCodeUtils.addResourceReference(codeProgramType, "", resources)); + } + + // convert to ODPS_MR + String command = Joiner.on(" ").join( + "jar", "-resources", + Optional.ofNullable(parameter.getMainJar().getName()).orElse(""), + "-classpath", + Joiner.on(",").join(resources), + Optional.ofNullable(parameter.getMainClass()).orElse(""), + Optional.ofNullable(parameter.getMainArgs()).orElse(""), + Optional.ofNullable(parameter.getOthers()).orElse("") + ); + codeLines.add(command); + + String code = Joiner.on("\n").join(codeLines); + code = EmrCodeUtils.toEmrCode(codeProgramType, taskName, code); + EmrCode emrCode = EmrCodeUtils.asEmrCode(codeProgramType.getName(), code); + Optional.ofNullable(emrCode).map(EmrCode::getLauncher) + .map(EmrLauncher::getAllocationSpec) + .map(EmrAllocationSpec::of) + .ifPresent(spec -> { + spec.setQueue(parameter.getQueue()); + emrCode.getLauncher().setAllocationSpec(spec.toMap()); + }); + return emrCode.getContent(); + } else { + throw new RuntimeException("not support type " + codeProgramType.getName()); + } + } + + protected String buildCommand(MapReduceParameters mapreduceParameters) { + // hadoop jar [mainClass] [GENERIC_OPTIONS] args... + List args = new ArrayList<>(); + + // other parameters + args.addAll(buildArgs(mapreduceParameters)); + + String command = ParameterUtils.convertParameterPlaceholders(String.join(" ", args), + new HashMap<>()); + log.info("mapreduce task command: {}", command); + + return command; + } + + public static List buildArgs(MapReduceParameters param) { + List args = new ArrayList<>(); + + ResourceInfo mainJar = param.getMainJar(); + if (mainJar != null) { + DolphinSchedulerV2Context context = DolphinSchedulerV2Context.getContext(); + String resourceName = CollectionUtils.emptyIfNull(context.getResources()) + .stream() + .filter(r -> r.getId() == mainJar.getId()) + .findAny() + .map(r -> r.getName()) + .orElse(""); + String resource = DataStudioCodeUtils.addResourceReference(CodeProgramType.EMR_MR, "", Arrays.asList(resourceName)); + args.add(resource + resourceName); + } + + ProgramType programType = param.getProgramType(); + String mainClass = param.getMainClass(); + if (programType != null && programType != ProgramType.PYTHON && StringUtils.isNotEmpty(mainClass)) { + args.add(mainClass); + } + + String appName = param.getAppName(); + if (StringUtils.isNotEmpty(appName)) { + args.add(String.format("%s%s=%s", D, MR_NAME, ArgsUtils.escape(appName))); + } + + String others = param.getOthers(); + if (StringUtils.isEmpty(others) || !others.contains(MR_QUEUE)) { + String queue = param.getQueue(); + if (StringUtils.isNotEmpty(queue)) { + args.add(String.format("%s%s=%s", D, MR_QUEUE, queue)); + } + } + + // -conf -archives -files -libjars -D + if (StringUtils.isNotEmpty(others)) { + args.add(others); + } + + String mainArgs = param.getMainArgs(); + if (StringUtils.isNotEmpty(mainArgs)) { + args.add(mainArgs); + } + return args; + } + + private String getConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_MR_NODE_TYPE_AS); + String defaultConvertType = CodeProgramType.EMR_MR.name(); + return getConverterType(convertType, defaultConvertType); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/ProcedureParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/ProcedureParameterConverter.java new file mode 100644 index 0000000..4ceafd8 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/ProcedureParameterConverter.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.entity.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.DbType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.procedure.ProcedureParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.migrationx.common.utils.GsonUtils; + +import com.google.gson.reflect.TypeToken; +import org.apache.commons.collections4.ListUtils; + +public class ProcedureParameterConverter extends AbstractParameterConverter { + + public ProcedureParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_SHELL_NODE_TYPE_AS, CodeProgramType.DIDE_SHELL.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(convertType); + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + //todo + //String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + //script.setContent(resourceReference + parameter.getRawScript()); + script.setContent(getCode()); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String getCode() { + String sqlNodeMapStr = properties.getProperty( + Constants.CONVERTER_TARGET_SQL_NODE_TYPE_MAP, "{}"); + Map sqlTypeNodeTypeMapping = GsonUtils.fromJsonString(sqlNodeMapStr, + new TypeToken>() {}.getType()); + sqlTypeNodeTypeMapping = Optional.ofNullable(sqlTypeNodeTypeMapping).orElse(new HashMap<>(1)); + + String defaultNodeTypeIfNotSupport = getConverterType(); + + DbType codeProgramType = sqlTypeNodeTypeMapping.get(parameter.getType()); + + //add ref datasource + if (parameter.getDatasource() > 0) { + List datasources = DolphinSchedulerV2Context.getContext().getDataSources(); + + //todo + //CollectionUtils.emptyIfNull(datasources).stream() + // .filter(s -> s.getId() == parameter.getDatasource()) + // .findFirst() + // .ifPresent(s -> dwNode.setConnection(s.getName())); + } + String code = parameter.getMethod(); + return code; + } + + private String getConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_UNSUPPORTED_NODE_TYPE_AS); + return getConverterType(convertType, CodeProgramType.VIRTUAL.name()); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/PythonParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/PythonParameterConverter.java new file mode 100644 index 0000000..8d5e412 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/PythonParameterConverter.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.python.PythonParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; + +import org.apache.commons.collections4.ListUtils; + +public class PythonParameterConverter extends AbstractParameterConverter { + + public PythonParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_PYTHON_NODE_TYPE_AS, CodeProgramType.PYTHON.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(convertType); + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + + String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + String pathImportCode = buildPathImportCode(specNode); + script.setContent(resourceReference + pathImportCode + parameter.getRawScript()); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + private String buildPathImportCode(SpecNode specNode) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("import os\n").append("import sys\n\n"); + Optional.ofNullable(specNode).map(SpecNode::getFileResources).ifPresent(fileResources -> + fileResources.forEach(fileResource -> { + String fileName = fileResource.getName(); + stringBuilder.append(String.format("sys.path.append(os.path.dirname(os.path.abspath('%s')))%n", fileName)); + })); + stringBuilder.append("\n"); + return stringBuilder.toString(); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/ShellParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/ShellParameterConverter.java new file mode 100644 index 0000000..0ddc7d1 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/ShellParameterConverter.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.shell.ShellParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; + +import org.apache.commons.collections4.ListUtils; + +public class ShellParameterConverter extends AbstractParameterConverter { + + public ShellParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_SHELL_NODE_TYPE_AS, CodeProgramType.DIDE_SHELL.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(convertType); + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + script.setContent(resourceReference + parameter.getRawScript()); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SparkParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SparkParameterConverter.java new file mode 100644 index 0000000..0936ffe --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SparkParameterConverter.java @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.OdpsSparkCode; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.utils.ArgsUtils; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.ProgramType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.process.ResourceInfo; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.spark.SparkParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.spark.SparkConstants; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class SparkParameterConverter extends AbstractParameterConverter { + + private static final String SQL_EXECUTION_TYPE_FILE = "FILE"; + + private static final String RESOURCE_REFERENCE_SQL_PREFIX = "--"; + + public SparkParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + /** + * Each node translates the specific logic of the parameters + */ + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + + String type = getConverterType(); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + script.setPath(getScriptPath(specNode)); + script.setContent(convertCode(codeProgramType)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String convertCode(CodeProgramType codeProgramType) { + if (CodeProgramType.EMR_SPARK.equals(codeProgramType)) { + List cmd = populateSparkOptions(parameter); + String code = String.join(" ", cmd); + return code; + } else if (CodeProgramType.ODPS_SPARK.equals(codeProgramType)) { + OdpsSparkCode odpsSparkCode = populateSparkOdpsCode(); + return odpsSparkCode.toString(); + } + throw new IllegalArgumentException("Unsupported code program type: " + codeProgramType); + } + + private List populateSparkOptions(SparkParameters sparkParameters) { + List args = new ArrayList<>(); + + ProgramType programType = sparkParameters.getProgramType(); + ResourceInfo mainJar = sparkParameters.getMainJar(); + if (programType != ProgramType.SQL) { + String resource = mainJar.getName(); + if (resource != null) { + String[] resources = resource.split("/"); + if (resources.length > 0) { + resource = resources[resources.length - 1]; + } + } else { + DolphinSchedulerV2Context context = DolphinSchedulerV2Context.getContext(); + resource = CollectionUtils.emptyIfNull(context.getResources()) + .stream() + .filter(r -> r.getId() == mainJar.getId()) + .findAny() + .map(r -> r.getName()) + .orElse(null); + mainJar.setName(resource); + } + String dwResource = "##@resource_reference{\"" + resource + "\"} \n"; + args.add(dwResource + SparkConstants.SPARK_SUBMIT_COMMAND); + } else { + args.add(SparkConstants.SPARK_SUBMIT_COMMAND); + } + + String deployMode = StringUtils.isNotEmpty(sparkParameters.getDeployMode()) ? sparkParameters.getDeployMode() + : SparkConstants.DEPLOY_MODE_LOCAL; + + if (!SparkConstants.DEPLOY_MODE_LOCAL.equals(deployMode)) { + args.add(SparkConstants.MASTER); + String masterUrl = SparkConstants.SPARK_ON_YARN; + args.add(masterUrl); + } + args.add(SparkConstants.DEPLOY_MODE); + args.add(deployMode); + + String mainClass = sparkParameters.getMainClass(); + if (programType != ProgramType.PYTHON && programType != ProgramType.SQL && StringUtils.isNotEmpty(mainClass)) { + args.add(SparkConstants.MAIN_CLASS); + args.add(mainClass); + } + + populateSparkResourceDefinitions(args, sparkParameters); + + String appName = sparkParameters.getAppName(); + if (StringUtils.isNotEmpty(appName)) { + args.add(SparkConstants.SPARK_NAME); + args.add(ArgsUtils.escape(appName)); + } + + String others = sparkParameters.getOthers(); + + // --conf --files --jars --packages + if (StringUtils.isNotEmpty(others)) { + args.add(others); + } + + //jar + if (programType != ProgramType.SQL && mainJar != null) { + args.add(mainJar.getName()); + } + + String mainArgs = sparkParameters.getMainArgs(); + if (programType != ProgramType.SQL && StringUtils.isNotEmpty(mainArgs)) { + args.add(mainArgs); + } + + return args; + } + + private OdpsSparkCode populateSparkOdpsCode() { + OdpsSparkCode odpsSparkCode = new OdpsSparkCode(); + odpsSparkCode.setResourceReferences(new ArrayList<>()); + odpsSparkCode.setSparkJson(new OdpsSparkCode.CodeJson()); + + ResourceInfo mainJar = parameter.getMainJar(); + String resource = mainJar.getName(); + if (StringUtils.isEmpty(resource)) { + resource = getResourceName(mainJar.getId()); + } + + if (resource != null) { + String[] resources = resource.split("/"); + if (resources.length > 0) { + resource = resources[resources.length - 1]; + } + mainJar.setName(resource); + //String dwResource = "##@resource_reference{\"" + resource + "\"} \n"; + odpsSparkCode.getResourceReferences().add(resource); + odpsSparkCode.getSparkJson().setMainJar(resource); + } + String mainClass = parameter.getMainClass(); + odpsSparkCode.getSparkJson().setMainClass(mainClass); + odpsSparkCode.getSparkJson().setVersion("2.x"); + odpsSparkCode.getSparkJson().setLanguage("java"); + + String mainArgs = parameter.getMainArgs(); + odpsSparkCode.getSparkJson().setArgs(mainArgs); + List confs = new ArrayList<>(); + populateOdpsSparkResourceDefinitions(confs, parameter); + odpsSparkCode.getSparkJson().setConfigs(confs); + return odpsSparkCode; + } + + private void populateOdpsSparkResourceDefinitions(List args, SparkParameters sparkParameters) { + int driverCores = sparkParameters.getDriverCores(); + if (driverCores > 0) { + args.add(String.format("spark.driver.cores=%d", driverCores)); + } + + String driverMemory = sparkParameters.getDriverMemory(); + if (StringUtils.isNotEmpty(driverMemory)) { + args.add(String.format("spark.driver.memory=%s", driverMemory)); + } + + int numExecutors = sparkParameters.getNumExecutors(); + if (numExecutors > 0) { + args.add(String.format("spark.executor.instances=%d", numExecutors)); + } + + int executorCores = sparkParameters.getExecutorCores(); + if (executorCores > 0) { + args.add(String.format("spark.executor.cores=%d", executorCores)); + } + + String executorMemory = sparkParameters.getExecutorMemory(); + if (StringUtils.isNotEmpty(executorMemory)) { + args.add(String.format("spark.executor.memory=%s", executorMemory)); + } + } + + private void populateSparkResourceDefinitions(List args, SparkParameters sparkParameters) { + int driverCores = sparkParameters.getDriverCores(); + if (driverCores > 0) { + args.add(String.format(SparkConstants.DRIVER_CORES, driverCores)); + } + + String driverMemory = sparkParameters.getDriverMemory(); + if (StringUtils.isNotEmpty(driverMemory)) { + args.add(String.format(SparkConstants.DRIVER_MEMORY, driverMemory)); + } + + int numExecutors = sparkParameters.getNumExecutors(); + if (numExecutors > 0) { + args.add(String.format(SparkConstants.NUM_EXECUTORS, numExecutors)); + } + + int executorCores = sparkParameters.getExecutorCores(); + if (executorCores > 0) { + args.add(String.format(SparkConstants.EXECUTOR_CORES, executorCores)); + } + + String executorMemory = sparkParameters.getExecutorMemory(); + if (StringUtils.isNotEmpty(executorMemory)) { + args.add(String.format(SparkConstants.EXECUTOR_MEMORY, executorMemory)); + } + } + + private String getResourceName(Integer id) { + if (id == null) { + return null; + } + DolphinSchedulerV2Context context = DolphinSchedulerV2Context.getContext(); + return CollectionUtils.emptyIfNull(context.getResources()) + .stream() + .filter(r -> r.getId() == id) + .findAny() + .map(r -> { + String name = r.getName(); + if (StringUtils.isEmpty(name)) { + name = r.getResourceName(); + } + if (StringUtils.isEmpty(name)) { + name = r.getFullName(); + } + return name; + }) + .orElse(null); + } + + private String getConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_SPARK_SUBMIT_TYPE_AS); + String defaultConvertType = CodeProgramType.EMR_SPARK_SHELL.name(); + return getConverterType(convertType, defaultConvertType); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SqlParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SqlParameterConverter.java new file mode 100644 index 0000000..5c0d0e4 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SqlParameterConverter.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.entity.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.DbType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.sql.SqlParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; +import com.aliyun.migrationx.common.utils.GsonUtils; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import com.google.gson.reflect.TypeToken; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang.StringUtils; + +@Slf4j +public class SqlParameterConverter extends AbstractParameterConverter { + + public SqlParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + String sqlNodeMapStr = properties.getProperty( + Constants.CONVERTER_TARGET_SQL_NODE_TYPE_MAP, "{}"); + Map sqlTypeNodeTypeMapping = GsonUtils.fromJsonString(sqlNodeMapStr, + new TypeToken>() {}.getType()); + + String type = Optional.ofNullable(sqlTypeNodeTypeMapping) + .map(s -> s.get(parameter.getType())) + .orElseGet(() -> { + if (DbType.HIVE.name().equalsIgnoreCase(parameter.getType())) { + return CodeProgramType.EMR_HIVE.name(); + } else if (DbType.SPARK.name().equalsIgnoreCase(parameter.getType())) { + return CodeProgramType.EMR_SPARK.name(); + } else if (DbType.ofType(parameter.getType()) != null) { + return parameter.getType(); + } else { + String defaultNodeTypeIfNotSupport = getSQLConverterType(); + log.warn("using default node Type {} for node {}", defaultNodeTypeIfNotSupport, taskDefinition.getName()); + return defaultNodeTypeIfNotSupport; + } + }); + + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String content = parameter.getSql(); + + if (CodeProgramType.EMR_HIVE.name().equals(codeProgramType) || CodeProgramType.EMR_SPARK.name().equals(codeProgramType)) { + content = EmrCodeUtils.toEmrCode(codeProgramType, taskDefinition.getName(), content); + } + + script.setContent(content); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + private String getConnectionName(String codeProgramType) { + String mappingJson = properties.getProperty(Constants.WORKFLOW_CONVERTER_CONNECTION_MAPPING); + if (StringUtils.isNotEmpty(mappingJson)) { + Map connectionMapping = JSONUtils.parseObject(mappingJson, Map.class); + if (connectionMapping == null) { + log.error("parse connection mapping with {} error", mappingJson); + } else { + String connectionName = connectionMapping.get(codeProgramType); + log.info("Got connectionName {} by {}", connectionName, codeProgramType); + return connectionName; + } + } + + if (!CodeProgramType.EMR_HIVE.name().equals(codeProgramType) && !CodeProgramType.EMR_SPARK.name().equals(codeProgramType)) { + //add ref datasource + List datasources = DolphinSchedulerV2Context.getContext().getDataSources(); + if (parameter.getDatasource() > 0) { + return CollectionUtils.emptyIfNull(datasources).stream() + .filter(s -> s.getId() == parameter.getDatasource()) + .findFirst() + .map(s -> s.getName()) + .orElse(null); + } + } + return null; + } + + private String getSQLConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_COMMAND_SQL_TYPE_AS); + return getConverterType(convertType, CodeProgramType.SQL_COMPONENT.name()); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SqoopParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SqoopParameterConverter.java new file mode 100644 index 0000000..62d05e2 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SqoopParameterConverter.java @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.process.ResourceInfo; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.sqoop.SqoopParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.sqoop.sources.SourceHdfsParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.sqoop.sources.SourceHiveParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.sqoop.sources.SourceMysqlParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.sqoop.targets.TargetHdfsParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.sqoop.targets.TargetHiveParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.sqoop.targets.TargetMysqlParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.CodeModeType; +import com.aliyun.dataworks.migrationx.transformer.core.sqoop.DIConfigTemplate; +import com.aliyun.dataworks.migrationx.transformer.core.sqoop.DIJsonProcessor; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import com.google.common.base.Joiner; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +public class SqoopParameterConverter extends AbstractParameterConverter { + + public SqoopParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + CodeProgramType codeProgramType = CodeProgramType.DI; + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String content = convertCode(); + script.setContent(content); + //todo + //String codeMode = CodeModeType.CODE.getValue(); + + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String convertCode() { + DIJsonProcessor diProcessor = DIJsonProcessor.from(DIConfigTemplate.DI_CODE_TEMPLATE); + DIJsonProcessor readerJp = diProcessor.getConfiguration("steps[0]"); + DIJsonProcessor writerJp = diProcessor.getConfiguration("steps[1]"); + processSqoopSource(readerJp); + processSqoopTarget(writerJp); + diProcessor.set("extend.mode", CodeModeType.CODE.getValue()); + return diProcessor.toJSON(); + //CodeModeType.CODE.getValue() + } + + private void processSqoopTarget(DIJsonProcessor writerJp) { + List resourceInfos = DolphinSchedulerV2Context.getContext().getResources(); + writerJp.set("stepType", StringUtils.lowerCase(parameter.getTargetType())); + if (StringUtils.equalsIgnoreCase(parameter.getTargetType(), "mysql")) { + TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(parameter.getTargetParams(), + TargetMysqlParameter.class); + Optional.ofNullable(targetMysqlParameter).ifPresent(p -> { + ListUtils.emptyIfNull(resourceInfos).stream() + .filter(ds -> Objects.equals(ds.getId(), targetMysqlParameter.getTargetDatasource())) + .findFirst().ifPresent(ds -> writerJp.set("parameter.datasource", ds.getName())); + writerJp.set("parameter.table", targetMysqlParameter.getTargetTable()); + writerJp.set("parameter.preSql", Optional.ofNullable(StringUtils.split( + targetMysqlParameter.getPreQuery(), ",")).orElse(new String[]{})); + writerJp.set("parameter.column", StringUtils.isBlank(targetMysqlParameter.getTargetColumns()) ? + new String[]{"*"} : StringUtils.split(targetMysqlParameter.getTargetColumns(), ",")); + if (StringUtils.equalsIgnoreCase(targetMysqlParameter.getTargetUpdateMode(), "updateonly")) { + writerJp.set("parameter.writeMode", "update"); + } else { + writerJp.set("parameter.writeMode", "replace"); + } + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getTargetType(), "hive")) { + TargetHiveParameter targetHiveParameter = JSONUtils.parseObject(parameter.getTargetParams(), + TargetHiveParameter.class); + Optional.ofNullable(targetHiveParameter).ifPresent(p -> { + ListUtils.emptyIfNull(resourceInfos).stream() + .filter(ds -> StringUtils.equals(java.lang.String.valueOf(ds.getId()), targetHiveParameter.getHiveDatabase())) + .findFirst().ifPresent(ds -> writerJp.set("parameter.datasource", ds.getName())); + writerJp.set("parameter.table", targetHiveParameter.getHiveTable()); + String[] keys = Optional.ofNullable(StringUtils.split( + targetHiveParameter.getHivePartitionKey(), ",")).orElse(new String[]{}); + String[] values = Optional.ofNullable(StringUtils.split( + targetHiveParameter.getHivePartitionValue(), ",")).orElse(new String[]{}); + List partitions = new ArrayList<>(); + for (int i = 0; i < Math.min(keys.length, values.length); i++) { + partitions.add(Joiner.on("=").join(keys[i], values[i])); + } + writerJp.set("parameter.partition", Joiner.on(",").join(partitions)); + writerJp.set("parameter.hdfsUsername", "hdfs"); + writerJp.set("parameter.writeMode", "append"); + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getTargetType(), "hdfs")) { + TargetHdfsParameter targetHdfsParameter = JSONUtils.parseObject(parameter.getSourceParams(), + TargetHdfsParameter.class); + Optional.ofNullable(targetHdfsParameter).ifPresent(p -> { + writerJp.set("parameter.path", p.getTargetPath()); + writerJp.set("parameter.compress", targetHdfsParameter.getCompressionCodec()); + writerJp.set("parameter.datasource", "hdfs"); + writerJp.set("parameter.fileType", targetHdfsParameter.getFileType()); + if (StringUtils.equalsIgnoreCase("parquet", targetHdfsParameter.getFileType())) { + writerJp.set("parameter.writeMode", "noConflict"); + } else { + writerJp.set("parameter.writeMode", "append"); + } + }); + } + } + + private void processSqoopSource(DIJsonProcessor readerJp) { + readerJp.set("stepType", StringUtils.lowerCase(parameter.getSourceType())); + List resourceInfos = DolphinSchedulerV2Context.getContext().getResources(); + + if (StringUtils.equalsIgnoreCase(parameter.getSourceType(), "mysql")) { + SourceMysqlParameter mysqlParameter = JSONUtils.parseObject(parameter.getSourceParams(), + SourceMysqlParameter.class); + Optional.ofNullable(mysqlParameter).ifPresent(p -> { + ListUtils.emptyIfNull(resourceInfos).stream() + .filter(ds -> Objects.equals(ds.getId(), mysqlParameter.getSrcDatasource())) + .findFirst().ifPresent(ds -> readerJp.set("parameter.datasource", ds.getName())); + readerJp.set("parameter.table", mysqlParameter.getSrcTable()); + readerJp.set("parameter.where", StringUtils.substringAfter( + StringUtils.lowerCase(mysqlParameter.getSrcQuerySql()), "where")); + readerJp.set("parameter.column", StringUtils.isBlank(mysqlParameter.getSrcColumns()) ? + new String[]{"*"} : StringUtils.split(mysqlParameter.getSrcColumns(), ",")); + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getSourceType(), "hive")) { + SourceHiveParameter sourceHiveParameter = JSONUtils.parseObject(parameter.getSourceParams(), + SourceHiveParameter.class); + Optional.ofNullable(sourceHiveParameter).ifPresent(p -> { + ListUtils.emptyIfNull(resourceInfos).stream() + .filter(ds -> StringUtils.equals(String.valueOf(ds.getId()), sourceHiveParameter.getHiveDatabase())) + .findFirst().ifPresent(ds -> readerJp.set("parameter.datasource", ds.getName())); + readerJp.set("parameter.table", sourceHiveParameter.getHiveTable()); + String[] keys = Optional.ofNullable(StringUtils.split( + sourceHiveParameter.getHivePartitionKey(), ",")).orElse(new String[]{}); + String[] values = Optional.ofNullable(StringUtils.split( + sourceHiveParameter.getHivePartitionValue(), ",")).orElse(new String[]{}); + List partitions = new ArrayList<>(); + for (int i = 0; i < Math.min(keys.length, values.length); i++) { + partitions.add(Joiner.on("=").join(keys[i], values[i])); + } + readerJp.set("parameter.partition", Joiner.on(",").join(partitions)); + readerJp.set("parameter.readMode", "hdfs"); + readerJp.set("parameter.hdfsUsername", "hdfs"); + List> columns = Arrays.stream(keys).map(key -> { + Map column = new HashMap<>(); + column.put("type", "string"); + column.put("value", key); + return column; + }).collect(Collectors.toList()); + readerJp.set("parameter.hivePartitionColumn", columns); + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getSourceType(), "hdfs")) { + SourceHdfsParameter sourceHdfsParameter = JSONUtils.parseObject(parameter.getSourceParams(), + SourceHdfsParameter.class); + Optional.ofNullable(sourceHdfsParameter).ifPresent( + p -> readerJp.set("parameter.path", p.getExportDir())); + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SubProcessParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SubProcessParameterConverter.java new file mode 100644 index 0000000..170dd8b --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SubProcessParameterConverter.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.subprocess.SubProcessParameters; + +import org.apache.commons.collections4.ListUtils; + +public class SubProcessParameterConverter extends AbstractParameterConverter { + + public SubProcessParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + CodeProgramType codeProgramType = CodeProgramType.SUB_PROCESS; + SpecScript script = new SpecScript(); + SpecScriptRuntime runtime = new SpecScriptRuntime(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + script.setPath(getScriptPath(specNode)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + + long depProcessCode = parameter.getProcessDefinitionCode(); + DolphinSchedulerV2Context.getContext().getSubProcessCodeNodeMap().put(depProcessCode, specNode); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SwitchParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SwitchParameterConverter.java new file mode 100644 index 0000000..145c03e --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v2/workflow/parameters/SwitchParameterConverter.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.parameters; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.noref.SpecBranch; +import com.aliyun.dataworks.common.spec.domain.noref.SpecBranches; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.DolphinSchedulerV2Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.switchtask.SwitchParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.switchtask.SwitchResultVo; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; + +@Slf4j +public class SwitchParameterConverter extends AbstractParameterConverter { + + public SwitchParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + CodeProgramType codeProgramType = CodeProgramType.CONTROLLER_BRANCH; + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + script.setPath(getScriptPath(specNode)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + List branchList = convertCode(); + SpecBranch branch = new SpecBranch(); + branch.setBranches(branchList); + specNode.setBranch(branch); + } + + public List convertCode() { + JsonNode param = JSONUtils.parseObject(taskDefinition.getTaskParams(), JsonNode.class); + SwitchParameters switchParameters = null; + if (param.get("switchResult") != null) { + switchParameters = JSONUtils.parseObject(param.get("switchResult"), SwitchParameters.class); + } + if (switchParameters == null || switchParameters.getDependTaskList() == null || switchParameters.getDependTaskList().isEmpty()) { + log.warn("no dependence param {}", taskDefinition.getTaskParams()); + return null; + } + + Long defaultNextCode = switchParameters.getNextNode(); + List branchList = new ArrayList<>(); + + for (SwitchResultVo switchResultVo : switchParameters.getDependTaskList()) { + String condition = switchResultVo.getCondition(); + //task code + Long nextNodeCode = switchResultVo.getNextNode(); + TaskDefinition branchTask = DolphinSchedulerV2Context.getContext().getTaskCodeMap().get(nextNodeCode); + if (branchTask == null) { + continue; + } + /// + SpecBranches b = new SpecBranches(); + //b.setDesc(switchResultVo); + SpecNodeOutput output = new SpecNodeOutput(); + //When the dependent task ends， replace with specNode id + output.setData(String.valueOf(branchTask.getCode())); + b.setOutput(output); + b.setWhen(condition); + branchList.add(b); + } + return branchList; + } +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DolphinSchedulerV3Converter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/DolphinSchedulerV3Converter.java similarity index 63% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DolphinSchedulerV3Converter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/DolphinSchedulerV3Converter.java index 780026f..5154ac2 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DolphinSchedulerV3Converter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/DolphinSchedulerV3Converter.java @@ -13,13 +13,10 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes; import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -39,23 +36,17 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.ProcessDefinition; -import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.entity.DataSource; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.entity.ResourceComponent; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.entity.UdfFunc; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.DbType; -import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.subprocess.SubProcessParameters; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.Asset; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwDatasource; -import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwNode; -import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwNodeIo; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwResource; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwWorkflow; -import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.Node; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.Resource; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.connection.JdbcConnection; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.AssetType; -import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.NodeUseType; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.tenant.EnvType; import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; import com.aliyun.dataworks.migrationx.transformer.core.loader.ProjectAssetLoader; @@ -69,11 +60,9 @@ import com.google.common.base.Joiner; import com.google.gson.JsonObject; -import com.google.gson.reflect.TypeToken; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -299,115 +288,6 @@ protected String getDefaultNodeOutput(ProcessDefinition processMeta, String task taskName); } - /** - * SubProcess dependents handling logic - */ - private void processSubProcessDefinitionDepends() { - // for SubProcess Type - ListUtils.emptyIfNull(dwWorkflowList).forEach(workflow -> ListUtils.emptyIfNull(workflow.getNodes()).stream() - .filter(n -> StringUtils.equalsIgnoreCase(TaskType.SUB_PROCESS.name(), ((DwNode) n).getRawNodeType())) - .forEach(subProcessNode -> { - processSubProcessDefinitionDependNode(subProcessNode, workflow); - })); - } - - private void processSubProcessDefinitionDependNode(Node subProcessNode, DwWorkflow workflow) { - SubProcessParameters subProcessParameter = GsonUtils.fromJsonString(subProcessNode.getCode(), new TypeToken() {}.getType()); - if (subProcessParameter == null) { - LOGGER.warn("subProcessParameter null, with node code {}", subProcessNode.getCode()); - return; - } - this.dagDataList.stream().map(dag -> dag.getProcessDefinition()) - .filter(processDef -> Objects.equals(processDef.getCode(), subProcessParameter.getProcessDefinitionCode())) - .findFirst() - .flatMap(processDef -> dwWorkflowList.stream() - .filter(wf -> StringUtils.equals(V3ProcessDefinitionConverter.toWorkflowName(processDef), wf.getName())) - .findFirst() - .map(wf -> addStartEndNodeToDependedWorkflow(subProcessNode, processDef, wf))) - .filter(node -> node.getData() != null) - .ifPresent(endNodeOut -> addSubProcess(subProcessNode, workflow, endNodeOut)); - } - - private void addSubProcess(Node subProcessNode, DwWorkflow workflow, DwNodeIo endNodeOut) { - ListUtils.emptyIfNull(workflow.getNodes()).stream() - // set children of sub process node depends on end node of depend workflow - .filter(n -> ListUtils.emptyIfNull(n.getInputs()).stream() - .anyMatch(in -> ListUtils.emptyIfNull(subProcessNode.getOutputs()) - .stream().anyMatch(out -> StringUtils.equalsIgnoreCase(out.getData(), in.getData())))) - .forEach(child -> ListUtils.emptyIfNull(child.getInputs()).stream() - .filter(in -> ListUtils.emptyIfNull(subProcessNode.getOutputs()).stream().anyMatch(depOut -> - StringUtils.equalsIgnoreCase(in.getData(), depOut.getData()))) - .forEach(in -> in.setData(endNodeOut.getData()))); - } - - /** - * add start, end node for sub process workflow - set start as parent of all nodes that has no parents - set end as - * child of all nodes that has no children - set start as child of sub process node - * - * @param subProcessNode - * @param proDef - * @param wf - * @return output of end node - */ - private DwNodeIo addStartEndNodeToDependedWorkflow(Node subProcessNode, ProcessDefinition proDef, DwWorkflow wf) { - DwNode startNode = new DwNode(); - startNode.setDescription("node added by dataworks migration service"); - startNode.setRawNodeType(CodeProgramType.VIRTUAL.name()); - startNode.setDependentType(0); - startNode.setCycleType(0); - startNode.setNodeUseType(NodeUseType.SCHEDULED); - startNode.setCronExpress("day"); - startNode.setName(Joiner.on("_").join("start", proDef.getProjectName(), proDef.getName())); - startNode.setType(CodeProgramType.VIRTUAL.name()); - startNode.setWorkflowRef(wf); - DwNodeIo startNodeOutput = new DwNodeIo(); - startNodeOutput.setData(Joiner.on(".") - .join(project.getName(), proDef.getProjectName(), proDef.getName(), "start")); - startNodeOutput.setParseType(1); - startNode.setOutputs(Collections.singletonList(startNodeOutput)); - startNode.setInputs(new ArrayList<>()); - ListUtils.emptyIfNull(subProcessNode.getOutputs()).stream().findFirst().ifPresent( - depOut -> startNode.getInputs().add(depOut)); - - DwNode endNode = new DwNode(); - endNode.setDescription("node added by dataworks migration service"); - endNode.setRawNodeType(CodeProgramType.VIRTUAL.name()); - endNode.setDependentType(0); - endNode.setCycleType(0); - endNode.setNodeUseType(NodeUseType.SCHEDULED); - endNode.setCronExpress("day"); - endNode.setName(Joiner.on("_").join("end", proDef.getProjectName(), proDef.getName())); - endNode.setType(CodeProgramType.VIRTUAL.name()); - endNode.setWorkflowRef(wf); - DwNodeIo endNodeOutput = new DwNodeIo(); - endNodeOutput.setData( - Joiner.on(".").join(project.getName(), proDef.getProjectName(), proDef.getName(), "end")); - endNodeOutput.setParseType(1); - endNode.setOutputs(Collections.singletonList(endNodeOutput)); - endNode.setInputs(new ArrayList<>()); - - ListUtils.emptyIfNull(wf.getNodes()).forEach(node -> { - String prefix = Joiner.on(".").join(project.getName(), proDef.getProjectName()); - if (ListUtils.emptyIfNull(node.getInputs()).stream() - .noneMatch(in -> StringUtils.startsWithIgnoreCase(in.getData(), prefix))) { - node.getInputs().add(startNodeOutput); - } - - if (ListUtils.emptyIfNull(wf.getNodes()).stream() - .map(Node::getInputs) - .flatMap(List::stream) - .noneMatch(in -> ListUtils.emptyIfNull(node.getOutputs()).stream().anyMatch(out -> - StringUtils.equalsIgnoreCase(in.getData(), out.getData())))) { - ListUtils.emptyIfNull(node.getOutputs()).stream().findFirst().ifPresent( - out -> endNode.getInputs().add(out)); - } - }); - wf.getNodes().add(startNode); - wf.getNodes().add(endNode); - - return endNodeOutput; - } - private void setJdbcConnection(BaseDataSource datasource, DwDatasource dwDatasource) { JdbcConnection conn = new JdbcConnection(); conn.setUsername(datasource.getUser()); @@ -417,47 +297,4 @@ private void setJdbcConnection(BaseDataSource datasource, DwDatasource dwDatasou conn.setTag("public"); dwDatasource.setConnection(GsonUtils.defaultGson.toJson(conn)); } - - private List convertDagDataTasks(DwWorkflow dwWorkflow, DagData dagData, List dagDataList) { - return ListUtils.emptyIfNull(dagData.getTaskDefinitionList()).stream() - .map(taskDefinition -> convertDagDataTaskDefinitionToDwNode(dwWorkflow, taskDefinition, dagData, - dagDataList)) - .collect(Collectors.toList()); - } - - private DwNode convertDagDataTaskDefinitionToDwNode(DwWorkflow dwWorkflow, TaskDefinition taskDefinition, - DagData dagData, List dagDataList) { - DwNode dwNode = new DwNode(); - dwNode.setName(taskDefinition.getName()); - dwNode.setWorkflowRef(dwWorkflow); - dwNode.setType(convertDagDataTaskDefinitionType(taskDefinition, dagData)); - return dwNode; - } - - private String convertDagDataTaskDefinitionType(TaskDefinition taskDefinition, DagData dagData) { - taskDefinition.getTaskParams(); - return null; - } - - private List readDagDataList(File rootDir) { - if (!rootDir.exists()) { - log.info("root directory not exits: {}", rootDir); - return ListUtils.emptyIfNull(null); - } - - return Optional.ofNullable(rootDir.listFiles(f -> f.isFile() && f.getName().endsWith(".json"))) - .map(Arrays::asList) - .map(files -> files.stream().map(this::readDagDataJson).collect(Collectors.toList())) - .orElse(ListUtils.emptyIfNull(null)); - } - - private DagData readDagDataJson(File jsonFile) { - try { - String json = FileUtils.readFileToString(jsonFile, StandardCharsets.UTF_8); - return GsonUtils.fromJsonString(json, new TypeToken() {}.getType()); - } catch (IOException e) { - log.error("read json file: {} error: ", jsonFile, e); - throw new RuntimeException(e); - } - } } diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/TaskConverterFactoryV3.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/TaskConverterFactoryV3.java similarity index 62% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/TaskConverterFactoryV3.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/TaskConverterFactoryV3.java index 2465259..5406d0a 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/TaskConverterFactoryV3.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/TaskConverterFactoryV3.java @@ -13,12 +13,29 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.TaskType; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.AbstractParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.ConditionsParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.CustomParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.DLCParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.DataxParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.DependentParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.HiveCliParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.HttpParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.MrParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.ProcedureParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.PythonParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.ShellParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.SparkParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.SqlParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.SqoopParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.SubProcessParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.SwitchParameterConverter; import com.aliyun.migrationx.common.exception.UnSupportedTypeException; import com.aliyun.migrationx.common.utils.Config; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/V3ProcessDefinitionConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/V3ProcessDefinitionConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/V3ProcessDefinitionConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/V3ProcessDefinitionConverter.java index 8bb479a..b42b3b8 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/V3ProcessDefinitionConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/V3ProcessDefinitionConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes; import java.util.ArrayList; import java.util.Collections; @@ -46,7 +46,8 @@ import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.StoreWriter; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.filters.DolphinSchedulerConverterFilter; -import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.ProcessDefinitionConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.ProcessDefinitionConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.AbstractParameterConverter; import com.aliyun.migrationx.common.context.TransformerContext; import com.aliyun.migrationx.common.exception.UnSupportedTypeException; import com.aliyun.migrationx.common.metrics.DolphinMetrics; @@ -210,6 +211,7 @@ private List convertTaskToWorkFlow(TaskDefinition taskDefinition) { } catch (UnSupportedTypeException e) { markFailedProcess(taskDefinition, e.getMessage()); if (Config.INSTANCE.isSkipUnSupportType()) { + log.warn("task {} with type {} unsupported, skip", taskDefinition.getTaskType(), taskDefinition.getName()); return Collections.emptyList(); } else { throw e; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/AbstractParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/AbstractParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/AbstractParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/AbstractParameterConverter.java index e6f6341..40d4884 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/AbstractParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/AbstractParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.io.IOException; import java.util.ArrayList; @@ -66,7 +66,7 @@ import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.AbstractBaseConverter; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.utils.ConverterTypeUtils; -import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.ConditionsParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.parameters.ConditionsParameterConverter; import com.aliyun.migrationx.common.context.TransformerContext; import com.aliyun.migrationx.common.metrics.DolphinMetrics; import com.aliyun.migrationx.common.utils.GsonUtils; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/ConditionsParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/ConditionsParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/ConditionsParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/ConditionsParameterConverter.java index 38f6fae..8d9b007 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/ConditionsParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/ConditionsParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.io.IOException; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/CustomParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/CustomParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/CustomParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/CustomParameterConverter.java index 077225a..080db28 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/CustomParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/CustomParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.io.IOException; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DLCParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/DLCParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DLCParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/DLCParameterConverter.java index 9afed3d..e4d5680 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DLCParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/DLCParameterConverter.java @@ -1,4 +1,4 @@ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.io.File; import java.nio.charset.StandardCharsets; @@ -295,7 +295,7 @@ private String getConnectionName(String codeProgramType) { .filter(s -> s.getId() == parameter.getDatasource()) .findFirst() .map(s -> s.getName()) - .orElseGet(null); + .orElse(null); } } return null; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DataxParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/DataxParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DataxParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/DataxParameterConverter.java index b24d944..bdb3976 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DataxParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/DataxParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.util.Arrays; import java.util.HashMap; @@ -285,6 +285,6 @@ private DataSource getDataSource(int datasourceId) { return CollectionUtils.emptyIfNull(datasources).stream() .filter(s -> s.getId() == datasourceId) .findFirst() - .orElseGet(null); + .orElse(null); } } diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DependentParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/DependentParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DependentParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/DependentParameterConverter.java index 9e0f57b..513547b 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DependentParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/DependentParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/HiveCliParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/HiveCliParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/HiveCliParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/HiveCliParameterConverter.java index f48b65d..6c6643b 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/HiveCliParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/HiveCliParameterConverter.java @@ -1,4 +1,4 @@ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.io.File; import java.io.IOException; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/HttpParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/HttpParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/HttpParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/HttpParameterConverter.java index cbe6649..78cd717 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/HttpParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/HttpParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.io.IOException; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/MrParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/MrParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/MrParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/MrParameterConverter.java index 2bd35da..4f1c2c9 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/MrParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/MrParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/ProcedureParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/ProcedureParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/ProcedureParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/ProcedureParameterConverter.java index f67f653..140ff64 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/ProcedureParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/ProcedureParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.util.Arrays; import java.util.HashMap; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/PythonParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/PythonParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/PythonParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/PythonParameterConverter.java index f404b39..50e5575 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/PythonParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/PythonParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.io.File; import java.io.IOException; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/ShellParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/ShellParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/ShellParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/ShellParameterConverter.java index e664420..b16a259 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/ShellParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/ShellParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SparkParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SparkParameterConverter.java similarity index 95% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SparkParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SparkParameterConverter.java index e4850db..697882c 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SparkParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SparkParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.io.IOException; import java.util.ArrayList; @@ -149,7 +149,7 @@ private List populateSparkOptions(SparkParameters sparkParameters) { private OdpsSparkCode populateSparkOdpsCode() { OdpsSparkCode odpsSparkCode = new OdpsSparkCode(); - odpsSparkCode.setResourceReferences(odpsSparkCode.getResourceReferences()); + odpsSparkCode.setResourceReferences(new ArrayList<>()); odpsSparkCode.setSparkJson(new OdpsSparkCode.CodeJson()); ResourceInfo mainJar = parameter.getMainJar(); @@ -244,8 +244,17 @@ private String getResourceName(Integer id) { .stream() .filter(r -> r.getId() == id) .findAny() - .map(r -> r.getName()) - .orElseGet(null); + .map(r -> { + String name = r.getName(); + if (StringUtils.isEmpty(name)) { + name = r.getFileName(); + } + if (StringUtils.isEmpty(name)) { + name = r.getFullName(); + } + return name; + }) + .orElse(null); } private String getSparkConverterType() { diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SqlParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SqlParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SqlParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SqlParameterConverter.java index 306c911..09abc39 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SqlParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SqlParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.util.Arrays; import java.util.List; @@ -111,7 +111,7 @@ private String getConnectionName(String codeProgramType) { .filter(s -> s.getId() == parameter.getDatasource()) .findFirst() .map(s -> s.getName()) - .orElseGet(null); + .orElse(null); } } return null; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SqoopParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SqoopParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SqoopParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SqoopParameterConverter.java index 1739520..44fb0b6 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SqoopParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SqoopParameterConverter.java @@ -1,4 +1,4 @@ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SubProcessParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SubProcessParameterConverter.java similarity index 99% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SubProcessParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SubProcessParameterConverter.java index 66d837b..b45531e 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SubProcessParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SubProcessParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.util.ArrayList; import java.util.Arrays; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SwitchParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SwitchParameterConverter.java similarity index 98% rename from client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SwitchParameterConverter.java rename to client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SwitchParameterConverter.java index f08225d..18e2e3f 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/SwitchParameterConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/nodes/parameters/SwitchParameterConverter.java @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3; +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters; import java.io.IOException; import java.util.ArrayList; diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/DolphinSchedulerV3WorkflowConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/DolphinSchedulerV3WorkflowConverter.java new file mode 100644 index 0000000..a65977c --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/DolphinSchedulerV3WorkflowConverter.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.DataWorksWorkflowSpec; +import com.aliyun.dataworks.common.spec.domain.Specification; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.enums.SpecKind; +import com.aliyun.dataworks.common.spec.domain.interfaces.Output; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecSubFlow; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.DolphinSchedulerPackage; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.Project; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.entity.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.entity.ResourceComponent; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.entity.UdfFunc; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +@Slf4j +public class DolphinSchedulerV3WorkflowConverter { + public static final String SPEC_VERSION = "1.2.0"; + + private DolphinSchedulerPackage dolphinSchedulerPackage; + + private final List dagDataList; + private final Properties converterProperties; + + public DolphinSchedulerV3WorkflowConverter(DolphinSchedulerPackage dolphinSchedulerPackage, + Properties converterProperties) { + this.dolphinSchedulerPackage = dolphinSchedulerPackage; + this.converterProperties = converterProperties; + this.dagDataList = dolphinSchedulerPackage.getProcessDefinitions().values().stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + if (dagDataList.isEmpty()) { + throw new RuntimeException("process list empty"); + } + } + + public List> convert() { + List> specifications = new ArrayList<>(); + for (DagData dagData : dagDataList) { + List workflows = new ArrayList<>(); + //convert process to workflow + Specification specification = new Specification<>(); + specification.setKind(SpecKind.CYCLE_WORKFLOW.getLabel()); + specification.setVersion(SPEC_VERSION); + DataWorksWorkflowSpec spec = new DataWorksWorkflowSpec(); + String processName = dagData.getProcessDefinition().getName(); + spec.setName(processName); + V3ProcessDefinitionConverter converter = new V3ProcessDefinitionConverter(dagData, this.converterProperties); + SpecWorkflow workflow = converter.convert(); + workflows.add(workflow); + spec.setWorkflows(workflows); + specification.setSpec(spec); + specifications.add(specification); + } + handleSubprocess(); + handleDependents(specifications); + return specifications; + } + + /** + * subprocess + */ + private void handleSubprocess() { + Map codeWorkflowMap = DolphinSchedulerV3Context.getContext().getSubProcessCodeWorkflowMap(); + Map codeNodeMap = DolphinSchedulerV3Context.getContext().getSubProcessCodeNodeMap(); + //find subprocess + for (Map.Entry entry : codeNodeMap.entrySet()) { + //find workflow + SpecWorkflow specWorkflow = (SpecWorkflow) codeWorkflowMap.get(entry.getKey()); + SpecNodeOutput specNodeOutput; + if (specWorkflow.getOutputs().isEmpty()) { + specNodeOutput = new SpecNodeOutput(); + specNodeOutput.setData(specWorkflow.getId()); + specWorkflow.getOutputs().add(specNodeOutput); + } else { + specNodeOutput = (SpecNodeOutput) specWorkflow.getOutputs().get(0); + } + + SpecNode subprocess = (SpecNode) entry.getValue(); + SpecSubFlow subflow = new SpecSubFlow(); + subflow.setOutput(specNodeOutput.getData()); + subprocess.setSubflow(subflow); + } + } + + /** + * dependent process + * + * @param specifications + */ + private void handleDependents(List> specifications) { + Map nodeIdWorkflowMap = new HashMap<>(); + for (Specification specification : specifications) { + specification.getSpec().getWorkflows().forEach(workflow -> { + workflow.getNodes().forEach(node -> { + nodeIdWorkflowMap.put(node.getId(), workflow); + }); + }); + } + Map taskCodeSpecNodeMap = DolphinSchedulerV3Context.getContext().getTaskCodeSpecNodeMap(); + Map> depMap = DolphinSchedulerV3Context.getContext().getSpecNodeProcessCodeMap(); + for (Map.Entry> entry : depMap.entrySet()) { + SpecNode specNode = (SpecNode) entry.getKey(); + SpecWorkflow workflow = nodeIdWorkflowMap.get(specNode.getId()); + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + + List dependents = entry.getValue(); + List depends = new ArrayList<>(); + for (Long code : dependents) { + SpecNode depNode = (SpecNode) taskCodeSpecNodeMap.get(code); + if (depNode == null) { + log.warn("can not find spec node: {}", code); + continue; + } + SpecNodeOutput specNodeOutput = new SpecNodeOutput(); + List outputs = depNode.getOutputs(); + String data = depNode.getId(); + if (CollectionUtils.isNotEmpty(outputs)) { + data = ((SpecNodeOutput) outputs.get(0)).getData(); + } + specNodeOutput.setData(data); + SpecDepend specDepend = new SpecDepend(null, DependencyType.NORMAL, specNodeOutput); + depends.add(specDepend); + } + specFlowDepend.setDepends(depends); + specFlowDepend.setNodeId(specNode); + workflow.getDependencies().add(specFlowDepend); + } + mergeDeps(nodeIdWorkflowMap); + } + + private void mergeDeps(Map nodeIdWorkflowMap) { + nodeIdWorkflowMap.values().stream().forEach(workflow -> { + List depends = workflow.getDependencies(); + + if (CollectionUtils.isNotEmpty(depends)) { + Map> map = new HashMap<>(); + depends.stream().forEach(dep -> { + List specNodes = map.get(dep.getNodeId()); + if (CollectionUtils.isNotEmpty(specNodes)) { + specNodes.addAll(dep.getDepends()); + } else { + specNodes = new ArrayList<>(); + specNodes.addAll(dep.getDepends()); + map.put(dep.getNodeId(), specNodes); + } + }); + List specFlowDepends = new ArrayList<>(); + for (Map.Entry> entry : map.entrySet()) { + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + specFlowDepend.setNodeId(entry.getKey()); + specFlowDepend.setDepends(entry.getValue()); + specFlowDepends.add(specFlowDepend); + } + workflow.setDependencies(specFlowDepends); + } + }); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/ParamListConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/ParamListConverter.java new file mode 100644 index 0000000..3a2c89e --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/ParamListConverter.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.enums.VariableScopeType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.Direct; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.model.Property; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import org.apache.commons.collections4.ListUtils; + +public class ParamListConverter { + + public static final String SYSTEM_VARIABLE_TAG = "$"; + + private final List paramList; + + private final TaskDefinition taskDefinition; + + public ParamListConverter(List paramList) { + this(paramList, null); + } + + public ParamListConverter(List paramList, TaskDefinition taskDefinition) { + super(); + this.paramList = paramList; + this.taskDefinition = taskDefinition; + } + + public List convert() { + return ListUtils.emptyIfNull(paramList).stream().map(p -> { + // don't convert global out param + if (Objects.isNull(taskDefinition) && Direct.OUT.equals(p.getDirect())) { + return null; + } + + SpecVariable specVariable = new SpecVariable(); + + specVariable.setId(UuidGenerators.generateUuid()); + specVariable.setName(p.getProp()); + specVariable.setValue(p.getValue()); + specVariable.setDescription(p.getType().name()); + if (Direct.IN.equals(p.getDirect())) { + if (specVariable.getValue().startsWith(SYSTEM_VARIABLE_TAG)) { + specVariable.setType(VariableType.SYSTEM); + } else { + specVariable.setType(VariableType.CONSTANT); + } + specVariable.setScope(Objects.isNull(taskDefinition) ? VariableScopeType.FLOW : VariableScopeType.NODE_PARAMETER); + } else { + specVariable.setType(VariableType.NODE_OUTPUT); + specVariable.setScope(VariableScopeType.NODE_CONTEXT); + } + return specVariable; + } + ).filter(Objects::nonNull).collect(Collectors.toList()); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/SpecFlowDependConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/SpecFlowDependConverter.java new file mode 100644 index 0000000..604b83d --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/SpecFlowDependConverter.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.DataWorksWorkflowSpec; +import com.aliyun.dataworks.common.spec.domain.enums.ArtifactType; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.domain.ref.SpecArtifact; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.ProcessTaskRelation; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; + + +@Slf4j +public class SpecFlowDependConverter { + + private static final Map> ARTIFACT_TYPE_CLASS_MAP = new EnumMap<>(ArtifactType.class); + + private final DataWorksWorkflowSpec spec; + + private final SpecWorkflow specWorkflow; + + private final List processTaskRelationList; + + static { + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.TABLE, SpecTable.class); + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.VARIABLE, SpecVariable.class); + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.NODE_OUTPUT, SpecNodeOutput.class); + ARTIFACT_TYPE_CLASS_MAP.put(ArtifactType.FILE, SpecArtifact.class); + } + + public SpecFlowDependConverter(DataWorksWorkflowSpec spec, SpecWorkflow specWorkflow, List processTaskRelationList) { + this.spec = spec; + this.specWorkflow = specWorkflow; + this.processTaskRelationList = processTaskRelationList; + } + + + public List convert() { + if (Objects.nonNull(specWorkflow)) { + specWorkflow.setDependencies(convertTaskRelationList(processTaskRelationList)); + return specWorkflow.getDependencies(); + } + spec.setFlow(convertTaskRelationList(processTaskRelationList)); + return spec.getFlow(); + } + + private List convertTaskRelationList(List taskRelationList) { + List flow = Optional.ofNullable(specWorkflow).map(SpecWorkflow::getDependencies).orElse(Optional.ofNullable(spec) + .map(DataWorksWorkflowSpec::getFlow).orElse(new ArrayList<>())); + Map> nodeIdDependMap = flow.stream().collect( + Collectors.toMap(o -> o.getNodeId().getId(), SpecFlowDepend::getDepends)); + + Map taskCodeNodeDataMap = DolphinSchedulerV3Context.getContext().getTaskCodeNodeDataMap(); + Map taskCodeNodeIdMap = DolphinSchedulerV3Context.getContext().getTaskCodeNodeIdMap(); + for (ProcessTaskRelation processTaskRelation : ListUtils.emptyIfNull(taskRelationList)) { + long preTaskCode = processTaskRelation.getPreTaskCode(); + if (preTaskCode == 0L) { + continue; + } + long postTaskCode = processTaskRelation.getPostTaskCode(); + String nodeId = taskCodeNodeIdMap.get(postTaskCode); + SpecNode currentNode; + if (nodeId == null) { + log.warn("can not find nodeId {}", postTaskCode); + continue; + } else { + currentNode = specWorkflow.getNodes().stream().filter(node -> nodeId.equals(node.getId())) + .findAny().orElse(null); + } + final SpecNode finalCurrentNode = currentNode; + + String data = taskCodeNodeDataMap.get(preTaskCode); + if (data != null) { + specWorkflow.getNodes().stream() + .filter(node -> { + return CollectionUtils.emptyIfNull(node.getOutputs()) + .stream() + .map(output -> (SpecNodeOutput) output) + .anyMatch(output -> data.equals(output.getData())); + }) + .findAny() + .ifPresent(node -> { + SpecNodeOutput specNodeOutput = (SpecNodeOutput) node.getOutputs().get(0); + SpecDepend specDepend = new SpecDepend(null, DependencyType.NORMAL, specNodeOutput); + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + specFlowDepend.setDepends(Arrays.asList(specDepend)); + specFlowDepend.setNodeId(finalCurrentNode); + //flow.add(specFlowDepend); + specWorkflow.getDependencies().add(specFlowDepend); + }); + } + } + return flow; + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/TaskConverterFactoryV3.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/TaskConverterFactoryV3.java new file mode 100644 index 0000000..33efbf5 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/TaskConverterFactoryV3.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow; + +import java.util.Properties; + +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.TaskType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.AbstractParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.CustomParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.DataxParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.DependentParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.HiveCliParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.MrParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.ProcedureParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.PythonParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.ShellParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.SparkParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.SqlParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.SqoopParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.SubProcessParameterConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.SwitchParameterConverter; +import com.aliyun.migrationx.common.exception.UnSupportedTypeException; +import com.aliyun.migrationx.common.utils.Config; + +public class TaskConverterFactoryV3 { + public static AbstractParameterConverter create( + Properties properties, SpecWorkflow specWorkflow, + DagData processMeta, TaskDefinition taskDefinition) throws Throwable { + if (Config.INSTANCE.getTempTaskTypes().contains(taskDefinition.getTaskType())) { + return new CustomParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + } + TaskType taskType = TaskType.of(taskDefinition.getTaskType()); + if (taskType == null) { + throw new UnSupportedTypeException(taskDefinition.getTaskType()); + } + + switch (taskType) { + case SHELL: + return new ShellParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case PYTHON: + return new PythonParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SQL: + return new SqlParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case MR: + return new MrParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SPARK: + return new SparkParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SUB_PROCESS: + return new SubProcessParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case HIVECLI: + return new HiveCliParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case DEPENDENT: + return new DependentParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SWITCH: + return new SwitchParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case SQOOP: + return new SqoopParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case DATAX: + return new DataxParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + case PROCEDURE: + return new ProcedureParameterConverter(properties, specWorkflow, processMeta, taskDefinition); + default: + throw new UnSupportedTypeException(taskDefinition.getTaskType()); + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/TriggerConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/TriggerConverter.java new file mode 100644 index 0000000..c8ffe91 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/TriggerConverter.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow; + +import java.text.ParseException; +import java.time.Duration; +import java.util.Objects; + +import com.aliyun.dataworks.common.spec.domain.enums.NodeRecurrenceType; +import com.aliyun.dataworks.common.spec.domain.enums.TriggerType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.Schedule; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.Flag; +import com.aliyun.dataworks.migrationx.domain.dataworks.utils.CronExpressUtil; +import com.aliyun.migrationx.common.utils.DateUtils; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class TriggerConverter { + + private final Schedule schedule; + + private final SpecTrigger currTrigger; + + private final TaskDefinition taskDefinition; + + private SpecTrigger convertRes; + + public TriggerConverter(Schedule schedule) { + super(); + this.schedule = schedule; + this.currTrigger = null; + this.taskDefinition = null; + this.convertRes = new SpecTrigger(); + } + + public TriggerConverter(SpecTrigger trigger, TaskDefinition taskDefinition) { + super(); + this.schedule = null; + this.currTrigger = trigger; + this.taskDefinition = taskDefinition; + this.convertRes = new SpecTrigger(); + } + + public SpecTrigger convert() { + if (Objects.nonNull(schedule)) { + convertBySchedule(); + } else if (Objects.nonNull(currTrigger) && Objects.nonNull(taskDefinition)) { + convertByTaskDefinition(); + } else { + return null; + } + return convertRes; + } + + /** + * convert workflow trigger + */ + private void convertBySchedule() { + if (Objects.isNull(schedule)) { + convertRes.setType(TriggerType.MANUAL); + } else { + convertRes.setId(UuidGenerators.generateUuid(Long.valueOf(schedule.getId()))); + convertRes.setType(TriggerType.SCHEDULER); + convertRes.setStartTime(DateUtils.convertDateToString(schedule.getStartTime())); + convertRes.setEndTime(DateUtils.convertDateToString(schedule.getEndTime())); + convertRes.setCron(dolphinCron2SpecCron(schedule.getCrontab())); + convertRes.setTimezone(schedule.getTimezoneId()); + convertRes.setDelaySeconds(0); + } + } + + /** + * convert task trigger, especially for delay seconds. + * Because node will use same trigger with parent workflow except delay seconds. + */ + private void convertByTaskDefinition() { + if (currTrigger == null || taskDefinition == null) { + throw new RuntimeException("currTrigger or taskDefinition null"); + } + //convertRes = BeanUtils.deepCopy(currTrigger, SpecTrigger.class); + convertRes = currTrigger; + convertRes.setDelaySeconds((int) Duration.ofMinutes(taskDefinition.getDelayTime()).getSeconds()); + convertRes.setRecurrence(Flag.YES.equals(taskDefinition.getFlag()) ? NodeRecurrenceType.NORMAL : NodeRecurrenceType.PAUSE); + convertRes.setId(UuidGenerators.generateUuid()); + } + + private String dolphinCron2SpecCron(String dolphinCron) { + try { + return CronExpressUtil.quartzCronExpressionToDwCronExpress(dolphinCron); + } catch (ParseException e) { + log.warn("dolphin cron parse error: {}", dolphinCron); + return dolphinCron; + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/V3ProcessDefinitionConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/V3ProcessDefinitionConverter.java new file mode 100644 index 0000000..a2e9f90 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/V3ProcessDefinitionConverter.java @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.ControllerBranchCode.Branch; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.FailureStrategy; +import com.aliyun.dataworks.common.spec.domain.enums.NodeInstanceModeType; +import com.aliyun.dataworks.common.spec.domain.enums.NodeRerunModeType; +import com.aliyun.dataworks.common.spec.domain.noref.SpecBranches; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScheduleStrategy; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.ProcessDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.ProcessTaskRelation; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.Schedule; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.Priority; +import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.CheckPoint; +import com.aliyun.dataworks.migrationx.transformer.core.checkpoint.StoreWriter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.filters.DolphinSchedulerConverterFilter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters.AbstractParameterConverter; +import com.aliyun.migrationx.common.context.TransformerContext; +import com.aliyun.migrationx.common.exception.UnSupportedTypeException; +import com.aliyun.migrationx.common.metrics.DolphinMetrics; +import com.aliyun.migrationx.common.utils.Config; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import com.google.common.base.Joiner; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class V3ProcessDefinitionConverter { + private List specNodes = new ArrayList<>(); + + private static final SpecScriptRuntime WORKFLOW_RUNTIME = new SpecScriptRuntime(); + + private static final SpecScriptRuntime MANUAL_WORKFLOW_RUNTIME = new SpecScriptRuntime(); + + static { + WORKFLOW_RUNTIME.setEngine(CodeProgramType.VIRTUAL_WORKFLOW.getCalcEngineType().getLabel()); + WORKFLOW_RUNTIME.setCommand("WORKFLOW"); + + MANUAL_WORKFLOW_RUNTIME.setEngine(CodeProgramType.VIRTUAL_WORKFLOW.getCalcEngineType().getLabel()); + MANUAL_WORKFLOW_RUNTIME.setCommand("MANUAL_WORKFLOW"); + } + + private ProcessDefinition processDefinition; + private DagData dagData; + private Properties converterProperties; + private List taskDefinitionList; + + private DolphinSchedulerConverterFilter filter; + + public V3ProcessDefinitionConverter(DagData dagData, Properties converterProperties) { + this.dagData = dagData; + this.converterProperties = converterProperties; + this.processDefinition = dagData.getProcessDefinition(); + this.taskDefinitionList = dagData.getTaskDefinitionList(); + this.filter = new DolphinSchedulerConverterFilter(); + } + + public static String toWorkflowName(ProcessDefinition processDefinition) { + if (processDefinition.getProjectName() == null) { + return processDefinition.getName(); + } + return com.aliyun.dataworks.migrationx.domain.dataworks.utils.StringUtils.toValidName(Joiner.on("_").join( + processDefinition.getProjectName(), processDefinition.getName())); + } + + private SpecWorkflow initWorkflow() { + SpecWorkflow specWorkflow = new SpecWorkflow(); + specWorkflow.setDependencies(new ArrayList<>()); + specWorkflow.setNodes(new ArrayList<>()); + specWorkflow.setInputs(new ArrayList<>()); + specWorkflow.setOutputs(new ArrayList<>()); + return specWorkflow; + } + + public SpecWorkflow convert() { + SpecWorkflow specWorkflow = convertProcess(processDefinition); + DolphinSchedulerV3Context.getContext().getSubProcessCodeWorkflowMap().put(processDefinition.getCode(), specWorkflow); + convertTaskDefinitions(specWorkflow); + convertTrigger(specWorkflow); + convertTaskRelations(specWorkflow); + handleBranch(specWorkflow); + return specWorkflow; + } + + protected SpecWorkflow convertProcess(ProcessDefinition processDefinition) { + log.info("convert workflow,processDefinition name: {}, code: {}", + processDefinition.getName(), processDefinition.getCode()); + SpecWorkflow specWorkflow = initWorkflow(); + specWorkflow.setId(UuidGenerators.generateUuid(processDefinition.getCode())); + specWorkflow.setName(toWorkflowName(processDefinition)); + specWorkflow.setDescription(processDefinition.getDescription()); + + List specVariableList = new ParamListConverter(processDefinition.getGlobalParamList()).convert(); + log.info("convert workflow,global params: {}", specVariableList); + + SpecScript script = new SpecScript(); + script.setParameters(specVariableList); + script.setRuntime(WORKFLOW_RUNTIME); + script.setPath(getScriptPath(specWorkflow.getName())); + specWorkflow.setScript(script); + + specWorkflow.getOutputs().add(buildDefaultOutput(specWorkflow)); + return specWorkflow; + } + + protected String getScriptPath(String name) { + String defaultPath = StringUtils.defaultString(Config.INSTANCE.getBasePath(), StringUtils.EMPTY); + return FilenameUtils.concat(defaultPath, name); + } + + protected List convertTaskDefinitions(SpecWorkflow specWorkflow) { + return convertTasks(specWorkflow); + } + + protected void convertTrigger(SpecWorkflow specWorkflow) { + Schedule schedule = dagData.getSchedule(); + if (Objects.nonNull(schedule)) { + SpecTrigger trigger = new TriggerConverter(schedule).convert(); + specWorkflow.setTrigger(trigger); + specWorkflow.setStrategy(buildSpecScheduleStrategy(processDefinition, schedule)); + log.info("convert workflow,schedule: {}", schedule); + } + } + + /** + * when handling task, spec node maybe not exists, post handle branch + * change branch output data (task code) to specNode id + */ + private void handleBranch(SpecWorkflow specWorkflow) { + Map taskCodeSpecNodeMap = DolphinSchedulerV3Context.getContext().getTaskCodeSpecNodeMap(); + for (SpecNode specNode : specWorkflow.getNodes()) { + if (specNode.getBranch() != null && specNode.getBranch().getBranches() != null) { + List specBranches = specNode.getBranch().getBranches(); + List branchList = new ArrayList<>(); + for (SpecBranches specBranch : specBranches) { + SpecNodeOutput specNodeOutput = specBranch.getOutput(); + if (specNodeOutput != null && specNodeOutput.getData() != null) { + String data = specNodeOutput.getData(); + //current data is task code, convert to specNode id + Long taskCode = Long.parseLong(data); + SpecNode branchSpecNode = (SpecNode) taskCodeSpecNodeMap.get(taskCode); + specNodeOutput.setData(branchSpecNode.getId()); + Branch branch = new Branch(); + branch.setCondition(specBranch.getWhen()); + branch.setNodeoutput(branchSpecNode.getId()); + branchList.add(branch); + } + } + if (branchList.size() > 0) { + String content = com.aliyun.dataworks.common.spec.utils.GsonUtils.toJsonString(branchList); + specNode.getScript().setContent(content); + } + } + } + } + + protected void convertTaskRelations(SpecWorkflow specWorkflow) { + List processTaskRelationList = dagData.getProcessTaskRelationList(); + log.info("convert workflow,processTaskRelationList: {}", processTaskRelationList); + new SpecFlowDependConverter(null, specWorkflow, processTaskRelationList).convert(); + } + + private SpecScheduleStrategy buildSpecScheduleStrategy(ProcessDefinition processDefinition, Schedule schedule) { + SpecScheduleStrategy strategy = new SpecScheduleStrategy(); + strategy.setPriority(convertPriority(schedule.getProcessInstancePriority())); + strategy.setTimeout(processDefinition.getTimeout()); + strategy.setInstanceMode(NodeInstanceModeType.T_PLUS_1); + strategy.setRerunMode(NodeRerunModeType.ALL_ALLOWED); + strategy.setRerunTimes(0); + strategy.setRerunInterval(0); + strategy.setIgnoreBranchConditionSkip(false); + strategy.setFailureStrategy( + com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.FailureStrategy.CONTINUE.equals(schedule.getFailureStrategy()) + ? FailureStrategy.CONTINUE + : FailureStrategy.BREAK); + return strategy; + } + + protected Integer convertPriority(Priority priority) { + return Priority.LOWEST.getCode() - priority.getCode(); + } + + private SpecNodeOutput buildDefaultOutput(SpecWorkflow specWorkflow) { + SpecNodeOutput specNodeOutput = new SpecNodeOutput(); + specNodeOutput.setIsDefault(true); + //String data = String.valueOf(processDefinition.getCode()); + String data = String.format("%s.%s", processDefinition.getProjectName(), processDefinition.getName()); + specNodeOutput.setData(data); + specNodeOutput.setRefTableName(specWorkflow.getName()); + return specNodeOutput; + } + + public List convertTasks(SpecWorkflow specWorkflow) { + CheckPoint checkPoint = CheckPoint.getInstance(); + String projectName = processDefinition.getProjectName(); + String processName = processDefinition.getName(); + + Map> loadedTasks = checkPoint.loadFromCheckPoint(projectName, processName); + + final Function> processFunc = (StoreWriter writer) -> + ListUtils.emptyIfNull(taskDefinitionList) + .stream() + .filter(s -> { + if (Thread.currentThread().isInterrupted()) { + throw new RuntimeException(new InterruptedException()); + } + return true; + }) + .filter(task -> { + boolean passed = filter.filter(projectName, processName, task.getName()); + if (!passed) { + log.warn("task filtered {}", task.getName()); + } + return passed; + }) + .map(task -> { + List specNodes = convertTaskToWorkflowWithLoadedTask(specWorkflow, task, loadedTasks); + checkPoint.doCheckpoint(writer, specNodes, processName, task.getName()); + return specNodes; + }) + .flatMap(List::stream) + .collect(Collectors.toList()); + + specNodes = checkPoint.doWithCheckpoint(processFunc, projectName); + + log.info("successfully converted {}, size {}", processDefinition.getName(), specNodes.size()); + return specNodes; + } + + private List convertTaskToWorkflowWithLoadedTask(SpecWorkflow specWorkflow, TaskDefinition taskDefinition, Map> loadedTasks) { + List specNodes = loadedTasks.get(taskDefinition.getName()); + if (specNodes != null) { + markSuccessProcess(specWorkflow, taskDefinition); + log.info("loaded task {} from checkpoint", taskDefinition.getName()); + return specNodes; + } + return convertTaskDefinition(specWorkflow, taskDefinition); + } + + private List convertTaskDefinition(SpecWorkflow specWorkflow, TaskDefinition taskDefinition) { + if (inSkippedList(taskDefinition)) { + log.warn("task {} in skipped list", taskDefinition.getName()); + return Collections.emptyList(); + } + + try { + AbstractParameterConverter converter = TaskConverterFactoryV3.create(converterProperties, specWorkflow, dagData, taskDefinition); + SpecNode specNode = converter.convert(); + if (specNode != null) { + DolphinSchedulerV3Context.getContext().getTaskCodeSpecNodeMap().put(taskDefinition.getCode(), specNode); + return Arrays.asList(specNode); + } else { + return Collections.emptyList(); + } + } catch (UnSupportedTypeException e) { + markFailedProcess(taskDefinition, e.getMessage()); + if (Config.INSTANCE.isSkipUnSupportType()) { + log.warn("task {} with type {} unsupported, skip", taskDefinition.getTaskType(), taskDefinition.getName()); + return Collections.emptyList(); + } else { + throw e; + } + } catch (Throwable e) { + log.error("task converter error, taskName {} ", taskDefinition.getName(), e); + if (Config.INSTANCE.isTransformContinueWithError()) { + return Collections.emptyList(); + } else { + throw new RuntimeException(e); + } + } + + } + + protected void markSuccessProcess(SpecWorkflow workflow, TaskDefinition taskDefinition) { + for (SpecNode node : workflow.getNodes()) { + DolphinMetrics metrics = DolphinMetrics.builder() + .projectName(taskDefinition.getProjectName()) + .projectCode(taskDefinition.getProjectCode()) + .processName(processDefinition.getName()) + .processCode(processDefinition.getCode()) + .taskName(taskDefinition.getName()) + .taskCode(taskDefinition.getCode()) + .taskType(taskDefinition.getTaskType()) + .build(); + metrics.setWorkflowName(workflow.getName()); + metrics.setDwName(node.getName()); + String type = node.getScript().getRuntime().getCommand(); + metrics.setDwType(type); + TransformerContext.getCollector().markSuccessMiddleProcess(metrics); + } + } + + private boolean inSkippedList(TaskDefinition taskDefinition) { + if (Config.INSTANCE.getSkipTypes().contains(taskDefinition.getTaskType()) + || Config.INSTANCE.getSkipTaskCodes().contains(String.valueOf(taskDefinition.getCode()))) { + log.warn("task name {} code {} in skipped list", taskDefinition.getName(), taskDefinition.getCode()); + markSkippedProcess(taskDefinition); + return true; + } else { + return false; + } + } + + protected void markFailedProcess(TaskDefinition taskDefinition, String errorMsg) { + DolphinMetrics metrics = DolphinMetrics.builder() + .projectName(taskDefinition.getProjectName()) + .projectCode(taskDefinition.getProjectCode()) + .processName(processDefinition.getName()) + .processCode(processDefinition.getCode()) + .taskName(taskDefinition.getName()) + .taskCode(taskDefinition.getCode()) + .taskType(taskDefinition.getTaskType()) + .build(); + metrics.setErrorMsg(errorMsg); + TransformerContext.getCollector().markFailedMiddleProcess(metrics); + } + + protected void markSkippedProcess(TaskDefinition taskDefinition) { + DolphinMetrics metrics = DolphinMetrics.builder() + .projectName(taskDefinition.getProjectName()) + .projectCode(taskDefinition.getProjectCode()) + .processName(processDefinition.getName()) + .processCode(processDefinition.getCode()) + .taskName(taskDefinition.getName()) + .taskCode(taskDefinition.getCode()) + .taskType(taskDefinition.getTaskType()) + .build(); + TransformerContext.getCollector().markSkippedProcess(metrics); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/AbstractParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/AbstractParameterConverter.java new file mode 100644 index 0000000..dda02e3 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/AbstractParameterConverter.java @@ -0,0 +1,460 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.io.File; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.dw.types.LanguageEnum; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.enums.NodeRecurrenceType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableScopeType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.interfaces.Output; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.domain.ref.SpecFileResource; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScheduleStrategy; +import com.aliyun.dataworks.common.spec.domain.ref.SpecTrigger; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.enums.TaskType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.ProcessDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.Flag; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.Priority; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.datax.DataxParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.flink.FlinkParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.http.HttpParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.mr.MapReduceParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.parameters.AbstractParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.procedure.ProcedureParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.python.PythonParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.shell.ShellParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.spark.SparkParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.sql.SqlParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.sqoop.SqoopParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.subprocess.SubProcessParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.switchs.SwitchParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.v320.task.condition.ConditionsParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.v320.task.dependent.DependentParameters; +import com.aliyun.dataworks.migrationx.transformer.core.utils.SpecFileResourceTypeUtils; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.utils.ConverterTypeUtils; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.ParamListConverter; +import com.aliyun.migrationx.common.exception.BizException; +import com.aliyun.migrationx.common.exception.ErrorCode; +import com.aliyun.migrationx.common.utils.BeanUtils; +import com.aliyun.migrationx.common.utils.Config; +import com.aliyun.migrationx.common.utils.GsonUtils; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import com.google.gson.reflect.TypeToken; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public abstract class AbstractParameterConverter { + + protected static final String RESOURCE_REFERENCE_FORMAT = "%s@resource_reference{\"%s\"}"; + protected static final String RESOURCE_REFERENCE_PREFIX = "##"; + + protected final TaskDefinition taskDefinition; + protected final DagData processMeta; + protected final ProcessDefinition processDefinition; + protected final Properties properties; + + protected static Map> taskTypeClassMap; + + protected T parameter; + + /** + * The purpose of setting this field private is to mask the differences in how subclasses perceive spec and workflow + */ + protected final SpecWorkflow specWorkflow; + + static { + taskTypeClassMap = new EnumMap<>(TaskType.class); + taskTypeClassMap.put(TaskType.SQL, SqlParameters.class); + taskTypeClassMap.put(TaskType.DEPENDENT, DependentParameters.class); + taskTypeClassMap.put(TaskType.FLINK, FlinkParameters.class); + taskTypeClassMap.put(TaskType.SPARK, SparkParameters.class); + taskTypeClassMap.put(TaskType.DATAX, DataxParameters.class); + taskTypeClassMap.put(TaskType.SHELL, ShellParameters.class); + taskTypeClassMap.put(TaskType.HTTP, HttpParameters.class); + taskTypeClassMap.put(TaskType.PROCEDURE, ProcedureParameters.class); + taskTypeClassMap.put(TaskType.CONDITIONS, ConditionsParameters.class); + taskTypeClassMap.put(TaskType.SQOOP, SqoopParameters.class); + taskTypeClassMap.put(TaskType.SUB_PROCESS, SubProcessParameters.class); + taskTypeClassMap.put(TaskType.PYTHON, PythonParameters.class); + taskTypeClassMap.put(TaskType.MR, MapReduceParameters.class); + taskTypeClassMap.put(TaskType.SWITCH, SwitchParameters.class); + } + + protected AbstractParameterConverter(Properties properties, + SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(); + this.properties = properties; + //this.spec = spec; + this.specWorkflow = specWorkflow; + this.processMeta = processMeta; + this.processDefinition = processMeta.getProcessDefinition(); + this.taskDefinition = taskDefinition; + + TaskType taskType = TaskType.valueOf(taskDefinition.getTaskType()); + try { + this.parameter = GsonUtils.fromJsonString( + taskDefinition.getTaskParams(), TypeToken.get(taskTypeClassMap.get(taskType)).getType()); + } catch (Exception ex) { + log.error("parse task {}, parameter {} error: ", taskType, taskTypeClassMap.get(taskType), ex); + } + } + + /** + * Each node translates the specific logic of the parameters + */ + protected abstract void convertParameter(SpecNode specNode); + + public SpecNode convert() { + SpecNode specNode = newSpecNode(taskDefinition); + convertParameter(specNode); + this.specWorkflow.getNodes().add(specNode); + + // hint: the node returned may not be the final result of the conversion + return specNode; + } + + private SpecNode initSpecNode() { + SpecNode specNode = new SpecNode(); + specNode.setInputs(new ArrayList<>()); + specNode.setOutputs(new ArrayList<>()); + specNode.setFileResources(new ArrayList<>()); + specNode.setFunctions(new ArrayList<>()); + return specNode; + } + + /** + * common new SpecNode method, almost all nodes use this method + * + * @param taskDefinition taskDefinition + * @return SpecNode + */ + protected SpecNode newSpecNode(TaskDefinition taskDefinition) { + SpecNode specNode = initSpecNode(); + specNode.setId(UuidGenerators.generateUuid(taskDefinition.getCode())); + specNode.setName(taskDefinition.getName()); + specNode.setDescription(taskDefinition.getDescription()); + specNode.setRerunTimes(taskDefinition.getFailRetryTimes()); + // Unit conversion, minutes to milliseconds + specNode.setRerunInterval((int) Duration.ofMinutes(taskDefinition.getFailRetryInterval()).toMillis()); + specNode.setTimeout(taskDefinition.getTimeout()); + specNode.setPriority(convertPriority(taskDefinition.getTaskPriority())); + resetNodeStrategy(specNode); + + SpecNodeOutput defaultOutput = buildDefaultNodeOutput(specNode); + DolphinSchedulerV3Context.getContext().getTaskCodeNodeDataMap().put(taskDefinition.getCode(), defaultOutput.getData()); + DolphinSchedulerV3Context.getContext().getTaskCodeNodeIdMap().put(taskDefinition.getCode(), specNode.getId()); + //specNode.getOutputs().add(BeanUtils.deepCopy(defaultOutput, SpecNodeOutput.class)); + + specNode.getOutputs().add(defaultOutput); + + specNode.setTrigger(convertByTaskDefinition()); + + return specNode; + } + + private SpecTrigger convertByTaskDefinition() { + SpecTrigger specTrigger = new SpecTrigger(); + specTrigger.setDelaySeconds((int) Duration.ofMinutes(taskDefinition.getDelayTime()).getSeconds()); + specTrigger.setRecurrence(Flag.YES.equals(taskDefinition.getFlag()) ? NodeRecurrenceType.NORMAL : NodeRecurrenceType.PAUSE); + specTrigger.setId(UuidGenerators.generateUuid()); + return specTrigger; + } + + protected Integer convertPriority(Priority priority) { + return Priority.LOWEST.getCode() - priority.getCode(); + } + + protected SpecNodeOutput buildDefaultNodeOutput(SpecNode specNode) { + SpecNodeOutput specNodeOutput = new SpecNodeOutput(); + specNodeOutput.setIsDefault(true); + //specNodeOutput.setId(generateUuid()); + //String data = String.format("%s.%s", processDefinition.getCode(), taskDefinition.getCode()); + String data = String.format("%s.%s.%s", processDefinition.getProjectName(), processDefinition.getName(), taskDefinition.getName()); + specNodeOutput.setData(data); + specNodeOutput.setRefTableName(specNode.getName()); + return specNodeOutput; + } + + protected SpecNodeOutput getDefaultOutput(SpecNode specNode) { + return getDefaultOutput(Optional.ofNullable(specNode).map(SpecNode::getOutputs).orElse(null)); + } + + protected SpecNodeOutput getDefaultOutput(SpecWorkflow specWorkflow, boolean throwException) { + return getDefaultOutput(Optional.ofNullable(specWorkflow).map(SpecWorkflow::getOutputs).orElse(null), throwException); + } + + protected SpecNodeOutput getDefaultOutput(SpecNode specNode, boolean throwException) { + return getDefaultOutput(Optional.ofNullable(specNode).map(SpecNode::getOutputs).orElse(null), throwException); + } + + protected SpecNodeOutput getDefaultOutput(List outputList) { + return getDefaultOutput(outputList, false); + } + + protected SpecNodeOutput getDefaultOutput(List outputList, boolean throwException) { + Optional first = ListUtils.emptyIfNull(outputList).stream().filter( + output -> output instanceof SpecNodeOutput && ((SpecNodeOutput) output).getIsDefault()) + .findFirst(); + if (throwException && !first.isPresent()) { + throw new BizException(ErrorCode.PARAMETER_NOT_SET, "defaultOutput"); + } + return (SpecNodeOutput) first.orElse(null); + } + + protected List getContextOutputs(SpecNode specNode) { + return specNode.getOutputs().stream() + .filter(v -> v instanceof SpecVariable && VariableScopeType.NODE_CONTEXT.equals(((SpecVariable) v).getScope()) + && VariableType.NODE_OUTPUT.equals(((SpecVariable) v).getType())) + .map(v -> { + SpecVariable variable = BeanUtils.deepCopy(v, SpecVariable.class); + variable.setNode(new SpecDepend(specNode, DependencyType.NORMAL, null)); + return variable; + }).collect(Collectors.toList()); + } + + protected List convertSpecNodeParam(SpecNode specNode) { + ParamListConverter paramListConverter = new ParamListConverter(taskDefinition.getTaskParamList(), taskDefinition); + List specVariableList = paramListConverter.convert(); + for (SpecVariable specVariable : specVariableList) { + // all outputs are context output, all inputs are all script inputs + if (VariableType.NODE_OUTPUT.equals(specVariable.getType())) { + SpecDepend nodeDepend = new SpecDepend(); + nodeDepend.setNodeId(specNode); + nodeDepend.setOutput(getDefaultOutput(specNode)); + specVariable.setNode(nodeDepend); + specNode.getOutputs().add(specVariable); + } + } + return specVariableList; + } + + /** + * convert resource with fileResources info in workflow, if not exists in fileResources, create and add in fileResources + * + * @param specNode node need to convert + */ + protected void convertFileResourceList(SpecNode specNode) { + ListUtils.emptyIfNull(parameter.getResourceFilesList()).forEach(resourceInfo -> { + SpecFileResource specFileResource = new SpecFileResource(); + specFileResource.setRuntimeResource(specNode.getRuntimeResource()); + if (resourceInfo.getResourceName() != null) { + specFileResource.setName(getFileNameByPath(resourceInfo.getResourceName())); + } + + specFileResource.setType(SpecFileResourceTypeUtils.getResourceTypeBySuffix(specFileResource.getName())); + checkFileSameName(specFileResource.getName(), resourceInfo.getResourceName()); + specNode.getFileResources().add(specFileResource); + }); + } + + private void checkFileSameName(String fileName, String fullName) { + //String fullNameIn = context.getFileNameMap().get(fileName); + String fullNameIn = null; + if (Objects.nonNull(fullNameIn) && !fullNameIn.equals(fullName)) { + log.warn("存在同名资源冲突风险, {} 和 {} 导入后会同名", fullNameIn, fullName); + } else { + //context.getFileNameMap().put(fileName, fullName); + } + } + + protected String buildFileResourceReference(SpecNode specNode, String prefix) { + StringBuilder stringBuilder = new StringBuilder(); + Optional.ofNullable(specNode).map(SpecNode::getFileResources) + .ifPresent(fileResources -> + fileResources.forEach(fileResource -> + stringBuilder.append(String.format(RESOURCE_REFERENCE_FORMAT, prefix, fileResource.getName())).append("\n"))); + return stringBuilder.append("\n").toString(); + } + + /** + * add relation before join node. if the node depend on a whole workflow, need depend on workflow output + * + * @param postNode post join node + * @param preNodeList pre node list + * @param preNodeOutputList pre workflow output list + */ + protected void addRelation(SpecNode postNode, List preNodeList, List preNodeOutputList) { + SpecFlowDepend specFlowDepend = newSpecFlowDepend(); + specFlowDepend.setNodeId(postNode); + ListUtils.emptyIfNull(preNodeList).forEach(preNode -> { + SpecNodeOutput preNodeOutput = getDefaultOutput(preNode); + postNode.getInputs().add(preNodeOutput); + postNode.getInputs().addAll(getContextOutputs(preNode)); + specFlowDepend.getDepends().add(new SpecDepend(preNode, DependencyType.NORMAL, preNodeOutput)); + }); + + ListUtils.emptyIfNull(preNodeOutputList).forEach(preNodeOutput -> { + postNode.getInputs().add(preNodeOutput); + SpecDepend specDepend = new SpecDepend(); + specDepend.setType(DependencyType.NORMAL); + specDepend.setOutput(preNodeOutput); + specFlowDepend.getDepends().add(specDepend); + }); + getWorkflowDependencyList().add(specFlowDepend); + } + + protected SpecFlowDepend newSpecFlowDepend() { + SpecFlowDepend specFlowDepend = new SpecFlowDepend(); + specFlowDepend.setDepends(new ArrayList<>()); + return specFlowDepend; + } + + protected void addRelation(SpecNode postNode, List preNodeList) { + addRelation(postNode, preNodeList, null); + } + + protected String getFileNameByPath(String path) { + File file = new File(path); + return file.getName(); + } + + protected String getScriptPath(SpecNode specNode) { + if (Objects.isNull(specNode)) { + return StringUtils.EMPTY; + } + String defaultPath = StringUtils.defaultString(Config.INSTANCE.getBasePath(), StringUtils.EMPTY); + String workFlowPath = Optional.ofNullable(specWorkflow) + .map(SpecWorkflow::getName) + .orElse(StringUtils.EMPTY); + return FilenameUtils.concat(FilenameUtils.concat(defaultPath, workFlowPath), specNode.getName()); + } + + protected List getWorkflowDependencyList() { + if (Objects.nonNull(specWorkflow)) { + return specWorkflow.getDependencies(); + } else { + return Collections.emptyList(); + } + } + + protected SpecTrigger getWorkflowTrigger() { + if (Objects.nonNull(specWorkflow)) { + return specWorkflow.getTrigger(); + } + // may be manual flow + return null; + } + + /** + * get workflow, but it will be null in spec version < 1.2.0 + * + * @return SpecWorkflow if it is not null + */ + protected SpecWorkflow getWorkFlow() { + return specWorkflow; + } + + protected void resetNodeStrategy(SpecNode specNode) { + if (specNode.getStrategy() == null) { + specNode.setStrategy(new SpecScheduleStrategy()); + } + SpecScheduleStrategy strategy = specNode.getStrategy(); + strategy.setPriority(specNode.getPriority()); + strategy.setTimeout(specNode.getTimeout()); + strategy.setRerunInterval(specNode.getRerunInterval()); + strategy.setRerunTimes(specNode.getRerunTimes()); + strategy.setIgnoreBranchConditionSkip(specNode.getIgnoreBranchConditionSkip()); + strategy.setInstanceMode(specNode.getInstanceMode()); + strategy.setRerunMode(specNode.getRerunMode()); + + Optional.ofNullable(getWorkFlow()) + .map(SpecWorkflow::getStrategy) + .map(SpecScheduleStrategy::getFailureStrategy) + .ifPresent(strategy::setFailureStrategy); + } + + protected String codeToLanguageIdentifier(CodeProgramType nodeType) { + LanguageEnum languageEnum = codeToLanguage(nodeType); + if (languageEnum == null) { + log.warn("can not find language by {}", nodeType); + return null; + } + return languageEnum.getIdentifier(); + } + + protected LanguageEnum codeToLanguage(CodeProgramType nodeType) { + switch (nodeType) { + case SHELL: + case DIDE_SHELL: + case CDH_SHELL: + case EMR_SPARK_SHELL: + case CDH_SPARK_SHELL: + case EMR_SHELL: + case EMR_HIVE_CLI: + case PERL: + return LanguageEnum.SHELL_SCRIPT; + case EMR_SPARK_SQL: + return LanguageEnum.SPARK_SQL; + case CDH_HIVE: + case HIVE: + case EMR_HIVE: + return LanguageEnum.HIVE_SQL; + case EMR_IMPALA: + case CDH_IMPALA: + return LanguageEnum.IMPALA_SQL; + case CLICK_SQL: + return LanguageEnum.CLICKHOUSE_SQL; + case ODPS_SQL: + case ODPS_PERL: + return LanguageEnum.ODPS_SQL; + case ODPS_SCRIPT: + return LanguageEnum.ODPS_SCRIPT; + case EMR_PRESTO: + case CDH_PRESTO: + return LanguageEnum.PRESTO_SQL; + case PYODPS: + return LanguageEnum.PYTHON2; + case PYODPS3: + case PYTHON: + return LanguageEnum.PYTHON3; + case DATAX2: + case DATAX: + case RI: + case DI: + return LanguageEnum.JSON; + case HOLOGRES_SQL: + return LanguageEnum.HOLOGRES_SQL; + default: + return null; + } + } + + protected String getConverterType(String convertType, String defaultConvertType) { + String projectName = processDefinition.getProjectName(); + String processName = processDefinition.getName(); + String taskName = taskDefinition.getName(); + return ConverterTypeUtils.getConverterType(convertType, projectName, processName, taskName, defaultConvertType); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/CustomParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/CustomParameterConverter.java new file mode 100644 index 0000000..ff5e622 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/CustomParameterConverter.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.parameters.AbstractParameters; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import org.apache.commons.collections4.ListUtils; + +public class CustomParameterConverter extends AbstractParameterConverter { + public CustomParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + + SpecScript script = new SpecScript(); + CodeProgramType codeProgramType = CodeProgramType.CUSTOM; + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + script.setContent(JSONUtils.toJsonString(taskDefinition)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/DataxParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/DataxParameterConverter.java new file mode 100644 index 0000000..dd23484 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/DataxParameterConverter.java @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.entity.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.DbType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.Flag; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.model.Property; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.datax.DataxParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.datax.DataxUtils; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.utils.ParameterUtils; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.entity.Connection; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.entity.Parameter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.entity.Step; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; +import com.alibaba.druid.sql.ast.statement.SQLSelect; +import com.alibaba.druid.sql.ast.statement.SQLSelectItem; +import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; +import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; +import com.alibaba.druid.sql.parser.SQLStatementParser; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.tuple.Pair; + +@Slf4j +public class DataxParameterConverter extends AbstractParameterConverter { + /** + * select all + */ + private static final String SELECT_ALL_CHARACTER = "*"; + + private static final int DATAX_CHANNEL_COUNT = 1; + + public DataxParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + String type = properties.getProperty(Constants.CONVERTER_TARGET_DATAX_NODE_TYPE_AS, CodeProgramType.DI.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String json = buildDataxJsonFile(new HashMap<>()); + script.setContent(json); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + private String buildDataxJsonFile(Map paramsMap) { + String json; + + if (parameter.getCustomConfig() == Flag.YES.ordinal()) { + json = parameter.getJson().replaceAll("\\r\\n", System.lineSeparator()); + } else { + ObjectNode job = JSONUtils.createObjectNode(); + job.put("transform", false); + job.put("type", "job"); + job.put("version", "2.0"); + ArrayNode steps = buildDataxJobSteps(); + job.putArray("steps").addAll(steps); + job.set("setting", buildDataxJobSettingJson()); + json = job.toString(); + } + + // replace placeholder + json = ParameterUtils.convertParameterPlaceholders(json, ParameterUtils.convert(paramsMap)); + + log.debug("datax job json : {}", json); + return json; + } + + private ArrayNode buildDataxJobSteps() { + DataSource source = getDataSource(parameter.getDataSource()); + DataSource target = getDataSource(parameter.getDataTarget()); + Step reader = new Step(); + Step writer = new Step(); + reader.setName("Reader"); + reader.setCategory("reader"); + reader.setStepType(source.getType().name().toLowerCase()); + + writer.setName("Writer"); + writer.setCategory("writer"); + writer.setStepType(target.getType().name().toLowerCase()); + + Parameter readParameter = new Parameter(); + readParameter.setEncoding("UTF-8"); + readParameter.setEnvType(1); + readParameter.setUseSpecialSecret(false); + + Parameter writeParameter = new Parameter(); + writeParameter.setEncoding("UTF-8"); + writeParameter.setEnvType(1); + writeParameter.setUseSpecialSecret(false); + + Pair pair = tryGrammaticalAnalysisSqlColumnNames(source.getType(), parameter.getSql()); + String[] srcColumnNames = pair.getRight(); + if (srcColumnNames != null && srcColumnNames.length > 0) { + readParameter.setColumn(Arrays.asList(srcColumnNames)); + String[] tgtColumnNames = DataxUtils.convertKeywordsColumns(target.getType(), srcColumnNames); + if (tgtColumnNames != null && tgtColumnNames.length > 0) { + writeParameter.setColumn(Arrays.asList(srcColumnNames)); + } + } + Connection sourceConnection = new Connection(); + sourceConnection.setDatasource(source.getName()); + sourceConnection.setTable(Arrays.asList(pair.getLeft())); + readParameter.setConnection(Arrays.asList(sourceConnection)); + Connection targetConnection = new Connection(); + targetConnection.setDatasource(target.getName()); + targetConnection.setTable(Arrays.asList(parameter.getTargetTable())); + writeParameter.setConnection(Arrays.asList(targetConnection)); + reader.setParameter(readParameter); + writer.setParameter(writeParameter); + + ObjectNode steps = JSONUtils.createObjectNode(); + ArrayNode tableArr = steps.putArray("steps"); + JsonNode readerNode = JSONUtils.toJsonNode(reader); + tableArr.add(readerNode); + JsonNode writerNode = JSONUtils.toJsonNode(writer); + tableArr.add(writerNode); + return tableArr; + } + + /** + * build datax setting config + * + * @return datax setting config JSONObject + */ + private ObjectNode buildDataxJobSettingJson() { + + ObjectNode speed = JSONUtils.createObjectNode(); + + speed.put("channel", DATAX_CHANNEL_COUNT); + + if (parameter.getJobSpeedByte() > 0) { + speed.put("byte", parameter.getJobSpeedByte()); + } + + if (parameter.getJobSpeedRecord() > 0) { + speed.put("record", parameter.getJobSpeedRecord()); + } + + ObjectNode errorLimit = JSONUtils.createObjectNode(); + errorLimit.put("record", 0); + errorLimit.put("percentage", 0); + + ObjectNode setting = JSONUtils.createObjectNode(); + setting.set("speed", speed); + setting.set("errorLimit", errorLimit); + + return setting; + } + + /** + * try grammatical parsing column + * + * @param dbType database type + * @param sql sql for data synchronization + * @return column name array + * @throws RuntimeException if error throws RuntimeException + */ + private Pair tryGrammaticalAnalysisSqlColumnNames(DbType dbType, String sql) { + String[] columnNames; + String sourceTable = null; + try { + SQLStatementParser parser = DataxUtils.getSqlStatementParser(dbType, sql); + if (parser == null) { + log.warn("database driver [{}] is not support grammatical analysis sql", dbType); + return Pair.of(null, new String[0]); + } + + SQLStatement sqlStatement = parser.parseStatement(); + SQLSelectStatement sqlSelectStatement = (SQLSelectStatement) sqlStatement; + SQLSelect sqlSelect = sqlSelectStatement.getSelect(); + + List selectItemList = null; + + if (sqlSelect.getQuery() instanceof SQLSelectQueryBlock) { + SQLSelectQueryBlock block = (SQLSelectQueryBlock) sqlSelect.getQuery(); + selectItemList = block.getSelectList(); + if (block.getFrom() instanceof SQLExprTableSource) { + SQLExprTableSource expr = (SQLExprTableSource) block.getFrom(); + if (expr.getExpr() instanceof SQLIdentifierExpr) { + sourceTable = ((SQLIdentifierExpr) expr.getExpr()).getName(); + } + } + } else if (sqlSelect.getQuery() instanceof SQLUnionQuery) { + SQLUnionQuery unionQuery = (SQLUnionQuery) sqlSelect.getQuery(); + SQLSelectQueryBlock block = (SQLSelectQueryBlock) unionQuery.getRight(); + selectItemList = block.getSelectList(); + if (block.getFrom() instanceof SQLExprTableSource) { + SQLExprTableSource expr = (SQLExprTableSource) block.getFrom(); + if (expr.getExpr() instanceof SQLIdentifierExpr) { + sourceTable = ((SQLIdentifierExpr) expr.getExpr()).getName(); + } + } + } + + if (selectItemList == null) { + throw new RuntimeException(String.format("select query type [%s] is not support", sqlSelect.getQuery().toString())); + } + + columnNames = new String[selectItemList.size()]; + for (int i = 0; i < selectItemList.size(); i++) { + SQLSelectItem item = selectItemList.get(i); + + String columnName = null; + + if (item.getAlias() != null) { + columnName = item.getAlias(); + } else if (item.getExpr() != null) { + if (item.getExpr() instanceof SQLPropertyExpr) { + SQLPropertyExpr expr = (SQLPropertyExpr) item.getExpr(); + columnName = expr.getName(); + } else if (item.getExpr() instanceof SQLIdentifierExpr) { + SQLIdentifierExpr expr = (SQLIdentifierExpr) item.getExpr(); + columnName = expr.getName(); + } + } else { + throw new RuntimeException( + String.format("grammatical analysis sql column [ %s ] failed", item)); + } + + if (SELECT_ALL_CHARACTER.equals(item.toString())) { + log.info("sql contains *, grammatical analysis failed"); + return Pair.of(sourceTable, new String[]{"*"}); + } + + if (columnName == null) { + throw new RuntimeException( + String.format("grammatical analysis sql column [ %s ] failed", item)); + } + + columnNames[i] = columnName; + } + } catch (Exception e) { + log.warn(e.getMessage(), e); + return Pair.of(null, new String[0]); + } + + return Pair.of(sourceTable, columnNames); + } + + private DataSource getDataSource(int datasourceId) { + List datasources = DolphinSchedulerV3Context.getContext().getDataSources(); + return CollectionUtils.emptyIfNull(datasources).stream() + .filter(s -> s.getId() == datasourceId) + .findFirst() + .orElse(null); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/DependentParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/DependentParameterConverter.java new file mode 100644 index 0000000..3e58d9a --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/DependentParameterConverter.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.model.DependentItem; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.model.DependentTaskModel; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.dependent.DependentParameters; +import com.aliyun.migrationx.common.utils.GsonUtils; + +import com.google.gson.JsonObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; + +@Slf4j +public class DependentParameterConverter extends AbstractParameterConverter { + + public DependentParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + CodeProgramType codeProgramType = CodeProgramType.VIRTUAL; + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + DolphinSchedulerV3Context context = DolphinSchedulerV3Context.getContext(); + List deps = convertDeps(); + context.getSpecNodeProcessCodeMap().put(specNode, deps); + } + + public List convertDeps() { + log.info("params : {}", taskDefinition.getTaskParams()); + JsonObject param = GsonUtils.fromJsonString(taskDefinition.getTaskParams(), JsonObject.class); + DependentParameters dependentParameters = null; + if (param.get("dependence") != null) { + dependentParameters = GsonUtils.fromJson(param.getAsJsonObject("dependence"), DependentParameters.class); + } + if (dependentParameters == null || dependentParameters.getDependTaskList() == null || dependentParameters.getDependTaskList().isEmpty()) { + log.warn("no dependence param {}", taskDefinition.getTaskParams()); + return Collections.emptyList(); + } + DolphinSchedulerV3Context context = DolphinSchedulerV3Context.getContext(); + + // 本节点的条件依赖 + List dependencies = dependentParameters.getDependTaskList(); + List codes = new ArrayList<>(); + ListUtils.emptyIfNull(dependencies).forEach(dependModel -> + ListUtils.emptyIfNull(dependModel.getDependItemList()).forEach(depItem -> { + long depCode = depItem.getDepTaskCode(); + if (depCode == 0L) { + //all leaf task of process definition + //1. get all relation of process + // preTaskCode -> postTaskCode + //Map> relations = findRelations(depItem); + + List taskDefinitions = context.getProcessCodeTaskRelationMap().get(depItem.getDefinitionCode()); + for (TaskDefinition task : CollectionUtils.emptyIfNull(taskDefinitions)) { + //2. find all taskCode not in preTaskCode (not as a pre dependent, leaf task) + if (!codes.contains(task.getCode())) { + codes.add(task.getCode()); + } + } + } else { + if (!codes.contains(depCode)) { + codes.add(depCode); + } + } + })); + return codes; + } + + private Map> findRelations(DependentItem depItem) { + Map> relations = new HashMap<>(); + DolphinSchedulerV3Context.getContext().getDagDatas().stream() + .filter(dag -> dag.getProcessDefinition().getCode() == depItem.getDefinitionCode()) + .map(dag -> dag.getProcessTaskRelationList()) + .flatMap(List::stream) + .forEach(s -> { + List postCodes = relations.get(s.getPreTaskCode()); + if (postCodes == null) { + relations.put(s.getPreTaskCode(), new ArrayList<>()); + } + relations.get(s.getPreTaskCode()).add(s.getPostTaskCode()); + }); + return relations; + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/HiveCliParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/HiveCliParameterConverter.java new file mode 100644 index 0000000..6bec6f6 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/HiveCliParameterConverter.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.model.ResourceInfo; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.hivecli.HiveCliConstants; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.hivecli.HiveCliParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.utils.DataStudioCodeUtils; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; + +import com.google.common.base.Joiner; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +public class HiveCliParameterConverter extends AbstractParameterConverter { + public HiveCliParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + + SpecScript script = new SpecScript(); + //script.setId(generateUuid()); + String type = getConverterType(); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + //String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + String cmd = buildCommand(codeProgramType, this.parameter); + String code = EmrCodeUtils.toEmrCode(codeProgramType, taskDefinition.getName(), cmd); + String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + + //String code = convertCode(codeProgramType, taskDefinition.getName()); + script.setContent(resourceReference + code); + script.setContent(code); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + protected String buildFileResourceReference(SpecNode specNode, String prefix) { + StringBuilder stringBuilder = new StringBuilder(); + Optional.ofNullable(specNode).map(SpecNode::getFileResources) + .ifPresent(fileResources -> + fileResources.forEach(fileResource -> + stringBuilder.append(String.format(RESOURCE_REFERENCE_FORMAT, prefix, fileResource.getName())).append("\n"))); + return stringBuilder.append("\n").toString(); + } + + private String buildPathImportCode(SpecNode specNode) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("import os\n").append("import sys\n\n"); + Optional.ofNullable(specNode).map(SpecNode::getFileResources).ifPresent(fileResources -> + fileResources.forEach(fileResource -> { + String fileName = fileResource.getName(); + stringBuilder.append(String.format("sys.path.append(os.path.dirname(os.path.abspath('%s')))%n", fileName)); + })); + stringBuilder.append("\n"); + return stringBuilder.toString(); + } + + private String buildCommand(CodeProgramType codeProgramType, HiveCliParameters hiveCliParameters) { + final String type = hiveCliParameters.getHiveCliTaskExecutionType(); + + String resName = ""; + if (HiveCliConstants.TYPE_FILE.equals(type)) { + List resourceInfos = hiveCliParameters.getResourceList(); + if (resourceInfos != null && resourceInfos.size() > 0) { + resName = resourceInfos.get(0).getResourceName(); + } + } else { + String sqlContent = hiveCliParameters.getHiveSqlScript(); + resName = Joiner.on("_").join(processDefinition.getName(), taskDefinition.getName()) + ".sql"; + } + + final List args = new ArrayList<>(); + List resources = new ArrayList<>(); + resources.add(resName); + String resourceRef = DataStudioCodeUtils.addResourceReference(codeProgramType, "", resources); + args.add(resourceRef + HiveCliConstants.HIVE_CLI_EXECUTE_FILE); + args.add(resName); + final String hiveCliOptions = hiveCliParameters.getHiveCliOptions(); + if (StringUtils.isNotEmpty(hiveCliOptions)) { + args.add(hiveCliOptions); + } + + String command = String.join(" ", args); + + return command; + } + + private String getConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_SHELL_NODE_TYPE_AS); + String defaultConvertType = CodeProgramType.EMR_SHELL.name(); + return getConverterType(convertType, defaultConvertType); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/MrParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/MrParameterConverter.java new file mode 100644 index 0000000..afcfff4 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/MrParameterConverter.java @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.EmrAllocationSpec; +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.EmrCode; +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.EmrLauncher; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.utils.ArgsUtils; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.model.ProgramType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.model.ResourceInfo; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.mr.MapReduceParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.utils.DataStudioCodeUtils; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; + +import com.google.common.base.Joiner; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +import static com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.task.mr.MapReduceTaskConstants.MR_NAME; +import static com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v2.utils.TaskConstants.D; + +@Slf4j +public class MrParameterConverter extends AbstractParameterConverter { + public static final String MR_YARN_QUEUE = "mapreduce.job.queuename"; + + public MrParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + convertFileResourceList(specNode); + + SpecScript script = new SpecScript(); + String type = getConverterType(); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + //todo + //String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + String code = convertCode(codeProgramType, taskDefinition.getName()); + //script.setContent(resourceReference + parameter.getRawScript()); + script.setContent(code); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String convertCode(CodeProgramType type, String taskName) { + // convert to EMR_MR + if (CodeProgramType.EMR_MR.equals(type)) { + String code = buildCommand(parameter); + return EmrCodeUtils.toEmrCode(type, taskName, code); + } else if (CodeProgramType.ODPS_MR.name().equals(type)) { + List resources = ListUtils.emptyIfNull(parameter.getResourceFilesList()).stream() + .filter(Objects::nonNull) + .map(ResourceInfo::getResourceName) + .filter(name -> StringUtils.isNotEmpty(name)) + .distinct().collect(Collectors.toList()); + + List codeLines = new ArrayList<>(); + codeLines.add(DataStudioCodeUtils.addResourceReference(type, "", resources)); + + // convert to ODPS_MR + String command = Joiner.on(" ").join( + "jar", "-resources", + Optional.ofNullable(parameter.getMainJar().getResourceName()).orElse(""), + "-classpath", + Joiner.on(",").join(resources), + Optional.ofNullable(parameter.getMainClass()).orElse(""), + Optional.ofNullable(parameter.getMainArgs()).orElse(""), + Optional.ofNullable(parameter.getOthers()).orElse("") + ); + codeLines.add(command); + String code = Joiner.on("\n").join(codeLines); + code = EmrCodeUtils.toEmrCode(type, taskName, code); + EmrCode emrCode = EmrCodeUtils.asEmrCode(type.name(), code); + Optional.ofNullable(emrCode).map(EmrCode::getLauncher) + .map(EmrLauncher::getAllocationSpec) + .map(EmrAllocationSpec::of) + .ifPresent(spec -> { + spec.setQueue(parameter.getYarnQueue()); + emrCode.getLauncher().setAllocationSpec(spec.toMap()); + }); + return emrCode.getContent(); + } else { + throw new RuntimeException("not support type " + type); + } + } + + private String buildCommand(MapReduceParameters mapreduceParameters) { + List args = buildArgs(mapreduceParameters); + String command = String.join(" ", args); + log.info("mapreduce task command: {}", command); + + return command; + } + + private static List buildArgs(MapReduceParameters param) { + List args = new ArrayList<>(); + ResourceInfo mainJar = param.getMainJar(); + if (mainJar != null) { + String resourceName = mainJar.getResourceName(); + if (StringUtils.isNotEmpty(resourceName)) { + String[] resourceNames = resourceName.split("/"); + if (resourceNames.length > 0) { + resourceName = resourceNames[resourceNames.length - 1]; + } + String resource = DataStudioCodeUtils.addResourceReference(CodeProgramType.EMR_MR, "", Arrays.asList(resourceName)); + args.add(resource + resourceName); + } + } + + ProgramType programType = param.getProgramType(); + String mainClass = param.getMainClass(); + if (programType != null && programType != ProgramType.PYTHON && StringUtils.isNotEmpty(mainClass)) { + args.add(mainClass); + } + + String appName = param.getAppName(); + if (StringUtils.isNotEmpty(appName)) { + args.add(String.format("%s%s=%s", D, MR_NAME, ArgsUtils.escape(appName))); + } + + String others = param.getOthers(); + if (StringUtils.isEmpty(others) || !others.contains(MR_YARN_QUEUE)) { + String yarnQueue = param.getYarnQueue(); + if (StringUtils.isNotEmpty(yarnQueue)) { + args.add(String.format("%s%s=%s", D, MR_YARN_QUEUE, yarnQueue)); + } + } + + // -conf -archives -files -libjars -D + if (StringUtils.isNotEmpty(others)) { + args.add(others); + } + + String mainArgs = param.getMainArgs(); + if (StringUtils.isNotEmpty(mainArgs)) { + args.add(mainArgs); + } + return args; + } + + private String getConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_MR_NODE_TYPE_AS); + String defaultConvertType = CodeProgramType.EMR_MR.name(); + return getConverterType(convertType, defaultConvertType); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/ProcedureParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/ProcedureParameterConverter.java new file mode 100644 index 0000000..5b3e4ae --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/ProcedureParameterConverter.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.entity.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.DbType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.procedure.ProcedureParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.migrationx.common.utils.GsonUtils; + +import com.google.gson.reflect.TypeToken; +import org.apache.commons.collections4.ListUtils; + +public class ProcedureParameterConverter extends AbstractParameterConverter { + + public ProcedureParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_SHELL_NODE_TYPE_AS, CodeProgramType.DIDE_SHELL.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(convertType); + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + //todo + //String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + //script.setContent(resourceReference + parameter.getRawScript()); + script.setContent(getCode()); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String getCode() { + String sqlNodeMapStr = properties.getProperty( + Constants.CONVERTER_TARGET_SQL_NODE_TYPE_MAP, "{}"); + Map sqlTypeNodeTypeMapping = GsonUtils.fromJsonString(sqlNodeMapStr, + new TypeToken>() {}.getType()); + sqlTypeNodeTypeMapping = Optional.ofNullable(sqlTypeNodeTypeMapping).orElse(new HashMap<>(1)); + + String defaultNodeTypeIfNotSupport = getConverterType(); + + DbType codeProgramType = sqlTypeNodeTypeMapping.get(parameter.getType()); + + //add ref datasource + List datasources = DolphinSchedulerV3Context.getContext().getDataSources(); + if (parameter.getDatasource() > 0) { + //todo + //CollectionUtils.emptyIfNull(datasources).stream() + // .filter(s -> s.getId() == parameter.getDatasource()) + // .findFirst() + // .ifPresent(s -> dwNode.setConnection(s.getName())); + } + String code = parameter.getMethod(); + return code; + } + + private String getConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_UNSUPPORTED_NODE_TYPE_AS); + return getConverterType(convertType, CodeProgramType.VIRTUAL.name()); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/PythonParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/PythonParameterConverter.java new file mode 100644 index 0000000..5af3add --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/PythonParameterConverter.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.python.PythonParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; + +import org.apache.commons.collections4.ListUtils; + +public class PythonParameterConverter extends AbstractParameterConverter { + + public PythonParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_PYTHON_NODE_TYPE_AS, CodeProgramType.PYTHON.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(convertType); + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + + String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + String pathImportCode = buildPathImportCode(specNode); + script.setContent(resourceReference + pathImportCode + parameter.getRawScript()); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + private String buildPathImportCode(SpecNode specNode) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("import os\n").append("import sys\n\n"); + Optional.ofNullable(specNode).map(SpecNode::getFileResources).ifPresent(fileResources -> + fileResources.forEach(fileResource -> { + String fileName = fileResource.getName(); + stringBuilder.append(String.format("sys.path.append(os.path.dirname(os.path.abspath('%s')))%n", fileName)); + })); + stringBuilder.append("\n"); + return stringBuilder.toString(); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/ShellParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/ShellParameterConverter.java new file mode 100644 index 0000000..9880e90 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/ShellParameterConverter.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.shell.ShellParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; + +import org.apache.commons.collections4.ListUtils; + + +public class ShellParameterConverter extends AbstractParameterConverter { + + + public ShellParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_SHELL_NODE_TYPE_AS, CodeProgramType.DIDE_SHELL.name()); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(convertType); + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String resourceReference = buildFileResourceReference(specNode, RESOURCE_REFERENCE_PREFIX); + script.setContent(resourceReference + parameter.getRawScript()); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SparkParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SparkParameterConverter.java new file mode 100644 index 0000000..24c091e --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SparkParameterConverter.java @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.codemodel.OdpsSparkCode; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.utils.ArgsUtils; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.model.ProgramType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.model.ResourceInfo; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.spark.SparkConstants; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.spark.SparkParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class SparkParameterConverter extends AbstractParameterConverter { + + private static final String SQL_EXECUTION_TYPE_FILE = "FILE"; + + private static final String RESOURCE_REFERENCE_SQL_PREFIX = "--"; + + public SparkParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + /** + * Each node translates the specific logic of the parameters + */ + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + + convertFileResourceList(specNode); + String type = getSparkConverterType(); + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + //runtime + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + script.setPath(getScriptPath(specNode)); + + String code = convertCode(codeProgramType); + script.setContent(code); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String convertCode(CodeProgramType codeProgramType) { + if (CodeProgramType.EMR_SPARK.equals(codeProgramType)) { + List cmd = populateSparkOptions(parameter); + String code = String.join(" ", cmd); + return code; + } else if (CodeProgramType.ODPS_SPARK.equals(codeProgramType)) { + OdpsSparkCode odpsSparkCode = populateSparkOdpsCode(); + return odpsSparkCode.toString(); + } else { + throw new IllegalArgumentException("Unsupported code program type: " + codeProgramType); + } + } + + private List populateSparkOptions(SparkParameters sparkParameters) { + List args = new ArrayList<>(); + + ProgramType programType = sparkParameters.getProgramType(); + ResourceInfo mainJar = sparkParameters.getMainJar(); + if (programType != ProgramType.SQL) { + String resource = mainJar.getResourceName(); + if (StringUtils.isEmpty(resource)) { + resource = getResourceName(mainJar.getId()); + } + + if (resource != null) { + String[] resources = resource.split("/"); + if (resources.length > 0) { + resource = resources[resources.length - 1]; + } + mainJar.setResourceName(resource); + String dwResource = "##@resource_reference{\"" + resource + "\"} \n"; + args.add(dwResource + SparkConstants.SPARK_SUBMIT_COMMAND); + } else { + args.add(SparkConstants.SPARK_SUBMIT_COMMAND); + } + } else { + args.add(SparkConstants.SPARK_SUBMIT_COMMAND); + } + + String deployMode = StringUtils.isNotEmpty(sparkParameters.getDeployMode()) ? sparkParameters.getDeployMode() + : SparkConstants.DEPLOY_MODE_LOCAL; + + if (!SparkConstants.DEPLOY_MODE_LOCAL.equals(deployMode)) { + args.add(SparkConstants.MASTER); + String masterUrl = SparkConstants.SPARK_ON_YARN; + args.add(masterUrl); + } + args.add(SparkConstants.DEPLOY_MODE); + args.add(deployMode); + + String mainClass = sparkParameters.getMainClass(); + if (programType != ProgramType.PYTHON && programType != ProgramType.SQL && StringUtils.isNotEmpty(mainClass)) { + args.add(SparkConstants.MAIN_CLASS); + args.add(mainClass); + } + + populateSparkResourceDefinitions(args, sparkParameters); + + String appName = sparkParameters.getAppName(); + if (StringUtils.isNotEmpty(appName)) { + args.add(SparkConstants.SPARK_NAME); + args.add(ArgsUtils.escape(appName)); + } + + String others = sparkParameters.getOthers(); + if (!SparkConstants.DEPLOY_MODE_LOCAL.equals(deployMode) + && (StringUtils.isEmpty(others) || !others.contains(SparkConstants.SPARK_YARN_QUEUE))) { + String yarnQueue = sparkParameters.getYarnQueue(); + if (StringUtils.isNotEmpty(yarnQueue)) { + args.add(SparkConstants.SPARK_YARN_QUEUE); + args.add(yarnQueue); + } + } + + // --conf --files --jars --packages + if (StringUtils.isNotEmpty(others)) { + args.add(others); + } + + //jar + if (programType != ProgramType.SQL && mainJar != null) { + args.add(mainJar.getResourceName()); + } + + String mainArgs = sparkParameters.getMainArgs(); + if (programType != ProgramType.SQL && StringUtils.isNotEmpty(mainArgs)) { + args.add(mainArgs); + } + + return args; + } + + private OdpsSparkCode populateSparkOdpsCode() { + OdpsSparkCode odpsSparkCode = new OdpsSparkCode(); + odpsSparkCode.setResourceReferences(new ArrayList<>()); + odpsSparkCode.setSparkJson(new OdpsSparkCode.CodeJson()); + + ResourceInfo mainJar = parameter.getMainJar(); + String resource = mainJar.getResourceName(); + if (StringUtils.isEmpty(resource)) { + resource = getResourceName(mainJar.getId()); + } + + if (resource != null) { + String[] resources = resource.split("/"); + if (resources.length > 0) { + resource = resources[resources.length - 1]; + } + mainJar.setResourceName(resource); + //String dwResource = "##@resource_reference{\"" + resource + "\"} \n"; + //odpsSparkCode.getResourceReferences().add(resource); + odpsSparkCode.getSparkJson().setMainJar(resource); + } + String mainClass = parameter.getMainClass(); + odpsSparkCode.getSparkJson().setMainClass(mainClass); + odpsSparkCode.getSparkJson().setVersion("2.x"); + odpsSparkCode.getSparkJson().setLanguage("java"); + + String mainArgs = parameter.getMainArgs(); + odpsSparkCode.getSparkJson().setArgs(mainArgs); + List confs = new ArrayList<>(); + populateOdpsSparkResourceDefinitions(confs, parameter); + odpsSparkCode.getSparkJson().setConfigs(confs); + return odpsSparkCode; + } + + private void populateSparkResourceDefinitions(List args, SparkParameters sparkParameters) { + int driverCores = sparkParameters.getDriverCores(); + if (driverCores > 0) { + args.add(String.format(SparkConstants.DRIVER_CORES, driverCores)); + } + + String driverMemory = sparkParameters.getDriverMemory(); + if (StringUtils.isNotEmpty(driverMemory)) { + args.add(String.format(SparkConstants.DRIVER_MEMORY, driverMemory)); + } + + int numExecutors = sparkParameters.getNumExecutors(); + if (numExecutors > 0) { + args.add(String.format(SparkConstants.NUM_EXECUTORS, numExecutors)); + } + + int executorCores = sparkParameters.getExecutorCores(); + if (executorCores > 0) { + args.add(String.format(SparkConstants.EXECUTOR_CORES, executorCores)); + } + + String executorMemory = sparkParameters.getExecutorMemory(); + if (StringUtils.isNotEmpty(executorMemory)) { + args.add(String.format(SparkConstants.EXECUTOR_MEMORY, executorMemory)); + } + } + + private void populateOdpsSparkResourceDefinitions(List args, SparkParameters sparkParameters) { + int driverCores = sparkParameters.getDriverCores(); + if (driverCores > 0) { + args.add(String.format("spark.driver.cores=%d", driverCores)); + } + + String driverMemory = sparkParameters.getDriverMemory(); + if (StringUtils.isNotEmpty(driverMemory)) { + args.add(String.format("spark.driver.memory=%s", driverMemory)); + } + + int numExecutors = sparkParameters.getNumExecutors(); + if (numExecutors > 0) { + args.add(String.format("spark.executor.instances=%d", numExecutors)); + } + + int executorCores = sparkParameters.getExecutorCores(); + if (executorCores > 0) { + args.add(String.format("spark.executor.cores=%d", executorCores)); + } + + String executorMemory = sparkParameters.getExecutorMemory(); + if (StringUtils.isNotEmpty(executorMemory)) { + args.add(String.format("spark.executor.memory=%s", executorMemory)); + } + } + + private String getResourceName(Integer id) { + if (id == null) { + return null; + } + DolphinSchedulerV3Context context = DolphinSchedulerV3Context.getContext(); + return CollectionUtils.emptyIfNull(context.getResources()) + .stream() + .filter(r -> r.getId() == id) + .findAny() + .map(r -> { + String name = r.getName(); + if (StringUtils.isEmpty(name)) { + name = r.getFileName(); + } + if (StringUtils.isEmpty(name)) { + name = r.getFullName(); + } + return name; + }) + .orElse(""); + } + + private String getSparkConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_SPARK_SUBMIT_TYPE_AS); + String defaultConvertType = CodeProgramType.EMR_SPARK_SHELL.name(); + return getConverterType(convertType, defaultConvertType); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SqlParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SqlParameterConverter.java new file mode 100644 index 0000000..50fa515 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SqlParameterConverter.java @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.SpecRefEntity; +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.DependencyType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.interfaces.Output; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.enums.DbType; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.sql.SqlParameters; +import com.aliyun.dataworks.migrationx.transformer.core.common.Constants; +import com.aliyun.dataworks.migrationx.transformer.core.utils.EmrCodeUtils; +import com.aliyun.migrationx.common.utils.BeanUtils; +import com.aliyun.migrationx.common.utils.GsonUtils; +import com.aliyun.migrationx.common.utils.UuidGenerators; + +import com.google.gson.reflect.TypeToken; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; + +@Slf4j +public class SqlParameterConverter extends AbstractParameterConverter { + + + public SqlParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + String sqlNodeMapStr = properties.getProperty( + Constants.CONVERTER_TARGET_SQL_NODE_TYPE_MAP, "{}"); + Map sqlTypeNodeTypeMapping = GsonUtils.fromJsonString(sqlNodeMapStr, + new TypeToken>() {}.getType()); + + String type = Optional.ofNullable(sqlTypeNodeTypeMapping) + .map(s -> s.get(parameter.getType())) + .orElseGet(() -> { + if (DbType.HIVE.name().equalsIgnoreCase(parameter.getType())) { + return CodeProgramType.EMR_HIVE.name(); + } else if (DbType.SPARK.name().equalsIgnoreCase(parameter.getType())) { + return CodeProgramType.EMR_SPARK.name(); + } else if (DbType.CLICKHOUSE.name().equalsIgnoreCase(parameter.getType())) { + return CodeProgramType.CLICK_SQL.name(); + } else if (DbType.ofName(parameter.getType()) != null) { + return parameter.getType(); + } else { + String defaultNodeTypeIfNotSupport = getSQLConverterType(); + log.warn("using default node Type {} for node {}", defaultNodeTypeIfNotSupport, specNode.getName()); + return defaultNodeTypeIfNotSupport; + } + }); + + CodeProgramType codeProgramType = CodeProgramType.getNodeTypeByName(type); + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String content = parameter.getSql(); + if (CodeProgramType.EMR_HIVE.name().equals(codeProgramType) || CodeProgramType.EMR_SPARK.name().equals(codeProgramType)) { + content = EmrCodeUtils.toEmrCode(codeProgramType, taskDefinition.getName(), content); + } + + script.setContent(content); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + + //todo + //SpecDatasource datasource = MapUtils.emptyIfNull(context.getDataSourceMap()).get(String.valueOf(parameter.getDatasource())); + //if (Objects.nonNull(datasource)) { + // specNode.setDatasource(datasource); + //} + + //dealPreAndPostSql(specNode, parameter.getPreStatements(), parameter.getPostStatements()); + } + + /** + * parse pre sql and post sql to extra node + * + * @param specNodeRef origin node + * @param preSqlList pre sql list + * @param postSqlList post sql list + */ + private void dealPreAndPostSql(SpecNode specNodeRef, List preSqlList, List postSqlList) { + List specNodeList = new ArrayList<>(); + + //if (CollectionUtils.isNotEmpty(preSqlList)) { + // for (int i = 0; i < preSqlList.size(); i++) { + // String sql = preSqlList.get(i); + // SpecNode preNode = copySpecNode(specNodeRef, sql, "_pre_" + i); + // specNodeList.add(preNode); + // if (i == 0) { + // headList.add(newWrapper(preNode)); + // } + // } + //} + specNodeList.add(specNodeRef); + //if (CollectionUtils.isNotEmpty(postSqlList)) { + // for (int i = 0; i < postSqlList.size(); i++) { + // String sql = postSqlList.get(i); + // SpecNode postNode = copySpecNode(specNodeRef, sql, "_post_" + i); + // specNodeList.add(postNode); + // if (i == postSqlList.size() - 1) { + // tailList.add(newWrapper(postNode)); + // } + // } + //} + if (specNodeList.size() > 1) { + SpecNode pre = specNodeList.get(0); + for (int i = 1; i < specNodeList.size(); i++) { + SpecNode specNode = specNodeList.get(i); + specNode.getInputs().add(getDefaultOutput(pre)); + specNode.getInputs().addAll(getContextOutputs(pre)); + SpecFlowDepend specFlowDepend = newSpecFlowDepend(); + specFlowDepend.setNodeId(specNode); + specFlowDepend.getDepends().add(new SpecDepend(pre, DependencyType.NORMAL, getDefaultOutput(pre))); + getWorkflowDependencyList().add(specFlowDepend); + pre = specNode; + } + } + } + + /** + * copy node, only used in pre and post sql. + * + * @param specNode origin node + * @param sql new sql + * @param suffix new suffix + * @return copied node + */ + private SpecNode copySpecNode(SpecNode specNode, String sql, String suffix) { + SpecNode specNodeCopy = BeanUtils.deepCopy(specNode, SpecNode.class); + specNodeCopy.setId(UuidGenerators.generateUuid()); + specNodeCopy.setName(specNodeCopy.getName() + suffix); + for (Output output : specNodeCopy.getOutputs()) { + if (output instanceof SpecNodeOutput && Boolean.TRUE.equals(((SpecNodeOutput) output).getIsDefault())) { + ((SpecNodeOutput) output).setId(UuidGenerators.generateUuid()); + ((SpecNodeOutput) output).setData(specNodeCopy.getId()); + ((SpecNodeOutput) output).setRefTableName(specNodeCopy.getName()); + } else if (output instanceof SpecRefEntity) { + ((SpecRefEntity) output).setId(UuidGenerators.generateUuid()); + } + } + specWorkflow.getNodes().add(specNodeCopy); + + SpecScript scriptCopy = BeanUtils.deepCopy(specNodeCopy.getScript(), SpecScript.class); + scriptCopy.setId(UuidGenerators.generateUuid()); + scriptCopy.setPath(scriptCopy.getPath() + suffix); + scriptCopy.setContent(sql); + + specNodeCopy.setScript(scriptCopy); + return specNodeCopy; + } + + private String getSQLConverterType() { + String convertType = properties.getProperty(Constants.CONVERTER_TARGET_COMMAND_SQL_TYPE_AS); + return getConverterType(convertType, CodeProgramType.SQL_COMPONENT.name()); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SqoopParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SqoopParameterConverter.java new file mode 100644 index 0000000..51e9cc2 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SqoopParameterConverter.java @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.entity.DataSource; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.sqoop.SqoopParameters; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.sqoop.sources.SourceHdfsParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.sqoop.sources.SourceHiveParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.sqoop.sources.SourceMysqlParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.sqoop.targets.TargetHdfsParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.sqoop.targets.TargetHiveParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.sqoop.targets.TargetMysqlParameter; +import com.aliyun.dataworks.migrationx.domain.dataworks.objects.types.CodeModeType; +import com.aliyun.dataworks.migrationx.transformer.core.sqoop.DIConfigTemplate; +import com.aliyun.dataworks.migrationx.transformer.core.sqoop.DIJsonProcessor; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import com.google.common.base.Joiner; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +public class SqoopParameterConverter extends AbstractParameterConverter { + + public SqoopParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + CodeProgramType codeProgramType = CodeProgramType.DI; + + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + + script.setPath(getScriptPath(specNode)); + String content = convertCode(); + script.setContent(content); + //todo + //String codeMode = CodeModeType.CODE.getValue(); + + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + } + + public String convertCode() { + CodeProgramType codeProgramType = CodeProgramType.DI; + DIJsonProcessor diProcessor = DIJsonProcessor.from(DIConfigTemplate.DI_CODE_TEMPLATE); + DIJsonProcessor readerJp = diProcessor.getConfiguration("steps[0]"); + DIJsonProcessor writerJp = diProcessor.getConfiguration("steps[1]"); + processSqoopSource(readerJp); + processSqoopTarget(writerJp); + diProcessor.set("extend.mode", CodeModeType.CODE.getValue()); + return diProcessor.toJSON(); + } + + private void processSqoopTarget(DIJsonProcessor writerJp) { + List dataSources = DolphinSchedulerV3Context.getContext().getDataSources(); + writerJp.set("stepType", StringUtils.lowerCase(parameter.getTargetType())); + if (StringUtils.equalsIgnoreCase(parameter.getTargetType(), "mysql")) { + TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(parameter.getTargetParams(), + TargetMysqlParameter.class); + Optional.ofNullable(targetMysqlParameter).ifPresent(p -> { + ListUtils.emptyIfNull(dataSources).stream() + .filter(ds -> Objects.equals(ds.getId(), targetMysqlParameter.getTargetDatasource())) + .findFirst().ifPresent(ds -> writerJp.set("parameter.datasource", ds.getName())); + writerJp.set("parameter.table", targetMysqlParameter.getTargetTable()); + writerJp.set("parameter.preSql", Optional.ofNullable(StringUtils.split( + targetMysqlParameter.getPreQuery(), ",")).orElse(new String[]{})); + writerJp.set("parameter.column", StringUtils.isBlank(targetMysqlParameter.getTargetColumns()) ? + new String[]{"*"} : StringUtils.split(targetMysqlParameter.getTargetColumns(), ",")); + if (StringUtils.equalsIgnoreCase(targetMysqlParameter.getTargetUpdateMode(), "updateonly")) { + writerJp.set("parameter.writeMode", "update"); + } else { + writerJp.set("parameter.writeMode", "replace"); + } + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getTargetType(), "hive")) { + TargetHiveParameter targetHiveParameter = JSONUtils.parseObject(parameter.getTargetParams(), + TargetHiveParameter.class); + Optional.ofNullable(targetHiveParameter).ifPresent(p -> { + ListUtils.emptyIfNull(dataSources).stream() + .filter(ds -> StringUtils.equals(java.lang.String.valueOf(ds.getId()), targetHiveParameter.getHiveDatabase())) + .findFirst().ifPresent(ds -> writerJp.set("parameter.datasource", ds.getName())); + writerJp.set("parameter.table", targetHiveParameter.getHiveTable()); + String[] keys = Optional.ofNullable(StringUtils.split( + targetHiveParameter.getHivePartitionKey(), ",")).orElse(new String[]{}); + String[] values = Optional.ofNullable(StringUtils.split( + targetHiveParameter.getHivePartitionValue(), ",")).orElse(new String[]{}); + List partitions = new ArrayList<>(); + for (int i = 0; i < Math.min(keys.length, values.length); i++) { + partitions.add(Joiner.on("=").join(keys[i], values[i])); + } + writerJp.set("parameter.partition", Joiner.on(",").join(partitions)); + writerJp.set("parameter.hdfsUsername", "hdfs"); + writerJp.set("parameter.writeMode", "append"); + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getTargetType(), "hdfs")) { + TargetHdfsParameter targetHdfsParameter = JSONUtils.parseObject(parameter.getSourceParams(), + TargetHdfsParameter.class); + Optional.ofNullable(targetHdfsParameter).ifPresent(p -> { + writerJp.set("parameter.path", p.getTargetPath()); + writerJp.set("parameter.compress", targetHdfsParameter.getCompressionCodec()); + writerJp.set("parameter.datasource", "hdfs"); + writerJp.set("parameter.fileType", targetHdfsParameter.getFileType()); + if (StringUtils.equalsIgnoreCase("parquet", targetHdfsParameter.getFileType())) { + writerJp.set("parameter.writeMode", "noConflict"); + } else { + writerJp.set("parameter.writeMode", "append"); + } + }); + } + } + + private void processSqoopSource(DIJsonProcessor readerJp) { + List dataSources = DolphinSchedulerV3Context.getContext().getDataSources(); + + readerJp.set("stepType", StringUtils.lowerCase(parameter.getSourceType())); + if (StringUtils.equalsIgnoreCase(parameter.getSourceType(), "mysql")) { + SourceMysqlParameter mysqlParameter = JSONUtils.parseObject(parameter.getSourceParams(), + SourceMysqlParameter.class); + Optional.ofNullable(mysqlParameter).ifPresent(p -> { + ListUtils.emptyIfNull(dataSources).stream() + .filter(ds -> Objects.equals(ds.getId(), mysqlParameter.getSrcDatasource())) + .findFirst().ifPresent(ds -> readerJp.set("parameter.datasource", ds.getName())); + readerJp.set("parameter.table", mysqlParameter.getSrcTable()); + readerJp.set("parameter.where", StringUtils.substringAfter( + StringUtils.lowerCase(mysqlParameter.getSrcQuerySql()), "where")); + readerJp.set("parameter.column", StringUtils.isBlank(mysqlParameter.getSrcColumns()) ? + new String[]{"*"} : StringUtils.split(mysqlParameter.getSrcColumns(), ",")); + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getSourceType(), "hive")) { + SourceHiveParameter sourceHiveParameter = JSONUtils.parseObject(parameter.getSourceParams(), + SourceHiveParameter.class); + Optional.ofNullable(sourceHiveParameter).ifPresent(p -> { + ListUtils.emptyIfNull(dataSources).stream() + .filter(ds -> StringUtils.equals(String.valueOf(ds.getId()), sourceHiveParameter.getHiveDatabase())) + .findFirst().ifPresent(ds -> readerJp.set("parameter.datasource", ds.getName())); + readerJp.set("parameter.table", sourceHiveParameter.getHiveTable()); + String[] keys = Optional.ofNullable(StringUtils.split( + sourceHiveParameter.getHivePartitionKey(), ",")).orElse(new String[]{}); + String[] values = Optional.ofNullable(StringUtils.split( + sourceHiveParameter.getHivePartitionValue(), ",")).orElse(new String[]{}); + List partitions = new ArrayList<>(); + for (int i = 0; i < Math.min(keys.length, values.length); i++) { + partitions.add(Joiner.on("=").join(keys[i], values[i])); + } + readerJp.set("parameter.partition", Joiner.on(",").join(partitions)); + readerJp.set("parameter.readMode", "hdfs"); + readerJp.set("parameter.hdfsUsername", "hdfs"); + List> columns = Arrays.stream(keys).map(key -> { + Map column = new HashMap<>(); + column.put("type", "string"); + column.put("value", key); + return column; + }).collect(Collectors.toList()); + readerJp.set("parameter.hivePartitionColumn", columns); + }); + } + + if (StringUtils.equalsIgnoreCase(parameter.getSourceType(), "hdfs")) { + SourceHdfsParameter sourceHdfsParameter = JSONUtils.parseObject(parameter.getSourceParams(), + SourceHdfsParameter.class); + Optional.ofNullable(sourceHdfsParameter).ifPresent( + p -> readerJp.set("parameter.path", p.getExportDir())); + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SubProcessParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SubProcessParameterConverter.java new file mode 100644 index 0000000..f966d55 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SubProcessParameterConverter.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.subprocess.SubProcessParameters; + +import org.apache.commons.collections4.ListUtils; + +public class SubProcessParameterConverter extends AbstractParameterConverter { + + public SubProcessParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + CodeProgramType codeProgramType = CodeProgramType.SUB_PROCESS; + SpecScript script = new SpecScript(); + SpecScriptRuntime runtime = new SpecScriptRuntime(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + script.setPath(getScriptPath(specNode)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + + long depProcessCode = parameter.getProcessDefinitionCode(); + DolphinSchedulerV3Context.getContext().getSubProcessCodeNodeMap().put(depProcessCode, specNode); + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SwitchParameterConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SwitchParameterConverter.java new file mode 100644 index 0000000..3a975a8 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/workflow/parameters/SwitchParameterConverter.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2024. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.parameters; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; +import com.aliyun.dataworks.common.spec.domain.enums.VariableType; +import com.aliyun.dataworks.common.spec.domain.noref.SpecBranch; +import com.aliyun.dataworks.common.spec.domain.noref.SpecBranches; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNode; +import com.aliyun.dataworks.common.spec.domain.ref.SpecNodeOutput; +import com.aliyun.dataworks.common.spec.domain.ref.SpecScript; +import com.aliyun.dataworks.common.spec.domain.ref.SpecVariable; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.common.spec.domain.ref.runtime.SpecScriptRuntime; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DagData; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.DolphinSchedulerV3Context; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.TaskDefinition; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.model.SwitchResultVo; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.v3.task.switchs.SwitchParameters; +import com.aliyun.migrationx.common.utils.JSONUtils; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; + +@Slf4j +public class SwitchParameterConverter extends AbstractParameterConverter { + + public SwitchParameterConverter(Properties properties, SpecWorkflow specWorkflow, DagData processMeta, TaskDefinition taskDefinition) { + super(properties, specWorkflow, processMeta, taskDefinition); + } + + @Override + protected void convertParameter(SpecNode specNode) { + List specVariableList = convertSpecNodeParam(specNode); + CodeProgramType codeProgramType = CodeProgramType.CONTROLLER_BRANCH; + SpecScript script = new SpecScript(); + String language = codeToLanguageIdentifier(codeProgramType); + script.setLanguage(language); + + SpecScriptRuntime runtime = new SpecScriptRuntime(); + runtime.setEngine(codeProgramType.getCalcEngineType().getLabel()); + runtime.setCommand(codeProgramType.getName()); + script.setRuntime(runtime); + script.setPath(getScriptPath(specNode)); + script.setParameters(ListUtils.emptyIfNull(specVariableList).stream().filter(v -> !VariableType.NODE_OUTPUT.equals(v.getType())) + .collect(Collectors.toList())); + specNode.setScript(script); + List branchList = convertParameter(); + SpecBranch branch = new SpecBranch(); + branch.setBranches(branchList); + specNode.setBranch(branch); + } + + public List convertParameter() { + JsonNode param = JSONUtils.parseObject(taskDefinition.getTaskParams(), JsonNode.class); + SwitchParameters switchParameters = null; + if (param.get("switchResult") != null) { + switchParameters = JSONUtils.parseObject(param.get("switchResult"), SwitchParameters.class); + } + if (switchParameters == null || switchParameters.getDependTaskList() == null || switchParameters.getDependTaskList().isEmpty()) { + log.warn("no dependence param {}", taskDefinition.getTaskParams()); + return null; + } + + Long defaultNextCode = switchParameters.getNextNode(); + List branchList = new ArrayList<>(); + + for (SwitchResultVo switchResultVo : switchParameters.getDependTaskList()) { + String condition = switchResultVo.getCondition(); + //task code + Long nextNodeCode = switchResultVo.getNextNode(); + TaskDefinition branchTask = DolphinSchedulerV3Context.getContext().getTaskCodeMap().get(nextNodeCode); + if (branchTask == null) { + continue; + } + /// + SpecBranches b = new SpecBranches(); + //b.setDesc(switchResultVo); + SpecNodeOutput output = new SpecNodeOutput(); + //When the dependent task ends， replace with specNode id + output.setData(String.valueOf(branchTask.getCode())); + b.setOutput(output); + b.setWhen(condition); + branchList.add(b); + } + return branchList; + } + +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/ide/IdeExportProjectConverter.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/ide/IdeExportProjectConverter.java index 8d9cf17..53a332a 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/ide/IdeExportProjectConverter.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/ide/IdeExportProjectConverter.java @@ -69,6 +69,7 @@ import com.aliyun.dataworks.migrationx.transformer.core.report.ReportItemType; import com.aliyun.dataworks.migrationx.transformer.core.report.ReportRiskLevel; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.AbstractBaseConverter; +import com.aliyun.migrationx.common.utils.BeanUtils; import com.aliyun.migrationx.common.utils.GsonUtils; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -84,7 +85,6 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.BeanUtils; /** * @author sam.liux @@ -364,7 +364,7 @@ private DwNode convertIdeFileToNode(IdeFile ideFile) { .filter(io -> Integer.valueOf(0).equals(io.getType())) .map(io -> { NodeContext ctx = new NodeContext(); - BeanUtils.copyProperties(io, ctx); + com.aliyun.migrationx.common.utils.BeanUtils.copyProperties(io, ctx); return ctx; }).collect(Collectors.toList())); node.setOutputContexts(CollectionUtils.emptyIfNull(ideFile.getNodeInputOutputContexts()).stream() diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/DataWorksDolphinSchedulerTransformer.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/DataWorksDolphinSchedulerTransformer.java index 6febb2c..e57876c 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/DataWorksDolphinSchedulerTransformer.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/DataWorksDolphinSchedulerTransformer.java @@ -17,7 +17,6 @@ import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -37,20 +36,15 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.service.impl.DataWorksSpecPackageFileService; import com.aliyun.dataworks.migrationx.transformer.core.transformer.AbstractPackageTransformer; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.AbstractDolphinSchedulerConverter; -import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.DolphinSchedulerV1Converter; -import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.DolphinSchedulerV2Converter; -import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.DolphinSchedulerV3Converter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.nodes.DolphinSchedulerV1Converter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.nodes.DolphinSchedulerV2Converter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.DolphinSchedulerV3Converter; import com.aliyun.migrationx.common.context.TransformerContext; import com.aliyun.migrationx.common.exception.BizException; import com.aliyun.migrationx.common.exception.ErrorCode; -import com.aliyun.migrationx.common.utils.Config; -import com.aliyun.migrationx.common.utils.GsonUtils; -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.reflect.TypeToken; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,21 +57,22 @@ @Slf4j public class DataWorksDolphinSchedulerTransformer extends AbstractPackageTransformer { private static final Logger LOGGER = LoggerFactory.getLogger(DataWorksDolphinSchedulerTransformer.class); - private File packageFile; - private DwProject dwProject; - private DataWorksTransformerConfig dataWorksTransformerConfig; - private Properties converterProperties; + protected File packageFile; + protected DwProject dwProject; + protected DataWorksTransformerConfig dataWorksTransformerConfig; + protected Properties converterProperties; - public DataWorksDolphinSchedulerTransformer(File configFile, DolphinSchedulerPackage sourcePacakgeFile, + public DataWorksDolphinSchedulerTransformer(DataWorksTransformerConfig dataWorksTransformerConfig, DolphinSchedulerPackage sourcePacakgeFile, DataWorksPackage targetPackageFile) { - super(configFile, sourcePacakgeFile, targetPackageFile); + super(null, sourcePacakgeFile, targetPackageFile); + this.dataWorksTransformerConfig = dataWorksTransformerConfig; } @Override public void init() throws Exception { this.sourcePackageFileService = new DolphinSchedulerPackageFileService(); - initConfig(this.configFile); + initConfig(); log.info("target package format: {}", this.dataWorksTransformerConfig.getFormat()); switch (this.dataWorksTransformerConfig.getFormat()) { @@ -85,6 +80,7 @@ public void init() throws Exception { this.targetPackageFileService = new DataWorksDwmaPackageFileService(); break; case SPEC: + case WORKFLOW: this.targetPackageFileService = new DataWorksSpecPackageFileService(); break; default: @@ -95,22 +91,7 @@ public void init() throws Exception { this.targetPackageFileService.setLocale(this.dataWorksTransformerConfig.getLocale()); } - private void initConfig(File configFile) throws IOException { - if (!configFile.exists()) { - log.error("config file not exists: {}", configFile); - throw new BizException(ErrorCode.FILE_NOT_FOUND).with(configFile); - } - - String config = FileUtils.readFileToString(configFile, StandardCharsets.UTF_8); - this.dataWorksTransformerConfig - = JSONUtils.parseObject(config, new TypeReference() {}); - if (this.dataWorksTransformerConfig == null) { - log.error("config file: {}, config class: {}", configFile, DataWorksTransformerConfig.class); - throw new BizException(ErrorCode.PARSE_CONFIG_FILE_FAILED).with(configFile); - } - Config dwConfig = GsonUtils.fromJsonString(config, new TypeToken() {}.getType()); - Config.init(dwConfig); - + private void initConfig() throws IOException { this.dwProject = Optional.ofNullable(this.dataWorksTransformerConfig.getProject()).orElseGet(() -> { DwProject p = new DwProject(); p.setName("tmp_transform_project"); diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/DataWorksPackageFormat.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/DataWorksPackageFormat.java index 010ed60..f3b6304 100644 --- a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/DataWorksPackageFormat.java +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/DataWorksPackageFormat.java @@ -28,5 +28,7 @@ public enum DataWorksPackageFormat { /** * Spec格式 */ - SPEC + SPEC, + + WORKFLOW } diff --git a/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/WorkflowDolphinSchedulerTransformer.java b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/WorkflowDolphinSchedulerTransformer.java new file mode 100644 index 0000000..b661bd1 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/main/java/com/aliyun/dataworks/migrationx/transformer/dataworks/transformer/WorkflowDolphinSchedulerTransformer.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2024, Alibaba Cloud; + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.aliyun.dataworks.migrationx.transformer.dataworks.transformer; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; + +import com.aliyun.dataworks.common.spec.SpecUtil; +import com.aliyun.dataworks.common.spec.domain.DataWorksWorkflowSpec; +import com.aliyun.dataworks.common.spec.domain.Specification; +import com.aliyun.dataworks.common.spec.domain.ref.SpecWorkflow; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.DolphinSchedulerPackage; +import com.aliyun.dataworks.migrationx.domain.dataworks.dolphinscheduler.DolphinSchedulerVersion; +import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DataWorksPackage; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v1.workflow.DolphinSchedulerV1WorkflowConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v2.workflow.DolphinSchedulerV2WorkflowConverter; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.workflow.DolphinSchedulerV3WorkflowConverter; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class WorkflowDolphinSchedulerTransformer extends DataWorksDolphinSchedulerTransformer { + + public WorkflowDolphinSchedulerTransformer(DataWorksTransformerConfig config, DolphinSchedulerPackage sourcePacakgeFile, DataWorksPackage targetPackageFile) { + super(config, sourcePacakgeFile, targetPackageFile); + } + + private List> specifications; + + @Override + public void transform() throws Exception { + this.targetPackage.setDwProject(this.dwProject); + this.packageFile = this.sourcePackage.getPackageFile(); + + DolphinSchedulerVersion version = sourcePackageFileService.getPackage().getPackageInfo().getDolphinSchedulerVersion(); + switch (version) { + case V1: + DolphinSchedulerV1WorkflowConverter v1Converter = + new DolphinSchedulerV1WorkflowConverter(sourcePackageFileService.getPackage(), this.converterProperties); + specifications = v1Converter.convert(); + break; + case V2: + DolphinSchedulerV2WorkflowConverter v2Converter = + new DolphinSchedulerV2WorkflowConverter(sourcePackageFileService.getPackage(), this.converterProperties); + specifications = v2Converter.convert(); + break; + case V3: + DolphinSchedulerV3WorkflowConverter v3Converter = + new DolphinSchedulerV3WorkflowConverter(sourcePackageFileService.getPackage(), this.converterProperties); + specifications = v3Converter.convert(); + break; + default: + throw new RuntimeException("Unsupported version"); + } + } + + @Override + public void write() throws Exception { + File target = targetPackage.getPackageFile(); + doWrite(target); + } + + public void doWrite(File target) { + log.info("write workflow package to {}", target.getAbsolutePath()); + //clear files + if (target.exists()) { + clearDir(target); + } else { + target.mkdirs(); + } + + for (Specification specification : specifications) { + DataWorksWorkflowSpec spec = specification.getSpec(); + for (SpecWorkflow workflow : spec.getWorkflows()) { + String workflowName = workflow.getName(); + File workflowFile = new File(target, workflowName); + if (!workflowFile.exists()) { + workflowFile.mkdir(); + } + File file = new File(workflowFile, specification.getSpec().getName() + ".json"); + try { + Files.write(file.toPath(), SpecUtil.writeToSpec(specification).getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + log.error("write to target file error", e); + throw new RuntimeException(e); + } + } + } + } + + private void clearDir(File target) { + if (target.isDirectory()) { + File[] files = target.listFiles(); + if (files != null && files.length > 0) { + for (File file : target.listFiles()) { + clearDir(file); + } + } else { + if (!target.delete()) { + throw new RuntimeException("delete file " + target.getAbsolutePath() + " failed"); + } + } + } else { + if (!target.delete()) { + throw new RuntimeException("delete file " + target.getAbsolutePath() + " failed"); + } + } + } +} diff --git a/client/migrationx/migrationx-transformer/src/main/python/python2sql/py2sql.py b/client/migrationx/migrationx-transformer/src/main/python/python2sql/py2sql.py index 7b434ac..a6abd6f 100644 --- a/client/migrationx/migrationx-transformer/src/main/python/python2sql/py2sql.py +++ b/client/migrationx/migrationx-transformer/src/main/python/python2sql/py2sql.py @@ -17,7 +17,7 @@ 'ad': 'xingye_ad' } comment_annotation = '-- ' -sql_names = ['sql'] +sql_names = ['sql', 'TARGET_SQL'] parameters = [] diff --git a/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/DataWorksDolphinschedulerTransformerAppTest.java b/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/DataWorksDolphinschedulerTransformerAppTest.java index 0057a5c..0ddbb8a 100644 --- a/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/DataWorksDolphinschedulerTransformerAppTest.java +++ b/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/DataWorksDolphinschedulerTransformerAppTest.java @@ -26,4 +26,18 @@ public void test1() { }; transformerApp.run(args); } + + @Test + public void test2() { + BaseTransformerApp transformerApp = new DataWorksDolphinschedulerTransformerApp(); + String[] args = new String[]{ + "-c", "../../../temp/conf/transformer2.json", + //"-s", "../../temp/13666515015680/.tmp", + //"-s", "../../../temp/shell2", + "-s", "../../../temp/test1111", + //"-s", "../../temp/datax", + "-t", "../../../temp/specs4" + }; + transformerApp.run(args); + } } \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/dwd_xingye_did_client_chat_stats_di.py b/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/dwd_xingye_did_client_chat_stats_di.py deleted file mode 100644 index e66450c..0000000 --- a/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/dwd_xingye_did_client_chat_stats_di.py +++ /dev/null @@ -1,462 +0,0 @@ -from pyspark.sql import SparkSession -from da_utils import * -import sys - -if __name__ == '__main__': - p_date = sys.argv[1] - P_DATE = date_upper(p_date) - # spark = SparkSession.builder.getOrCreate() - spark = SparkSession.builder.config('spark.sql.shuffle.partitions', '2000').getOrCreate() - - - - sql = f""" - -WITH base_data AS ( - SELECT - message_id, - BIGINT(device_id) AS device_id, - BIGINT(npc_id) AS npc_id, - event_name, - event_page, - page, - GET_JSON_OBJECT(properties,'$.chat_id') AS chat_id, - -- CASE - -- WHEN UPPER(properties_lib) = 'ANDROID' THEN IF(GET_JSON_OBJECT(properties,'$.rec_type') IS NOT NULL, 'rec_reply', 'normal') - -- WHEN UPPER(properties_lib) = 'IOS' THEN COALESCE(GET_JSON_OBJECT(properties,'$.rec_type'),'normal') - -- ELSE GET_JSON_OBJECT(properties,'$.rec_type') - -- END AS rec_type, - IF(GET_JSON_OBJECT(properties,'$.rec_type') IS NOT NULL, 'rec_reply', 'normal') AS rec_type, - GET_JSON_OBJECT(properties,'$.chat_type') AS chat_type, - GET_JSON_OBJECT(properties,'$.clk_type') AS clk_type, - GET_JSON_OBJECT(properties,'$.call_status') AS call_status, - GET_JSON_OBJECT(properties,'$.user_or_npc') AS user_or_npc, - GET_JSON_OBJECT(properties,'$.msg_type') AS msg_type, - GET_JSON_OBJECT(properties,'$.base_free_duration') AS base_free_duration, - GET_JSON_OBJECT(properties,'$.deck_bonus_duration') AS deck_bonus_duration, - GET_JSON_OBJECT(properties,'$.voice_call_id') AS voice_call_id, - GET_JSON_OBJECT(properties,'$.call_round_id') AS call_round_id, - GET_JSON_OBJECT(properties,'$.left_call_time') AS left_call_time, - GET_JSON_OBJECT(properties,'$.call_duration') AS call_duration, - GET_JSON_OBJECT(properties,'$.action_type') AS action_type, - GET_JSON_OBJECT(properties,'$.feedback_type') AS feedback_type, - GET_JSON_OBJECT(properties,'$.feedback_id') AS feedback_id, - GET_JSON_OBJECT(properties,'$.item_type') AS item_type, - GET_JSON_OBJECT(properties, '$.duration') AS duration, - CASE GET_JSON_OBJECT(properties, '$.chat_scene') - WHEN 0 THEN '单聊' - WHEN 1 THEN '剧情聊天' - WHEN 2 THEN '群聊' - END AS chat_scene - FROM - dwd_xingye.xingye_app_log_di - WHERE - ymd = '{p_date}' - AND event_name IN ( - 'message_send', - 'retalk_pick_click', - 'retalk_write_comfort_click', - 'msg_send', - 'call_end_status', - 'chat_rec_click', - 'retalk_msg_click', - 'retalk_write_click', - 'msg_action', - 'chat_reset_success', - 'msg_score_click', - 'message_receive', - 'chat_rec_result' - ) - AND app_id = 600 -) -INSERT OVERWRITE TABLE dialogue.dwd_xingye_did_client_chat_stats_di PARTITION(ymd = '{P_DATE}') -SELECT - device_id, - NVL(SUM(IF(metric_name = 'normal_msg_cnt_1d', BIGINT(metric_value), 0)), 0) AS normal_msg_cnt_1d, - NVL(SUM(IF(metric_name = 'single_chat_user_msg_cnt_1d', BIGINT(metric_value), 0)), 0) AS single_chat_user_msg_cnt_1d, - NVL(SUM(IF(metric_name = 'group_chat_user_msg_cnt_1d', BIGINT(metric_value), 0)), 0) AS group_chat_user_msg_cnt_1d, - NVL(SUM(IF(metric_name = 'story_chat_user_msg_cnt_1d', BIGINT(metric_value), 0)), 0) AS story_chat_user_msg_cnt_1d, - NVL(SUM(IF(metric_name = 'sgst_msg_cnt_1d', BIGINT(metric_value), 0)), 0) AS sgst_msg_cnt_1d, - NVL(SUM(IF(metric_name = 'resay_msg_cnt_1d', BIGINT(metric_value), 0)), 0) AS resay_msg_cnt_1d, - NVL(SUM(IF(metric_name = 'rewrite_msg_cnt_1d', BIGINT(metric_value), 0)), 0) AS rewrite_msg_cnt_1d, - NVL(SUM(IF(metric_name = 'voice_msg_cnt_1d', BIGINT(metric_value), 0)), 0) AS voice_msg_cnt_1d, - NVL(SUM(IF(metric_name = 'voice_cnt_1d', BIGINT(metric_value), 0)), 0) AS voice_cnt_1d, - NVL(SUM(IF(metric_name = 'voice_suc_cnt_1d', BIGINT(metric_value), 0)), 0) AS voice_suc_cnt_1d, - NVL(SUM(IF(metric_name = 'voice_user_start_cnt_1d', BIGINT(metric_value), 0)), 0) AS voice_user_start_cnt_1d, - NVL(SUM(IF(metric_name = 'voice_free_duration_cost_1d', BIGINT(metric_value), 0)), 0) AS voice_free_duration_cost_1d, - -- NVL(SUM(IF(metric_name = 'voice_free_bonus_duration_cost_1d', BIGINT(metric_value), 0)), 0) AS voice_free_bonus_duration_cost_1d, - NVL(SUM(IF(metric_name = 'voice_call_duration_1d', BIGINT(metric_value), 0)), 0) AS voice_call_duration_1d, - NVL(SUM(IF(metric_name = 'group_cnt_1d', BIGINT(metric_value), 0)), 0) AS group_cnt_1d, - NVL(SUM(IF(metric_name = 'group_chat_ai_msg_cnt_1d', BIGINT(metric_value), 0)), 0) AS group_chat_ai_msg_cnt_1d, - SPLIT(NVL(MAX(IF(metric_name = 'single_chat_user_msg_cnt_list_1d', metric_value, '')), ''), ',') AS single_chat_user_msg_cnt_list_1d, - SPLIT(NVL(MAX(IF(metric_name = 'group_chat_user_msg_cnt_list_1d', metric_value, '')), ''), ',') AS group_chat_user_msg_cnt_list_1d, - SPLIT(NVL(MAX(IF(metric_name = 'group_chat_total_msg_cnt_list_1d', metric_value, '')), ''), ',') AS group_chat_total_msg_cnt_list_1d, - SPLIT(NVL(MAX(IF(metric_name = 'story_chat_user_msg_cnt_list_1d', metric_value, '')), ''), ',') AS story_chat_user_msg_cnt_list_1d, - SPLIT(NVL(MAX(IF(metric_name = 'voice_chat_user_msg_cnt_list_1d', metric_value, '')), ''), ',') AS voice_chat_user_msg_cnt_list_1d, - NVL(SUM(IF(metric_name = 'story_cnt_1d', BIGINT(metric_value), 0)), 0) AS story_cnt_1d, - NVL(SUM(IF(metric_name = 'sgst_clk_cnt_1d', BIGINT(metric_value), 0)), 0) AS sgst_clk_cnt_1d, - NVL(SUM(IF(metric_name = 'resay_clk_cnt_1d', BIGINT(metric_value), 0)), 0) AS resay_clk_cnt_1d, - NVL(SUM(IF(metric_name = 'rewrite_clk_cnt_1d', BIGINT(metric_value), 0)), 0) AS rewrite_clk_cnt_1d, - NVL(SUM(IF(metric_name = 'backtrack_cnt_1d', BIGINT(metric_value), 0)), 0) AS backtrack_cnt_1d, - NVL(SUM(IF(metric_name = 'reset_cnt_1d', BIGINT(metric_value), 0)), 0) AS reset_cnt_1d, - NVL(SUM(IF(metric_name = 'msg_score_1_cnt_1d', BIGINT(metric_value), 0)), 0) AS msg_score_1_cnt_1d, - NVL(SUM(IF(metric_name = 'msg_score_2_cnt_1d', BIGINT(metric_value), 0)), 0) AS msg_score_2_cnt_1d, - NVL(SUM(IF(metric_name = 'msg_score_3_cnt_1d', BIGINT(metric_value), 0)), 0) AS msg_score_3_cnt_1d, - NVL(SUM(IF(metric_name = 'msg_score_4_cnt_1d', BIGINT(metric_value), 0)), 0) AS msg_score_4_cnt_1d, - NVL(SUM(IF(metric_name = 'msg_score_5_cnt_1d', BIGINT(metric_value), 0)), 0) AS msg_score_5_cnt_1d, - NVL(SUM(IF(metric_name = 'msg_score_6_cnt_1d', BIGINT(metric_value), 0)), 0) AS msg_score_6_cnt_1d, - NVL(SUM(IF(metric_name = 'voice_score_1_cnt_1d', BIGINT(metric_value), 0)), 0) AS voice_score_1_cnt_1d, - NVL(SUM(IF(metric_name = 'voice_score_2_cnt_1d', BIGINT(metric_value), 0)), 0) AS voice_score_2_cnt_1d, - NVL(SUM(IF(metric_name = 'feedback_cnt_1d', BIGINT(metric_value), 0)), 0) AS feedback_cnt_1d, - NVL(SUM(IF(metric_name = 'share_cnt_1d', BIGINT(metric_value), 0)), 0) AS share_cnt_1d, - NVL(SUM(IF(metric_name = 'chat_npc_cnt_1d', BIGINT(metric_value), 0)), 0) AS chat_npc_cnt_1d, - NVL(SUM(IF(metric_name = 'single_ai_reply_ts', BIGINT(metric_value), 0)), 0) AS single_ai_reply_ts, - NVL(SUM(IF(metric_name = 'single_ai_reply_cnt', BIGINT(metric_value), 0)), 0) AS single_ai_reply_cnt, - NVL(SUM(IF(metric_name = 'single_ai_reply_ts_drt_5', BIGINT(metric_value), 0)), 0) AS single_ai_reply_ts_drt_5, - NVL(SUM(IF(metric_name = 'single_ai_reply_ts_drt_20', BIGINT(metric_value), 0)), 0) AS single_ai_reply_ts_drt_20, - NVL(SUM(IF(metric_name = 'single_ai_reply_ts_drt_30', BIGINT(metric_value), 0)), 0) AS single_ai_reply_ts_drt_30, - NVL(SUM(IF(metric_name = 'single_ai_reply_ts_drt_50', BIGINT(metric_value), 0)), 0) AS single_ai_reply_ts_drt_50, - NVL(SUM(IF(metric_name = 'single_ai_reply_ts_drt_80', BIGINT(metric_value), 0)), 0) AS single_ai_reply_ts_drt_80, - NVL(SUM(IF(metric_name = 'single_ai_reply_ts_drt_90', BIGINT(metric_value), 0)), 0) AS single_ai_reply_ts_drt_90, - NVL(SUM(IF(metric_name = 'single_ai_reply_ts_drt_95', BIGINT(metric_value), 0)), 0) AS single_ai_reply_ts_drt_95, - NVL(SUM(IF(metric_name = 'story_ai_reply_ts', BIGINT(metric_value), 0)), 0) AS story_ai_reply_ts, - NVL(SUM(IF(metric_name = 'story_ai_reply_cnt', BIGINT(metric_value), 0)), 0) AS story_ai_reply_cnt, - NVL(SUM(IF(metric_name = 'story_ai_reply_ts_drt_5', BIGINT(metric_value), 0)), 0) AS story_ai_reply_ts_drt_5, - NVL(SUM(IF(metric_name = 'story_ai_reply_ts_drt_20', BIGINT(metric_value), 0)), 0) AS story_ai_reply_ts_drt_20, - NVL(SUM(IF(metric_name = 'story_ai_reply_ts_drt_30', BIGINT(metric_value), 0)), 0) AS story_ai_reply_ts_drt_30, - NVL(SUM(IF(metric_name = 'story_ai_reply_ts_drt_50', BIGINT(metric_value), 0)), 0) AS story_ai_reply_ts_drt_50, - NVL(SUM(IF(metric_name = 'story_ai_reply_ts_drt_80', BIGINT(metric_value), 0)), 0) AS story_ai_reply_ts_drt_80, - NVL(SUM(IF(metric_name = 'story_ai_reply_ts_drt_90', BIGINT(metric_value), 0)), 0) AS story_ai_reply_ts_drt_90, - NVL(SUM(IF(metric_name = 'story_ai_reply_ts_drt_95', BIGINT(metric_value), 0)), 0) AS story_ai_reply_ts_drt_95, - NVL(SUM(IF(metric_name = 'group_ai_reply_ts', BIGINT(metric_value), 0)), 0) AS group_ai_reply_ts, - NVL(SUM(IF(metric_name = 'group_ai_reply_cnt', BIGINT(metric_value), 0)), 0) AS group_ai_reply_cnt, - NVL(SUM(IF(metric_name = 'group_ai_reply_ts_drt_5', BIGINT(metric_value), 0)), 0) AS group_ai_reply_ts_drt_5, - NVL(SUM(IF(metric_name = 'group_ai_reply_ts_drt_20', BIGINT(metric_value), 0)), 0) AS group_ai_reply_ts_drt_20, - NVL(SUM(IF(metric_name = 'group_ai_reply_ts_drt_30', BIGINT(metric_value), 0)), 0) AS group_ai_reply_ts_drt_30, - NVL(SUM(IF(metric_name = 'group_ai_reply_ts_drt_50', BIGINT(metric_value), 0)), 0) AS group_ai_reply_ts_drt_50, - NVL(SUM(IF(metric_name = 'group_ai_reply_ts_drt_80', BIGINT(metric_value), 0)), 0) AS group_ai_reply_ts_drt_80, - NVL(SUM(IF(metric_name = 'group_ai_reply_ts_drt_90', BIGINT(metric_value), 0)), 0) AS group_ai_reply_ts_drt_90, - NVL(SUM(IF(metric_name = 'group_ai_reply_ts_drt_95', BIGINT(metric_value), 0)), 0) AS group_ai_reply_ts_drt_95, - NVL(SUM(IF(metric_name = 'suggest_reply_ts', BIGINT(metric_value), 0)), 0) AS suggest_reply_ts, - NVL(SUM(IF(metric_name = 'suggest_reply_cnt', BIGINT(metric_value), 0)), 0) AS suggest_reply_cnt, - NVL(SUM(IF(metric_name = 'suggest_reply_ts_drt_5', BIGINT(metric_value), 0)), 0) AS suggest_reply_ts_drt_5, - NVL(SUM(IF(metric_name = 'suggest_reply_ts_drt_20', BIGINT(metric_value), 0)), 0) AS suggest_reply_ts_drt_20, - NVL(SUM(IF(metric_name = 'suggest_reply_ts_drt_30', BIGINT(metric_value), 0)), 0) AS suggest_reply_ts_drt_30, - NVL(SUM(IF(metric_name = 'suggest_reply_ts_drt_50', BIGINT(metric_value), 0)), 0) AS suggest_reply_ts_drt_50, - NVL(SUM(IF(metric_name = 'suggest_reply_ts_drt_80', BIGINT(metric_value), 0)), 0) AS suggest_reply_ts_drt_80, - NVL(SUM(IF(metric_name = 'suggest_reply_ts_drt_90', BIGINT(metric_value), 0)), 0) AS suggest_reply_ts_drt_90, - NVL(SUM(IF(metric_name = 'suggest_reply_ts_drt_95', BIGINT(metric_value), 0)), 0) AS suggest_reply_ts_drt_95 -FROM -( - SELECT - device_id, - metric_name, - STRING(metric_value) AS metric_value - FROM - ( - SELECT - device_id, - MAP( - 'normal_msg_cnt_1d' ,normal_msg_cnt_1d, - 'single_chat_user_msg_cnt_1d' ,single_chat_user_msg_cnt_1d, - 'group_chat_user_msg_cnt_1d' ,group_chat_user_msg_cnt_1d, - 'story_chat_user_msg_cnt_1d' ,story_chat_user_msg_cnt_1d, - 'sgst_msg_cnt_1d' ,sgst_msg_cnt_1d, - 'resay_msg_cnt_1d' ,resay_msg_cnt_1d, - 'rewrite_msg_cnt_1d' ,rewrite_msg_cnt_1d, - 'voice_msg_cnt_1d' ,voice_msg_cnt_1d, - 'voice_cnt_1d' ,voice_cnt_1d, - 'voice_suc_cnt_1d' ,voice_suc_cnt_1d, - 'voice_user_start_cnt_1d' ,voice_user_start_cnt_1d, - 'group_cnt_1d' ,group_cnt_1d, - 'group_chat_ai_msg_cnt_1d' ,group_chat_ai_msg_cnt_1d, - 'group_chat_user_msg_cnt_list_1d' ,group_chat_user_msg_cnt_list_1d, - 'group_chat_msg_cnt_list_1d' ,group_chat_msg_cnt_list_1d, - 'story_cnt_1d' ,story_cnt_1d, - 'sgst_clk_cnt_1d' ,sgst_clk_cnt_1d, - 'resay_clk_cnt_1d' ,resay_clk_cnt_1d, - 'rewrite_clk_cnt_1d' ,rewrite_clk_cnt_1d, - 'backtrack_cnt_1d' ,backtrack_cnt_1d, - 'reset_cnt_1d' ,reset_cnt_1d, - 'msg_score_1_cnt_1d' ,msg_score_1_cnt_1d, - 'msg_score_2_cnt_1d' ,msg_score_2_cnt_1d, - 'msg_score_3_cnt_1d' ,msg_score_3_cnt_1d, - 'msg_score_4_cnt_1d' ,msg_score_4_cnt_1d, - 'msg_score_5_cnt_1d' ,msg_score_5_cnt_1d, - 'msg_score_6_cnt_1d' ,msg_score_6_cnt_1d, - 'voice_score_1_cnt_1d' ,voice_score_1_cnt_1d, - 'voice_score_2_cnt_1d' ,voice_score_2_cnt_1d, - 'feedback_cnt_1d' ,feedback_cnt_1d, - 'share_cnt_1d' ,share_cnt_1d, - 'chat_npc_cnt_1d' ,chat_npc_cnt_1d, - 'single_ai_reply_ts' ,single_ai_reply_ts, - 'single_ai_reply_cnt' ,single_ai_reply_cnt, - 'single_ai_reply_ts_drt_5' ,single_ai_reply_ts_drt_5, - 'single_ai_reply_ts_drt_20' ,single_ai_reply_ts_drt_20, - 'single_ai_reply_ts_drt_30' ,single_ai_reply_ts_drt_30, - 'single_ai_reply_ts_drt_50' ,single_ai_reply_ts_drt_50, - 'single_ai_reply_ts_drt_80' ,single_ai_reply_ts_drt_80, - 'single_ai_reply_ts_drt_90' ,single_ai_reply_ts_drt_90, - 'single_ai_reply_ts_drt_95' ,single_ai_reply_ts_drt_95, - 'story_ai_reply_ts' ,story_ai_reply_ts, - 'story_ai_reply_cnt' ,story_ai_reply_cnt, - 'story_ai_reply_ts_drt_5' ,story_ai_reply_ts_drt_5, - 'story_ai_reply_ts_drt_20' ,story_ai_reply_ts_drt_20, - 'story_ai_reply_ts_drt_30' ,story_ai_reply_ts_drt_30, - 'story_ai_reply_ts_drt_50' ,story_ai_reply_ts_drt_50, - 'story_ai_reply_ts_drt_80' ,story_ai_reply_ts_drt_80, - 'story_ai_reply_ts_drt_90' ,story_ai_reply_ts_drt_90, - 'story_ai_reply_ts_drt_95' ,story_ai_reply_ts_drt_95, - 'group_ai_reply_ts' ,group_ai_reply_ts, - 'group_ai_reply_cnt' ,group_ai_reply_cnt, - 'group_ai_reply_ts_drt_5' ,group_ai_reply_ts_drt_5, - 'group_ai_reply_ts_drt_20' ,group_ai_reply_ts_drt_20, - 'group_ai_reply_ts_drt_30' ,group_ai_reply_ts_drt_30, - 'group_ai_reply_ts_drt_50' ,group_ai_reply_ts_drt_50, - 'group_ai_reply_ts_drt_80' ,group_ai_reply_ts_drt_80, - 'group_ai_reply_ts_drt_90' ,group_ai_reply_ts_drt_90, - 'group_ai_reply_ts_drt_95' ,group_ai_reply_ts_drt_95, - 'suggest_reply_ts' ,suggest_reply_ts, - 'suggest_reply_cnt' ,suggest_reply_cnt, - 'suggest_reply_ts_drt_5' ,suggest_reply_ts_drt_5, - 'suggest_reply_ts_drt_20' ,suggest_reply_ts_drt_20, - 'suggest_reply_ts_drt_30' ,suggest_reply_ts_drt_30, - 'suggest_reply_ts_drt_50' ,suggest_reply_ts_drt_50, - 'suggest_reply_ts_drt_80' ,suggest_reply_ts_drt_80, - 'suggest_reply_ts_drt_90' ,suggest_reply_ts_drt_90, - 'suggest_reply_ts_drt_95' ,suggest_reply_ts_drt_95 - ) AS value_map - FROM - ( - SELECT - device_id, - - COUNT(DISTINCT IF(event_name = 'message_send' AND COALESCE(rec_type,'normal') = 'normal' AND COALESCE(msg_type, 'ugc') != 'aigc', message_id, NULL)) AS normal_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') NOT IN ('group_chat','story_chat') AND COALESCE(page, '') NOT IN ('groupchat_page','story_chat_page')), message_id, NULL)) AS single_chat_user_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') = 'group_chat' OR page in ('groupchat_page')) AND COALESCE(msg_type, 'ugc') = 'ugc', message_id, NULL)) AS group_chat_user_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') = 'story_chat' OR page in ('story_chat_page')), message_id, NULL)) AS story_chat_user_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND rec_type IN ('rec_reply','rec_aside','rec_bubble','rec_asside'), message_id, NULL)) AS sgst_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'retalk_pick_click', message_id, NULL)) AS resay_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'retalk_write_comfort_click' AND clk_type = '1', message_id, NULL)) AS rewrite_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_send', call_round_id, NULL)) AS voice_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'call_end_status', voice_call_id, NULL)) AS voice_cnt_1d, - COUNT(DISTINCT IF(event_name = 'call_end_status' AND call_status = 'success', voice_call_id, NULL)) AS voice_suc_cnt_1d, - COUNT(DISTINCT IF(event_name = 'call_end_status' AND user_or_npc = 'user', voice_call_id, NULL)) AS voice_user_start_cnt_1d, - SUM(IF(event_name = 'call_end_status', base_free_duration, NULL)) AS voice_free_duration_cost_1d, - -- SUM(IF(event_name = 'call_end_status', deck_bonus_duration, NULL)) AS voice_free_bonus_duration_cost_1d, - SUM(IF(event_name = 'call_end_status', call_duration, NULL)) AS voice_call_duration_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') = 'group_chat' OR page in ('groupchat_page')), chat_id, NULL)) AS group_cnt_1d, - COUNT(IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') = 'group_chat' OR page in ('groupchat_page')) AND msg_type = 'aigc', 1, NULL)) AS group_chat_ai_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') = 'group_chat' OR page in ('groupchat_page')) AND msg_type = 'ugc', message_id, NULL)) AS group_chat_user_msg_cnt_list_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') = 'group_chat' OR page in ('groupchat_page')), message_id, NULL)) AS group_chat_msg_cnt_list_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') = 'story_chat' OR page in ('story_chat_page')), chat_id, NULL)) AS story_cnt_1d, - COUNT(IF(event_name = 'chat_rec_click', 1, NULL)) AS sgst_clk_cnt_1d, - COUNT(IF(event_name = 'retalk_msg_click', 1, NULL)) AS resay_clk_cnt_1d, - COUNT(IF(event_name = 'retalk_write_click', 1, NULL)) AS rewrite_clk_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_action' AND action_type = 'backtrack', message_id, NULL)) AS backtrack_cnt_1d, - COUNT(IF(event_name = 'chat_reset_success', 1, NULL)) AS reset_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_score_click' AND feedback_type = 'normal_chat' AND feedback_id = '1' AND COALESCE(item_type, 'confirm') = 'confirm', message_id, NULL)) AS msg_score_1_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_score_click' AND feedback_type = 'normal_chat' AND feedback_id = '2' AND COALESCE(item_type, 'confirm') = 'confirm', message_id, NULL)) AS msg_score_2_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_score_click' AND feedback_type = 'normal_chat' AND feedback_id = '3' AND COALESCE(item_type, 'confirm') = 'confirm', message_id, NULL)) AS msg_score_3_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_score_click' AND feedback_type = 'normal_chat' AND feedback_id = '4' AND COALESCE(item_type, 'confirm') = 'confirm', message_id, NULL)) AS msg_score_4_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_score_click' AND feedback_type = 'normal_chat' AND feedback_id = '5' AND COALESCE(item_type, 'confirm') = 'confirm', message_id, NULL)) AS msg_score_5_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_score_click' AND feedback_type = 'normal_chat' AND feedback_id = '6' AND COALESCE(item_type, 'confirm') = 'confirm', message_id, NULL)) AS msg_score_6_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_score_click' AND feedback_type = 'call_chat' AND feedback_id = '1' AND COALESCE(item_type, 'confirm') = 'confirm', voice_call_id, NULL)) AS voice_score_1_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_score_click' AND feedback_type = 'call_chat' AND feedback_id = '2' AND COALESCE(item_type, 'confirm') = 'confirm', voice_call_id, NULL)) AS voice_score_2_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_action' AND action_type IN ('2', 'feedback'), message_id, NULL)) AS feedback_cnt_1d, - COUNT(DISTINCT IF(event_name = 'msg_action' AND action_type = 'share', message_id, NULL)) AS share_cnt_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND BIGINT(npc_id) > 0, npc_id, NULL)) AS chat_npc_cnt_1d, - - SUM(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '单聊', duration, NULL)) AS single_ai_reply_ts, - COUNT(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '单聊', duration, NULL)) AS single_ai_reply_cnt, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '单聊', duration, NULL), 0.05) AS single_ai_reply_ts_drt_5, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '单聊', duration, NULL), 0.2) AS single_ai_reply_ts_drt_20, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '单聊', duration, NULL), 0.30) AS single_ai_reply_ts_drt_30, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '单聊', duration, NULL), 0.50) AS single_ai_reply_ts_drt_50, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '单聊', duration, NULL), 0.80) AS single_ai_reply_ts_drt_80, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '单聊', duration, NULL), 0.90) AS single_ai_reply_ts_drt_90, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '单聊', duration, NULL), 0.95) AS single_ai_reply_ts_drt_95, - - SUM(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '剧情聊天', duration, NULL)) AS story_ai_reply_ts, - COUNT(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '剧情聊天', duration, NULL)) AS story_ai_reply_cnt, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '剧情聊天', duration, NULL), 0.05) AS story_ai_reply_ts_drt_5, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '剧情聊天', duration, NULL), 0.2) AS story_ai_reply_ts_drt_20, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '剧情聊天', duration, NULL), 0.30) AS story_ai_reply_ts_drt_30, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '剧情聊天', duration, NULL), 0.50) AS story_ai_reply_ts_drt_50, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '剧情聊天', duration, NULL), 0.80) AS story_ai_reply_ts_drt_80, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '剧情聊天', duration, NULL), 0.90) AS story_ai_reply_ts_drt_90, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '剧情聊天', duration, NULL), 0.95) AS story_ai_reply_ts_drt_95, - - SUM(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '群聊', duration, NULL)) AS group_ai_reply_ts, - COUNT(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '群聊', duration, NULL)) AS group_ai_reply_cnt, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '群聊', duration, NULL), 0.05) AS group_ai_reply_ts_drt_5, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '群聊', duration, NULL), 0.2) AS group_ai_reply_ts_drt_20, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '群聊', duration, NULL), 0.30) AS group_ai_reply_ts_drt_30, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '群聊', duration, NULL), 0.50) AS group_ai_reply_ts_drt_50, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '群聊', duration, NULL), 0.80) AS group_ai_reply_ts_drt_80, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '群聊', duration, NULL), 0.90) AS group_ai_reply_ts_drt_90, - APPROX_PERCENTILE(IF(event_name = 'message_receive' AND duration < 10000 AND chat_scene = '群聊', duration, NULL), 0.95) AS group_ai_reply_ts_drt_95, - - SUM(IF(event_name = 'chat_rec_result' AND duration < 10000, duration, NULL)) AS suggest_reply_ts, - COUNT(IF(event_name = 'chat_rec_result' AND duration < 10000, duration, NULL)) AS suggest_reply_cnt, - APPROX_PERCENTILE(IF(event_name = 'chat_rec_result' AND duration < 10000, duration, NULL), 0.05) AS suggest_reply_ts_drt_5, - APPROX_PERCENTILE(IF(event_name = 'chat_rec_result' AND duration < 10000, duration, NULL), 0.2) AS suggest_reply_ts_drt_20, - APPROX_PERCENTILE(IF(event_name = 'chat_rec_result' AND duration < 10000, duration, NULL), 0.30) AS suggest_reply_ts_drt_30, - APPROX_PERCENTILE(IF(event_name = 'chat_rec_result' AND duration < 10000, duration, NULL), 0.50) AS suggest_reply_ts_drt_50, - APPROX_PERCENTILE(IF(event_name = 'chat_rec_result' AND duration < 10000, duration, NULL), 0.80) AS suggest_reply_ts_drt_80, - APPROX_PERCENTILE(IF(event_name = 'chat_rec_result' AND duration < 10000, duration, NULL), 0.90) AS suggest_reply_ts_drt_90, - APPROX_PERCENTILE(IF(event_name = 'chat_rec_result' AND duration < 10000, duration, NULL), 0.95) AS suggest_reply_ts_drt_95 - - FROM - base_data - WHERE - event_name IN ( - 'message_send', - 'retalk_pick_click', - 'retalk_write_comfort_click', - 'msg_send', - 'call_end_status', - 'chat_rec_click', - 'retalk_msg_click', - 'retalk_write_click', - 'msg_action', - 'chat_reset_success', - 'msg_score_click', - 'message_receive', - 'chat_rec_result' - ) - GROUP BY - device_id - ) - - ) LATERAL VIEW EXPLODE(value_map) AS metric_name , metric_value - - UNION all - - SELECT - device_id, - metric_name, - STRING(metric_value) AS metric_value - FROM - ( - SELECT - device_id, - MAP( - 'voice_free_duration_cost_1d' ,300000 - MIN(base_free_duration), - -- 'voice_free_bonus_duration_cost_1d' ,SUM(deck_bonus_duration), - 'voice_call_duration_1d' ,SUM(call_duration) - ) AS value_map - FROM - ( - SELECT - device_id, - voice_call_id, - MAX(base_free_duration) AS base_free_duration, - -- MAX(deck_bonus_duration) AS deck_bonus_duration, - MAX(call_duration) AS call_duration - FROM - base_data - WHERE - event_name IN ( - 'call_end_status' - ) - AND BIGINT(DOUBLE(voice_call_id)) > 0 - GROUP BY - device_id, - voice_call_id - ) - GROUP BY - device_id - ) LATERAL VIEW EXPLODE(value_map) AS metric_name , metric_value - - UNION all - - SELECT - device_id, - metric_name, - ARRAY_JOIN(metric_value, ',') AS metric_value - FROM - ( - SELECT - device_id, - MAP( - 'single_chat_user_msg_cnt_list_1d' ,array_remove(COLLECT_LIST(single_chat_user_msg_cnt_1d),0), - 'group_chat_user_msg_cnt_list_1d' ,array_remove(COLLECT_LIST(group_chat_user_msg_cnt_1d),0), - 'group_chat_total_msg_cnt_list_1d' ,array_remove(COLLECT_LIST(group_chat_total_msg_cnt_1d),0), - 'story_chat_user_msg_cnt_list_1d' ,array_remove(COLLECT_LIST(story_chat_user_msg_cnt_1d),0) - ) AS value_map - FROM - ( - SELECT - device_id, - chat_id, - - COUNT(DISTINCT IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') NOT IN ('group_chat','story_chat') AND COALESCE(page, '') NOT IN ('groupchat_page','story_chat_page')), message_id, NULL)) AS single_chat_user_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') = 'group_chat' OR page in ('groupchat_page')) AND COALESCE(msg_type, 'ugc') = 'ugc', message_id, NULL)) AS group_chat_user_msg_cnt_1d, - COUNT(IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') = 'group_chat' OR page in ('groupchat_page')), 1, NULL)) AS group_chat_total_msg_cnt_1d, - COUNT(DISTINCT IF(event_name = 'message_send' AND (COALESCE(chat_type, 'single_chat') = 'story_chat' OR page in ('story_chat_page')), message_id, NULL)) AS story_chat_user_msg_cnt_1d - - FROM - base_data - WHERE - event_name IN ( - 'message_send' - ) - AND COALESCE(chat_id,'') !='' - GROUP BY - device_id, - chat_id - ) - GROUP BY - device_id - ) LATERAL VIEW EXPLODE(value_map) AS metric_name , metric_value - - UNION all - - SELECT - device_id, - metric_name, - ARRAY_JOIN(metric_value, ',') AS metric_value - FROM - ( - SELECT - device_id, - MAP( - 'voice_chat_user_msg_cnt_list_1d' ,array_remove(COLLECT_LIST(voice_chat_user_msg_cnt_1d),0) - ) AS value_map - FROM - ( - SELECT - device_id, - voice_call_id, - COUNT(DISTINCT call_round_id) AS voice_chat_user_msg_cnt_1d - FROM - base_data - WHERE - event_name IN ( - 'msg_send' - ) - AND COALESCE(voice_call_id,'') !='' - GROUP BY - device_id, - voice_call_id - ) - GROUP BY - device_id - ) LATERAL VIEW EXPLODE(value_map) AS metric_name , metric_value -) -GROUP BY - device_id -; - """ - - print(sql) - spark.sql(sql) - - standardized_quality_check({ - 'key_columns': ['device_id'], - 'partition_date': P_DATE, - 'table_name': 'dialogue.dwd_xingye_did_client_chat_stats_di' - }, spark) - - spark.stop() \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/py2sql.py b/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/py2sql.py deleted file mode 100644 index 507414b..0000000 --- a/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/apps/py2sql.py +++ /dev/null @@ -1,95 +0,0 @@ -import ast -import os -import sys -import warnings - -warnings.filterwarnings("ignore", category=DeprecationWarning) - -formatted_mapping = {} -engine_mapping = {} -comment_annotation = '-- ' -sql_names = ['sql'] - - -def in_sql_names(name): - if name in sql_names: - return True - else: - return False - - -def replace_formatted_name(source): - try: - return formatted_mapping[source] - except KeyError: - return source - - -def replace_engine_name(source): - try: - return engine_mapping[source] - except KeyError: - return source - - -def extract_sql(file): - with open(file, 'r', encoding='utf-8') as f: - node = ast.parse(f.read(), file) - - values = [] - for item in node.body: - if isinstance(item, ast.If) and isinstance(item.test, ast.Compare): - if (isinstance(item.test.left, ast.Name) and item.test.left.id == '__name__' and - any(isinstance(op, ast.Eq) for op in item.test.ops) and - any(isinstance(comp, ast.Str) and comp.s == '__main__' for comp in item.test.comparators)): - for main_item in item.body: - if isinstance(main_item, ast.Assign): - for target in main_item.targets: - if isinstance(target, ast.Name): - if in_sql_names(target.id): - if isinstance(main_item.value, ast.JoinedStr): - for v in main_item.value.values: - if isinstance(v, ast.Constant): - values.append(v.value) - elif isinstance(v, ast.FormattedValue): - if isinstance(v.value, ast.Name): - values.append("${" + replace_formatted_name(v.value.id) + "}") - return values - - -def parse_python(file): - if os.path.isdir(file): - for name in os.listdir(file): - try: - file_path = file + os.sep + name - print(f'parsing python file {file_path}') - sql = extract_sql(file_path) - comment = read_python_as_comment(file_path) - data = ''.join(sql) - data = comment + '\n\n' + data - print(f'parsed python file {file_path}') - print(f'name {name} : \n sql {data}') - except Exception as e: - print(e) - - -def read_python_as_comment(file): - lines = [] - with open(file, 'r', encoding='utf-8') as f: - for line in f: - line = comment_annotation + line - lines.append(line) - return ''.join(lines) - - -def test_read_python(): - parse_python("sources") - - -if __name__ == '__main__': - filename = sys.argv[1] - sql = extract_sql(filename) - comment = read_python_as_comment(filename) - data = ''.join(sql) - data = comment + '\n\n' + data - print(data) diff --git a/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DLCParameterConverterTest.java b/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DLCParameterConverterTest.java index 4dd1009..8139cbb 100644 --- a/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DLCParameterConverterTest.java +++ b/client/migrationx/migrationx-transformer/src/test/java/com/aliyun/dataworks/migrationx/transformer/dataworks/converter/dolphinscheduler/v3/DLCParameterConverterTest.java @@ -17,6 +17,7 @@ import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwNode; import com.aliyun.dataworks.migrationx.domain.dataworks.objects.entity.DwWorkflow; import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.DolphinSchedulerConverterContext; +import com.aliyun.dataworks.migrationx.transformer.dataworks.converter.dolphinscheduler.v3.nodes.parameters.DLCParameterConverter; import com.aliyun.migrationx.common.context.TransformerContext; import com.aliyun.migrationx.common.utils.JSONUtils; @@ -66,7 +67,7 @@ public void testDLCParameterConverter() throws Exception { TransformerContext.getContext().setCustomResourceDir("src/test/resources/py2sql"); TransformerContext.getContext().setScriptDir("src/main/python/python2sql"); DLCParameterConverter converter = new DLCParameterConverter(processMeta, taskDefinition, converterContext); - List nodes = converter.doConvert(); + List nodes = converter.convert(); Assert.assertNotNull(nodes); Assert.assertTrue(nodes.size() > 0); } catch (Exception e) { diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/conf/transformer-spec-odps.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/conf/transformer-spec-odps.json new file mode 100644 index 0000000..53d2916 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/conf/transformer-spec-odps.json @@ -0,0 +1,19 @@ +{ + "format": "SPEC", + "locale": "zh_CN", + "skipUnSupportType": true, + "transformContinueWithError": true, + "specContinueWithError": true, + "settings": { + "workflow.converter.shellNodeType": "DIDE_SHELL", + "workflow.converter.commandSqlAs": "ODPS_SQL", + "workflow.converter.sparkSubmitAs": "ODPS_SPARK", + "workflow.converter.target.unknownNodeTypeAs": "DIDE_SHELL", + "workflow.converter.mrNodeType": "ODPS_MR", + "workflow.converter.target.engine.type": "ODPS", + "workflow.converter.dolphinscheduler.sqlNodeTypeMapping": { + "POSTGRESQL": "POSTGRESQL", + "MYSQL": "MYSQL" + } + } +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/conf/transformer-workflow-odps.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/conf/transformer-workflow-odps.json new file mode 100644 index 0000000..e7fb3c6 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/conf/transformer-workflow-odps.json @@ -0,0 +1,19 @@ +{ + "format": "WORKFLOW", + "locale": "zh_CN", + "skipUnSupportType": true, + "transformContinueWithError": true, + "specContinueWithError": true, + "settings": { + "workflow.converter.shellNodeType": "DIDE_SHELL", + "workflow.converter.commandSqlAs": "ODPS_SQL", + "workflow.converter.sparkSubmitAs": "ODPS_SPARK", + "workflow.converter.target.unknownNodeTypeAs": "DIDE_SHELL", + "workflow.converter.mrNodeType": "ODPS_MR", + "workflow.converter.target.engine.type": "ODPS", + "workflow.converter.dolphinscheduler.sqlNodeTypeMapping": { + "POSTGRESQL": "POSTGRESQL", + "MYSQL": "MYSQL" + } + } +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/datasource/datasource_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/datasource/datasource_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/package_info.json new file mode 100644 index 0000000..2cc32aa --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "1.0" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/projects.json new file mode 100644 index 0000000..6208dbe --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/projects.json @@ -0,0 +1,13 @@ +[ + { + "id": 2, + "userId": 1, + "name": "wl_test4", + "description": "", + "createTime": "2025-01-08T13:48:52.240+0800", + "updateTime": "2025-01-08T13:48:52.240+0800", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/projects/wl_test4/processDefinition/process_definitions_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/projects/wl_test4/processDefinition/process_definitions_page_1.json new file mode 100644 index 0000000..9d89363 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/projects/wl_test4/processDefinition/process_definitions_page_1.json @@ -0,0 +1,15 @@ +[ { + "processDefinitionConnects" : "[]", + "processDefinitionJson" : "{\"tenantId\":1,\"globalParams\":[],\"tasks\":[{\"taskInstancePriority\":\"MEDIUM\",\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"name\":\"test2\",\"description\":\"\",\"dependence\":{},\"preTasks\":[],\"id\":\"tasks-27657\",\"runFlag\":\"NORMAL\",\"workerGroup\":\"default\",\"type\":\"SUB_PROCESS\",\"params\":{\"processDefinitionId\":1},\"timeout\":{\"enable\":false,\"strategy\":\"\"}},{\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"description\":\"\",\"runFlag\":\"NORMAL\",\"type\":\"DEPENDENT\",\"params\":{},\"timeout\":{\"enable\":false,\"strategy\":\"\"},\"maxRetryTimes\":\"0\",\"taskInstancePriority\":\"MEDIUM\",\"name\":\"dep\",\"dependence\":{\"dependTaskList\":[{\"dependItemList\":[{\"dateValue\":\"today\",\"definitionName\":\"test1\",\"depTasks\":\"ALL\",\"projectName\":\"wl_test4\",\"projectId\":2,\"cycle\":\"day\",\"definitionId\":1}],\"relation\":\"AND\"}],\"relation\":\"AND\"},\"retryInterval\":\"1\",\"preTasks\":[],\"id\":\"tasks-90667\",\"workerGroup\":\"default\"}],\"timeout\":0}", + "processDefinitionLocations" : "{\"tasks-27657\":{\"name\":\"test2\",\"targetarr\":\"\",\"nodenumber\":\"0\",\"x\":327,\"y\":136},\"tasks-90667\":{\"name\":\"dep\",\"targetarr\":\"\",\"nodenumber\":\"0\",\"x\":497,\"y\":179}}", + "processDefinitionName" : "sub", + "projectName" : "wl_test4", + "processDefinitionId" : 2 +}, { + "processDefinitionConnects" : "[]", + "processDefinitionJson" : "{\"tenantId\":1,\"globalParams\":[],\"tasks\":[{\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"description\":\"\",\"runFlag\":\"NORMAL\",\"type\":\"SHELL\",\"params\":{\"rawScript\":\"#!/bin/bash\\n# 这是一个简单的shell脚本示例\\n\\n# 定义函数\\nfunction greet {\\n echo \\\"欢迎使用我的脚本！\\\"\\n}\\n\\nfunction show_menu {\\n echo \\\"请选择一个选项：\\\"\\n echo \\\"1) 显示当前日期和时间\\\"\\n echo \\\"2) 显示当前用户名\\\"\\n echo \\\"3) 退出程序\\\"\\n}\\n\\nfunction get_date {\\n date\\n}\\n\\nfunction get_username {\\n whoami\\n}\\n\\n# 主程序开始\\ngreet\\nwhile true; do\\n show_menu\\n read -p \\\"请输入你的选择 (1-3): \\\" choice\\n \\n case $choice in\\n 1)\\n get_date\\n ;;\\n 2)\\n get_username\\n ;;\\n 3)\\n echo \\\"再见！\\\"\\n exit 0\\n ;;\\n *)\\n echo \\\"无效的选择，请重新输入！\\\"\\n ;;\\n esac\\ndone\\n\",\"localParams\":[],\"resourceList\":[]},\"timeout\":{\"enable\":false,\"strategy\":\"\"},\"maxRetryTimes\":\"1\",\"taskInstancePriority\":\"MEDIUM\",\"name\":\"shell\",\"dependence\":{},\"retryInterval\":\"1\",\"preTasks\":[],\"id\":\"tasks-69675\",\"workerGroup\":\"default\"},{\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"description\":\"\",\"runFlag\":\"NORMAL\",\"type\":\"PYTHON\",\"params\":{\"rawScript\":\"print(\\\"hello\\\");\",\"localParams\":[],\"resourceList\":[]},\"timeout\":{\"enable\":false,\"strategy\":\"\"},\"maxRetryTimes\":\"0\",\"taskInstancePriority\":\"MEDIUM\",\"name\":\"python\",\"dependence\":{},\"retryInterval\":\"1\",\"preTasks\":[],\"id\":\"tasks-49410\",\"workerGroup\":\"default\"}],\"timeout\":0}", + "processDefinitionLocations" : "{\"tasks-69675\":{\"name\":\"shell\",\"targetarr\":\"\",\"nodenumber\":\"0\",\"x\":239,\"y\":69},\"tasks-49410\":{\"name\":\"python\",\"targetarr\":\"\",\"nodenumber\":\"0\",\"x\":400,\"y\":163}}", + "processDefinitionName" : "test1", + "projectName" : "wl_test4", + "processDefinitionId" : 1 +} ] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/resource/resources.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/resource/resources.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v1/all/udfFunction/udf_function_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/datasource/datasource_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/datasource/datasource_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/package_info.json new file mode 100644 index 0000000..e3e7a7f --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "2.0.5" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/projects.json new file mode 100644 index 0000000..6214e65 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/projects.json @@ -0,0 +1,14 @@ +[ + { + "id": 4, + "userId": 1, + "code": 16230615659328, + "name": "wl_test3", + "description": "", + "createTime": "2025-01-07T14:41:24.838+0800", + "updateTime": "2025-01-07T14:41:24.838+0800", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/projects/wl_test3/processDefinition/process_definitions_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/projects/wl_test3/processDefinition/process_definitions_page_1.json new file mode 100644 index 0000000..edeb0cf --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/projects/wl_test3/processDefinition/process_definitions_page_1.json @@ -0,0 +1,514 @@ +[ { + "processDefinition" : { + "id" : 6, + "code" : 16230666336448, + "name" : "test1", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 16230615659328, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2025-01-07 14:48:00", + "updateTime" : "2025-01-07 14:48:00", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":16230617831360,\"x\":55,\"y\":98},{\"taskCode\":16230644290496,\"x\":249,\"y\":240},{\"taskCode\":16230662974016,\"x\":79,\"y\":327}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0 + }, + "processTaskRelationList" : [ { + "id" : 14, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 16230615659328, + "processDefinitionCode" : 16230666336448, + "preTaskCode" : 16230617831360, + "preTaskVersion" : 1, + "postTaskCode" : 16230644290496, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2025-01-07 14:48:00", + "updateTime" : "2025-01-07 14:48:00", + "operator" : 1, + "operateTime" : "2025-01-07 14:48:00" + }, { + "id" : 15, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 16230615659328, + "processDefinitionCode" : 16230666336448, + "preTaskCode" : 16230617831360, + "preTaskVersion" : 1, + "postTaskCode" : 16230662974016, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2025-01-07 14:48:00", + "updateTime" : "2025-01-07 14:48:00", + "operator" : 1, + "operateTime" : "2025-01-07 14:48:00" + }, { + "id" : 16, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 16230615659328, + "processDefinitionCode" : 16230666336448, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 16230617831360, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2025-01-07 14:48:00", + "updateTime" : "2025-01-07 14:48:00", + "operator" : 1, + "operateTime" : "2025-01-07 14:48:00" + } ], + "taskDefinitionList" : [ { + "id" : 13, + "code" : 16230617831360, + "name" : "shell", + "version" : 1, + "description" : "", + "projectCode" : 16230615659328, + "userId" : 1, + "taskType" : "SHELL", + "taskParams" : { + "resourceList" : [ ], + "localParams" : [ { + "prop" : "id", + "direct" : "IN", + "type" : "VARCHAR", + "value" : "3" + } ], + "rawScript" : "#!/bin/bash\n# 这是一个简单的shell脚本示例\n\n# 定义函数\nfunction greet {\n echo \"欢迎使用我的脚本！\"\n}\n\nfunction show_menu {\n echo \"请选择一个选项：\"\n echo \"1) 显示当前日期和时间\"\n echo \"2) 显示当前用户名\"\n echo \"3) 退出程序\"\n}\n\nfunction get_date {\n date\n}\n\nfunction get_username {\n whoami\n}\n\n# 主程序开始\ngreet\nwhile true; do\n show_menu\n read -p \"请输入你的选择 (1-3): \" choice\n \n case $choice in\n 1)\n get_date\n ;;\n 2)\n get_username\n ;;\n 3)\n echo \"再见！\"\n exit 0\n ;;\n *)\n echo \"无效的选择，请重新输入！\"\n ;;\n esac\ndone\n", + "dependence" : { }, + "conditionResult" : { + "successNode" : [ ], + "failedNode" : [ ] + }, + "waitStartTimeout" : { }, + "switchResult" : { } + }, + "taskParamList" : [ { + "prop" : "id", + "direct" : "IN", + "type" : "VARCHAR", + "value" : "3" + } ], + "taskParamMap" : { + "id" : "3" + }, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2025-01-07 14:48:00", + "updateTime" : "2025-01-07 14:48:00", + "modifyBy" : null, + "operator" : 1, + "operateTime" : "2025-01-07 14:48:00" + }, { + "id" : 14, + "code" : 16230644290496, + "name" : "python", + "version" : 1, + "description" : "", + "projectCode" : 16230615659328, + "userId" : 1, + "taskType" : "PYTHON", + "taskParams" : { + "resourceList" : [ ], + "localParams" : [ ], + "rawScript" : "print(\"hello\");", + "dependence" : { }, + "conditionResult" : { + "successNode" : [ ], + "failedNode" : [ ] + }, + "waitStartTimeout" : { }, + "switchResult" : { } + }, + "taskParamList" : [ ], + "taskParamMap" : { }, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 3, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 2, + "resourceIds" : "", + "createTime" : "2025-01-07 14:48:00", + "updateTime" : "2025-01-07 14:48:00", + "modifyBy" : null, + "operator" : 1, + "operateTime" : "2025-01-07 14:48:00" + }, { + "id" : 15, + "code" : 16230662974016, + "name" : "http", + "version" : 1, + "description" : "", + "projectCode" : 16230615659328, + "userId" : 1, + "taskType" : "HTTP", + "taskParams" : { + "localParams" : [ ], + "httpParams" : [ ], + "url" : "http://www.baidu.com", + "httpMethod" : "GET", + "httpCheckCondition" : "STATUS_CODE_DEFAULT", + "condition" : "", + "connectTimeout" : 60000, + "socketTimeout" : 60000, + "dependence" : { }, + "conditionResult" : { + "successNode" : [ ], + "failedNode" : [ ] + }, + "waitStartTimeout" : { }, + "switchResult" : { } + }, + "taskParamList" : [ ], + "taskParamMap" : { }, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2025-01-07 14:48:00", + "updateTime" : "2025-01-07 14:48:00", + "modifyBy" : null, + "operator" : 1, + "operateTime" : "2025-01-07 14:48:00" + } ], + "schedule" : null +}, { + "processDefinition" : { + "id" : 7, + "code" : 16230680321344, + "name" : "test2", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 16230615659328, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2025-01-07 14:49:50", + "updateTime" : "2025-01-07 14:49:50", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":16230670128448,\"x\":129,\"y\":142},{\"taskCode\":16230677723712,\"x\":346,\"y\":77}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0 + }, + "processTaskRelationList" : [ { + "id" : 17, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 16230615659328, + "processDefinitionCode" : 16230680321344, + "preTaskCode" : 16230670128448, + "preTaskVersion" : 1, + "postTaskCode" : 16230677723712, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2025-01-07 14:49:50", + "updateTime" : "2025-01-07 14:49:50", + "operator" : 1, + "operateTime" : "2025-01-07 14:49:50" + }, { + "id" : 18, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 16230615659328, + "processDefinitionCode" : 16230680321344, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 16230670128448, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2025-01-07 14:49:50", + "updateTime" : "2025-01-07 14:49:50", + "operator" : 1, + "operateTime" : "2025-01-07 14:49:50" + } ], + "taskDefinitionList" : [ { + "id" : 16, + "code" : 16230670128448, + "name" : "dep", + "version" : 1, + "description" : "", + "projectCode" : 16230615659328, + "userId" : 1, + "taskType" : "DEPENDENT", + "taskParams" : { + "dependence" : { + "relation" : "AND", + "dependTaskList" : [ { + "relation" : "AND", + "dependItemList" : [ { + "projectCode" : 16230615659328, + "definitionCode" : 16230666336448, + "depTaskCode" : 0, + "cycle" : "day", + "dateValue" : "today" + } ] + } ] + }, + "conditionResult" : { + "successNode" : [ ], + "failedNode" : [ ] + }, + "waitStartTimeout" : { + "strategy" : "FAILED", + "interval" : null, + "checkInterval" : null, + "enable" : false + }, + "switchResult" : { } + }, + "taskParamList" : null, + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2025-01-07 14:49:50", + "updateTime" : "2025-01-07 14:49:50", + "modifyBy" : null, + "operator" : 1, + "operateTime" : "2025-01-07 14:49:50" + }, { + "id" : 17, + "code" : 16230677723712, + "name" : "sub", + "version" : 1, + "description" : "sub", + "projectCode" : 16230615659328, + "userId" : 1, + "taskType" : "SUB_PROCESS", + "taskParams" : { + "processDefinitionCode" : 16230666336448, + "dependence" : { }, + "conditionResult" : { + "successNode" : [ ], + "failedNode" : [ ] + }, + "waitStartTimeout" : { }, + "switchResult" : { } + }, + "taskParamList" : null, + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2025-01-07 14:49:50", + "updateTime" : "2025-01-07 14:49:50", + "modifyBy" : null, + "operator" : 1, + "operateTime" : "2025-01-07 14:49:50" + } ], + "schedule" : null +}, { + "processDefinition" : { + "id" : 8, + "code" : 16230685917632, + "name" : "test3", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 16230615659328, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2025-01-07 14:50:33", + "updateTime" : "2025-01-07 14:50:33", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":16230681788992,\"x\":119,\"y\":120},{\"taskCode\":16230684374720,\"x\":237,\"y\":260}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0 + }, + "processTaskRelationList" : [ { + "id" : 19, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 16230615659328, + "processDefinitionCode" : 16230685917632, + "preTaskCode" : 16230681788992, + "preTaskVersion" : 1, + "postTaskCode" : 16230684374720, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2025-01-07 14:50:33", + "updateTime" : "2025-01-07 14:50:33", + "operator" : 1, + "operateTime" : "2025-01-07 14:50:33" + }, { + "id" : 20, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 16230615659328, + "processDefinitionCode" : 16230685917632, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 16230681788992, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2025-01-07 14:50:33", + "updateTime" : "2025-01-07 14:50:33", + "operator" : 1, + "operateTime" : "2025-01-07 14:50:33" + } ], + "taskDefinitionList" : [ { + "id" : 18, + "code" : 16230681788992, + "name" : "pthon", + "version" : 1, + "description" : "", + "projectCode" : 16230615659328, + "userId" : 1, + "taskType" : "PYTHON", + "taskParams" : { + "resourceList" : [ ], + "localParams" : [ ], + "rawScript" : "print(\"hello\");", + "dependence" : { }, + "conditionResult" : { + "successNode" : [ ], + "failedNode" : [ ] + }, + "waitStartTimeout" : { }, + "switchResult" : { } + }, + "taskParamList" : [ ], + "taskParamMap" : { }, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2025-01-07 14:50:33", + "updateTime" : "2025-01-07 14:50:33", + "modifyBy" : null, + "operator" : 1, + "operateTime" : "2025-01-07 14:50:33" + }, { + "id" : 19, + "code" : 16230684374720, + "name" : "test3", + "version" : 1, + "description" : "", + "projectCode" : 16230615659328, + "userId" : 1, + "taskType" : "SUB_PROCESS", + "taskParams" : { + "processDefinitionCode" : 16230680321344, + "dependence" : { }, + "conditionResult" : { + "successNode" : [ ], + "failedNode" : [ ] + }, + "waitStartTimeout" : { }, + "switchResult" : { } + }, + "taskParamList" : null, + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2025-01-07 14:50:33", + "updateTime" : "2025-01-07 14:50:33", + "modifyBy" : null, + "operator" : 1, + "operateTime" : "2025-01-07 14:50:33" + } ], + "schedule" : null +} ] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/resource/resources.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/resource/resources.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v2/all/udfFunction/udf_function_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/datasource/datasource_page_1.json new file mode 100644 index 0000000..51d5708 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/datasource/datasource_page_1.json @@ -0,0 +1,13 @@ +[ + { + "id": 1, + "userId": 1, + "userName": "admin", + "name": "pg1", + "note": "", + "type": "POSTGRESQL", + "connectionParams": "{\"user\":\"feilun\",\"password\":\"******\",\"address\":\"jdbc:postgresql://pgm-2ze601n0c1h5ks05go.pg.rds.aliyuncs.com:5432\",\"database\":\"test1234\",\"jdbcUrl\":\"jdbc:postgresql://pgm-2ze601n0c1h5ks05go.pg.rds.aliyuncs.com:5432/test1234\",\"driverClassName\":\"org.postgresql.Driver\",\"validationQuery\":\"select version()\"}", + "createTime": "2024-12-19 06:41:27", + "updateTime": "2024-12-19 06:41:27" + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/package_info.json new file mode 100644 index 0000000..a4d3938 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "3.1.5" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/projects.json new file mode 100644 index 0000000..296e6e4 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/projects.json @@ -0,0 +1,14 @@ +[ + { + "id": 2, + "userId": 1, + "code": 15866588694784, + "name": "test2", + "description": "", + "createTime": "2024-12-05 08:42:04", + "updateTime": "2024-12-05 08:42:04", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/projects/test2/processDefinition/process_definitions_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/projects/test2/processDefinition/process_definitions_page_1.json new file mode 100644 index 0000000..ea7ecb2 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/projects/test2/processDefinition/process_definitions_page_1.json @@ -0,0 +1,458 @@ +[ { + "processDefinition" : { + "id" : 6, + "code" : 15866592686592, + "name" : "shell1", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 15866588694784, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2024-12-05 08:42:35", + "updateTime" : "2024-12-05 08:42:35", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":15866590220288,\"x\":423.9176025390625,\"y\":133.01136779785156}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0, + "executionType" : "PARALLEL" + }, + "processTaskRelationList" : [ { + "id" : 11, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 15866592686592, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 15866590220288, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-05 08:42:35", + "updateTime" : "2024-12-05 08:42:35", + "operator" : 1, + "operateTime" : "2024-12-05 08:42:35" + } ], + "taskDefinitionList" : [ { + "id" : 11, + "code" : 15866590220288, + "name" : "shell2", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "SHELL", + "taskParams" : { + "localParams" : [ ], + "rawScript" : "echo 'test2'", + "resourceList" : [ ] + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-05 08:42:35", + "updateTime" : "2024-12-05 08:42:35", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-05 08:42:35" + } ], + "schedule" : null +}, { + "processDefinition" : { + "id" : 8, + "code" : 16010045991936, + "name" : "python1", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 15866588694784, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2024-12-18 08:01:24", + "updateTime" : "2024-12-18 08:01:24", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":16010040398976,\"x\":481.9176025390625,\"y\":121.01136779785156},{\"taskCode\":16010043196288,\"x\":478.9176025390625,\"y\":274.0113830566406}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0, + "executionType" : "PARALLEL" + }, + "processTaskRelationList" : [ { + "id" : 14, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 16010045991936, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 16010040398976, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-18 08:01:24", + "updateTime" : "2024-12-18 08:01:24", + "operator" : 1, + "operateTime" : "2024-12-18 08:01:24" + }, { + "id" : 15, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 16010045991936, + "preTaskCode" : 16010040398976, + "preTaskVersion" : 1, + "postTaskCode" : 16010043196288, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-18 08:01:24", + "updateTime" : "2024-12-18 08:01:24", + "operator" : 1, + "operateTime" : "2024-12-18 08:01:24" + } ], + "taskDefinitionList" : [ { + "id" : 14, + "code" : 16010040398976, + "name" : "python_task1", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "PYTHON", + "taskParams" : { + "localParams" : [ ], + "rawScript" : "print('test1')", + "resourceList" : [ ] + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-18 08:01:24", + "updateTime" : "2024-12-18 08:01:24", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-18 08:01:24" + }, { + "id" : 15, + "code" : 16010043196288, + "name" : "shell_task2", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "SHELL", + "taskParams" : { + "localParams" : [ ], + "rawScript" : "echo 'tes222'", + "resourceList" : [ ] + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-18 08:01:24", + "updateTime" : "2024-12-18 08:01:24", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-18 08:01:24" + } ], + "schedule" : null +}, { + "processDefinition" : { + "id" : 9, + "code" : 16020498428160, + "name" : "sql_p1", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 15866588694784, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2024-12-19 06:42:23", + "updateTime" : "2024-12-19 06:42:23", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":16020493200256,\"x\":502.9176025390625,\"y\":236.01136779785156}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0, + "executionType" : "PARALLEL" + }, + "processTaskRelationList" : [ { + "id" : 16, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 16020498428160, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 16020493200256, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-19 06:42:23", + "updateTime" : "2024-12-19 06:42:23", + "operator" : 1, + "operateTime" : "2024-12-19 06:42:23" + } ], + "taskDefinitionList" : [ { + "id" : 16, + "code" : 16020493200256, + "name" : "sql_task1", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "SQL", + "taskParams" : { + "localParams" : [ ], + "resourceList" : [ ], + "type" : "POSTGRESQL", + "datasource" : 1, + "sql" : "select * from some;", + "sqlType" : "0", + "preStatements" : [ ], + "postStatements" : [ ], + "segmentSeparator" : "", + "displayRows" : 10 + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-19 06:42:23", + "updateTime" : "2024-12-19 06:42:23", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-19 06:42:23" + } ], + "schedule" : null +}, { + "processDefinition" : { + "id" : 10, + "code" : 16021372891264, + "name" : "subprocess1", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 15866588694784, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2024-12-19 08:36:15", + "updateTime" : "2024-12-19 08:36:15", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":16021366072576,\"x\":618.9176025390625,\"y\":322.0113830566406},{\"taskCode\":16021368344576,\"x\":574.9176025390625,\"y\":59.01136779785156}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0, + "executionType" : "PARALLEL" + }, + "processTaskRelationList" : [ { + "id" : 17, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 16021372891264, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 16021368344576, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-19 08:36:15", + "updateTime" : "2024-12-19 08:36:15", + "operator" : 1, + "operateTime" : "2024-12-19 08:36:15" + }, { + "id" : 18, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 16021372891264, + "preTaskCode" : 16021368344576, + "preTaskVersion" : 1, + "postTaskCode" : 16021366072576, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-19 08:36:15", + "updateTime" : "2024-12-19 08:36:15", + "operator" : 1, + "operateTime" : "2024-12-19 08:36:15" + } ], + "taskDefinitionList" : [ { + "id" : 17, + "code" : 16021366072576, + "name" : "sub_task1", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "SUB_PROCESS", + "taskParams" : { + "localParams" : [ ], + "resourceList" : [ ], + "processDefinitionCode" : 15866592686592 + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-19 08:36:15", + "updateTime" : "2024-12-19 08:36:15", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-19 08:36:15" + }, { + "id" : 18, + "code" : 16021368344576, + "name" : "sub_task_shell1", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "SHELL", + "taskParams" : { + "localParams" : [ ], + "rawScript" : "echo 'test1'", + "resourceList" : [ ] + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-19 08:36:15", + "updateTime" : "2024-12-19 08:36:15", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-19 08:36:15" + } ], + "schedule" : null +} ] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/projects/test2/processDefinition/workflow_1734682376191.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/projects/test2/processDefinition/workflow_1734682376191.json new file mode 100644 index 0000000..809b26a --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/projects/test2/processDefinition/workflow_1734682376191.json @@ -0,0 +1 @@ +[{"processDefinition":{"id":12,"code":16032087046528,"name":"switch_proc1","version":2,"releaseState":"OFFLINE","projectCode":15866588694784,"description":"","globalParams":"[]","globalParamList":[],"globalParamMap":{},"createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 08:12:51","flag":"YES","userId":1,"userName":null,"projectName":null,"locations":"[{\"taskCode\":16032043474944,\"x\":530,\"y\":350},{\"taskCode\":16032048684672,\"x\":567.9176025390625,\"y\":601.0113525390625},{\"taskCode\":16032052121728,\"x\":894.9176025390625,\"y\":494.0113525390625}]","scheduleReleaseState":null,"timeout":0,"tenantId":-1,"tenantCode":null,"modifyBy":null,"warningGroupId":0,"executionType":"PARALLEL"},"processTaskRelationList":[{"id":23,"name":"","processDefinitionVersion":2,"projectCode":15866588694784,"processDefinitionCode":16032087046528,"preTaskCode":0,"preTaskVersion":0,"postTaskCode":16032043474944,"postTaskVersion":2,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-20 08:12:51","updateTime":"2024-12-20 08:12:51","operator":1,"operateTime":"2024-12-20 08:12:51"},{"id":24,"name":"","processDefinitionVersion":2,"projectCode":15866588694784,"processDefinitionCode":16032087046528,"preTaskCode":16032043474944,"preTaskVersion":2,"postTaskCode":16032048684672,"postTaskVersion":1,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-20 08:12:51","updateTime":"2024-12-20 08:12:51","operator":1,"operateTime":"2024-12-20 08:12:51"},{"id":25,"name":"","processDefinitionVersion":2,"projectCode":15866588694784,"processDefinitionCode":16032087046528,"preTaskCode":16032043474944,"preTaskVersion":2,"postTaskCode":16032052121728,"postTaskVersion":1,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-20 08:12:51","updateTime":"2024-12-20 08:12:51","operator":1,"operateTime":"2024-12-20 08:12:51"}],"taskDefinitionList":[{"id":21,"code":16032048684672,"name":"switch_shell1","version":1,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SHELL","taskParams":{"localParams":[],"rawScript":"eco1","resourceList":[]},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"","createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 07:51:20","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-20 07:51:20"},{"id":22,"code":16032052121728,"name":"switch_shell2","version":1,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SHELL","taskParams":{"localParams":[],"rawScript":"eww","resourceList":[]},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"","createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 07:51:20","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-20 07:51:20"},{"id":23,"code":16032043474944,"name":"switch_task1","version":2,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SWITCH","taskParams":{"localParams":[],"rawScript":"","resourceList":[],"switchResult":{"dependTaskList":[{"condition":"if $test==1","nextNode":16032048684672},{"condition":"if $test==2","nextNode":16032052121728}],"nextNode":16032048684672}},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"","createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 08:12:51","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-20 08:12:51"}],"schedule":null}] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/resource/resources.json new file mode 100644 index 0000000..a686ec0 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/resource/resources.json @@ -0,0 +1,107 @@ +[ + { + "id": 7, + "pid": -1, + "alias": "python的脚本模板1 copy.py", + "fullName": "/python的脚本模板1 copy.py", + "description": "", + "fileName": "python的脚本模板1 copy.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:14", + "updateTime": "2024-11-19 06:58:14", + "userName": "admin", + "directory": false + }, + { + "id": 6, + "pid": -1, + "alias": "python的脚本模板1 copy 6.py", + "fullName": "/python的脚本模板1 copy 6.py", + "description": "", + "fileName": "python的脚本模板1 copy 6.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:09", + "updateTime": "2024-11-19 06:58:09", + "userName": "admin", + "directory": false + }, + { + "id": 5, + "pid": -1, + "alias": "python的脚本模板1 copy 5.py", + "fullName": "/python的脚本模板1 copy 5.py", + "description": "", + "fileName": "python的脚本模板1 copy 5.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:03", + "updateTime": "2024-11-19 06:58:03", + "userName": "admin", + "directory": false + }, + { + "id": 4, + "pid": -1, + "alias": "python的脚本模板1 copy 4.py", + "fullName": "/python的脚本模板1 copy 4.py", + "description": "", + "fileName": "python的脚本模板1 copy 4.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:57", + "updateTime": "2024-11-19 06:57:57", + "userName": "admin", + "directory": false + }, + { + "id": 3, + "pid": -1, + "alias": "python的脚本模板1 copy 3.py", + "fullName": "/python的脚本模板1 copy 3.py", + "description": "", + "fileName": "python的脚本模板1 copy 3.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:50", + "updateTime": "2024-11-19 06:57:50", + "userName": "admin", + "directory": false + }, + { + "id": 2, + "pid": -1, + "alias": "python的脚本模板1 copy 2.py", + "fullName": "/python的脚本模板1 copy 2.py", + "description": "", + "fileName": "python的脚本模板1 copy 2.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:44", + "updateTime": "2024-11-19 06:57:44", + "userName": "admin", + "directory": false + }, + { + "id": 1, + "pid": -1, + "alias": "python的脚本模板1.py", + "fullName": "/python的脚本模板1.py", + "description": "", + "fileName": "python的脚本模板1.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:34", + "updateTime": "2024-11-19 06:57:34", + "userName": "admin", + "directory": false + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/dependent/udfFunction/udf_function_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/datasource/datasource_page_1.json new file mode 100644 index 0000000..e69de29 diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/package_info.json new file mode 100644 index 0000000..e63a756 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "3.2.0" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/projects.json new file mode 100644 index 0000000..a947920 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/projects.json @@ -0,0 +1,14 @@ +[ + { + "id": 1, + "userId": 1, + "code": 13942964612128, + "name": "ff", + "description": "", + "createTime": "2024-06-14 18:10:11", + "updateTime": "2024-06-14 18:10:11", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/projects/13942964612128/processDefinition/process_definitions_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/projects/13942964612128/processDefinition/process_definitions_page_1.json new file mode 100644 index 0000000..8ad11fc --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/projects/13942964612128/processDefinition/process_definitions_page_1.json @@ -0,0 +1,97 @@ +[ + { + "processDefinition": { + "id": 10, + "code": 14140238584736, + "name": "mr12", + "version": 1, + "releaseState": "OFFLINE", + "projectCode": 13942964612128, + "description": "", + "globalParams": "[]", + "globalParamList": [], + "globalParamMap": {}, + "createTime": "2024-07-02 14:16:53", + "updateTime": "2024-07-02 14:16:53", + "flag": "YES", + "userId": 1, + "userName": null, + "projectName": null, + "locations": "[{\"taskCode\":14140233200032,\"x\":616.9176025390625,\"y\":200.1889190673828}]", + "scheduleReleaseState": null, + "timeout": 0, + "modifyBy": null, + "warningGroupId": null, + "executionType": "PARALLEL" + }, + "processTaskRelationList": [ + { + "id": 18, + "name": "", + "processDefinitionVersion": 1, + "projectCode": 13942964612128, + "processDefinitionCode": 14140238584736, + "preTaskCode": 0, + "preTaskVersion": 0, + "postTaskCode": 14140233200032, + "postTaskVersion": 1, + "conditionType": "NONE", + "conditionParams": {}, + "createTime": "2024-07-02 14:16:53", + "updateTime": "2024-07-02 14:16:53", + "operator": 1, + "operateTime": "2024-07-02 14:16:53" + } + ], + "taskDefinitionList": [ + { + "id": 18, + "code": 14140233200032, + "name": "mr2", + "version": 1, + "description": "", + "projectCode": 13942964612128, + "userId": 1, + "taskType": "MR", + "taskParams": { + "localParams": [], + "resourceList": [], + "programType": "JAVA", + "mainClass": "org.apache.hadoop.examples.WordCount", + "mainJar": { + "resourceName": "dolphinscheduler/default/resources/hadoop-mapreduce-examples-3.2.1.jar" + }, + "appName": "test1", + "yarnQueue": "" + }, + "taskParamList": [], + "taskParamMap": null, + "flag": "YES", + "isCache": "NO", + "taskPriority": "MEDIUM", + "userName": null, + "projectName": null, + "workerGroup": "default", + "environmentCode": -1, + "failRetryTimes": 0, + "failRetryInterval": 1, + "timeoutFlag": "CLOSE", + "timeoutNotifyStrategy": null, + "timeout": 0, + "delayTime": 0, + "resourceIds": null, + "createTime": "2024-07-02 14:16:53", + "updateTime": "2024-07-02 14:16:53", + "modifyBy": null, + "taskGroupId": 0, + "taskGroupPriority": 0, + "cpuQuota": -1, + "memoryMax": -1, + "taskExecuteType": "BATCH", + "operator": 1, + "operateTime": "2024-07-02 14:16:53" + } + ], + "schedule": null + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/resource/resources.json new file mode 100644 index 0000000..56d2351 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/resource/resources.json @@ -0,0 +1,22 @@ +[ + { + "id": 0, + "pid": "dolphinscheduler/default/resources/", + "name": "hadoop-mapreduce-examples-3.2.1.jar", + "fullName": "dolphinscheduler/default/resources/hadoop-mapreduce-examples-3.2.1.jar", + "children": [], + "type": "FILE", + "idValue": "0_0", + "dirctory": false + }, + { + "id": 0, + "pid": "dolphinscheduler/default/resources/", + "name": "original-spark-examples_2.12-3.2.0.jar", + "fullName": "dolphinscheduler/default/resources/original-spark-examples_2.12-3.2.0.jar", + "children": [], + "type": "FILE", + "idValue": "0_0", + "dirctory": false + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/mr/udfFunction/udf_function_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/datasource/datasource_page_1.json new file mode 100644 index 0000000..6ff028c --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/datasource/datasource_page_1.json @@ -0,0 +1,13 @@ +[ + { + "id": 1, + "userId": 1, + "userName": "admin", + "name": "pg1", + "note": "pg1 test", + "type": "POSTGRESQL", + "connectionParams": "{\"user\":\"feilun\",\"password\":\"******\",\"address\":\"jdbc:postgresql://pgm-2ze5n0y3qv3v3wr4fo.pg.rds.aliyuncs.com:5432\",\"database\":\"test1\",\"jdbcUrl\":\"jdbc:postgresql://pgm-2ze5n0y3qv3v3wr4fo.pg.rds.aliyuncs.com:5432/test1\",\"driverClassName\":\"org.postgresql.Driver\",\"validationQuery\":\"select version()\"}", + "createTime": "2024-05-20 20:10:54", + "updateTime": "2024-05-20 20:10:54" + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/package_info.json new file mode 100644 index 0000000..b87ab15 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "3.0.5" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/projects.json new file mode 100644 index 0000000..3514753 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/projects.json @@ -0,0 +1,14 @@ +[ + { + "id": 1, + "userId": 1, + "code": 13666515015680, + "name": "proj1", + "description": "", + "createTime": "2024-05-20T18:14:08.559+0800", + "updateTime": "2024-05-20T18:14:08.559+0800", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/projects/13666515015680/processDefinition/process_definitions_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/projects/13666515015680/processDefinition/process_definitions_page_1.json new file mode 100644 index 0000000..8611bcc --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/projects/13666515015680/processDefinition/process_definitions_page_1.json @@ -0,0 +1,113 @@ +[ + { + "processDefinition": { + "id": 13, + "code": 13753369295360, + "name": "python1", + "version": 2, + "releaseState": "OFFLINE", + "projectCode": 13666515015680, + "description": "dd", + "globalParams": "[]", + "globalParamList": [], + "globalParamMap": {}, + "createTime": "2024-05-28 14:43:17", + "updateTime": "2024-06-07 14:17:38", + "flag": "YES", + "userId": 1, + "userName": null, + "projectName": null, + "locations": "[{\"taskCode\":13753360391040,\"x\":494.00286865234375,\"y\":160.8181915283203}]", + "scheduleReleaseState": null, + "timeout": 0, + "tenantId": -1, + "tenantCode": null, + "modifyBy": null, + "warningGroupId": 0 + }, + "processTaskRelationList": [ + { + "id": 94, + "name": "", + "processDefinitionVersion": 2, + "projectCode": 13666515015680, + "processDefinitionCode": 13753369295360, + "preTaskCode": 0, + "preTaskVersion": 0, + "postTaskCode": 13753360391040, + "postTaskVersion": 2, + "conditionType": "NONE", + "conditionParams": {}, + "createTime": "2024-06-07 14:17:38", + "updateTime": "2024-06-07 14:17:38", + "operator": 1, + "operateTime": "2024-06-07 14:17:38" + } + ], + "taskDefinitionList": [ + { + "id": 62, + "code": 13753360391040, + "name": "python1", + "version": 2, + "description": "", + "projectCode": 13666515015680, + "userId": 1, + "taskType": "PYTHON", + "taskParams": { + "resourceList": [ + { + "id": 3 + } + ], + "localParams": [ + { + "prop": "weee", + "direct": "IN", + "type": "VARCHAR", + "value": "wwwwa" + } + ], + "rawScript": "print('test')", + "dependence": {}, + "conditionResult": { + "successNode": [], + "failedNode": [] + }, + "waitStartTimeout": {}, + "switchResult": {} + }, + "taskParamList": [ + { + "prop": "weee", + "direct": "IN", + "type": "VARCHAR", + "value": "wwwwa" + } + ], + "taskParamMap": { + "weee": "wwwwa" + }, + "flag": "YES", + "taskPriority": "MEDIUM", + "userName": null, + "projectName": null, + "workerGroup": "default", + "environmentCode": -1, + "failRetryTimes": 0, + "failRetryInterval": 1, + "timeoutFlag": "CLOSE", + "timeoutNotifyStrategy": "WARN", + "timeout": 0, + "delayTime": 0, + "resourceIds": "3", + "createTime": "2024-05-28 14:43:17", + "updateTime": "2024-06-07 14:17:38", + "modifyBy": null, + "operator": 1, + "operateTime": "2024-06-07 14:17:38" + } + ], + "schedule": null + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/resource/resources.json new file mode 100644 index 0000000..8477cd7 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/resource/resources.json @@ -0,0 +1,36 @@ +[ + { + "id": 5, + "pid": -1, + "name": "udf-dir1", + "fullName": "udf-dir1", + "description": "ddd", + "children": [ + { + "id": 7, + "pid": 5, + "name": "HiveUDFs-0.0.1-SNAPSHOT.jar", + "fullName": "udf-dir1HiveUDFs-0.0.1-SNAPSHOT.jar", + "description": "", + "children": [], + "type": "UDF", + "idValue": "7_0", + "dirctory": false + }, + { + "id": 6, + "pid": 5, + "name": "commons-cli-1.4.jar", + "fullName": "udf-dir1/commons-cli-1.4.jar", + "description": "", + "children": [], + "type": "UDF", + "idValue": "6_0", + "dirctory": false + } + ], + "type": "UDF", + "idValue": "5_1", + "dirctory": true + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..44944d9 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/python1/udfFunction/udf_function_page_1.json @@ -0,0 +1,14 @@ +[ + { + "id": 1, + "userId": 1, + "funcName": "Strip", + "className": "org.hardik.letsdobigdata.Strip", + "description": "", + "resourceId": 7, + "resourceName": "udf-dir1/HiveUDFs-0.0.1-SNAPSHOT.jar", + "type": "HIVE", + "createTime": "2024-05-20 20:55:35", + "updateTime": "2024-05-20 20:55:35" + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/datasource/datasource_page_1.json new file mode 100644 index 0000000..51d5708 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/datasource/datasource_page_1.json @@ -0,0 +1,13 @@ +[ + { + "id": 1, + "userId": 1, + "userName": "admin", + "name": "pg1", + "note": "", + "type": "POSTGRESQL", + "connectionParams": "{\"user\":\"feilun\",\"password\":\"******\",\"address\":\"jdbc:postgresql://pgm-2ze601n0c1h5ks05go.pg.rds.aliyuncs.com:5432\",\"database\":\"test1234\",\"jdbcUrl\":\"jdbc:postgresql://pgm-2ze601n0c1h5ks05go.pg.rds.aliyuncs.com:5432/test1234\",\"driverClassName\":\"org.postgresql.Driver\",\"validationQuery\":\"select version()\"}", + "createTime": "2024-12-19 06:41:27", + "updateTime": "2024-12-19 06:41:27" + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/package_info.json new file mode 100644 index 0000000..a4d3938 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "3.1.5" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/projects.json new file mode 100644 index 0000000..296e6e4 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/projects.json @@ -0,0 +1,14 @@ +[ + { + "id": 2, + "userId": 1, + "code": 15866588694784, + "name": "test2", + "description": "", + "createTime": "2024-12-05 08:42:04", + "updateTime": "2024-12-05 08:42:04", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/projects/test2/processDefinition/workflow_1734937013861.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/projects/test2/processDefinition/workflow_1734937013861.json new file mode 100644 index 0000000..35447c7 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/projects/test2/processDefinition/workflow_1734937013861.json @@ -0,0 +1 @@ +[{"processDefinition":{"id":6,"code":15866592686592,"name":"shell1","version":1,"releaseState":"OFFLINE","projectCode":15866588694784,"description":"","globalParams":"[]","globalParamList":[],"globalParamMap":{},"createTime":"2024-12-05 08:42:35","updateTime":"2024-12-05 08:42:35","flag":"YES","userId":1,"userName":null,"projectName":null,"locations":"[{\"taskCode\":15866590220288,\"x\":423.9176025390625,\"y\":133.01136779785156}]","scheduleReleaseState":null,"timeout":0,"tenantId":-1,"tenantCode":null,"modifyBy":null,"warningGroupId":0,"executionType":"PARALLEL"},"processTaskRelationList":[{"id":11,"name":"","processDefinitionVersion":1,"projectCode":15866588694784,"processDefinitionCode":15866592686592,"preTaskCode":0,"preTaskVersion":0,"postTaskCode":15866590220288,"postTaskVersion":1,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-05 08:42:35","updateTime":"2024-12-05 08:42:35","operator":1,"operateTime":"2024-12-05 08:42:35"}],"taskDefinitionList":[{"id":11,"code":15866590220288,"name":"shell2","version":1,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SHELL","taskParams":{"localParams":[],"rawScript":"echo 'test2'","resourceList":[]},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"","createTime":"2024-12-05 08:42:35","updateTime":"2024-12-05 08:42:35","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-05 08:42:35"}],"schedule":null}] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/resource/resources.json new file mode 100644 index 0000000..a686ec0 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/resource/resources.json @@ -0,0 +1,107 @@ +[ + { + "id": 7, + "pid": -1, + "alias": "python的脚本模板1 copy.py", + "fullName": "/python的脚本模板1 copy.py", + "description": "", + "fileName": "python的脚本模板1 copy.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:14", + "updateTime": "2024-11-19 06:58:14", + "userName": "admin", + "directory": false + }, + { + "id": 6, + "pid": -1, + "alias": "python的脚本模板1 copy 6.py", + "fullName": "/python的脚本模板1 copy 6.py", + "description": "", + "fileName": "python的脚本模板1 copy 6.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:09", + "updateTime": "2024-11-19 06:58:09", + "userName": "admin", + "directory": false + }, + { + "id": 5, + "pid": -1, + "alias": "python的脚本模板1 copy 5.py", + "fullName": "/python的脚本模板1 copy 5.py", + "description": "", + "fileName": "python的脚本模板1 copy 5.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:03", + "updateTime": "2024-11-19 06:58:03", + "userName": "admin", + "directory": false + }, + { + "id": 4, + "pid": -1, + "alias": "python的脚本模板1 copy 4.py", + "fullName": "/python的脚本模板1 copy 4.py", + "description": "", + "fileName": "python的脚本模板1 copy 4.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:57", + "updateTime": "2024-11-19 06:57:57", + "userName": "admin", + "directory": false + }, + { + "id": 3, + "pid": -1, + "alias": "python的脚本模板1 copy 3.py", + "fullName": "/python的脚本模板1 copy 3.py", + "description": "", + "fileName": "python的脚本模板1 copy 3.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:50", + "updateTime": "2024-11-19 06:57:50", + "userName": "admin", + "directory": false + }, + { + "id": 2, + "pid": -1, + "alias": "python的脚本模板1 copy 2.py", + "fullName": "/python的脚本模板1 copy 2.py", + "description": "", + "fileName": "python的脚本模板1 copy 2.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:44", + "updateTime": "2024-11-19 06:57:44", + "userName": "admin", + "directory": false + }, + { + "id": 1, + "pid": -1, + "alias": "python的脚本模板1.py", + "fullName": "/python的脚本模板1.py", + "description": "", + "fileName": "python的脚本模板1.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:34", + "updateTime": "2024-11-19 06:57:34", + "userName": "admin", + "directory": false + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/shell/udfFunction/udf_function_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/datasource/datasource_page_1.json new file mode 100644 index 0000000..6ff028c --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/datasource/datasource_page_1.json @@ -0,0 +1,13 @@ +[ + { + "id": 1, + "userId": 1, + "userName": "admin", + "name": "pg1", + "note": "pg1 test", + "type": "POSTGRESQL", + "connectionParams": "{\"user\":\"feilun\",\"password\":\"******\",\"address\":\"jdbc:postgresql://pgm-2ze5n0y3qv3v3wr4fo.pg.rds.aliyuncs.com:5432\",\"database\":\"test1\",\"jdbcUrl\":\"jdbc:postgresql://pgm-2ze5n0y3qv3v3wr4fo.pg.rds.aliyuncs.com:5432/test1\",\"driverClassName\":\"org.postgresql.Driver\",\"validationQuery\":\"select version()\"}", + "createTime": "2024-05-20 20:10:54", + "updateTime": "2024-05-20 20:10:54" + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/package_info.json new file mode 100644 index 0000000..b87ab15 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "3.0.5" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/projects.json new file mode 100644 index 0000000..3514753 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/projects.json @@ -0,0 +1,14 @@ +[ + { + "id": 1, + "userId": 1, + "code": 13666515015680, + "name": "proj1", + "description": "", + "createTime": "2024-05-20T18:14:08.559+0800", + "updateTime": "2024-05-20T18:14:08.559+0800", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/projects/13666515015680/processDefinition/workflow_1735111947074.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/projects/13666515015680/processDefinition/workflow_1735111947074.json new file mode 100644 index 0000000..103ac5e --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/projects/13666515015680/processDefinition/workflow_1735111947074.json @@ -0,0 +1 @@ +[{"processDefinition":{"id":14,"code":16087237096448,"name":"spark_process1","version":1,"releaseState":"OFFLINE","projectCode":15866588694784,"description":"","globalParams":"[]","globalParamList":[],"globalParamMap":{},"createTime":"2024-12-25 07:32:19","updateTime":"2024-12-25 07:32:19","flag":"YES","userId":1,"userName":null,"projectName":null,"locations":"[{\"taskCode\":16087230095872,\"x\":525.9176025390625,\"y\":229.1889190673828}]","scheduleReleaseState":null,"timeout":0,"tenantId":-1,"tenantCode":null,"modifyBy":null,"warningGroupId":0,"executionType":"PARALLEL"},"processTaskRelationList":[{"id":26,"name":"","processDefinitionVersion":1,"projectCode":15866588694784,"processDefinitionCode":16087237096448,"preTaskCode":0,"preTaskVersion":0,"postTaskCode":16087230095872,"postTaskVersion":1,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-25 07:32:19","updateTime":"2024-12-25 07:32:19","operator":1,"operateTime":"2024-12-25 07:32:19"}],"taskDefinitionList":[{"id":24,"code":16087230095872,"name":"spark_task1","version":1,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SPARK","taskParams":{"localParams":[],"rawScript":"","resourceList":[{"id":1}],"programType":"SCALA","mainClass":"com.test1.main.test1","mainJar":{"id":9},"deployMode":"local","mainArgs":"-args ste223","others":"args 2333","sparkVersion":"SPARK2","driverCores":1,"driverMemory":"512M","numExecutors":2,"executorMemory":"2G","executorCores":2},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"1,9","createTime":"2024-12-25 07:32:19","updateTime":"2024-12-25 07:32:19","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-25 07:32:19"}],"schedule":null}] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/resource/resources.json new file mode 100644 index 0000000..8477cd7 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/resource/resources.json @@ -0,0 +1,36 @@ +[ + { + "id": 5, + "pid": -1, + "name": "udf-dir1", + "fullName": "udf-dir1", + "description": "ddd", + "children": [ + { + "id": 7, + "pid": 5, + "name": "HiveUDFs-0.0.1-SNAPSHOT.jar", + "fullName": "udf-dir1HiveUDFs-0.0.1-SNAPSHOT.jar", + "description": "", + "children": [], + "type": "UDF", + "idValue": "7_0", + "dirctory": false + }, + { + "id": 6, + "pid": 5, + "name": "commons-cli-1.4.jar", + "fullName": "udf-dir1/commons-cli-1.4.jar", + "description": "", + "children": [], + "type": "UDF", + "idValue": "6_0", + "dirctory": false + } + ], + "type": "UDF", + "idValue": "5_1", + "dirctory": true + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..44944d9 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/spark/udfFunction/udf_function_page_1.json @@ -0,0 +1,14 @@ +[ + { + "id": 1, + "userId": 1, + "funcName": "Strip", + "className": "org.hardik.letsdobigdata.Strip", + "description": "", + "resourceId": 7, + "resourceName": "udf-dir1/HiveUDFs-0.0.1-SNAPSHOT.jar", + "type": "HIVE", + "createTime": "2024-05-20 20:55:35", + "updateTime": "2024-05-20 20:55:35" + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/datasource/datasource_page_1.json new file mode 100644 index 0000000..8238cd5 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/datasource/datasource_page_1.json @@ -0,0 +1,79 @@ +[ + { + "id": 5, + "userId": 1, + "userName": "admin", + "name": "mysql", + "note": "", + "type": "MYSQL", + "connectionParams": "{\"user\":\"admin\",\"password\":\"******\",\"address\":\"jdbc:mysql://123:3306\",\"database\":\"dw\",\"jdbcUrl\":\"jdbc:mysql://123:3306/dw\",\"driverClassName\":\"com.mysql.cj.jdbc.Driver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-07-05 16:53:57", + "updateTime": "2024-07-08 15:50:27" + }, + { + "id": 7, + "userId": 1, + "userName": "admin", + "name": "mysql_workshop_log", + "note": "", + "type": "MYSQL", + "connectionParams": "{\"user\":\"workshop\",\"password\":\"******\",\"address\":\"jdbc:mysql://101.37.201.51:3306\",\"database\":\"workshop\",\"jdbcUrl\":\"jdbc:mysql://101.37.201.51:3306/workshop\",\"driverClassName\":\"com.mysql.cj.jdbc.Driver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-07-08 15:32:19", + "updateTime": "2024-07-08 15:32:19" + }, + { + "id": 6, + "userId": 1, + "userName": "admin", + "name": "pg", + "note": "", + "type": "POSTGRESQL", + "connectionParams": "{\"user\":\"admin\",\"password\":\"******\",\"address\":\"jdbc:postgresql://pg:5432\",\"database\":\"dw\",\"jdbcUrl\":\"jdbc:postgresql://pg:5432/dw\",\"driverClassName\":\"org.postgresql.Driver\",\"validationQuery\":\"select version()\"}", + "createTime": "2024-07-05 16:58:50", + "updateTime": "2024-07-05 16:58:50" + }, + { + "id": 4, + "userId": 1, + "userName": "admin", + "name": "sqlserver", + "note": "", + "type": "SQLSERVER", + "connectionParams": "{\"user\":\"admin\",\"password\":\"******\",\"address\":\"jdbc:sqlserver://test:1433\",\"database\":\"dw\",\"jdbcUrl\":\"jdbc:sqlserver://test:1433;databaseName\u003ddw\",\"driverClassName\":\"com.microsoft.sqlserver.jdbc.SQLServerDriver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-07-05 16:50:53", + "updateTime": "2024-07-05 16:50:53" + }, + { + "id": 3, + "userId": 1, + "userName": "admin", + "name": "test1", + "note": "", + "type": "CLICKHOUSE", + "connectionParams": "{\"user\":\"feilun\",\"password\":\"******\",\"address\":\"jdbc:clickhouse://pgm-2ze5n0y3qv3v3wr4fo.pg.rds.aliyuncs.com:5432\",\"database\":\"test1\",\"jdbcUrl\":\"jdbc:clickhouse://pgm-2ze5n0y3qv3v3wr4fo.pg.rds.aliyuncs.com:5432/test1\",\"driverClassName\":\"com.clickhouse.jdbc.ClickHouseDriver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-07-05 15:01:57", + "updateTime": "2024-07-05 15:01:57" + }, + { + "id": 2, + "userId": 1, + "userName": "admin", + "name": "spark1", + "note": "", + "type": "SPARK", + "connectionParams": "{\"user\":\"root\",\"password\":\"******\",\"address\":\"jdbc:hive2://172.16.3.197:10001\",\"database\":\"default\",\"jdbcUrl\":\"jdbc:hive2://172.16.3.197:10001/default\",\"driverClassName\":\"org.apache.hive.jdbc.HiveDriver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-06-25 15:38:30", + "updateTime": "2024-06-25 15:38:30" + }, + { + "id": 1, + "userId": 1, + "userName": "admin", + "name": "hive1", + "note": "", + "type": "HIVE", + "connectionParams": "{\"user\":\"root\",\"password\":\"******\",\"address\":\"jdbc:hive2://172.16.3.197:10000\",\"database\":\"default\",\"jdbcUrl\":\"jdbc:hive2://172.16.3.197:10000/default\",\"driverClassName\":\"org.apache.hive.jdbc.HiveDriver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-06-14 18:59:59", + "updateTime": "2024-06-14 18:59:59" + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/package_info.json new file mode 100644 index 0000000..e63a756 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "3.2.0" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/projects.json new file mode 100644 index 0000000..9dc9572 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/projects.json @@ -0,0 +1,38 @@ +[ + { + "id": 1, + "userId": 1, + "code": 13942964612128, + "name": "ff", + "description": "", + "createTime": "2024-06-14 18:10:11", + "updateTime": "2024-06-14 18:10:11", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + }, + { + "id": 2, + "userId": 1, + "code": 14149481615008, + "name": "wl_dolphin3", + "description": "", + "createTime": "2024-07-03 10:20:25", + "updateTime": "2024-07-03 10:20:25", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + }, + { + "id": 3, + "userId": 1, + "code": 14206985639328, + "name": "wl_test111", + "description": "", + "createTime": "2024-07-08 15:07:55", + "updateTime": "2024-07-08 15:07:55", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/projects/13942964612128/processDefinition/process_definitions_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/projects/13942964612128/processDefinition/process_definitions_page_1.json new file mode 100644 index 0000000..306643b --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/projects/13942964612128/processDefinition/process_definitions_page_1.json @@ -0,0 +1,97 @@ +[ + { + "processDefinition": { + "id": 73, + "code": 14438979030176, + "name": "sql2", + "version": 1, + "releaseState": "OFFLINE", + "projectCode": 13942964612128, + "description": "", + "globalParams": "[]", + "globalParamList": [], + "globalParamMap": {}, + "createTime": "2024-07-29 14:35:23", + "updateTime": "2024-07-29 14:35:23", + "flag": "YES", + "userId": 1, + "userName": null, + "projectName": null, + "locations": "[{\"taskCode\":14438974928288,\"x\":494.9176025390625,\"y\":308.1889343261719}]", + "scheduleReleaseState": null, + "timeout": 0, + "modifyBy": null, + "warningGroupId": null, + "executionType": "PARALLEL" + }, + "processTaskRelationList": [ + { + "id": 224, + "name": "", + "processDefinitionVersion": 1, + "projectCode": 13942964612128, + "processDefinitionCode": 14438979030176, + "preTaskCode": 0, + "preTaskVersion": 0, + "postTaskCode": 14438974928288, + "postTaskVersion": 1, + "conditionType": "NONE", + "conditionParams": {}, + "createTime": "2024-07-29 14:35:23", + "updateTime": "2024-07-29 14:35:23", + "operator": 1, + "operateTime": "2024-07-29 14:35:23" + } + ], + "taskDefinitionList": [ + { + "id": 103, + "code": 14438974928288, + "name": "sql2", + "version": 1, + "description": "", + "projectCode": 13942964612128, + "userId": 1, + "taskType": "SQL", + "taskParams": { + "localParams": [], + "resourceList": [], + "type": "MYSQL", + "datasource": 7, + "sql": "select 1;", + "sqlType": "0", + "preStatements": [], + "postStatements": [], + "displayRows": 10 + }, + "taskParamList": [], + "taskParamMap": null, + "flag": "YES", + "isCache": "NO", + "taskPriority": "MEDIUM", + "userName": null, + "projectName": null, + "workerGroup": "default", + "environmentCode": -1, + "failRetryTimes": 0, + "failRetryInterval": 1, + "timeoutFlag": "CLOSE", + "timeoutNotifyStrategy": null, + "timeout": 0, + "delayTime": 0, + "resourceIds": null, + "createTime": "2024-07-29 14:35:23", + "updateTime": "2024-07-29 14:35:23", + "modifyBy": null, + "taskGroupId": 0, + "taskGroupPriority": 0, + "cpuQuota": -1, + "memoryMax": -1, + "taskExecuteType": "BATCH", + "operator": 1, + "operateTime": "2024-07-29 14:35:23" + } + ], + "schedule": null + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/resource/resources.json new file mode 100644 index 0000000..56d2351 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/resource/resources.json @@ -0,0 +1,22 @@ +[ + { + "id": 0, + "pid": "dolphinscheduler/default/resources/", + "name": "hadoop-mapreduce-examples-3.2.1.jar", + "fullName": "dolphinscheduler/default/resources/hadoop-mapreduce-examples-3.2.1.jar", + "children": [], + "type": "FILE", + "idValue": "0_0", + "dirctory": false + }, + { + "id": 0, + "pid": "dolphinscheduler/default/resources/", + "name": "original-spark-examples_2.12-3.2.0.jar", + "fullName": "dolphinscheduler/default/resources/original-spark-examples_2.12-3.2.0.jar", + "children": [], + "type": "FILE", + "idValue": "0_0", + "dirctory": false + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sql/udfFunction/udf_function_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/datasource/datasource_page_1.json new file mode 100644 index 0000000..dc777e0 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/datasource/datasource_page_1.json @@ -0,0 +1,79 @@ +[ + { + "id": 8, + "userId": 1, + "userName": "admin", + "name": "feilun_test1", + "note": "", + "type": "MYSQL", + "connectionParams": "{\"user\":\"admin\",\"password\":\"******\",\"address\":\"jdbc:mysql://123:3306\",\"database\":\"dw\",\"jdbcUrl\":\"jdbc:mysql://123:3306/dw\",\"driverClassName\":\"com.mysql.cj.jdbc.Driver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-07-05 16:53:57", + "updateTime": "2024-07-08 15:50:27" + }, + { + "id": 7, + "userId": 1, + "userName": "admin", + "name": "mysql_workshop_log", + "note": "", + "type": "MYSQL", + "connectionParams": "{\"user\":\"workshop\",\"password\":\"******\",\"address\":\"jdbc:mysql://101.37.201.51:3306\",\"database\":\"workshop\",\"jdbcUrl\":\"jdbc:mysql://101.37.201.51:3306/workshop\",\"driverClassName\":\"com.mysql.cj.jdbc.Driver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-07-08 15:32:19", + "updateTime": "2024-07-08 15:32:19" + }, + { + "id": 6, + "userId": 1, + "userName": "admin", + "name": "pg", + "note": "", + "type": "POSTGRESQL", + "connectionParams": "{\"user\":\"admin\",\"password\":\"******\",\"address\":\"jdbc:postgresql://pg:5432\",\"database\":\"dw\",\"jdbcUrl\":\"jdbc:postgresql://pg:5432/dw\",\"driverClassName\":\"org.postgresql.Driver\",\"validationQuery\":\"select version()\"}", + "createTime": "2024-07-05 16:58:50", + "updateTime": "2024-07-05 16:58:50" + }, + { + "id": 4, + "userId": 1, + "userName": "admin", + "name": "sqlserver", + "note": "", + "type": "SQLSERVER", + "connectionParams": "{\"user\":\"admin\",\"password\":\"******\",\"address\":\"jdbc:sqlserver://test:1433\",\"database\":\"dw\",\"jdbcUrl\":\"jdbc:sqlserver://test:1433;databaseName\u003ddw\",\"driverClassName\":\"com.microsoft.sqlserver.jdbc.SQLServerDriver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-07-05 16:50:53", + "updateTime": "2024-07-05 16:50:53" + }, + { + "id": 3, + "userId": 1, + "userName": "admin", + "name": "test1", + "note": "", + "type": "CLICKHOUSE", + "connectionParams": "{\"user\":\"feilun\",\"password\":\"******\",\"address\":\"jdbc:clickhouse://pgm-2ze5n0y3qv3v3wr4fo.pg.rds.aliyuncs.com:5432\",\"database\":\"test1\",\"jdbcUrl\":\"jdbc:clickhouse://pgm-2ze5n0y3qv3v3wr4fo.pg.rds.aliyuncs.com:5432/test1\",\"driverClassName\":\"com.clickhouse.jdbc.ClickHouseDriver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-07-05 15:01:57", + "updateTime": "2024-07-05 15:01:57" + }, + { + "id": 2, + "userId": 1, + "userName": "admin", + "name": "spark1", + "note": "", + "type": "SPARK", + "connectionParams": "{\"user\":\"root\",\"password\":\"******\",\"address\":\"jdbc:hive2://172.16.3.197:10001\",\"database\":\"default\",\"jdbcUrl\":\"jdbc:hive2://172.16.3.197:10001/default\",\"driverClassName\":\"org.apache.hive.jdbc.HiveDriver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-06-25 15:38:30", + "updateTime": "2024-06-25 15:38:30" + }, + { + "id": 1, + "userId": 1, + "userName": "admin", + "name": "hive1", + "note": "", + "type": "HIVE", + "connectionParams": "{\"user\":\"root\",\"password\":\"******\",\"address\":\"jdbc:hive2://172.16.3.197:10000\",\"database\":\"default\",\"jdbcUrl\":\"jdbc:hive2://172.16.3.197:10000/default\",\"driverClassName\":\"org.apache.hive.jdbc.HiveDriver\",\"validationQuery\":\"select 1\"}", + "createTime": "2024-06-14 18:59:59", + "updateTime": "2024-06-14 18:59:59" + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/package_info.json new file mode 100644 index 0000000..e63a756 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "3.2.0" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/projects.json new file mode 100644 index 0000000..9dc9572 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/projects.json @@ -0,0 +1,38 @@ +[ + { + "id": 1, + "userId": 1, + "code": 13942964612128, + "name": "ff", + "description": "", + "createTime": "2024-06-14 18:10:11", + "updateTime": "2024-06-14 18:10:11", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + }, + { + "id": 2, + "userId": 1, + "code": 14149481615008, + "name": "wl_dolphin3", + "description": "", + "createTime": "2024-07-03 10:20:25", + "updateTime": "2024-07-03 10:20:25", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + }, + { + "id": 3, + "userId": 1, + "code": 14206985639328, + "name": "wl_test111", + "description": "", + "createTime": "2024-07-08 15:07:55", + "updateTime": "2024-07-08 15:07:55", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/projects/13942964612128/processDefinition/process_definitions_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/projects/13942964612128/processDefinition/process_definitions_page_1.json new file mode 100644 index 0000000..6c7f3b7 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/projects/13942964612128/processDefinition/process_definitions_page_1.json @@ -0,0 +1,115 @@ +[ + { + "processDefinition": { + "id": 76, + "code": 14603253125536, + "name": "sqoop_process1", + "version": 1, + "releaseState": "OFFLINE", + "projectCode": 13942964612128, + "description": "", + "globalParams": "[]", + "globalParamList": [], + "globalParamMap": {}, + "createTime": "2024-08-13 11:05:15", + "updateTime": "2024-08-13 11:05:15", + "flag": "YES", + "userId": 1, + "userName": null, + "projectName": null, + "locations": "[{\"taskCode\":14603239785888,\"x\":582.9176025390625,\"y\":187.1889190673828}]", + "scheduleReleaseState": null, + "timeout": 0, + "modifyBy": null, + "warningGroupId": null, + "executionType": "PARALLEL" + }, + "processTaskRelationList": [ + { + "id": 227, + "name": "", + "processDefinitionVersion": 1, + "projectCode": 13942964612128, + "processDefinitionCode": 14603253125536, + "preTaskCode": 0, + "preTaskVersion": 0, + "postTaskCode": 14603239785888, + "postTaskVersion": 1, + "conditionType": "NONE", + "conditionParams": {}, + "createTime": "2024-08-13 11:05:15", + "updateTime": "2024-08-13 11:05:15", + "operator": 1, + "operateTime": "2024-08-13 11:05:15" + } + ], + "taskDefinitionList": [ + { + "id": 106, + "code": 14603239785888, + "name": "sqoop1", + "version": 1, + "description": "", + "projectCode": 13942964612128, + "userId": 1, + "taskType": "SQOOP", + "taskParams": { + "localParams": [], + "resourceList": [], + "jobType": "TEMPLATE", + "jobName": "sqooptask", + "hadoopCustomParams": [ + { + "prop": "hadoopparam", + "direct": "IN", + "type": "VARCHAR", + "value": "3" + } + ], + "sqoopAdvancedParams": [ + { + "prop": "sqoopparam", + "direct": "IN", + "type": "VARCHAR", + "value": "2333" + } + ], + "concurrency": 2, + "splitBy": "id", + "modelType": "import", + "sourceType": "MYSQL", + "targetType": "HDFS", + "targetParams": "{\"targetPath\":\"/home/test\",\"deleteTargetDir\":true,\"compressionCodec\":\"snappy\",\"fileType\":\"--as-avrodatafile\",\"fieldsTerminated\":\",\",\"linesTerminated\":\"\\\\n\"}", + "sourceParams": "{\"srcTable\":\"\",\"srcColumnType\":\"0\",\"srcColumns\":\"\",\"srcQuerySql\":\"select * from table1;\",\"srcQueryType\":\"1\",\"srcType\":\"MYSQL\",\"srcDatasource\":7,\"mapColumnHive\":[{\"prop\":\"hivetype\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"string\"}],\"mapColumnJava\":[{\"prop\":\"java type\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"string\"}]}" + }, + "taskParamList": [], + "taskParamMap": null, + "flag": "YES", + "isCache": "NO", + "taskPriority": "MEDIUM", + "userName": null, + "projectName": null, + "workerGroup": "default", + "environmentCode": -1, + "failRetryTimes": 0, + "failRetryInterval": 1, + "timeoutFlag": "CLOSE", + "timeoutNotifyStrategy": null, + "timeout": 0, + "delayTime": 0, + "resourceIds": null, + "createTime": "2024-08-13 11:05:15", + "updateTime": "2024-08-13 11:05:15", + "modifyBy": null, + "taskGroupId": 0, + "taskGroupPriority": 0, + "cpuQuota": -1, + "memoryMax": -1, + "taskExecuteType": "BATCH", + "operator": 1, + "operateTime": "2024-08-13 11:05:15" + } + ], + "schedule": null + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/resource/resources.json new file mode 100644 index 0000000..56d2351 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/resource/resources.json @@ -0,0 +1,22 @@ +[ + { + "id": 0, + "pid": "dolphinscheduler/default/resources/", + "name": "hadoop-mapreduce-examples-3.2.1.jar", + "fullName": "dolphinscheduler/default/resources/hadoop-mapreduce-examples-3.2.1.jar", + "children": [], + "type": "FILE", + "idValue": "0_0", + "dirctory": false + }, + { + "id": 0, + "pid": "dolphinscheduler/default/resources/", + "name": "original-spark-examples_2.12-3.2.0.jar", + "fullName": "dolphinscheduler/default/resources/original-spark-examples_2.12-3.2.0.jar", + "children": [], + "type": "FILE", + "idValue": "0_0", + "dirctory": false + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/sqoop/udfFunction/udf_function_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/datasource/datasource_page_1.json new file mode 100644 index 0000000..51d5708 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/datasource/datasource_page_1.json @@ -0,0 +1,13 @@ +[ + { + "id": 1, + "userId": 1, + "userName": "admin", + "name": "pg1", + "note": "", + "type": "POSTGRESQL", + "connectionParams": "{\"user\":\"feilun\",\"password\":\"******\",\"address\":\"jdbc:postgresql://pgm-2ze601n0c1h5ks05go.pg.rds.aliyuncs.com:5432\",\"database\":\"test1234\",\"jdbcUrl\":\"jdbc:postgresql://pgm-2ze601n0c1h5ks05go.pg.rds.aliyuncs.com:5432/test1234\",\"driverClassName\":\"org.postgresql.Driver\",\"validationQuery\":\"select version()\"}", + "createTime": "2024-12-19 06:41:27", + "updateTime": "2024-12-19 06:41:27" + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/package_info.json new file mode 100644 index 0000000..a4d3938 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "3.1.5" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/projects.json new file mode 100644 index 0000000..296e6e4 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/projects.json @@ -0,0 +1,14 @@ +[ + { + "id": 2, + "userId": 1, + "code": 15866588694784, + "name": "test2", + "description": "", + "createTime": "2024-12-05 08:42:04", + "updateTime": "2024-12-05 08:42:04", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/projects/test2/processDefinition/process_definitions_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/projects/test2/processDefinition/process_definitions_page_1.json new file mode 100644 index 0000000..ea7ecb2 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/projects/test2/processDefinition/process_definitions_page_1.json @@ -0,0 +1,458 @@ +[ { + "processDefinition" : { + "id" : 6, + "code" : 15866592686592, + "name" : "shell1", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 15866588694784, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2024-12-05 08:42:35", + "updateTime" : "2024-12-05 08:42:35", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":15866590220288,\"x\":423.9176025390625,\"y\":133.01136779785156}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0, + "executionType" : "PARALLEL" + }, + "processTaskRelationList" : [ { + "id" : 11, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 15866592686592, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 15866590220288, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-05 08:42:35", + "updateTime" : "2024-12-05 08:42:35", + "operator" : 1, + "operateTime" : "2024-12-05 08:42:35" + } ], + "taskDefinitionList" : [ { + "id" : 11, + "code" : 15866590220288, + "name" : "shell2", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "SHELL", + "taskParams" : { + "localParams" : [ ], + "rawScript" : "echo 'test2'", + "resourceList" : [ ] + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-05 08:42:35", + "updateTime" : "2024-12-05 08:42:35", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-05 08:42:35" + } ], + "schedule" : null +}, { + "processDefinition" : { + "id" : 8, + "code" : 16010045991936, + "name" : "python1", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 15866588694784, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2024-12-18 08:01:24", + "updateTime" : "2024-12-18 08:01:24", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":16010040398976,\"x\":481.9176025390625,\"y\":121.01136779785156},{\"taskCode\":16010043196288,\"x\":478.9176025390625,\"y\":274.0113830566406}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0, + "executionType" : "PARALLEL" + }, + "processTaskRelationList" : [ { + "id" : 14, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 16010045991936, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 16010040398976, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-18 08:01:24", + "updateTime" : "2024-12-18 08:01:24", + "operator" : 1, + "operateTime" : "2024-12-18 08:01:24" + }, { + "id" : 15, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 16010045991936, + "preTaskCode" : 16010040398976, + "preTaskVersion" : 1, + "postTaskCode" : 16010043196288, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-18 08:01:24", + "updateTime" : "2024-12-18 08:01:24", + "operator" : 1, + "operateTime" : "2024-12-18 08:01:24" + } ], + "taskDefinitionList" : [ { + "id" : 14, + "code" : 16010040398976, + "name" : "python_task1", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "PYTHON", + "taskParams" : { + "localParams" : [ ], + "rawScript" : "print('test1')", + "resourceList" : [ ] + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-18 08:01:24", + "updateTime" : "2024-12-18 08:01:24", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-18 08:01:24" + }, { + "id" : 15, + "code" : 16010043196288, + "name" : "shell_task2", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "SHELL", + "taskParams" : { + "localParams" : [ ], + "rawScript" : "echo 'tes222'", + "resourceList" : [ ] + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-18 08:01:24", + "updateTime" : "2024-12-18 08:01:24", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-18 08:01:24" + } ], + "schedule" : null +}, { + "processDefinition" : { + "id" : 9, + "code" : 16020498428160, + "name" : "sql_p1", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 15866588694784, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2024-12-19 06:42:23", + "updateTime" : "2024-12-19 06:42:23", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":16020493200256,\"x\":502.9176025390625,\"y\":236.01136779785156}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0, + "executionType" : "PARALLEL" + }, + "processTaskRelationList" : [ { + "id" : 16, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 16020498428160, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 16020493200256, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-19 06:42:23", + "updateTime" : "2024-12-19 06:42:23", + "operator" : 1, + "operateTime" : "2024-12-19 06:42:23" + } ], + "taskDefinitionList" : [ { + "id" : 16, + "code" : 16020493200256, + "name" : "sql_task1", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "SQL", + "taskParams" : { + "localParams" : [ ], + "resourceList" : [ ], + "type" : "POSTGRESQL", + "datasource" : 1, + "sql" : "select * from some;", + "sqlType" : "0", + "preStatements" : [ ], + "postStatements" : [ ], + "segmentSeparator" : "", + "displayRows" : 10 + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-19 06:42:23", + "updateTime" : "2024-12-19 06:42:23", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-19 06:42:23" + } ], + "schedule" : null +}, { + "processDefinition" : { + "id" : 10, + "code" : 16021372891264, + "name" : "subprocess1", + "version" : 1, + "releaseState" : "OFFLINE", + "projectCode" : 15866588694784, + "description" : "", + "globalParams" : "[]", + "globalParamList" : [ ], + "globalParamMap" : { }, + "createTime" : "2024-12-19 08:36:15", + "updateTime" : "2024-12-19 08:36:15", + "flag" : "YES", + "userId" : 1, + "userName" : null, + "projectName" : null, + "locations" : "[{\"taskCode\":16021366072576,\"x\":618.9176025390625,\"y\":322.0113830566406},{\"taskCode\":16021368344576,\"x\":574.9176025390625,\"y\":59.01136779785156}]", + "scheduleReleaseState" : null, + "timeout" : 0, + "tenantId" : -1, + "tenantCode" : null, + "modifyBy" : null, + "warningGroupId" : 0, + "executionType" : "PARALLEL" + }, + "processTaskRelationList" : [ { + "id" : 17, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 16021372891264, + "preTaskCode" : 0, + "preTaskVersion" : 0, + "postTaskCode" : 16021368344576, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-19 08:36:15", + "updateTime" : "2024-12-19 08:36:15", + "operator" : 1, + "operateTime" : "2024-12-19 08:36:15" + }, { + "id" : 18, + "name" : "", + "processDefinitionVersion" : 1, + "projectCode" : 15866588694784, + "processDefinitionCode" : 16021372891264, + "preTaskCode" : 16021368344576, + "preTaskVersion" : 1, + "postTaskCode" : 16021366072576, + "postTaskVersion" : 1, + "conditionType" : "NONE", + "conditionParams" : { }, + "createTime" : "2024-12-19 08:36:15", + "updateTime" : "2024-12-19 08:36:15", + "operator" : 1, + "operateTime" : "2024-12-19 08:36:15" + } ], + "taskDefinitionList" : [ { + "id" : 17, + "code" : 16021366072576, + "name" : "sub_task1", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "SUB_PROCESS", + "taskParams" : { + "localParams" : [ ], + "resourceList" : [ ], + "processDefinitionCode" : 15866592686592 + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-19 08:36:15", + "updateTime" : "2024-12-19 08:36:15", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-19 08:36:15" + }, { + "id" : 18, + "code" : 16021368344576, + "name" : "sub_task_shell1", + "version" : 1, + "description" : "", + "projectCode" : 15866588694784, + "userId" : 1, + "taskType" : "SHELL", + "taskParams" : { + "localParams" : [ ], + "rawScript" : "echo 'test1'", + "resourceList" : [ ] + }, + "taskParamList" : [ ], + "taskParamMap" : null, + "flag" : "YES", + "taskPriority" : "MEDIUM", + "userName" : null, + "projectName" : null, + "workerGroup" : "default", + "environmentCode" : -1, + "failRetryTimes" : 0, + "failRetryInterval" : 1, + "timeoutFlag" : "CLOSE", + "timeoutNotifyStrategy" : null, + "timeout" : 0, + "delayTime" : 0, + "resourceIds" : "", + "createTime" : "2024-12-19 08:36:15", + "updateTime" : "2024-12-19 08:36:15", + "modifyBy" : null, + "taskGroupId" : 0, + "taskGroupPriority" : 0, + "cpuQuota" : -1, + "memoryMax" : -1, + "taskExecuteType" : "BATCH", + "operator" : 1, + "operateTime" : "2024-12-19 08:36:15" + } ], + "schedule" : null +} ] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/projects/test2/processDefinition/workflow_1734682376191.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/projects/test2/processDefinition/workflow_1734682376191.json new file mode 100644 index 0000000..809b26a --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/projects/test2/processDefinition/workflow_1734682376191.json @@ -0,0 +1 @@ +[{"processDefinition":{"id":12,"code":16032087046528,"name":"switch_proc1","version":2,"releaseState":"OFFLINE","projectCode":15866588694784,"description":"","globalParams":"[]","globalParamList":[],"globalParamMap":{},"createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 08:12:51","flag":"YES","userId":1,"userName":null,"projectName":null,"locations":"[{\"taskCode\":16032043474944,\"x\":530,\"y\":350},{\"taskCode\":16032048684672,\"x\":567.9176025390625,\"y\":601.0113525390625},{\"taskCode\":16032052121728,\"x\":894.9176025390625,\"y\":494.0113525390625}]","scheduleReleaseState":null,"timeout":0,"tenantId":-1,"tenantCode":null,"modifyBy":null,"warningGroupId":0,"executionType":"PARALLEL"},"processTaskRelationList":[{"id":23,"name":"","processDefinitionVersion":2,"projectCode":15866588694784,"processDefinitionCode":16032087046528,"preTaskCode":0,"preTaskVersion":0,"postTaskCode":16032043474944,"postTaskVersion":2,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-20 08:12:51","updateTime":"2024-12-20 08:12:51","operator":1,"operateTime":"2024-12-20 08:12:51"},{"id":24,"name":"","processDefinitionVersion":2,"projectCode":15866588694784,"processDefinitionCode":16032087046528,"preTaskCode":16032043474944,"preTaskVersion":2,"postTaskCode":16032048684672,"postTaskVersion":1,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-20 08:12:51","updateTime":"2024-12-20 08:12:51","operator":1,"operateTime":"2024-12-20 08:12:51"},{"id":25,"name":"","processDefinitionVersion":2,"projectCode":15866588694784,"processDefinitionCode":16032087046528,"preTaskCode":16032043474944,"preTaskVersion":2,"postTaskCode":16032052121728,"postTaskVersion":1,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-20 08:12:51","updateTime":"2024-12-20 08:12:51","operator":1,"operateTime":"2024-12-20 08:12:51"}],"taskDefinitionList":[{"id":21,"code":16032048684672,"name":"switch_shell1","version":1,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SHELL","taskParams":{"localParams":[],"rawScript":"eco1","resourceList":[]},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"","createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 07:51:20","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-20 07:51:20"},{"id":22,"code":16032052121728,"name":"switch_shell2","version":1,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SHELL","taskParams":{"localParams":[],"rawScript":"eww","resourceList":[]},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"","createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 07:51:20","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-20 07:51:20"},{"id":23,"code":16032043474944,"name":"switch_task1","version":2,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SWITCH","taskParams":{"localParams":[],"rawScript":"","resourceList":[],"switchResult":{"dependTaskList":[{"condition":"if $test==1","nextNode":16032048684672},{"condition":"if $test==2","nextNode":16032052121728}],"nextNode":16032048684672}},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"","createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 08:12:51","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-20 08:12:51"}],"schedule":null}] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/resource/resources.json new file mode 100644 index 0000000..a686ec0 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/resource/resources.json @@ -0,0 +1,107 @@ +[ + { + "id": 7, + "pid": -1, + "alias": "python的脚本模板1 copy.py", + "fullName": "/python的脚本模板1 copy.py", + "description": "", + "fileName": "python的脚本模板1 copy.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:14", + "updateTime": "2024-11-19 06:58:14", + "userName": "admin", + "directory": false + }, + { + "id": 6, + "pid": -1, + "alias": "python的脚本模板1 copy 6.py", + "fullName": "/python的脚本模板1 copy 6.py", + "description": "", + "fileName": "python的脚本模板1 copy 6.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:09", + "updateTime": "2024-11-19 06:58:09", + "userName": "admin", + "directory": false + }, + { + "id": 5, + "pid": -1, + "alias": "python的脚本模板1 copy 5.py", + "fullName": "/python的脚本模板1 copy 5.py", + "description": "", + "fileName": "python的脚本模板1 copy 5.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:03", + "updateTime": "2024-11-19 06:58:03", + "userName": "admin", + "directory": false + }, + { + "id": 4, + "pid": -1, + "alias": "python的脚本模板1 copy 4.py", + "fullName": "/python的脚本模板1 copy 4.py", + "description": "", + "fileName": "python的脚本模板1 copy 4.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:57", + "updateTime": "2024-11-19 06:57:57", + "userName": "admin", + "directory": false + }, + { + "id": 3, + "pid": -1, + "alias": "python的脚本模板1 copy 3.py", + "fullName": "/python的脚本模板1 copy 3.py", + "description": "", + "fileName": "python的脚本模板1 copy 3.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:50", + "updateTime": "2024-11-19 06:57:50", + "userName": "admin", + "directory": false + }, + { + "id": 2, + "pid": -1, + "alias": "python的脚本模板1 copy 2.py", + "fullName": "/python的脚本模板1 copy 2.py", + "description": "", + "fileName": "python的脚本模板1 copy 2.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:44", + "updateTime": "2024-11-19 06:57:44", + "userName": "admin", + "directory": false + }, + { + "id": 1, + "pid": -1, + "alias": "python的脚本模板1.py", + "fullName": "/python的脚本模板1.py", + "description": "", + "fileName": "python的脚本模板1.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:34", + "updateTime": "2024-11-19 06:57:34", + "userName": "admin", + "directory": false + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/subprocess1/udfFunction/udf_function_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/datasource/datasource_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/datasource/datasource_page_1.json new file mode 100644 index 0000000..51d5708 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/datasource/datasource_page_1.json @@ -0,0 +1,13 @@ +[ + { + "id": 1, + "userId": 1, + "userName": "admin", + "name": "pg1", + "note": "", + "type": "POSTGRESQL", + "connectionParams": "{\"user\":\"feilun\",\"password\":\"******\",\"address\":\"jdbc:postgresql://pgm-2ze601n0c1h5ks05go.pg.rds.aliyuncs.com:5432\",\"database\":\"test1234\",\"jdbcUrl\":\"jdbc:postgresql://pgm-2ze601n0c1h5ks05go.pg.rds.aliyuncs.com:5432/test1234\",\"driverClassName\":\"org.postgresql.Driver\",\"validationQuery\":\"select version()\"}", + "createTime": "2024-12-19 06:41:27", + "updateTime": "2024-12-19 06:41:27" + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/package_info.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/package_info.json new file mode 100644 index 0000000..a4d3938 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/package_info.json @@ -0,0 +1,3 @@ +{ + "version": "3.1.5" +} \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/projects.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/projects.json new file mode 100644 index 0000000..296e6e4 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/projects.json @@ -0,0 +1,14 @@ +[ + { + "id": 2, + "userId": 1, + "code": 15866588694784, + "name": "test2", + "description": "", + "createTime": "2024-12-05 08:42:04", + "updateTime": "2024-12-05 08:42:04", + "perm": 0, + "defCount": 0, + "instRunningCount": 0 + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/projects/test2/processDefinition/workflow_1734937144406.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/projects/test2/processDefinition/workflow_1734937144406.json new file mode 100644 index 0000000..809b26a --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/projects/test2/processDefinition/workflow_1734937144406.json @@ -0,0 +1 @@ +[{"processDefinition":{"id":12,"code":16032087046528,"name":"switch_proc1","version":2,"releaseState":"OFFLINE","projectCode":15866588694784,"description":"","globalParams":"[]","globalParamList":[],"globalParamMap":{},"createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 08:12:51","flag":"YES","userId":1,"userName":null,"projectName":null,"locations":"[{\"taskCode\":16032043474944,\"x\":530,\"y\":350},{\"taskCode\":16032048684672,\"x\":567.9176025390625,\"y\":601.0113525390625},{\"taskCode\":16032052121728,\"x\":894.9176025390625,\"y\":494.0113525390625}]","scheduleReleaseState":null,"timeout":0,"tenantId":-1,"tenantCode":null,"modifyBy":null,"warningGroupId":0,"executionType":"PARALLEL"},"processTaskRelationList":[{"id":23,"name":"","processDefinitionVersion":2,"projectCode":15866588694784,"processDefinitionCode":16032087046528,"preTaskCode":0,"preTaskVersion":0,"postTaskCode":16032043474944,"postTaskVersion":2,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-20 08:12:51","updateTime":"2024-12-20 08:12:51","operator":1,"operateTime":"2024-12-20 08:12:51"},{"id":24,"name":"","processDefinitionVersion":2,"projectCode":15866588694784,"processDefinitionCode":16032087046528,"preTaskCode":16032043474944,"preTaskVersion":2,"postTaskCode":16032048684672,"postTaskVersion":1,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-20 08:12:51","updateTime":"2024-12-20 08:12:51","operator":1,"operateTime":"2024-12-20 08:12:51"},{"id":25,"name":"","processDefinitionVersion":2,"projectCode":15866588694784,"processDefinitionCode":16032087046528,"preTaskCode":16032043474944,"preTaskVersion":2,"postTaskCode":16032052121728,"postTaskVersion":1,"conditionType":"NONE","conditionParams":{},"createTime":"2024-12-20 08:12:51","updateTime":"2024-12-20 08:12:51","operator":1,"operateTime":"2024-12-20 08:12:51"}],"taskDefinitionList":[{"id":21,"code":16032048684672,"name":"switch_shell1","version":1,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SHELL","taskParams":{"localParams":[],"rawScript":"eco1","resourceList":[]},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"","createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 07:51:20","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-20 07:51:20"},{"id":22,"code":16032052121728,"name":"switch_shell2","version":1,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SHELL","taskParams":{"localParams":[],"rawScript":"eww","resourceList":[]},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"","createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 07:51:20","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-20 07:51:20"},{"id":23,"code":16032043474944,"name":"switch_task1","version":2,"description":"","projectCode":15866588694784,"userId":1,"taskType":"SWITCH","taskParams":{"localParams":[],"rawScript":"","resourceList":[],"switchResult":{"dependTaskList":[{"condition":"if $test==1","nextNode":16032048684672},{"condition":"if $test==2","nextNode":16032052121728}],"nextNode":16032048684672}},"taskParamList":[],"taskParamMap":null,"flag":"YES","taskPriority":"MEDIUM","userName":null,"projectName":null,"workerGroup":"default","environmentCode":-1,"failRetryTimes":0,"failRetryInterval":1,"timeoutFlag":"CLOSE","timeoutNotifyStrategy":null,"timeout":0,"delayTime":0,"resourceIds":"","createTime":"2024-12-20 07:51:20","updateTime":"2024-12-20 08:12:51","modifyBy":null,"taskGroupId":0,"taskGroupPriority":0,"cpuQuota":-1,"memoryMax":-1,"taskExecuteType":"BATCH","operator":1,"operateTime":"2024-12-20 08:12:51"}],"schedule":null}] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/resource/resources.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/resource/resources.json new file mode 100644 index 0000000..a686ec0 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/resource/resources.json @@ -0,0 +1,107 @@ +[ + { + "id": 7, + "pid": -1, + "alias": "python的脚本模板1 copy.py", + "fullName": "/python的脚本模板1 copy.py", + "description": "", + "fileName": "python的脚本模板1 copy.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:14", + "updateTime": "2024-11-19 06:58:14", + "userName": "admin", + "directory": false + }, + { + "id": 6, + "pid": -1, + "alias": "python的脚本模板1 copy 6.py", + "fullName": "/python的脚本模板1 copy 6.py", + "description": "", + "fileName": "python的脚本模板1 copy 6.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:09", + "updateTime": "2024-11-19 06:58:09", + "userName": "admin", + "directory": false + }, + { + "id": 5, + "pid": -1, + "alias": "python的脚本模板1 copy 5.py", + "fullName": "/python的脚本模板1 copy 5.py", + "description": "", + "fileName": "python的脚本模板1 copy 5.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:58:03", + "updateTime": "2024-11-19 06:58:03", + "userName": "admin", + "directory": false + }, + { + "id": 4, + "pid": -1, + "alias": "python的脚本模板1 copy 4.py", + "fullName": "/python的脚本模板1 copy 4.py", + "description": "", + "fileName": "python的脚本模板1 copy 4.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:57", + "updateTime": "2024-11-19 06:57:57", + "userName": "admin", + "directory": false + }, + { + "id": 3, + "pid": -1, + "alias": "python的脚本模板1 copy 3.py", + "fullName": "/python的脚本模板1 copy 3.py", + "description": "", + "fileName": "python的脚本模板1 copy 3.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:50", + "updateTime": "2024-11-19 06:57:50", + "userName": "admin", + "directory": false + }, + { + "id": 2, + "pid": -1, + "alias": "python的脚本模板1 copy 2.py", + "fullName": "/python的脚本模板1 copy 2.py", + "description": "", + "fileName": "python的脚本模板1 copy 2.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:44", + "updateTime": "2024-11-19 06:57:44", + "userName": "admin", + "directory": false + }, + { + "id": 1, + "pid": -1, + "alias": "python的脚本模板1.py", + "fullName": "/python的脚本模板1.py", + "description": "", + "fileName": "python的脚本模板1.py", + "userId": 1, + "type": "FILE", + "size": 1419, + "createTime": "2024-11-19 06:57:34", + "updateTime": "2024-11-19 06:57:34", + "userName": "admin", + "directory": false + } +] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/udfFunction/udf_function_page_1.json b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/udfFunction/udf_function_page_1.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/dolphin/v3/switch/udfFunction/udf_function_page_1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/client/migrationx/migrationx-transformer/src/test/resources/json/adf/workflow1.json b/client/migrationx/migrationx-transformer/src/test/resources/json/adf/workflow1.json new file mode 100644 index 0000000..3f491df --- /dev/null +++ b/client/migrationx/migrationx-transformer/src/test/resources/json/adf/workflow1.json @@ -0,0 +1,157 @@ +{ + "version":"1.1.0", + "kind":"CycleWorkflow", + "spec":{ + "workflows":[ + { + "script":{ + "path":"parallel_activity", + "runtime":{ + "command":"WORKFLOW" + } + }, + "id":"1774306292390895558", + "trigger":{ + "type":"Scheduler", + "cron":"15 30 12 * * ?", + "recurrence":"Normal", + "startTime":"2024-11-15T08:05:00", + "timezone":"China Standard Time" + }, + "type":"CycleWorkflow", + "name":"parallel_activity", + "outputs":{ + "nodeOutputs":[ + { + "data":"1774306292390895558", + "artifactType":"NodeOutput", + "refTableName":"parallel_activity" + } + ] + }, + "nodes":[ + { + "recurrence":"Normal", + "id":"6382923470645782090", + "timeout":12, + "instanceMode":"T+1", + "rerunMode":"Denied", + "rerunTimes":0, + "script":{ + "path":"Web1", + "runtime":{ + "command":"DIDE_SHELL", + "commandTypeId":6 + }, + "content":"GET http://baidu.com" + }, + "name":"Web1", + "description":"this is first activity", + "outputs":{ + "nodeOutputs":[ + { + "data":"6382923470645782090", + "artifactType":"NodeOutput", + "refTableName":"Web1" + } + ] + } + }, + { + "recurrence":"Normal", + "id":"8843999166546393415", + "timeout":12, + "instanceMode":"T+1", + "rerunMode":"Allowed", + "rerunTimes":1, + "script":{ + "path":"Web2", + "runtime":{ + "command":"DIDE_SHELL", + "commandTypeId":6 + }, + "content":"GET http://baidu.com" + }, + "name":"Web2", + "description":"this is web 2 activity", + "outputs":{ + "nodeOutputs":[ + { + "data":"8843999166546393415", + "artifactType":"NodeOutput", + "refTableName":"Web2" + } + ] + } + }, + { + "recurrence":"Skip", + "id":"6832020729022765258", + "timeout":12, + "instanceMode":"T+1", + "rerunMode":"Denied", + "rerunTimes":0, + "script":{ + "path":"Web3", + "runtime":{ + "command":"DIDE_SHELL", + "commandTypeId":6 + }, + "content":"GET http://baidu.com" + }, + "name":"Web3", + "description":"this is web3", + "outputs":{ + "nodeOutputs":[ + { + "data":"6832020729022765258", + "artifactType":"NodeOutput", + "refTableName":"Web3" + } + ] + } + }, + { + "recurrence":"Normal", + "id":"8822487438522928140", + "timeout":36, + "instanceMode":"T+1", + "rerunMode":"Allowed", + "rerunTimes":1, + "script":{ + "path":"Notebook1", + "runtime":{ + "command":"ODPS_SQL", + "commandTypeId":10 + }, + "content":"/notebook/a" + }, + "name":"Notebook1", + "description":"this is nb", + "outputs":{ + "nodeOutputs":[ + { + "data":"8822487438522928140", + "artifactType":"NodeOutput", + "refTableName":"Notebook1" + } + ] + } + } + ], + "dependencies":[ + { + "nodeId":"6832020729022765258", + "depends":[ + { + "type":"Normal", + "output":"8843999166546393415", + "refTableName":"Web2" + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git "a/client/migrationx/migrationx-transformer/src/test/resources/py2sql/dws_xingye_creator_metric_df\345\220\214\346\255\245ck.py" "b/client/migrationx/migrationx-transformer/src/test/resources/py2sql/dws_xingye_creator_metric_df\345\220\214\346\255\245ck.py" deleted file mode 100644 index d4e1baa..0000000 --- "a/client/migrationx/migrationx-transformer/src/test/resources/py2sql/dws_xingye_creator_metric_df\345\220\214\346\255\245ck.py" +++ /dev/null @@ -1,77 +0,0 @@ -from pyspark.sql import SparkSession -# from da_utils import * -import sys - -if __name__ == '__main__': - p_date = sys.argv[1] - # P_DATE = date_upper(p_date) - P_DATE = p_date - - spark = SparkSession.builder.getOrCreate() - - sql = f""" - WITH device_table AS -( - xxxx -), - -category_table AS -( - SELECT - user_id, - category AS category - FROM - ( - xxx - ) - WHERE - rank = 1 - AND category_percentage >=0.7 -), - -INSERT OVERWRITE da.dwd_xingye_user_portrait_df PARTITION(ymd = '{P_DATE}') -SELECT - 'xingye' AS app_name, - NVL(t10.nick_name,'') AS nick_name -FROM -( - SELECT - id AS user_id - FROM - ods_xingye.account_account_df - WHERE - ymd = GREATEST('{p_date}', '20230907') - AND FROM_UNIXTIME(created_at / 1000, 'yyyy-MM-dd') <= '{P_DATE}' - GROUP BY - id -)t -LEFT JOIN -( - SELECT - user_id, - FROM - da.dwm_xingye_npc_meta_metric_di - WHERE - ymd = '{P_DATE}' - )t - GROUP BY - user_id -)t1 -ON t.user_id = t1.user_id -LEFT JOIN ( - SELECT -LEFT JOIN - maintained_creator t10 -ON t.user_id = t10.user_id -; - """ - print(sql) - spark.sql(sql) - - # standardized_quality_check({ - # 'key_columns': ['user_id'], - # 'partition_date': P_DATE, - # 'table_name': 'da.dwd_xingye_user_portrait_df' - # }, spark) - - spark.stop() diff --git a/client/migrationx/migrationx-writer/pom.xml b/client/migrationx/migrationx-writer/pom.xml index 87a874a..5943aaa 100644 --- a/client/migrationx/migrationx-writer/pom.xml +++ b/client/migrationx/migrationx-writer/pom.xml @@ -21,7 +21,7 @@ com.aliyun.dataworks migrationx - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/client/migrationx/migrationx-writer/src/main/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationAssistWriter.java b/client/migrationx/migrationx-writer/src/main/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationAssistWriter.java index df3336f..dbde2b3 100644 --- a/client/migrationx/migrationx-writer/src/main/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationAssistWriter.java +++ b/client/migrationx/migrationx-writer/src/main/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationAssistWriter.java @@ -15,13 +15,16 @@ package com.aliyun.dataworks.migrationx.writer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.text.MessageFormat; import java.util.Arrays; import java.util.List; -import java.util.Optional; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import com.aliyun.dataworks.client.command.CommandApp; import com.aliyun.dataworks_public20200518.Client; @@ -33,6 +36,11 @@ import com.aliyun.dataworks_public20200518.models.GetMigrationSummaryResponse; import com.aliyun.dataworks_public20200518.models.StartMigrationRequest; import com.aliyun.dataworks_public20200518.models.StartMigrationResponse; +import com.aliyun.dataworks_public20240518.models.GetJobStatusRequest; +import com.aliyun.dataworks_public20240518.models.GetJobStatusResponse; +import com.aliyun.dataworks_public20240518.models.ImportWorkflowDefinitionRequest; +import com.aliyun.dataworks_public20240518.models.ImportWorkflowDefinitionResponse; +import com.aliyun.dataworks_public20240518.models.ImportWorkflowDefinitionResponseBody; import com.aliyun.migrationx.common.utils.DateUtils; import com.aliyun.migrationx.common.utils.GsonUtils; import com.aliyun.teaopenapi.models.Config; @@ -43,6 +51,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; @@ -59,6 +68,11 @@ public class DataWorksMigrationAssistWriter extends CommandApp { private static final String DATA_WORKS_ENDPOINT = "dataworks.%s.aliyuncs.com"; + private static final int IMPORT_STATUS_CHECKING_WAIT_SECONDS = 15; + private static final int IMPORT_STATUS_CHECKING_COUNT_THRESHOLD = 20; + + private static final String PACKAGE_TYPE = "SPEC"; + @Override public void run(String[] args) throws Exception { CommandLine commandLine = getCommandLine(getOptions(), args); @@ -68,7 +82,7 @@ public void run(String[] args) throws Exception { String regionId = commandLine.getOptionValue("r"); String projectId = commandLine.getOptionValue("p"); String file = commandLine.getOptionValue("f"); - String packageType = Optional.ofNullable(commandLine.getOptionValue("t")).orElse("SPEC"); + String packageType = commandLine.getOptionValue("t"); Config config = new Config(); config.setAccessKeyId(accessId); @@ -85,17 +99,109 @@ public void run(String[] args) throws Exception { throw new RuntimeException("no region or endpoint args"); } log.info("do import with endpoint {}, region {}, projectId {}, file {}", endpoint, regionId, projectId, file); - Client client = new Client(config); + Path path = Paths.get(file); + if (!Files.exists(path)) { + throw new RuntimeException("file " + file + " not found"); + } + if (packageType != null && "SPEC".equalsIgnoreCase(packageType)) { + Client client = new Client(config); + doImportNodes(file, projectId, regionId, client); + } else { + com.aliyun.dataworks_public20240518.Client client = new com.aliyun.dataworks_public20240518.Client(config); + importWorkflow(path, projectId, client); + } + } - doImport(packageType, file, projectId, regionId, client); + private void importWorkflow( + final Path dir, final String projectId, final com.aliyun.dataworks_public20240518.Client client + ) throws Exception { + log.info("Importing specifications under folder: {} to DataWorks Project Id: {}", dir, projectId); + try (Stream specFiles = Files.list(dir)) { + specFiles.forEachOrdered(specFile -> + { + if (Files.isDirectory(specFile)) { + try { + importWorkflow(specFile, projectId, client); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + importSingleFlowSpec(projectId, client, specFile); + } + }); + } + log.info("Finished specifications importing, server may need sometime to analyze them."); + } + + private boolean isJsonFile(Path path) { + String fileName = path.getFileName().toString(); + return fileName.endsWith(".json"); + } + + void importSingleFlowSpec(String projectId, com.aliyun.dataworks_public20240518.Client client, Path specFile) { + if (!isJsonFile(specFile) || Files.isDirectory(specFile)) { + log.warn("**Invalid workflow definition file: {}, skip importing it.**", specFile); + return; + } + log.info("Importing workflow definition from specification file: {}", specFile); + try { + String specContent = FileUtils.readFileToString(specFile.toFile(), StandardCharsets.UTF_8); + ImportWorkflowDefinitionRequest request = new ImportWorkflowDefinitionRequest(); + request.setProjectId(projectId); + request.setSpec(specContent); + ImportWorkflowDefinitionResponse response = client.importWorkflowDefinition(request); + log.info("importing workflow definition file : {} \n response: {}", specFile, GsonUtils.toJsonString(response)); + + if (null == response + || null == response.getBody() + || null == response.getBody().getAsyncJob()) { + throw new RuntimeException("workflow " + specFile.getFileName() + " response error "); + } + + ImportWorkflowDefinitionResponseBody body = response.getBody(); + String requestId = body.getRequestId(), asyncJobId = body.getAsyncJob().getId(); + if (null != response.getHeaders() && response.getHeaders().containsKey("x-acs-trace-id")) { + requestId = requestId + "(" + response.getHeaders().get("x-acs-trace-id") + ")"; + } + log.info("Specification submitted successfully with request ids: {}", requestId); + checkImportJobStatus(asyncJobId, client); + } catch (Exception ex) { + log.error("Failed importing {}, detailed exception is: ", specFile, ex); + } + } + + void checkImportJobStatus(String asyncJobId, com.aliyun.dataworks_public20240518.Client client) throws Exception { + log.info("\tChecking workflow importing job {}'s status", asyncJobId); + String jobStatus = "Fail"; + for (int i = 0; i < IMPORT_STATUS_CHECKING_COUNT_THRESHOLD; i++) { + GetJobStatusRequest getJobStatusRequest = new GetJobStatusRequest(); + getJobStatusRequest.setJobId(asyncJobId); + GetJobStatusResponse getJobStatusResponse = client.getJobStatus(getJobStatusRequest); + jobStatus = getJobStatusResponse.getBody().getJobStatus().getStatus(); + log.info("\tWorkflow importing job {}'s latest status is: {}", asyncJobId, jobStatus); + if ("Fail".equals(jobStatus)) { + log.error("Importing job {} failed due to: {}", asyncJobId, getJobStatusResponse.getBody().getJobStatus().getError()); + break; + } else if ("Success".equals(jobStatus)) { + break; + } else { + TimeUnit.SECONDS.sleep(IMPORT_STATUS_CHECKING_WAIT_SECONDS); + } + } + if ("Running".equals(jobStatus)) { + log.warn("Exceeded job status checking threshold: {} seconds!! The workflow importing job [{}] is still under running.", + IMPORT_STATUS_CHECKING_COUNT_THRESHOLD * IMPORT_STATUS_CHECKING_WAIT_SECONDS, asyncJobId); + } else { + log.info("Importing job {}'s final status is: {}\n", asyncJobId, jobStatus); + } } - private static void doImport(String packageType, String file, String projectId, String regionId, Client client) throws Exception { + private void doImportNodes(String file, String projectId, String regionId, Client client) throws Exception { log.info("Importing file: {} to DataWorks Project Id: {}", file, projectId); CreateImportMigrationAdvanceRequest createRequest = new CreateImportMigrationAdvanceRequest(); createRequest.setName("migrationx_import_" + projectId + System.currentTimeMillis()); createRequest.setProjectId(Long.valueOf(projectId)); - createRequest.setPackageType(packageType); + createRequest.setPackageType(PACKAGE_TYPE); createRequest.setPackageFileObject(Files.newInputStream(Paths.get(file))); createRequest.setDescription("MigrationX import, Package file: " + file); //set any mapping info to disable engine replacement diff --git a/client/migrationx/migrationx-writer/src/test/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationAssistWriterTest.java b/client/migrationx/migrationx-writer/src/test/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationAssistWriterTest.java deleted file mode 100644 index eb36d59..0000000 --- a/client/migrationx/migrationx-writer/src/test/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationAssistWriterTest.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.aliyun.dataworks.migrationx.writer; - -import org.junit.Ignore; -import org.junit.Test; - -@Ignore -public class DataWorksMigrationAssistWriterTest { - - @Test - public void test1() throws Exception { - DataWorksMigrationAssistWriter writer = new DataWorksMigrationAssistWriter(); - String baseDir = "/Users/xx/Documents/workspace/alibabacloud-dataworks-tool-migration"; - String source = String.format("%s/temp/target3.zip", baseDir); - String[] args = new String[]{ - //"-e", "dataworks.cn-shenzhen.aliyuncs.com", //endpoint - "-i", "xx", //accesasdsId - "-k", "xxx", //accessKey - "-r", "cn-shanghai", //regionId - "-p", "483776", //projectId - "-f", source, //file - "-t", "SPEC", //spect - }; - writer.run(args); - } -} \ No newline at end of file diff --git a/client/migrationx/migrationx-writer/src/test/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationSpecificationImportWriterTest.java b/client/migrationx/migrationx-writer/src/test/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationSpecificationImportWriterTest.java index a536c08..518b8b1 100644 --- a/client/migrationx/migrationx-writer/src/test/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationSpecificationImportWriterTest.java +++ b/client/migrationx/migrationx-writer/src/test/java/com/aliyun/dataworks/migrationx/writer/DataWorksMigrationSpecificationImportWriterTest.java @@ -16,6 +16,8 @@ import com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; @@ -23,9 +25,6 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - @RunWith(MockitoJUnitRunner.class) public class DataWorksMigrationSpecificationImportWriterTest { @@ -34,11 +33,11 @@ public void runApp() { URL url = Thread.currentThread().getContextClassLoader().getResource("airflow2/workflows/"); DataWorksMigrationSpecificationImportWriter writer = new DataWorksMigrationSpecificationImportWriter(); String[] args = new String[]{ - "-i", "xxx", //accessId - "-k", "xxx", //accessKey - "-r", "cn-shenzhen", //regionId - "-p", "-1", //projectId - "-f", url.getPath() + "-i", "xx", //accessId + "-k", "xx", //accessKey + "-r", "cn-shanghai", //regionId + "-p", "483776", //projectId + "-f", "../../../temp/specs_target" }; try { writer.run(args); diff --git a/client/migrationx/pom.xml b/client/migrationx/pom.xml index a9c6eb9..c8b5ffb 100644 --- a/client/migrationx/pom.xml +++ b/client/migrationx/pom.xml @@ -4,7 +4,7 @@ com.aliyun.dataworks client - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/client/pom.xml b/client/pom.xml index 76eff68..5579f5f 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ com.aliyun.dataworks dataworks-tool - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/pom.xml b/pom.xml index fb25f68..353f2a5 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 4.0.0 com.aliyun.dataworks dataworks-tool - 1.1.9-2 + 1.1.9-8 pom dataworks-tool diff --git a/spec/.gitignore b/spec/.gitignore deleted file mode 100644 index 621eeb2..0000000 --- a/spec/.gitignore +++ /dev/null @@ -1,39 +0,0 @@ -target/ -!.mvn/wrapper/maven-wrapper.jar -!**/src/main/**/target/ -!**/src/test/**/target/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ -build/ -!**/src/main/**/build/ -!**/src/test/**/build/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store -/src/test/java/com/aliyun/dataworks/common/spec/tospec/ diff --git a/spec/pom.xml b/spec/pom.xml index ad53795..eb5904e 100644 --- a/spec/pom.xml +++ b/spec/pom.xml @@ -19,7 +19,7 @@ dataworks-tool com.aliyun.dataworks - 1.1.9-2 + 1.1.9-8 ../pom.xml diff --git a/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/DefaultJsonFormCode.java b/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/DefaultJsonFormCode.java index bbbf241..55a6ce7 100644 --- a/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/DefaultJsonFormCode.java +++ b/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/DefaultJsonFormCode.java @@ -92,7 +92,7 @@ private DefaultJsonFormCode getDefaultCode(String code) { @Override public List getProgramTypes() { return Stream - .of(CodeProgramType.HOLOGRES_SYNC_DATA, CodeProgramType.HOLOGRES_SYNC_DDL) + .of(CodeProgramType.HOLOGRES_SYNC_DATA, CodeProgramType.HOLOGRES_SYNC_DDL, CodeProgramType.PAI_STUDIO) .map(CodeProgramType::name).distinct().collect(Collectors.toList()); } diff --git a/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/OdpsSparkCode.java b/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/OdpsSparkCode.java index 90ca023..42adcd6 100644 --- a/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/OdpsSparkCode.java +++ b/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/OdpsSparkCode.java @@ -204,10 +204,11 @@ public String getContent() { .collect(Collectors.toList()) ); code.append(refs); + code.append("\n"); } if (sparkJson != null) { - code.append("\n").append(GsonUtils.defaultGson.toJson(sparkJson)); + code.append(GsonUtils.defaultGson.toJson(sparkJson)); } return code.toString(); } @@ -222,10 +223,11 @@ public String toString() { .collect(Collectors.toList()) ); code.append(refs); + code.append("\n"); } if (sparkJson != null) { - code.append("\n" + GsonUtils.defaultGson.toJson(sparkJson)); + code.append(GsonUtils.defaultGson.toJson(sparkJson)); } return code.toString(); } diff --git a/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/types/CodeProgramType.java b/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/types/CodeProgramType.java index e50d074..c6bfde7 100644 --- a/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/types/CodeProgramType.java +++ b/spec/src/main/java/com/aliyun/dataworks/common/spec/domain/dw/types/CodeProgramType.java @@ -33,7 +33,7 @@ public enum CodeProgramType { SHELL(2, "SHELL", CalcEngineType.GENERAL, null, ".sh"), DIDE_SHELL(6, "DIDE_SHELL", CalcEngineType.GENERAL, null, ".sh"), PERL(31, "PERL", CalcEngineType.GENERAL, null, ".pl"), - SUB_PROCESS(1122, "SUB_PROCESS",CalcEngineType.GENERAL ,null ,null ), + SUB_PROCESS(1122, "SUB_PROCESS", CalcEngineType.GENERAL, null, null), VIRTUAL_WORKFLOW(97, "VIRTUAL_WORKFLOW", CalcEngineType.GENERAL, null, null), COMBINED_NODE(98, "COMBINED_NODE", CalcEngineType.GENERAL, null, null), VIRTUAL(99, "VIRTUAL", CalcEngineType.GENERAL, null, ".vi"), @@ -52,6 +52,7 @@ public enum CodeProgramType { FTP_CHECK(1320, "FTP_CHECK", CalcEngineType.GENERAL, null, ".json"), SSH(1321, "SSH", CalcEngineType.GENERAL, null, ".ssh.sh"), CHECK(19, "CHECK", CalcEngineType.GENERAL, null, ".json"), + CHECK_NODE(241, "CHECK_NODE", CalcEngineType.GENERAL, null, ".json"), OSS_INSPECT(239, "OSS_INSPECT", CalcEngineType.GENERAL, null, ".json"), CROSS_TENANTS(1089, "CROSS_TENANTS", CalcEngineType.GENERAL, null, ".json"), PYTHON(1322, "PYTHON", CalcEngineType.GENERAL, null, ".py"), @@ -168,7 +169,6 @@ public enum CodeProgramType { //only for temp CUSTOM(9999, "CUSTOM", CalcEngineType.CUSTOM, LabelType.DATA_PROCESS, ".json"); - private final int code; private final String name; private final CalcEngineType calcEngineType; diff --git a/spec/src/main/java/com/aliyun/dataworks/common/spec/writer/impl/SpecFlowDependWriter.java b/spec/src/main/java/com/aliyun/dataworks/common/spec/writer/impl/SpecFlowDependWriter.java index 956d61f..091e3cd 100644 --- a/spec/src/main/java/com/aliyun/dataworks/common/spec/writer/impl/SpecFlowDependWriter.java +++ b/spec/src/main/java/com/aliyun/dataworks/common/spec/writer/impl/SpecFlowDependWriter.java @@ -17,15 +17,14 @@ import java.util.List; -import com.aliyun.dataworks.common.spec.annotation.SpecWriter; -import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; -import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; -import com.aliyun.dataworks.common.spec.writer.SpecWriterContext; - import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; +import com.aliyun.dataworks.common.spec.annotation.SpecWriter; +import com.aliyun.dataworks.common.spec.domain.noref.SpecDepend; +import com.aliyun.dataworks.common.spec.domain.noref.SpecFlowDepend; +import com.aliyun.dataworks.common.spec.writer.SpecWriterContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; @@ -45,7 +44,7 @@ public SpecFlowDependWriter(SpecWriterContext context) { public JSONObject write(SpecFlowDepend specObj, SpecWriterContext context) { if (specObj.getNodeId() == null || specObj.getNodeId().getId() == null) { log.error("invalid spec flow depend: {}", JSON.toJSONString(specObj)); - throw new RuntimeException("invalid spec flow dependent"); + return null; } JSONObject json = writeJsonObject(specObj, true); @@ -61,8 +60,8 @@ private JSONArray writeDepends(List depends) { JSONArray jsonArray = new JSONArray(); ListUtils.emptyIfNull(depends).stream() - .map(this::writeByWriter) - .forEach(jsonArray::add); + .map(this::writeByWriter) + .forEach(jsonArray::add); return jsonArray; } } diff --git a/spec/src/test/java/com/aliyun/dataworks/common/spec/SpecUtilTest.java b/spec/src/test/java/com/aliyun/dataworks/common/spec/SpecUtilTest.java index 56e733c..4bd4ca0 100644 --- a/spec/src/test/java/com/aliyun/dataworks/common/spec/SpecUtilTest.java +++ b/spec/src/test/java/com/aliyun/dataworks/common/spec/SpecUtilTest.java @@ -2064,4 +2064,20 @@ public void testInnerNodes() { Assert.assertEquals(subnode.getId(), parsed.getSpec().getNodes().get(0).getSubflow().getNodes().get(0).getId()); Assert.assertEquals(subnode.getName(), parsed.getSpec().getNodes().get(0).getSubflow().getNodes().get(0).getName()); } + + @Test + public void testFlowDependWithNoNodeId() { + Specification sp = new Specification<>(); + sp.setKind(SpecKind.CYCLE_WORKFLOW.getLabel()); + sp.setVersion(SpecVersion.V_1_2_0.getLabel()); + DataWorksWorkflowSpec spec = new DataWorksWorkflowSpec(); + sp.setSpec(spec); + SpecNode node = new SpecNode(); + spec.setNodes(Collections.singletonList(node)); + spec.setFlow(Collections.singletonList(new SpecFlowDepend())); + JSONObject js = JSON.parseObject(SpecUtil.writeToSpec(sp)); + Assert.assertNotNull(js); + Assert.assertTrue(js.containsKey("spec")); + Assert.assertTrue(js.getJSONObject("spec").containsKey("flow")); + } } \ No newline at end of file diff --git a/spec/src/test/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/CodeModelFactoryTest.java b/spec/src/test/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/CodeModelFactoryTest.java index da7fe04..ceb361e 100644 --- a/spec/src/test/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/CodeModelFactoryTest.java +++ b/spec/src/test/java/com/aliyun/dataworks/common/spec/domain/dw/codemodel/CodeModelFactoryTest.java @@ -19,6 +19,9 @@ import java.util.Arrays; import java.util.Collections; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; + import com.aliyun.dataworks.common.spec.domain.dw.types.CodeProgramType; import com.aliyun.dataworks.common.spec.utils.GsonUtils; import lombok.extern.slf4j.Slf4j; @@ -192,4 +195,201 @@ public void testEscapeHtml() { Assert.assertTrue(StringUtils.indexOf(code.getCodeModel().getContent(), code.getCodeModel().getSourceCode()) > 0); } + + @Test + public void testPaiDesigner() { + String code + = "{\"content\":\"{\\\"appId\\\":609609,\\\"computeResource\\\":{\\\"MaxCompute\\\":\\\"execution_maxcompute\\\"}," + + "\\\"connectionType\\\":\\\"MaxCompute\\\",\\\"description\\\":\\\"机器学习算法计算出二氧化氮对于雾霾影响最大。\\\"," + + "\\\"flowUniqueCode\\\":\\\"42d966e5-20c3-4685-9ed2-de52d01912f9\\\",\\\"inputs\\\":[{\\\"type\\\":\\\"MaxComputeTable\\\"," + + "\\\"value\\\":\\\"pai_online_project.wumai_data\\\"}],\\\"name\\\":\\\"up雾霾天气预测001\\\",\\\"outputs\\\":[]," + + "\\\"paiflowArguments\\\":\\\"---\\\\narguments:\\\\n parameters:\\\\n - name: \\\\\\\"execution_maxcompute\\\\\\\"\\\\n " + + "value:\\\\n endpoint: \\\\\\\"http://service.cn-shanghai.maxcompute.aliyun-inc.com/api\\\\\\\"\\\\n odpsProject: " + + "\\\\\\\"mc1205004\\\\\\\"\\\\n spec:\\\\n endpoint: \\\\\\\"http://service.cn-shanghai.maxcompute.aliyun-inc" + + ".com/api\\\\\\\"\\\\n odpsProject: \\\\\\\"mc1205004\\\\\\\"\\\\n resourceType: \\\\\\\"MaxCompute\\\\\\\"\\\\n\\\"," + + "\\\"paiflowParameters\\\":{},\\\"paiflowPipeline\\\":\\\"---\\\\napiVersion: \\\\\\\"core/v1\\\\\\\"\\\\nmetadata:\\\\n provider: " + + "\\\\\\\"1107550004253538\\\\\\\"\\\\n version: \\\\\\\"v1\\\\\\\"\\\\n identifier: " + + "\\\\\\\"job-root-pipeline-identifier\\\\\\\"\\\\n annotations: {}\\\\nspec:\\\\n inputs:\\\\n artifacts: []\\\\n " + + "parameters:\\\\n - name: \\\\\\\"execution_maxcompute\\\\\\\"\\\\n type: \\\\\\\"Map\\\\\\\"\\\\n arguments:\\\\n " + + "artifacts: []\\\\n parameters: []\\\\n dependencies: []\\\\n initContainers: []\\\\n sideCarContainers: []\\\\n " + + "pipelines:\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n metadata:\\\\n provider: \\\\\\\"pai\\\\\\\"\\\\n version: " + + "\\\\\\\"v1\\\\\\\"\\\\n identifier: \\\\\\\"logisticregression_binary\\\\\\\"\\\\n name: " + + "\\\\\\\"id-9531-1608891073887-61143\\\\\\\"\\\\n displayName: \\\\\\\"逻辑回归二分类\\\\\\\"\\\\n annotations: {}\\\\n " + + "spec:\\\\n arguments:\\\\n artifacts:\\\\n - name: \\\\\\\"inputTable\\\\\\\"\\\\n from: " + + "\\\\\\\"{{pipelines.id-85ae-1608984392467-81946.outputs.artifacts.output1Table}}\\\\\\\"\\\\n parameters:\\\\n - name:" + + " \\\\\\\"featureColNames\\\\\\\"\\\\n value: \\\\\\\"pm10,so2,co,no2\\\\\\\"\\\\n - name: " + + "\\\\\\\"labelColName\\\\\\\"\\\\n value: \\\\\\\"_c2\\\\\\\"\\\\n - name: \\\\\\\"goodValue\\\\\\\"\\\\n " + + "value: \\\\\\\"1\\\\\\\"\\\\n - name: \\\\\\\"enableSparse\\\\\\\"\\\\n value: \\\\\\\"false\\\\\\\"\\\\n - " + + "name: \\\\\\\"generatePmml\\\\\\\"\\\\n value: false\\\\n - name: \\\\\\\"roleArn\\\\\\\"\\\\n value: " + + "\\\\\\\"acs:ram::1107550004253538:role/aliyunodpspaidefaultrole\\\\\\\"\\\\n - name: \\\\\\\"regularizedType\\\\\\\"\\\\n " + + " value: \\\\\\\"None\\\\\\\"\\\\n - name: \\\\\\\"maxIter\\\\\\\"\\\\n value: \\\\\\\"100\\\\\\\"\\\\n -" + + " name: \\\\\\\"regularizedLevel\\\\\\\"\\\\n value: \\\\\\\"1\\\\\\\"\\\\n - name: \\\\\\\"epsilon\\\\\\\"\\\\n " + + " value: \\\\\\\"0.000001\\\\\\\"\\\\n - name: \\\\\\\"execution\\\\\\\"\\\\n from: \\\\\\\"{{inputs.parameters" + + ".execution_maxcompute}}\\\\\\\"\\\\n dependencies:\\\\n - \\\\\\\"id-85ae-1608984392467-81946\\\\\\\"\\\\n " + + "initContainers: []\\\\n sideCarContainers: []\\\\n pipelines: []\\\\n volumes: []\\\\n - apiVersion: " + + "\\\\\\\"core/v1\\\\\\\"\\\\n metadata:\\\\n provider: \\\\\\\"pai\\\\\\\"\\\\n version: \\\\\\\"v1\\\\\\\"\\\\n " + + "identifier: \\\\\\\"random_forests_1\\\\\\\"\\\\n name: \\\\\\\"id-3d25-1608980864737-77856\\\\\\\"\\\\n displayName: " + + "\\\\\\\"随机森林\\\\\\\"\\\\n annotations: {}\\\\n spec:\\\\n arguments:\\\\n artifacts:\\\\n - name: " + + "\\\\\\\"inputTable\\\\\\\"\\\\n from: \\\\\\\"{{pipelines.id-85ae-1608984392467-81946.outputs.artifacts" + + ".output1Table}}\\\\\\\"\\\\n parameters:\\\\n - name: \\\\\\\"featureColNames\\\\\\\"\\\\n value: " + + "\\\\\\\"pm10,so2,co,no2\\\\\\\"\\\\n - name: \\\\\\\"labelColName\\\\\\\"\\\\n value: \\\\\\\"_c2\\\\\\\"\\\\n " + + " - name: \\\\\\\"generatePmml\\\\\\\"\\\\n value: false\\\\n - name: \\\\\\\"roleArn\\\\\\\"\\\\n value: " + + "\\\\\\\"acs:ram::1107550004253538:role/aliyunodpspaidefaultrole\\\\\\\"\\\\n - name: \\\\\\\"treeNum\\\\\\\"\\\\n " + + "value: \\\\\\\"100\\\\\\\"\\\\n - name: \\\\\\\"minNumObj\\\\\\\"\\\\n value: \\\\\\\"2\\\\\\\"\\\\n - name: " + + "\\\\\\\"minNumPer\\\\\\\"\\\\n value: \\\\\\\"0\\\\\\\"\\\\n - name: \\\\\\\"maxRecordSize\\\\\\\"\\\\n " + + "value: \\\\\\\"100000\\\\\\\"\\\\n - name: \\\\\\\"execution\\\\\\\"\\\\n from: \\\\\\\"{{inputs.parameters" + + ".execution_maxcompute}}\\\\\\\"\\\\n dependencies:\\\\n - \\\\\\\"id-85ae-1608984392467-81946\\\\\\\"\\\\n " + + "initContainers: []\\\\n sideCarContainers: []\\\\n pipelines: []\\\\n volumes: []\\\\n - apiVersion: " + + "\\\\\\\"core/v1\\\\\\\"\\\\n metadata:\\\\n provider: \\\\\\\"pai\\\\\\\"\\\\n version: \\\\\\\"v1\\\\\\\"\\\\n " + + "identifier: \\\\\\\"type_transform\\\\\\\"\\\\n name: \\\\\\\"id-2d88-1608982098027-91558\\\\\\\"\\\\n displayName: " + + "\\\\\\\"类型转换\\\\\\\"\\\\n annotations: {}\\\\n spec:\\\\n arguments:\\\\n artifacts:\\\\n - name: " + + "\\\\\\\"inputTable\\\\\\\"\\\\n from: \\\\\\\"{{pipelines.id-9309-1608992715826-47326.outputs.artifacts" + + ".outputTable}}\\\\\\\"\\\\n parameters:\\\\n - name: \\\\\\\"cols_to_double\\\\\\\"\\\\n value: \\\\\\\"time," + + "hour,pm2,pm10,so2,co,no2\\\\\\\"\\\\n - name: \\\\\\\"default_int_value\\\\\\\"\\\\n value: \\\\\\\"0\\\\\\\"\\\\n " + + " - name: \\\\\\\"reserveOldFeat\\\\\\\"\\\\n value: \\\\\\\"false\\\\\\\"\\\\n - name: " + + "\\\\\\\"execution\\\\\\\"\\\\n from: \\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n " + + "dependencies:\\\\n - \\\\\\\"id-9309-1608992715826-47326\\\\\\\"\\\\n initContainers: []\\\\n sideCarContainers: " + + "[]\\\\n pipelines: []\\\\n volumes: []\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n metadata:\\\\n provider: " + + "\\\\\\\"pai\\\\\\\"\\\\n version: \\\\\\\"v1\\\\\\\"\\\\n identifier: \\\\\\\"fe_meta_runner\\\\\\\"\\\\n name: " + + "\\\\\\\"id-2317-1608984201281-74996\\\\\\\"\\\\n displayName: \\\\\\\"数据视图\\\\\\\"\\\\n annotations: {}\\\\n spec:\\\\n" + + " arguments:\\\\n artifacts:\\\\n - name: \\\\\\\"inputTable\\\\\\\"\\\\n from: \\\\\\\"{{pipelines" + + ".id-73af-1608994414936-19117.outputs.artifacts.outputTable}}\\\\\\\"\\\\n parameters:\\\\n - name: " + + "\\\\\\\"selectedCols\\\\\\\"\\\\n value: \\\\\\\"pm10,so2,co,no2\\\\\\\"\\\\n - name: \\\\\\\"labelCol\\\\\\\"\\\\n " + + " value: \\\\\\\"_c2\\\\\\\"\\\\n - name: \\\\\\\"isSparse\\\\\\\"\\\\n value: \\\\\\\"false\\\\\\\"\\\\n " + + " - name: \\\\\\\"maxBins\\\\\\\"\\\\n value: \\\\\\\"100\\\\\\\"\\\\n - name: \\\\\\\"execution\\\\\\\"\\\\n " + + " from: \\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n dependencies:\\\\n - " + + "\\\\\\\"id-73af-1608994414936-19117\\\\\\\"\\\\n initContainers: []\\\\n sideCarContainers: []\\\\n pipelines: " + + "[]\\\\n volumes: []\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n metadata:\\\\n provider: \\\\\\\"pai\\\\\\\"\\\\n " + + " version: \\\\\\\"v1\\\\\\\"\\\\n identifier: \\\\\\\"split\\\\\\\"\\\\n name: " + + "\\\\\\\"id-85ae-1608984392467-81946\\\\\\\"\\\\n displayName: \\\\\\\"拆分\\\\\\\"\\\\n annotations: {}\\\\n spec:\\\\n " + + " arguments:\\\\n artifacts:\\\\n - name: \\\\\\\"inputTable\\\\\\\"\\\\n from: \\\\\\\"{{pipelines" + + ".id-9869-1608986331830-78910.outputs.artifacts.outputTable}}\\\\\\\"\\\\n parameters:\\\\n - name: " + + "\\\\\\\"_splitMethod\\\\\\\"\\\\n value: \\\\\\\"_fraction\\\\\\\"\\\\n - name: \\\\\\\"fraction\\\\\\\"\\\\n " + + " value: \\\\\\\"0.8\\\\\\\"\\\\n - name: \\\\\\\"_advanced\\\\\\\"\\\\n value: \\\\\\\"false\\\\\\\"\\\\n -" + + " name: \\\\\\\"execution\\\\\\\"\\\\n from: \\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n " + + "dependencies:\\\\n - \\\\\\\"id-9869-1608986331830-78910\\\\\\\"\\\\n initContainers: []\\\\n sideCarContainers: " + + "[]\\\\n pipelines: []\\\\n volumes: []\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n metadata:\\\\n provider: " + + "\\\\\\\"pai\\\\\\\"\\\\n version: \\\\\\\"v1\\\\\\\"\\\\n identifier: \\\\\\\"Prediction_1\\\\\\\"\\\\n name: " + + "\\\\\\\"id-4191-1608984751235-85036\\\\\\\"\\\\n displayName: \\\\\\\"预测\\\\\\\"\\\\n annotations: {}\\\\n spec:\\\\n " + + " arguments:\\\\n artifacts:\\\\n - name: \\\\\\\"model\\\\\\\"\\\\n from: \\\\\\\"{{pipelines" + + ".id-3d25-1608980864737-77856.outputs.artifacts.model}}\\\\\\\"\\\\n - name: \\\\\\\"inputTable\\\\\\\"\\\\n from: " + + "\\\\\\\"{{pipelines.id-85ae-1608984392467-81946.outputs.artifacts.output2Table}}\\\\\\\"\\\\n parameters:\\\\n - name:" + + " \\\\\\\"featureColNames\\\\\\\"\\\\n value: \\\\\\\"pm10,so2,co,no2\\\\\\\"\\\\n - name: " + + "\\\\\\\"appendColNames\\\\\\\"\\\\n value: \\\\\\\"time,hour,pm10,so2,co,no2,_c2\\\\\\\"\\\\n - name: " + + "\\\\\\\"resultColName\\\\\\\"\\\\n value: \\\\\\\"prediction_result\\\\\\\"\\\\n - name: " + + "\\\\\\\"scoreColName\\\\\\\"\\\\n value: \\\\\\\"prediction_score\\\\\\\"\\\\n - name: " + + "\\\\\\\"detailColName\\\\\\\"\\\\n value: \\\\\\\"prediction_detail\\\\\\\"\\\\n - name: " + + "\\\\\\\"enableSparse\\\\\\\"\\\\n value: \\\\\\\"false\\\\\\\"\\\\n - name: \\\\\\\"execution\\\\\\\"\\\\n " + + "from: \\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n dependencies:\\\\n - " + + "\\\\\\\"id-3d25-1608980864737-77856\\\\\\\"\\\\n - \\\\\\\"id-85ae-1608984392467-81946\\\\\\\"\\\\n initContainers: " + + "[]\\\\n sideCarContainers: []\\\\n pipelines: []\\\\n volumes: []\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n " + + " metadata:\\\\n provider: \\\\\\\"pai\\\\\\\"\\\\n version: \\\\\\\"v1\\\\\\\"\\\\n identifier: " + + "\\\\\\\"Prediction_1\\\\\\\"\\\\n name: \\\\\\\"id-6b26-1608984755520-07203\\\\\\\"\\\\n displayName: " + + "\\\\\\\"预测\\\\\\\"\\\\n annotations: {}\\\\n spec:\\\\n arguments:\\\\n artifacts:\\\\n - name: " + + "\\\\\\\"inputTable\\\\\\\"\\\\n from: \\\\\\\"{{pipelines.id-85ae-1608984392467-81946.outputs.artifacts" + + ".output2Table}}\\\\\\\"\\\\n - name: \\\\\\\"model\\\\\\\"\\\\n from: \\\\\\\"{{pipelines" + + ".id-9531-1608891073887-61143.outputs.artifacts.model}}\\\\\\\"\\\\n parameters:\\\\n - name: " + + "\\\\\\\"featureColNames\\\\\\\"\\\\n value: \\\\\\\"pm10,so2,co,no2\\\\\\\"\\\\n - name: " + + "\\\\\\\"appendColNames\\\\\\\"\\\\n value: \\\\\\\"time,hour,pm10,so2,co,no2,_c2\\\\\\\"\\\\n - name: " + + "\\\\\\\"resultColName\\\\\\\"\\\\n value: \\\\\\\"prediction_result\\\\\\\"\\\\n - name: " + + "\\\\\\\"scoreColName\\\\\\\"\\\\n value: \\\\\\\"prediction_score\\\\\\\"\\\\n - name: " + + "\\\\\\\"detailColName\\\\\\\"\\\\n value: \\\\\\\"prediction_detail\\\\\\\"\\\\n - name: " + + "\\\\\\\"enableSparse\\\\\\\"\\\\n value: \\\\\\\"false\\\\\\\"\\\\n - name: \\\\\\\"execution\\\\\\\"\\\\n " + + "from: \\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n dependencies:\\\\n - " + + "\\\\\\\"id-85ae-1608984392467-81946\\\\\\\"\\\\n - \\\\\\\"id-9531-1608891073887-61143\\\\\\\"\\\\n initContainers: " + + "[]\\\\n sideCarContainers: []\\\\n pipelines: []\\\\n volumes: []\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n " + + " metadata:\\\\n provider: \\\\\\\"pai\\\\\\\"\\\\n version: \\\\\\\"v1\\\\\\\"\\\\n identifier: " + + "\\\\\\\"evaluate_1\\\\\\\"\\\\n name: \\\\\\\"id-5dd4-1608984770722-54098\\\\\\\"\\\\n displayName: " + + "\\\\\\\"二分类评估\\\\\\\"\\\\n annotations: {}\\\\n spec:\\\\n arguments:\\\\n artifacts:\\\\n - name: " + + "\\\\\\\"inputTable\\\\\\\"\\\\n from: \\\\\\\"{{pipelines.id-4191-1608984751235-85036.outputs.artifacts" + + ".outputTable}}\\\\\\\"\\\\n parameters:\\\\n - name: \\\\\\\"labelColName\\\\\\\"\\\\n value: " + + "\\\\\\\"_c2\\\\\\\"\\\\n - name: \\\\\\\"scoreColName\\\\\\\"\\\\n value: \\\\\\\"prediction_score\\\\\\\"\\\\n " + + " - name: \\\\\\\"positiveLabel\\\\\\\"\\\\n value: \\\\\\\"1\\\\\\\"\\\\n - name: \\\\\\\"binCount\\\\\\\"\\\\n " + + " value: \\\\\\\"1000\\\\\\\"\\\\n - name: \\\\\\\"_advanced\\\\\\\"\\\\n value: \\\\\\\"false\\\\\\\"\\\\n " + + " - name: \\\\\\\"execution\\\\\\\"\\\\n from: \\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n " + + "dependencies:\\\\n - \\\\\\\"id-4191-1608984751235-85036\\\\\\\"\\\\n initContainers: []\\\\n sideCarContainers: " + + "[]\\\\n pipelines: []\\\\n volumes: []\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n metadata:\\\\n provider: " + + "\\\\\\\"pai\\\\\\\"\\\\n version: \\\\\\\"v1\\\\\\\"\\\\n identifier: \\\\\\\"evaluate_1\\\\\\\"\\\\n name: " + + "\\\\\\\"id-41c6-1608984773170-26269\\\\\\\"\\\\n displayName: \\\\\\\"二分类评估\\\\\\\"\\\\n annotations: {}\\\\n " + + "spec:\\\\n arguments:\\\\n artifacts:\\\\n - name: \\\\\\\"inputTable\\\\\\\"\\\\n from: " + + "\\\\\\\"{{pipelines.id-6b26-1608984755520-07203.outputs.artifacts.outputTable}}\\\\\\\"\\\\n parameters:\\\\n - name: " + + "\\\\\\\"labelColName\\\\\\\"\\\\n value: \\\\\\\"_c2\\\\\\\"\\\\n - name: \\\\\\\"scoreColName\\\\\\\"\\\\n " + + " value: \\\\\\\"prediction_score\\\\\\\"\\\\n - name: \\\\\\\"positiveLabel\\\\\\\"\\\\n value: " + + "\\\\\\\"1\\\\\\\"\\\\n - name: \\\\\\\"binCount\\\\\\\"\\\\n value: \\\\\\\"1000\\\\\\\"\\\\n - name: " + + "\\\\\\\"_advanced\\\\\\\"\\\\n value: \\\\\\\"false\\\\\\\"\\\\n - name: \\\\\\\"execution\\\\\\\"\\\\n " + + "from: \\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n dependencies:\\\\n - " + + "\\\\\\\"id-6b26-1608984755520-07203\\\\\\\"\\\\n initContainers: []\\\\n sideCarContainers: []\\\\n pipelines: " + + "[]\\\\n volumes: []\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n metadata:\\\\n provider: \\\\\\\"pai\\\\\\\"\\\\n " + + " version: \\\\\\\"v1\\\\\\\"\\\\n identifier: \\\\\\\"normalize_1\\\\\\\"\\\\n name: " + + "\\\\\\\"id-9869-1608986331830-78910\\\\\\\"\\\\n displayName: \\\\\\\"归一化\\\\\\\"\\\\n annotations: {}\\\\n spec:\\\\n " + + " arguments:\\\\n artifacts:\\\\n - name: \\\\\\\"inputTable\\\\\\\"\\\\n from: \\\\\\\"{{pipelines" + + ".id-73af-1608994414936-19117.outputs.artifacts.outputTable}}\\\\\\\"\\\\n parameters:\\\\n - name: " + + "\\\\\\\"selectedColNames\\\\\\\"\\\\n value: \\\\\\\"pm10,so2,co,no2\\\\\\\"\\\\n - name: " + + "\\\\\\\"keepOriginal\\\\\\\"\\\\n value: \\\\\\\"false\\\\\\\"\\\\n - name: \\\\\\\"execution\\\\\\\"\\\\n " + + "from: \\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n dependencies:\\\\n - " + + "\\\\\\\"id-73af-1608994414936-19117\\\\\\\"\\\\n initContainers: []\\\\n sideCarContainers: []\\\\n pipelines: " + + "[]\\\\n volumes: []\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n metadata:\\\\n provider: \\\\\\\"pai\\\\\\\"\\\\n " + + " version: \\\\\\\"v1\\\\\\\"\\\\n identifier: \\\\\\\"data_source\\\\\\\"\\\\n name: " + + "\\\\\\\"id-9309-1608992715826-47326\\\\\\\"\\\\n displayName: \\\\\\\"读数据源\\\\\\\"\\\\n annotations: {}\\\\n spec:\\\\n" + + " arguments:\\\\n artifacts: []\\\\n parameters:\\\\n - name: \\\\\\\"inputTableName\\\\\\\"\\\\n " + + "value: \\\\\\\"pai_online_project.wumai_data\\\\\\\"\\\\n - name: \\\\\\\"execution\\\\\\\"\\\\n from: " + + "\\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n dependencies: []\\\\n initContainers: []\\\\n " + + "sideCarContainers: []\\\\n pipelines: []\\\\n volumes: []\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n " + + "metadata:\\\\n provider: \\\\\\\"pai\\\\\\\"\\\\n version: \\\\\\\"v1\\\\\\\"\\\\n identifier: " + + "\\\\\\\"sql\\\\\\\"\\\\n name: \\\\\\\"id-73af-1608994414936-19117\\\\\\\"\\\\n displayName: \\\\\\\"SQL组件\\\\\\\"\\\\n " + + " annotations: {}\\\\n spec:\\\\n arguments:\\\\n artifacts:\\\\n - name: \\\\\\\"inputTable1\\\\\\\"\\\\n " + + " from: \\\\\\\"{{pipelines.id-2d88-1608982098027-91558.outputs.artifacts.outputTable}}\\\\\\\"\\\\n parameters:\\\\n " + + " - name: \\\\\\\"sql\\\\\\\"\\\\n value: \\\\\\\"select time,hour,(case when pm2>200 then 1 else 0 end),pm10,so2,co," + + "no2\\\\\\\\\\\\n \\\\\\\\ from ${t1}\\\\\\\"\\\\n - name: \\\\\\\"execution\\\\\\\"\\\\n from: " + + "\\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n dependencies:\\\\n - " + + "\\\\\\\"id-2d88-1608982098027-91558\\\\\\\"\\\\n initContainers: []\\\\n sideCarContainers: []\\\\n pipelines: " + + "[]\\\\n volumes: []\\\\n - apiVersion: \\\\\\\"core/v1\\\\\\\"\\\\n metadata:\\\\n provider: \\\\\\\"pai\\\\\\\"\\\\n " + + " version: \\\\\\\"v1\\\\\\\"\\\\n identifier: \\\\\\\"histograms\\\\\\\"\\\\n name: " + + "\\\\\\\"id-3906-1608996028711-50772\\\\\\\"\\\\n displayName: \\\\\\\"直方图\\\\\\\"\\\\n annotations: {}\\\\n spec:\\\\n " + + " arguments:\\\\n artifacts:\\\\n - name: \\\\\\\"inputTable\\\\\\\"\\\\n from: \\\\\\\"{{pipelines" + + ".id-2d88-1608982098027-91558.outputs.artifacts.outputTable}}\\\\\\\"\\\\n parameters:\\\\n - name: " + + "\\\\\\\"selectedColNames\\\\\\\"\\\\n value: \\\\\\\"hour,pm10,so2,co,no2,pm2\\\\\\\"\\\\n - name: " + + "\\\\\\\"intervalNum\\\\\\\"\\\\n value: \\\\\\\"100\\\\\\\"\\\\n - name: \\\\\\\"execution\\\\\\\"\\\\n " + + "from: \\\\\\\"{{inputs.parameters.execution_maxcompute}}\\\\\\\"\\\\n dependencies:\\\\n - " + + "\\\\\\\"id-2d88-1608982098027-91558\\\\\\\"\\\\n initContainers: []\\\\n sideCarContainers: []\\\\n pipelines: " + + "[]\\\\n volumes: []\\\\n volumes: []\\\\n\\\",\\\"paraValue\\\":\\\"--paiflow_endpoint=paiflow.cn-shanghai.aliyuncs.com " + + "--region=cn-shanghai\\\",\\\"prgType\\\":\\\"1000138\\\",\\\"requestId\\\":\\\"EBA73CBD-850E-53E4-BE5F-679410336F0C\\\"," + + "\\\"taskRelations\\\":[{\\\"childTaskUniqueCode\\\":\\\"id-3d25-1608980864737-77856\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-85ae-1608984392467-81946\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-9531-1608891073887-61143\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-85ae-1608984392467-81946\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-4191-1608984751235-85036\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-3d25-1608980864737-77856\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-4191-1608984751235-85036\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-85ae-1608984392467-81946\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-6b26-1608984755520-07203\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-85ae-1608984392467-81946\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-6b26-1608984755520-07203\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-9531-1608891073887-61143\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-5dd4-1608984770722-54098\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-4191-1608984751235-85036\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-41c6-1608984773170-26269\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-6b26-1608984755520-07203\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-85ae-1608984392467-81946\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-9869-1608986331830-78910\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-2d88-1608982098027-91558\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-9309-1608992715826-47326\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-73af-1608994414936-19117\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-2d88-1608982098027-91558\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-2317-1608984201281-74996\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-73af-1608994414936-19117\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-9869-1608986331830-78910\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-73af-1608994414936-19117\\\"},{\\\"childTaskUniqueCode\\\":\\\"id-3906-1608996028711-50772\\\"," + + "\\\"parentTaskUniqueCode\\\":\\\"id-2d88-1608982098027-91558\\\"}],\\\"tasks\\\":[{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"逻辑回归二分类\\\",\\\"taskUniqueCode\\\":\\\"id-9531-1608891073887-61143\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"随机森林\\\",\\\"taskUniqueCode\\\":\\\"id-3d25-1608980864737-77856\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"类型转换\\\",\\\"taskUniqueCode\\\":\\\"id-2d88-1608982098027-91558\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"数据视图\\\",\\\"taskUniqueCode\\\":\\\"id-2317-1608984201281-74996\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"拆分\\\",\\\"taskUniqueCode\\\":\\\"id-85ae-1608984392467-81946\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"预测\\\",\\\"taskUniqueCode\\\":\\\"id-4191-1608984751235-85036\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"预测\\\",\\\"taskUniqueCode\\\":\\\"id-6b26-1608984755520-07203\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"二分类评估\\\",\\\"taskUniqueCode\\\":\\\"id-5dd4-1608984770722-54098\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"二分类评估\\\",\\\"taskUniqueCode\\\":\\\"id-41c6-1608984773170-26269\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"归一化\\\",\\\"taskUniqueCode\\\":\\\"id-9869-1608986331830-78910\\\"},{\\\"root\\\":true," + + "\\\"taskName\\\":\\\"读数据源\\\",\\\"taskUniqueCode\\\":\\\"id-9309-1608992715826-47326\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"SQL组件\\\",\\\"taskUniqueCode\\\":\\\"id-73af-1608994414936-19117\\\"},{\\\"root\\\":false," + + "\\\"taskName\\\":\\\"直方图\\\",\\\"taskUniqueCode\\\":\\\"id-3906-1608996028711-50772\\\"}],\\\"workspaceId\\\":\\\"609609\\\"}\"," + + "\"extraContent\":\"{}\"}"; + CodeModel codeModel = CodeModelFactory.getCodeModel(CodeProgramType.PAI_STUDIO.name(), code); + log.info("content: {}", codeModel.getCodeModel().getContent()); + log.info("extraContent: {}", codeModel.getCodeModel().getExtraContent()); + JSONObject extraContentJs = JSON.parseObject(codeModel.getCodeModel().getExtraContent()); + extraContentJs.fluentPut("experimentId", "123123").fluentPut("name", "test"); + codeModel.getCodeModel().setExtraContent(extraContentJs.toString()); + log.info("extraContent: {}", codeModel.getCodeModel().getRawContent()); + Assert.assertTrue(StringUtils.indexOf(codeModel.getCodeModel().getRawContent(), "experimentId") > 0); + } }