From 47136c28807771a23dad157b33967cc79e435d2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=81=BF=E5=89=91?= <sam.liux@taobao.com>
Date: Wed, 3 Apr 2024 11:35:34 +0800
Subject: [PATCH] dataworks provider

---
 .vscode/settings.json                         |   3 +
 providers/README.md                           |   2 +
 .../airflow_provider_dataworks/README.md      |   3 +
 .../apache_airflow_provider.json              |  27 ++
 .../event/__init__.py                         |   0
 .../operator/__init__.py                      |   0
 .../operator/dataworks_node_operator.py       |   9 +
 .../operator/dataworks_trigger_node.py        |   7 +
 .../plugins/__init__.py                       |   0
 .../plugins/extra_link.py                     |   5 +
 .../requirement.txt                           |   0
 .../sensor/__init__.py                        |   0
 providers/airflow_provider_dataworks/setup.py |   0
 .../README.md                                 |   1 +
 .../pom.xml                                   | 438 ++++++++++++++++++
 15 files changed, 495 insertions(+)
 create mode 100644 .vscode/settings.json
 create mode 100644 providers/README.md
 create mode 100644 providers/airflow_provider_dataworks/README.md
 create mode 100644 providers/airflow_provider_dataworks/apache_airflow_provider.json
 create mode 100644 providers/airflow_provider_dataworks/event/__init__.py
 create mode 100644 providers/airflow_provider_dataworks/operator/__init__.py
 create mode 100644 providers/airflow_provider_dataworks/operator/dataworks_node_operator.py
 create mode 100644 providers/airflow_provider_dataworks/operator/dataworks_trigger_node.py
 create mode 100644 providers/airflow_provider_dataworks/plugins/__init__.py
 create mode 100644 providers/airflow_provider_dataworks/plugins/extra_link.py
 create mode 100644 providers/airflow_provider_dataworks/requirement.txt
 create mode 100644 providers/airflow_provider_dataworks/sensor/__init__.py
 create mode 100644 providers/airflow_provider_dataworks/setup.py
 create mode 100644 providers/dolphinscheduler-dataworks-provider/README.md
 create mode 100644 providers/dolphinscheduler-dataworks-provider/pom.xml

diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..7b016a8
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+    "java.compile.nullAnalysis.mode": "automatic"
+}
\ No newline at end of file
diff --git a/providers/README.md b/providers/README.md
new file mode 100644
index 0000000..69ce2b2
--- /dev/null
+++ b/providers/README.md
@@ -0,0 +1,2 @@
+# Introduction to providers
+Providers are a way to share code across multiple applications.
\ No newline at end of file
diff --git a/providers/airflow_provider_dataworks/README.md b/providers/airflow_provider_dataworks/README.md
new file mode 100644
index 0000000..8c5888d
--- /dev/null
+++ b/providers/airflow_provider_dataworks/README.md
@@ -0,0 +1,3 @@
+# Introduction
+airflow-dataworks-provider is an airflow provider for DataWorks, which provides the interaction functionality between Airflow and DataWorks.
+
diff --git a/providers/airflow_provider_dataworks/apache_airflow_provider.json b/providers/airflow_provider_dataworks/apache_airflow_provider.json
new file mode 100644
index 0000000..18336ce
--- /dev/null
+++ b/providers/airflow_provider_dataworks/apache_airflow_provider.json
@@ -0,0 +1,27 @@
+{
+    "package-name": "apcahe-airflow-provider-dataworks",
+    "version": "1.0.0",
+    "description": "Apache Airflow Provider for Alibaba Cloud DataWorks",
+    "url": "https://github.com/apache/airflow",
+    "license": "Apache License 2.0",
+    "author": "Alibaba Cloud, Inc.",
+    "extra-link": [
+        {
+            "name": "apache-airflow",
+            "url": "https://airflow.apache.org/"
+        }
+    ],
+    "task-decorators": [
+        {
+            "name": "dataworks_node",
+            "class": "airflow.providers.dataworks.decorators.DataWorksNode"
+        }
+    ],
+    "connection-types": [],
+    "secret-backends": [],
+    "auth-backends": [],
+    "nodifications": [],
+    "executors": [],
+    "logging": [],
+    "config": []
+}
\ No newline at end of file
diff --git a/providers/airflow_provider_dataworks/event/__init__.py b/providers/airflow_provider_dataworks/event/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/providers/airflow_provider_dataworks/operator/__init__.py b/providers/airflow_provider_dataworks/operator/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/providers/airflow_provider_dataworks/operator/dataworks_node_operator.py b/providers/airflow_provider_dataworks/operator/dataworks_node_operator.py
new file mode 100644
index 0000000..7694ce2
--- /dev/null
+++ b/providers/airflow_provider_dataworks/operator/dataworks_node_operator.py
@@ -0,0 +1,9 @@
+from airflow.models import BaseOperator
+
+class DataWorksNodeOperator(BaseOperator):
+    region_id = None
+    
+    
+    def __init__(self):
+        super.__init__(BaseOperator)
+        pass
\ No newline at end of file
diff --git a/providers/airflow_provider_dataworks/operator/dataworks_trigger_node.py b/providers/airflow_provider_dataworks/operator/dataworks_trigger_node.py
new file mode 100644
index 0000000..e4ee4d5
--- /dev/null
+++ b/providers/airflow_provider_dataworks/operator/dataworks_trigger_node.py
@@ -0,0 +1,7 @@
+from airflow.models import BaseOperator
+
+class DataWorksTriggerNode(BaseOperator):
+    def __init__(self):
+        pass
+
+    
\ No newline at end of file
diff --git a/providers/airflow_provider_dataworks/plugins/__init__.py b/providers/airflow_provider_dataworks/plugins/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/providers/airflow_provider_dataworks/plugins/extra_link.py b/providers/airflow_provider_dataworks/plugins/extra_link.py
new file mode 100644
index 0000000..ea5af7c
--- /dev/null
+++ b/providers/airflow_provider_dataworks/plugins/extra_link.py
@@ -0,0 +1,5 @@
+from airflow.models.baseoperatorlink import BaseOperatorLink
+
+class DataWorksLink(BaseOperatorLink):
+    name = 'DataWorks'
+    operators = [DataWorksOperator]
diff --git a/providers/airflow_provider_dataworks/requirement.txt b/providers/airflow_provider_dataworks/requirement.txt
new file mode 100644
index 0000000..e69de29
diff --git a/providers/airflow_provider_dataworks/sensor/__init__.py b/providers/airflow_provider_dataworks/sensor/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/providers/airflow_provider_dataworks/setup.py b/providers/airflow_provider_dataworks/setup.py
new file mode 100644
index 0000000..e69de29
diff --git a/providers/dolphinscheduler-dataworks-provider/README.md b/providers/dolphinscheduler-dataworks-provider/README.md
new file mode 100644
index 0000000..d79385c
--- /dev/null
+++ b/providers/dolphinscheduler-dataworks-provider/README.md
@@ -0,0 +1 @@
+Implementation of the DataWorks Task Provider for DolphinScheduler SPI interface.
\ No newline at end of file
diff --git a/providers/dolphinscheduler-dataworks-provider/pom.xml b/providers/dolphinscheduler-dataworks-provider/pom.xml
new file mode 100644
index 0000000..cf81504
--- /dev/null
+++ b/providers/dolphinscheduler-dataworks-provider/pom.xml
@@ -0,0 +1,438 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2024, Alibaba Cloud;
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <parent>
+        <groupId>com.aliyun.dataworks</groupId>
+        <artifactId>dataworks-tool</artifactId>
+        <version>1.1.2</version>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>dolphinscheduler-dataworks-provider</artifactId>
+    <version>0.0.1</version>
+    <packaging>pom</packaging>
+    <name>dolphinscheduler-dataworks-provider</name>
+    <properties>
+        <lombok.version>1.16.16</lombok.version>
+        <jackson.databind.version>2.13.4.2</jackson.databind.version>
+        <jackson.core.version>2.13.1</jackson.core.version>
+        <jackson.dataformat.xml.version>2.13.1</jackson.dataformat.xml.version>
+        <jackson.annotations.version>2.13.1</jackson.annotations.version>
+        <gson.version>2.8.9</gson.version>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <logback.version>1.2.13</logback.version>
+        <hive.version>2.1.0</hive.version>
+        <xml.apis.version>1.4.01</xml.apis.version>
+        <dom4j.version>1.6.1</dom4j.version>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <spark.launcher.version>2.4.3</spark.launcher.version>
+        <oozie.version>5.1.0</oozie.version>
+        <druid-version>1.2.9_preview_04</druid-version>
+        <azkaban.version>3.71.0-3</azkaban.version>
+        <powermock.version>2.0.9</powermock.version>
+    </properties>
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>com.aliyun.dataworks</groupId>
+                <artifactId>dw-common-spec</artifactId>
+                <version>1.1.2</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-reload4j</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.tomcat.embed</groupId>
+                <artifactId>tomcat-embed-core</artifactId>
+                <version>9.0.83</version>
+            </dependency>
+            <dependency>
+                <groupId>com.aliyun</groupId>
+                <artifactId>aliyun-java-sdk-dataworks-public</artifactId>
+                <version>3.3.10</version>
+            </dependency>
+            <dependency>
+                <groupId>com.linkedin.azkaban</groupId>
+                <artifactId>az-core</artifactId>
+                <version>${azkaban.version}-1</version>
+            </dependency>
+            <dependency>
+                <groupId>com.linkedin.azkaban</groupId>
+                <artifactId>azkaban-common</artifactId>
+                <version>${azkaban.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.linkedin.azkaban</groupId>
+                <artifactId>azkaban-spi</artifactId>
+                <version>${azkaban.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.reflections</groupId>
+                <artifactId>reflections</artifactId>
+                <version>0.9.11</version>
+            </dependency>
+            <!-- https://mvnrepository.com/artifact/com.aliyun.oss/aliyun-sdk-oss -->
+            <dependency>
+                <groupId>com.aliyun.oss</groupId>
+                <artifactId>aliyun-sdk-oss</artifactId>
+                <version>3.11.0</version>
+            </dependency>
+
+            <dependency>
+                <artifactId>json-smart</artifactId>
+                <groupId>net.minidev</groupId>
+                <version>2.4.9</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-lang3</artifactId>
+                <version>3.12.0</version>
+            </dependency>
+            <dependency>
+                <groupId>commons-lang</groupId>
+                <artifactId>commons-lang</artifactId>
+                <version>2.6</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-compress</artifactId>
+                <version>1.26.0</version>
+            </dependency>
+            <dependency>
+                <groupId>com.google.guava</groupId>
+                <artifactId>guava</artifactId>
+                <version>32.0.0-jre</version>
+            </dependency>
+            <dependency>
+                <groupId>com.netflix.archaius</groupId>
+                <artifactId>archaius-core</artifactId>
+                <version>0.6.0</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.poi</groupId>
+                <artifactId>poi</artifactId>
+                <version>4.1.1</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.poi</groupId>
+                <artifactId>poi-ooxml</artifactId>
+                <version>4.0.1</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-collections4</artifactId>
+                <version>4.3</version>
+            </dependency>
+            <!-- spring dependencies -->
+            <dependency>
+                <groupId>org.springframework</groupId>
+                <artifactId>spring-web</artifactId>
+                <version>6.0.17</version>
+            </dependency>
+            <dependency>
+                <groupId>org.springframework.shell</groupId>
+                <artifactId>spring-shell-starter</artifactId>
+                <version>2.0.1.RELEASE</version>
+            </dependency>
+            <dependency>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-starter-parent</artifactId>
+                <type>pom</type>
+                <scope>import</scope>
+                <version>2.7.6</version>
+            </dependency>
+            <dependency>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-starter-thymeleaf</artifactId>
+                <version>2.7.3</version>
+            </dependency>
+
+            <dependency>
+                <groupId>org.hibernate.validator</groupId>
+                <artifactId>hibernate-validator</artifactId>
+                <version>6.2.2.Final</version>
+            </dependency>
+            <dependency>
+                <groupId>javax.validation</groupId>
+                <artifactId>validation-api</artifactId>
+                <version>2.0.1.Final</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.httpcomponents</groupId>
+                <artifactId>httpclient</artifactId>
+                <version>4.5.13</version>
+            </dependency>
+            <dependency>
+                <groupId>commons-io</groupId>
+                <artifactId>commons-io</artifactId>
+                <version>2.7</version>
+            </dependency>
+            <dependency>
+                <groupId>commons-cli</groupId>
+                <artifactId>commons-cli</artifactId>
+                <version>1.4</version>
+            </dependency>
+            <dependency>
+                <groupId>com.aliyun</groupId>
+                <artifactId>aliyun-java-sdk-emr</artifactId>
+                <version>4.0.212-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>com.aliyun</groupId>
+                <artifactId>aliyun-java-sdk-core</artifactId>
+                <version>4.5.6</version>
+            </dependency>
+            <dependency>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+                <version>1.7.36</version>
+            </dependency>
+            <dependency>
+                <groupId>ch.qos.logback</groupId>
+                <artifactId>logback-classic</artifactId>
+                <version>${logback.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.google.code.gson</groupId>
+                <artifactId>gson</artifactId>
+                <version>${gson.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.dataformat</groupId>
+                <artifactId>jackson-dataformat-xml</artifactId>
+                <version>${jackson.dataformat.xml.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-core</artifactId>
+                <version>${jackson.core.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-annotations</artifactId>
+                <version>${jackson.annotations.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>${jackson.databind.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>dom4j</groupId>
+                <artifactId>dom4j</artifactId>
+                <version>${dom4j.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>xml-apis</groupId>
+                <artifactId>xml-apis</artifactId>
+                <version>${xml.apis.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.oozie</groupId>
+                <artifactId>oozie-core</artifactId>
+                <version>${oozie.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <artifactId>hive-hcatalog-server-extensions</artifactId>
+                        <groupId>org.apache.hive.hcatalog</groupId>
+                    </exclusion>
+                    <exclusion>
+                        <artifactId>hive-webhcat-java-client</artifactId>
+                        <groupId>org.apache.hive.hcatalog</groupId>
+                    </exclusion>
+                    <exclusion>
+                        <artifactId>hive-jdbc</artifactId>
+                        <groupId>org.apache.hive</groupId>
+                    </exclusion>
+                    <exclusion>
+                        <artifactId>jetty-server</artifactId>
+                        <groupId>org.eclipse.jetty</groupId>
+                    </exclusion>
+                    <exclusion>
+                        <artifactId>commons-logging</artifactId>
+                        <groupId>org.apache.commons</groupId>
+                    </exclusion>
+                    <exclusion>
+                        <artifactId>protobuf-java</artifactId>
+                        <groupId>com.google.protobuf</groupId>
+                    </exclusion>
+                    <exclusion>
+                        <artifactId>hive-exec</artifactId>
+                        <groupId>org.apache.hive</groupId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.oozie</groupId>
+                <artifactId>oozie-client</artifactId>
+                <version>${oozie.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.projectlombok</groupId>
+                <artifactId>lombok</artifactId>
+                <version>${lombok.version}</version>
+            </dependency>
+
+            <!--Test start-->
+            <dependency>
+                <groupId>org.powermock</groupId>
+                <artifactId>powermock-module-junit4-rule-agent</artifactId>
+                <version>${powermock.version}</version>
+                <scope>test</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.powermock</groupId>
+                <artifactId>powermock-module-junit4</artifactId>
+                <version>${powermock.version}</version>
+                <scope>test</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.powermock</groupId>
+                <artifactId>powermock-api-mockito2</artifactId>
+                <version>${powermock.version}</version>
+                <scope>test</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.mockito</groupId>
+                <artifactId>mockito-inline</artifactId>
+                <version>3.9.0</version>
+                <scope>test</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.mockito</groupId>
+                <artifactId>mockito-core</artifactId>
+                <version>3.9.0</version>
+                <scope>test</scope>
+            </dependency>
+            <dependency>
+                <groupId>junit</groupId>
+                <artifactId>junit</artifactId>
+                <version>4.13.2</version>
+                <scope>test</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+    <dependencies>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+            <version>4.13.2</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <artifactId>maven-antrun-plugin</artifactId>
+                    <version>1.7</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-compiler-plugin</artifactId>
+                    <version>3.8.0</version>
+                    <configuration>
+                        <source>1.8</source>
+                        <target>1.8</target>
+                    </configuration>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-clean-plugin</artifactId>
+                    <version>3.1.0</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-resources-plugin</artifactId>
+                    <version>3.0.2</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <version>2.22.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-jar-plugin</artifactId>
+                    <version>3.0.2</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-install-plugin</artifactId>
+                    <version>2.5.2</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-deploy-plugin</artifactId>
+                    <version>2.8.2</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-site-plugin</artifactId>
+                    <version>3.7.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-project-info-reports-plugin</artifactId>
+                    <version>3.0.0</version>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+        <plugins>
+            <plugin>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <version>2.8.2</version>
+            </plugin>
+            <plugin>
+                <artifactId>maven-enforcer-plugin</artifactId>
+                <version>3.0.0-M1</version>
+                <executions>
+                    <execution>
+                        <id>enforce-versions</id>
+                        <goals>
+                            <goal>enforce</goal>
+                        </goals>
+                        <configuration>
+                            <rules>
+                                <requireMavenVersion>
+                                    <version>3.0.0</version>
+                                </requireMavenVersion>
+                                <requireJavaVersion>
+                                    <version>1.8</version>
+                                </requireJavaVersion>
+                            </rules>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
+                <version>3.0.2</version>
+                <configuration>
+                    <!-- 以下类型的文件不会做占位符替换 -->
+                    <nonFilteredFileExtensions>
+                        <nonFilteredFileExtension>jar</nonFilteredFileExtension>
+                        <nonFilteredFileExtension>zip</nonFilteredFileExtension>
+                        <nonFilteredFileExtension>bz2</nonFilteredFileExtension>
+                        <nonFilteredFileExtension>gz</nonFilteredFileExtension>
+                        <nonFilteredFileExtension>doc</nonFilteredFileExtension>
+                        <nonFilteredFileExtension>docx</nonFilteredFileExtension>
+                        <nonFilteredFileExtension>xls</nonFilteredFileExtension>
+                        <nonFilteredFileExtension>xlsx</nonFilteredFileExtension>
+                    </nonFilteredFileExtensions>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>
\ No newline at end of file