[Refactor][Connector] Refactor connector comfig

datavane · Dec 6, 2024 · 8c3c9ec · 8c3c9ec
1 parent 2955262
commit 8c3c9ec
Show file tree

Hide file tree

Showing 76 changed files with 1,053 additions and 285 deletions.
diff --git a/datavines-common/src/main/java/io/datavines/common/ConfigConstants.java b/datavines-common/src/main/java/io/datavines/common/ConfigConstants.java
@@ -22,6 +22,7 @@ public class ConfigConstants {
     public static final String CONNECTOR_TYPE = "connector_type";
     public static final String DATASOURCE_ID = "datasource_id";
     public static final String TABLE = "table";
+    public static final String VIEW_SQL = "view_sql";
     public static final String TABLE_ALIAS = "table_alias";
     public static final String TABLE2_ALIAS = "table2_alias";
     public static final String TABLE_ALIAS_COLUMNS = "table_alias_columns";
@@ -37,6 +38,7 @@ public class ConfigConstants {
     public static final String ACTUAL_NAME = "actual_name";
     public static final String ACTUAL_EXECUTE_SQL = "actual_execute_sql";
     public static final String ACTUAL_AGGREGATE_SQL = "actual_aggregate_sql";
+    public static final String ACTUAL_CUSTOM_SQL = "actual_custom_sql";
     public static final String EXPECTED_NAME = "expected_name";
     public static final String EXPECTED_TYPE = "expected_type";
     public static final String EXPECTED_TABLE = "expected_table";
@@ -74,9 +76,12 @@ public class ConfigConstants {
     public static final String ERROR_DATA_FILE_NAME = "error_data_file_name";
     public static final String VALIDATE_RESULT_DATA_DIR = "validate_result_data_dir";
     public static final String INVALIDATE_ITEM_CAN_OUTPUT = "invalidate_item_can_output";
+
+    public static final String ERROR_DATA_STORAGE_ID = "error_data_storage_id";
     public static final String ERROR_DATA_OUTPUT_TO_DATASOURCE_DATABASE = "error_data_output_to_datasource_database";
     public static final String DATASOURCE = "datasource";
     public static final String DATABASE = "database";
+    public static final String SID = "sid";
     public static final String DATABASE_NAME = "database_name";
     public static final String TABLE_NAME = "table_name";
     public static final String COLUMN_NAME = "column_name";

diff --git a/datavines-common/src/main/java/io/datavines/common/config/enums/SourceType.java b/datavines-common/src/main/java/io/datavines/common/config/enums/SourceType.java
@@ -23,9 +23,9 @@
 public enum SourceType {
 
     /**
-     * 0 normal
-     * 1 invalidate items
-     * 2 actual value
+     * 0 source
+     * 1 target
+     * 2 metadata
      **/
     SOURCE(0, "source"),
     TARGET(1, "target"),

diff --git a/...ines-common/src/main/java/io/datavines/common/datasource/jdbc/BaseJdbcDataSourceInfo.java b/...ines-common/src/main/java/io/datavines/common/datasource/jdbc/BaseJdbcDataSourceInfo.java
@@ -16,12 +16,16 @@
  */
 package io.datavines.common.datasource.jdbc;
 
+import io.datavines.common.utils.Md5Utils;
 import io.datavines.common.utils.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
+import java.util.Map;
+
+import static io.datavines.common.ConfigConstants.*;
 
 /**
  * data source base class
@@ -30,26 +34,26 @@ public abstract class BaseJdbcDataSourceInfo {
 
     private static final Logger logger = LoggerFactory.getLogger(BaseJdbcDataSourceInfo.class);
 
-    protected final JdbcConnectionInfo jdbcConnectionInfo;
+    protected final Map<String,String> param;
 
-    public BaseJdbcDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) {
-        this.jdbcConnectionInfo = jdbcConnectionInfo;
+    public BaseJdbcDataSourceInfo(Map<String,String> param) {
+        this.param = param;
     }
 
     public String getUser() {
-        return jdbcConnectionInfo.getUser();
+        return param.get(USER);
     }
 
     public String getPassword() {
-        return jdbcConnectionInfo.getPassword();
+        return param.get(PASSWORD);
     }
 
     public String getHost() {
-        return jdbcConnectionInfo.getHost();
+        return param.get(HOST);
     }
 
     public String getPort() {
-        return jdbcConnectionInfo.getPort();
+        return param.get(PORT);
     }
 
     public String getValidationQuery() {
@@ -59,19 +63,19 @@ public String getValidationQuery() {
     public abstract String getAddress();
 
     public String getCatalog() {
-        return jdbcConnectionInfo.getCatalog();
+        return param.get(CATALOG);
     }
 
     public String getDatabase() {
-        return jdbcConnectionInfo.getDatabase();
+        return param.get(DATABASE);
     }
 
     public String getSchema() {
-        return jdbcConnectionInfo.getSchema();
+        return param.get(SCHEMA);
     }
 
     public String getProperties() {
-        return jdbcConnectionInfo.getProperties();
+        return param.get(PROPERTIES);
     }
 
     /**
@@ -159,6 +163,21 @@ public void loadClass() {
     }
 
     public String getUniqueKey() {
-        return jdbcConnectionInfo.getUniqueKey();
+        return Md5Utils.getMd5(paramToString(), false);
+    }
+
+    public String paramToString() {
+        return getHost().trim() +
+                "&" + getPort() +
+                "&" + getOrEmpty(getCatalog()) +
+                "&" + getOrEmpty(getDatabase()) +
+                "&" + getOrEmpty(getUser()) +
+                "&" + getOrEmpty(getPassword()) +
+                "&" + getProperties();
     }
+
+    private String getOrEmpty(String keyword) {
+        return StringUtils.isNotEmpty(keyword)? keyword.trim() : "";
+    }
+
 }
diff --git a/datavines-common/src/main/java/io/datavines/common/datasource/jdbc/IJdbcDataSourceInfo.java b/datavines-common/src/main/java/io/datavines/common/datasource/jdbc/IJdbcDataSourceInfo.java
@@ -16,7 +16,9 @@
  */
 package io.datavines.common.datasource.jdbc;
 
+import java.util.Map;
+
 public interface IJdbcDataSourceInfo {
 
-    BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo);
+    BaseJdbcDataSourceInfo getDatasourceInfo(Map<String,String> param);
 }
diff --git a/datavines-common/src/main/java/io/datavines/common/entity/QueryColumn.java b/datavines-common/src/main/java/io/datavines/common/entity/QueryColumn.java
@@ -18,8 +18,12 @@
 
 import lombok.Data;
 
+import java.io.Serializable;
+
 @Data
-public class QueryColumn {
+public class QueryColumn implements Serializable {
+
+    private static final long serialVersionUID = -2398995167525051291L;
 
     private String name;
 

diff --git a/datavines-common/src/main/java/io/datavines/common/enums/ExecutionStatus.java b/datavines-common/src/main/java/io/datavines/common/enums/ExecutionStatus.java
@@ -16,11 +16,11 @@
  */
 package io.datavines.common.enums;
 
-import java.util.HashMap;
-
 import com.baomidou.mybatisplus.annotation.EnumValue;
 import lombok.Getter;
 
+import java.util.HashMap;
+
 /**
  * running status for workflow and task nodes
  * 
@@ -55,7 +55,7 @@ public enum ExecutionStatus {
     WAITING_THREAD(10, "waiting thread", "等待线程"),
     WAITING_SUMMIT(11, "waiting_summit","待提交");
 
-    ExecutionStatus(int code, String description,String zhDescription){
+    ExecutionStatus(int code, String description, String zhDescription){
         this.code = code;
         this.description = description;
         this.zhDescription = zhDescription;

diff --git a/datavines-common/src/main/java/io/datavines/common/enums/IssueStatusType.java b/datavines-common/src/main/java/io/datavines/common/enums/IssueStatusType.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.common.enums;
+
+import com.baomidou.mybatisplus.annotation.EnumValue;
+
+import java.util.HashMap;
+
+/**
+ * job type
+ */
+public enum IssueStatusType {
+    /**
+     * wait_process
+     * processed
+     * ignored
+     */
+    WAIT_PROCESS("wait_process", "WAIT_PROCESS", "待处理"),
+    PROCESSED("processed", "PROCESSED","已处理"),
+    IGNORED("ignored", "IGNORED","已忽略");
+
+    IssueStatusType(String code, String description, String zhDescription) {
+        this.code = code;
+        this.description = description;
+        this.zhDescription = zhDescription;
+    }
+
+    @EnumValue
+    private final String code;
+
+    private final String description;
+
+    private final String zhDescription;
+
+    public String getCode() {
+        return code;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public String getZhDescription() {
+        return zhDescription;
+    }
+
+    private static final HashMap<String, IssueStatusType> JOB_TYPE_MAP = new HashMap<>();
+
+    static {
+        for (IssueStatusType jobType: IssueStatusType.values()){
+            JOB_TYPE_MAP.put(jobType.code, jobType);
+        }
+    }
+
+    public static IssueStatusType of(String jobType){
+        if(JOB_TYPE_MAP.containsKey(jobType)){
+            return JOB_TYPE_MAP.get(jobType);
+        }
+        throw new IllegalArgumentException("invalid job type : " + jobType);
+    }
+}
diff --git a/datavines-common/src/main/java/io/datavines/common/enums/JobType.java b/datavines-common/src/main/java/io/datavines/common/enums/JobType.java
@@ -41,6 +41,7 @@ public enum JobType {
 
     @EnumValue
     private final int code;
+
     private final String description;
 
     private final String zhDescription;

diff --git a/datavines-common/src/main/java/io/datavines/common/enums/MetricCategoryType.java b/datavines-common/src/main/java/io/datavines/common/enums/MetricCategoryType.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.datavines.common.enums;
+
+import com.baomidou.mybatisplus.annotation.EnumValue;
+import com.fasterxml.jackson.annotation.JsonValue;
+import lombok.Getter;
+
+import java.util.HashMap;
+
+public enum MetricCategoryType {
+    BASIC("basic", "基础规则"),
+    TEMPLATE("template", "规则模版"),
+    ;
+
+    @JsonValue
+    @EnumValue
+    @Getter
+    private final String code;
+
+    @Getter
+    private final String description;
+
+    MetricCategoryType(String code, String description) {
+        this.code = code;
+        this.description = description;
+    }
+
+    private static final HashMap<String, MetricCategoryType> METRIC_CATEGORY_TYPE_MAP = new HashMap<>();
+
+    static {
+        for (MetricCategoryType categoryType: MetricCategoryType.values()){
+            METRIC_CATEGORY_TYPE_MAP.put(categoryType.getCode(), categoryType);
+        }
+    }
+
+    public static MetricCategoryType of(String code){
+        if (METRIC_CATEGORY_TYPE_MAP.containsKey(code)){
+            return METRIC_CATEGORY_TYPE_MAP.get(code);
+        }
+        throw new IllegalArgumentException("invalid code : " + code);
+    }
+}