From 2ea72542c04ad5b6fd4113b6d10778ac6a482b3d Mon Sep 17 00:00:00 2001 From: GSHF <38513540+GSHF@users.noreply.github.com> Date: Thu, 26 Dec 2024 10:39:12 +0000 Subject: [PATCH] Add Flink execution engine support 2 --- .../flink/config/BaseFlinkConfigurationBuilder.java | 9 +++++++-- .../datavines/engine/flink/config/FlinkEngineConfig.java | 2 +- .../config/FlinkSingleTableConfigurationBuilder.java | 8 +++----- .../engine/flink/config/FlinkSinkSqlBuilder.java | 2 +- .../io.datavines.engine.api.engine.EngineExecutor | 1 + .../io.datavines.engine.config.JobConfigurationBuilder | 1 + .../io.datavines.engine.config.JobConfigurationBuilder | 1 - .../plugins/io.datavines.metric.api.ExpectedValue | 3 ++- .../plugins/io.datavines.metric.api.ExpectedValue | 3 ++- .../plugins/io.datavines.metric.api.ExpectedValue | 3 ++- .../plugins/io.datavines.metric.api.ExpectedValue | 3 ++- .../plugins/io.datavines.metric.api.ExpectedValue | 3 ++- .../plugins/io.datavines.metric.api.ExpectedValue | 3 ++- .../plugins/io.datavines.metric.api.ExpectedValue | 3 ++- .../plugins/io.datavines.metric.api.ExpectedValue | 3 ++- .../plugins/io.datavines.metric.api.ExpectedValue | 3 ++- datavines-server/pom.xml | 6 ++++++ .../server/api/controller/MetricController.java | 3 ++- 18 files changed, 40 insertions(+), 20 deletions(-) create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/BaseFlinkConfigurationBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/BaseFlinkConfigurationBuilder.java index 702302adb..f6e7df009 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/BaseFlinkConfigurationBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/BaseFlinkConfigurationBuilder.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.engine.flink.core.config; +package io.datavines.engine.flink.config; import io.datavines.common.config.EnvConfig; import io.datavines.common.config.SinkConfig; @@ -40,7 +40,12 @@ import static io.datavines.common.ConfigConstants.*; import static io.datavines.common.ConfigConstants.TABLE; - +/** + * + * + * @author dataVines + * @since 2021-07-01 + */ @Slf4j public abstract class BaseFlinkConfigurationBuilder extends BaseJobConfigurationBuilder { diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkEngineConfig.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkEngineConfig.java index cfd8a2287..d4e47eb33 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkEngineConfig.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkEngineConfig.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.engine.flink.core.config; +package io.datavines.engine.flink.config; import io.datavines.common.config.Config; import io.datavines.common.config.CheckResult; diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkSingleTableConfigurationBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkSingleTableConfigurationBuilder.java index af2c44dda..7fc1fcd92 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkSingleTableConfigurationBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkSingleTableConfigurationBuilder.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.engine.flink.core.config; +package io.datavines.engine.flink.config; -import io.datavines.common.config.Config; import io.datavines.common.config.EnvConfig; import io.datavines.common.config.SinkConfig; import io.datavines.common.config.SourceConfig; @@ -32,12 +31,11 @@ import java.util.List; import java.util.Map; -import static io.datavines.common.CommonConstants.*; import static io.datavines.common.ConfigConstants.*; -import static io.datavines.common.ConfigConstants.TABLE; public class FlinkSingleTableConfigurationBuilder extends BaseFlinkConfigurationBuilder { + @Override public void buildEnvConfig() { EnvConfig envConfig = new EnvConfig(); @@ -59,7 +57,7 @@ public void buildSinkConfigs() throws DataVinesException { } metricInputParameter.put(METRIC_UNIQUE_KEY, metricUniqueKey); - String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); + String expectedType = "local_" + parameter.getExpectedType(); ExpectedValue expectedValue = PluginLoader .getPluginLoader(ExpectedValue.class) .getNewPlugin(expectedType); diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkSinkSqlBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkSinkSqlBuilder.java index 81a157ff6..f8c5b13d3 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkSinkSqlBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/config/FlinkSinkSqlBuilder.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.engine.flink.core.config; +package io.datavines.engine.flink.config; public class FlinkSinkSqlBuilder { diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor new file mode 100644 index 000000000..68045cb12 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor @@ -0,0 +1 @@ +flink=io.datavines.engine.flink.FlinkEngineExecutor diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder new file mode 100644 index 000000000..e56682f6d --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder @@ -0,0 +1 @@ +flink_single_table=io.datavines.engine.flink.config.FlinkSingleTableConfigurationBuilder \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder deleted file mode 100644 index 2c8a8e39c..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder +++ /dev/null @@ -1 +0,0 @@ -io.datavines.engine.flink.core.config.FlinkSingleTableConfigurationBuilder diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index af948d983..bf539c12a 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,4 @@ local_daily_avg=io.datavines.metric.expected.plugin.DailyAvg spark_daily_avg=io.datavines.metric.expected.plugin.SparkDailyAvg -livy_daily_avg=io.datavines.metric.expected.plugin.SparkDailyAvg \ No newline at end of file +livy_daily_avg=io.datavines.metric.expected.plugin.SparkDailyAvg +flink_daily_avg=io.datavines.metric.expected.plugin.DailyAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index b53917884..bce9667bb 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,4 @@ local_fix_value=io.datavines.metric.expected.plugin.FixValue spark_fix_value=io.datavines.metric.expected.plugin.FixValue -livy_fix_value=io.datavines.metric.expected.plugin.FixValue \ No newline at end of file +livy_fix_value=io.datavines.metric.expected.plugin.FixValue +flink_fix_value=io.datavines.metric.expected.plugin.FixValue \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index 0031556b6..0814e8f87 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,4 @@ local_last_30d_avg=io.datavines.metric.expected.plugin.Last30DayAvg spark_last_30d_avg=io.datavines.metric.expected.plugin.SparkLast30DayAvg -livy_last_30d_avg=io.datavines.metric.expected.plugin.SparkLast30DayAvg \ No newline at end of file +livy_last_30d_avg=io.datavines.metric.expected.plugin.SparkLast30DayAvg +flink_last_30d_avg=io.datavines.metric.expected.plugin.Last30DayAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index 391573b28..da3004d96 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,4 @@ local_last_7d_avg=io.datavines.metric.expected.plugin.Last7DayAvg spark_last_7d_avg=io.datavines.metric.expected.plugin.SparkLast7DayAvg -livy_last_7d_avg=io.datavines.metric.expected.plugin.SparkLast7DayAvg \ No newline at end of file +livy_last_7d_avg=io.datavines.metric.expected.plugin.SparkLast7DayAvg +flink_last_7d_avg=io.datavines.metric.expected.plugin.Last7DayAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index c969402c4..585dcbe94 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,4 @@ local_monthly_avg=io.datavines.metric.expected.plugin.MonthlyAvg spark_monthly_avg=io.datavines.metric.expected.plugin.SparkMonthlyAvg -livy_monthly_avg=io.datavines.metric.expected.plugin.SparkMonthlyAvg \ No newline at end of file +livy_monthly_avg=io.datavines.metric.expected.plugin.SparkMonthlyAvg +flink_monthly_avg=io.datavines.metric.expected.plugin.MonthlyAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index c23fce078..7b6e28b26 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,4 @@ local_none=io.datavines.metric.expected.plugin.None spark_none=io.datavines.metric.expected.plugin.SparkNone -livy_none=io.datavines.metric.expected.plugin.SparkNone \ No newline at end of file +livy_none=io.datavines.metric.expected.plugin.SparkNone +flink_none=io.datavines.metric.expected.plugin.None \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index e0db566b2..f84be9653 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,4 @@ local_table_total_rows=io.datavines.metric.expected.plugin.TableTotalRows spark_table_total_rows=io.datavines.metric.expected.plugin.TableTotalRows -livy_table_total_rows=io.datavines.metric.expected.plugin.TableTotalRows \ No newline at end of file +livy_table_total_rows=io.datavines.metric.expected.plugin.TableTotalRows +flink_table_total_rows=io.datavines.metric.expected.plugin.TableTotalRows \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index f93c9dcc6..b105fbcda 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,4 @@ local_target_table_total_rows=io.datavines.metric.expected.plugin.TargetTableTotalRows spark_target_table_total_rows=io.datavines.metric.expected.plugin.TargetTableTotalRows -livy_target_table_total_rows=io.datavines.metric.expected.plugin.TargetTableTotalRows \ No newline at end of file +livy_target_table_total_rows=io.datavines.metric.expected.plugin.TargetTableTotalRows +flink_target_table_total_rows=io.datavines.metric.expected.plugin.TargetTableTotalRows \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue index d6784eaf2..c03459155 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue @@ -1,3 +1,4 @@ local_weekly_avg=io.datavines.metric.expected.plugin.WeeklyAvg spark_weekly_avg=io.datavines.metric.expected.plugin.SparkWeeklyAvg -livy_weekly_avg=io.datavines.metric.expected.plugin.SparkWeeklyAvg \ No newline at end of file +livy_weekly_avg=io.datavines.metric.expected.plugin.SparkWeeklyAvg +flink_weekly_avg=io.datavines.metric.expected.plugin.WeeklyAvg \ No newline at end of file diff --git a/datavines-server/pom.xml b/datavines-server/pom.xml index d132bf33c..b1ba8889d 100644 --- a/datavines-server/pom.xml +++ b/datavines-server/pom.xml @@ -376,6 +376,12 @@ + + io.datavines + datavines-engine-flink-core + ${project.version} + + io.datavines datavines-engine-local-config diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java index 197bdf6c8..03678af48 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java +++ b/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java @@ -158,7 +158,8 @@ public Object getExpectedTypeList(@PathVariable("type") String type) { afterFilterSet = expectedValueList.stream() .map(it ->it.replace("local_", "") .replace("spark_","") - .replace("livy_","")) + .replace("livy_","") + .replace("flink_","")) .collect(Collectors.toSet()); List items = new ArrayList<>();