[improvement](statistics)Change auto analyze max width to 300 and hea…

…lth threshold to 90. (#42104) ### What problem does this PR solve? Change auto analyze max width to 300 and health threshold to 90. This could auto analyze wider tables by default and collect table with data changes more frequently. Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None
apache · Nov 21, 2024 · dd4583f · dd4583f
1 parent 95e9765
commit dd4583f
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 9 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
@@ -35,6 +35,7 @@ public final class GlobalVariable {
 
     public static final int VARIABLE_VERSION_0 = 0;
     public static final int VARIABLE_VERSION_100 = 100;
+    public static final int VARIABLE_VERSION_101 = 101;
     public static final int VARIABLE_VERSION_200 = 200;
     public static final int VARIABLE_VERSION_300 = 300;
     public static final int CURRENT_VARIABLE_VERSION = VARIABLE_VERSION_300;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1921,7 +1921,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
                 "Maximum table width to enable auto analyze, "
                     + "table with more columns than this value will not be auto analyzed."},
             flag = VariableMgr.GLOBAL)
-    public int autoAnalyzeTableWidthThreshold = 100;
+    public int autoAnalyzeTableWidthThreshold = 300;
 
     @VariableMgr.VarAttr(name = AUTO_ANALYZE_START_TIME, needForward = true, checker = "checkAnalyzeTimeFormat",
             description = {"该参数定义自动ANALYZE例程的开始时间",
@@ -1986,7 +1986,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
                             + "exceeds (100 - table_stats_health_threshold)% since the last "
                             + "statistics collection operation, the statistics for this table are"
                             + "considered outdated."})
-    public int tableStatsHealthThreshold = 60;
+    public int tableStatsHealthThreshold = 90;
 
     @VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE, needForward = true,
             description = {"是否开启基于结构信息的物化视图透明改写",

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
@@ -34,6 +34,8 @@
 import org.apache.doris.common.util.SerializationUtils;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.persist.GlobalVarPersistInfo;
+import org.apache.doris.statistics.StatisticConstants;
+import org.apache.doris.statistics.util.StatisticsUtil;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
@@ -984,6 +986,20 @@ public static void forceUpdateVariables() {
                     SessionVariable.ENABLE_PIPELINE_X_ENGINE,
                     String.valueOf(true));
         }
+        if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_101) {
+            if (StatisticsUtil.getAutoAnalyzeTableWidthThreshold()
+                    < StatisticConstants.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD) {
+                VariableMgr.refreshDefaultSessionVariables("update variable version",
+                        SessionVariable.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD,
+                        String.valueOf(StatisticConstants.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD));
+            }
+            if (StatisticsUtil.getTableStatsHealthThreshold()
+                    < StatisticConstants.TABLE_STATS_HEALTH_THRESHOLD) {
+                VariableMgr.refreshDefaultSessionVariables("update variable version",
+                        SessionVariable.TABLE_STATS_HEALTH_THRESHOLD,
+                        String.valueOf(StatisticConstants.TABLE_STATS_HEALTH_THRESHOLD));
+            }
+        }
         if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_200) {
             // update from 3.0.2 or below to 3.0.3 or higher
             VariableMgr.refreshDefaultSessionVariables("update variable version",

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -93,13 +93,13 @@ public class StatisticConstants {
 
     public static final long EXTERNAL_TABLE_AUTO_ANALYZE_INTERVAL_IN_MILLIS = TimeUnit.HOURS.toMillis(24);
 
-    public static final int TABLE_STATS_HEALTH_THRESHOLD = 60;
+    public static final int TABLE_STATS_HEALTH_THRESHOLD = 90;
 
     public static final int ANALYZE_TIMEOUT_IN_SEC = 43200;
 
     public static final int TASK_QUEUE_CAP = 1;
 
-    public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 100;
+    public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 300;
 
     public static final int MSG_LEN_UPPER_BOUND = 1024;
 

diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
@@ -344,11 +344,11 @@ public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
         tableMeta.partitionChanged.set(false);
         Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
 
-        // Test update rows changed more than threshold.
+        // Test row count changed more than threshold.
         new MockUp<OlapTable>() {
             @Mock
             public long getRowCount() {
-                return 120;
+                return 111;
             }
         };
         new MockUp<TableStatsMeta>() {
@@ -358,12 +358,29 @@ public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
             }
         };
         tableMeta.partitionChanged.set(false);
-        tableMeta.updatedRows.set(200);
+        tableMeta.updatedRows.set(80);
         Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
 
-        // Test update rows changed less than threshold
+        // Test update rows changed more than threshold
+        new MockUp<OlapTable>() {
+            @Mock
+            public long getRowCount() {
+                return 101;
+            }
+        };
+        tableMeta.partitionChanged.set(false);
+        tableMeta.updatedRows.set(91);
+        Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
+
+        // Test row count and update rows changed less than threshold
+        new MockUp<OlapTable>() {
+            @Mock
+            public long getRowCount() {
+                return 100;
+            }
+        };
         tableMeta.partitionChanged.set(false);
-        tableMeta.updatedRows.set(100);
+        tableMeta.updatedRows.set(85);
         Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
 
     }