diff --git a/service/matching/poller/history.go b/service/matching/poller/history.go
index 30b55eeff2c..8bc93b44c75 100644
--- a/service/matching/poller/history.go
+++ b/service/matching/poller/history.go
@@ -46,6 +46,7 @@ type (
 	History interface {
 		UpdatePollerInfo(id Identity, info Info)
 		HasPollerAfter(earliestAccessTime time.Time) bool
+		GetPollerCount() int
 		GetPollerInfo(earliestAccessTime time.Time) []*types.PollerInfo
 		GetPollerIsolationGroups(earliestAccessTime time.Time) map[string]int
 	}
@@ -106,6 +107,10 @@ func (pollers *history) HasPollerAfter(earliestAccessTime time.Time) bool {
 	return false
 }
 
+func (pollers *history) GetPollerCount() int {
+	return pollers.historyCache.Size()
+}
+
 func (pollers *history) GetPollerInfo(earliestAccessTime time.Time) []*types.PollerInfo {
 	var result []*types.PollerInfo
 	// optimistic size get, it can change before Iterator call.
diff --git a/service/matching/poller/history_test.go b/service/matching/poller/history_test.go
index 88d1c4b3b75..986e55163e2 100644
--- a/service/matching/poller/history_test.go
+++ b/service/matching/poller/history_test.go
@@ -120,6 +120,16 @@ func TestHistory_HasPollerAfter(t *testing.T) {
 	})
 }
 
+func TestGetPollerCount(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	mockCache := cache.NewMockCache(mockCtrl)
+	mockCache.EXPECT().Size().Return(10)
+	p := &history{
+		historyCache: mockCache,
+	}
+	assert.Equal(t, 10, p.GetPollerCount())
+}
+
 func TestGetPollerInfo(t *testing.T) {
 	t.Run("with_time_filter", func(t *testing.T) {
 		mockCtrl := gomock.NewController(t)
diff --git a/service/matching/tasklist/task_list_manager.go b/service/matching/tasklist/task_list_manager.go
index ba968a1d563..85e1de1c69e 100644
--- a/service/matching/tasklist/task_list_manager.go
+++ b/service/matching/tasklist/task_list_manager.go
@@ -206,7 +206,7 @@ func NewManager(
 
 	tlMgr.pollerHistory = poller.NewPollerHistory(func() {
 		scope.UpdateGauge(metrics.PollerPerTaskListCounter,
-			float64(len(tlMgr.pollerHistory.GetPollerInfo(time.Time{}))))
+			float64(tlMgr.pollerHistory.GetPollerCount()))
 	}, timeSource)
 
 	livenessInterval := taskListConfig.IdleTasklistCheckInterval()
@@ -945,15 +945,7 @@ func (c *taskListManagerImpl) emitMisconfiguredPartitionMetrics() {
 	if c.config.NumReadPartitions() != c.config.NumWritePartitions() {
 		c.scope.UpdateGauge(metrics.TaskListReadWritePartitionMismatchGauge, 1)
 	}
-	pollerCount := len(c.pollerHistory.GetPollerInfo(time.Time{}))
-	if c.enableIsolation { // if isolation enabled, get the minimum poller count among the isolation groups
-		pollerCountsByIsolationGroup := c.pollerHistory.GetPollerIsolationGroups(time.Time{})
-		for _, count := range pollerCountsByIsolationGroup {
-			if count < pollerCount {
-				pollerCount = count
-			}
-		}
-	}
+	pollerCount := c.pollerHistory.GetPollerCount()
 	if pollerCount < c.config.NumReadPartitions() || pollerCount < c.config.NumWritePartitions() {
 		c.scope.Tagged(metrics.IsolationEnabledTag(c.enableIsolation)).UpdateGauge(metrics.TaskListPollerPartitionMismatchGauge, 1)
 	}