Enhanced DistributedObjectAdmin to compare for inconsistencies betwee…

…n instances (#426)

Co-authored-by: Anselm McClain <[email protected]>
Co-authored-by: lbwexler <[email protected]>

CHANGELOG.md

@@ -2,10 +2,17 @@
 
 ## 27.0-SNAPSHOT - unreleased
 
+### 💥 Breaking Changes (upgrade difficulty: 🟢 LOW - Hoist React update)
+
+* Requires `hoist-react >= 71` to support enhanced Distributed Objects page.
+
 ### 🎁 New Features
 
 * Added server-side APIs for the new Hoist React `ViewManager` component. Note this Hoist Core
   release will be required by `hoist-react >= 71`.
+* `DistributedObjectAdminService` now compares certain `adminState` fields of distributed objects
+  between instances. Implement `BaseService.getComparisonFields()` to enumerate custom fields to
+  compare.
 
 ## 26.0.0 - 2024-12-02
 

grails-app/controllers/io/xh/hoist/admin/cluster/ClusterAdminController.groovy

@@ -15,7 +15,6 @@ import io.xh.hoist.util.Utils
 import static io.xh.hoist.util.DateTimeUtils.SECONDS
 
 import static grails.async.Promises.task
-
 import static java.lang.Thread.sleep
 
 @Access(['HOIST_ADMIN_READER'])

grails-app/controllers/io/xh/hoist/admin/cluster/ClusterObjectsAdminController.groovy

@@ -0,0 +1,32 @@
+/*
+ * This file belongs to Hoist, an application development toolkit
+ * developed by Extremely Heavy Industries (www.xh.io | [email protected])
+ *
+ * Copyright © 2025 Extremely Heavy Industries Inc.
+ */
+package io.xh.hoist.admin.cluster
+
+import io.xh.hoist.BaseController
+import io.xh.hoist.security.Access
+
+@Access(['HOIST_ADMIN_READER'])
+class ClusterObjectsAdminController extends BaseController {
+    def clusterObjectsService
+
+    def getClusterObjectsReport() {
+        renderJSON(clusterObjectsService.getClusterObjectsReport())
+    }
+
+    @Access(['HOIST_ADMIN'])
+    def clearHibernateCaches() {
+        def req = parseRequestJSON()
+        clusterObjectsService.clearHibernateCaches(req.names)
+        renderJSON([success: true])
+    }
+
+    @Access(['HOIST_ADMIN'])
+    def clearAllHibernateCaches() {
+        clusterObjectsService.clearHibernateCaches()
+        renderJSON([success: true])
+    }
+}

grails-app/services/io/xh/hoist/admin/ClusterObjectsService.groovy

@@ -0,0 +1,81 @@
+/*
+ * This file belongs to Hoist, an application development toolkit
+ * developed by Extremely Heavy Industries (www.xh.io | [email protected])
+ *
+ * Copyright © 2025 Extremely Heavy Industries Inc.
+ */
+package io.xh.hoist.admin
+
+import com.hazelcast.cache.impl.CacheProxy
+import com.hazelcast.executor.impl.ExecutorServiceProxy
+import io.xh.hoist.AdminStats
+import io.xh.hoist.BaseService
+import io.xh.hoist.cluster.ClusterRequest
+
+import static io.xh.hoist.util.Utils.appContext
+import static java.lang.System.currentTimeMillis
+
+class ClusterObjectsService extends BaseService {
+    def grailsApplication
+
+    ClusterObjectsReport getClusterObjectsReport() {
+        def startTimestamp = currentTimeMillis(),
+            info = clusterService
+                .submitToAllInstances(new ListClusterObjects())
+                .collectMany { it.value.value }
+
+        return new ClusterObjectsReport(
+            info: info,
+            startTimestamp: startTimestamp,
+            endTimestamp: currentTimeMillis()
+        )
+    }
+
+    /**
+     * Clear all Hibernate caches, or a specific list of caches by name.
+     */
+    void clearHibernateCaches(List<String> names = null) {
+        def caches = clusterService.distributedObjects.findAll {it instanceof CacheProxy}
+        names ?= caches*.name
+        names.each { name ->
+            def obj = caches.find { it.name == name }
+            if (obj) {
+                obj.clear()
+                logInfo('Cleared ' + name)
+            } else {
+                logWarn('Cannot find cache', name)
+            }
+        }
+    }
+
+    //--------------------
+    // Implementation
+    //--------------------
+    private List<ClusterObjectInfo> listClusterObjects() {
+        // Services and their AdminStat implementing resources
+        Map<String, BaseService> svcs = grailsApplication.mainContext.getBeansOfType(BaseService.class, false, false)
+        def hoistObjs = svcs.collectMany { _, svc ->
+            [
+                new ClusterObjectInfo(name: svc.class.name, type: 'Service', target: svc),
+                *svc.resources
+                    .findAll { k, v -> v instanceof AdminStats }
+                    .collect { k, v -> new ClusterObjectInfo(name: svc.hzName(k), target: v) }
+            ]
+        }
+
+        // Hazelcast built-ins
+        def hzObjs = clusterService
+            .hzInstance
+            .distributedObjects
+            .findAll { !(it instanceof ExecutorServiceProxy) }
+            .collect { new ClusterObjectInfo(target: new HzAdminStats(it)) }
+
+        return (hzObjs + hoistObjs) as List<ClusterObjectInfo>
+    }
+
+    static class ListClusterObjects extends ClusterRequest<List<ClusterObjectInfo>> {
+        List<ClusterObjectInfo> doCall() {
+            appContext.clusterObjectsService.listClusterObjects()
+        }
+    }
+}

grails-app/services/io/xh/hoist/admin/ConnectionPoolMonitoringService.groovy

@@ -9,7 +9,6 @@ package io.xh.hoist.admin
 
 import io.xh.hoist.BaseService
 import io.xh.hoist.exception.DataNotAvailableException
-import io.xh.hoist.util.DateTimeUtils
 import org.apache.tomcat.jdbc.pool.DataSource as PooledDataSource
 import org.apache.tomcat.jdbc.pool.PoolConfiguration
 import org.springframework.boot.jdbc.DataSourceUnwrapper
@@ -37,7 +36,7 @@ class ConnectionPoolMonitoringService extends BaseService {
         createTimer(
             name: 'takeSnapshot',
             runFn: this.&takeSnapshot,
-            interval: {enabled ? config.snapshotInterval * DateTimeUtils.SECONDS: -1}
+            interval: {enabled ? config.snapshotInterval * SECONDS: -1}
         )
     }
 

grails-app/services/io/xh/hoist/admin/ServiceManagerService.groovy

@@ -8,12 +8,12 @@
 package io.xh.hoist.admin
 
 import com.hazelcast.core.DistributedObject
+import io.xh.hoist.AdminStats
 import io.xh.hoist.BaseService
 
 class ServiceManagerService extends BaseService {
 
-    def grailsApplication,
-        clusterAdminService
+    def grailsApplication
 
     Collection<Map> listServices() {
         getServicesInternal().collect { name, svc ->
@@ -47,19 +47,19 @@ class ServiceManagerService extends BaseService {
     // Implementation
     //----------------------
     private List getResourceStats(BaseService svc) {
+        def ret = []
         svc.resources
             .findAll { !it.key.startsWith('xh_') }  // skip hoist implementation objects
-            .collect { k, v ->
-                Map stats = v instanceof DistributedObject ?
-                    clusterAdminService.getAdminStatsForObject(v) :
-                    v.adminStats
-
+            .each { k, v ->
+                AdminStats stats = null
+                if (v instanceof AdminStats) stats = v
+                if (v instanceof DistributedObject) stats = new HzAdminStats(v)
                 // rely on the name (key) service knows, i.e avoid HZ prefix
-                return [*: stats, name: k]
+                if (stats) ret << [*: stats.adminStats, name: k]
             }
+        return ret
     }
 
-
     private Map<String, BaseService> getServicesInternal() {
         return grailsApplication.mainContext.getBeansOfType(BaseService.class, false, false)
     }