[Response Ops] Onweek APM WIP

x-pack/plugins/actions/common/monitoring/types.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { ValidMetricSet } from '@kbn/monitoring-collection-plugin/common/types';
+
+export interface RuleMonitoringMetrics {
+  [key: string]: {
+    data: Array<{ timestamp: number; value: number }>;
+  };
+}
+
+export interface NodeLevelMetricsType {
+  kibana_alerting_node_action_executions: number;
+  kibana_alerting_node_action_execution_time: number;
+  kibana_alerting_node_action_failures: number;
+  kibana_alerting_node_action_timeouts: number;
+}
+
+export enum NodeLevelMetricsEnum {
+  kibana_alerting_node_action_executions = 'kibana_alerting_node_action_executions',
+  kibana_alerting_node_action_execution_time = 'kibana_alerting_node_action_execution_time',
+  kibana_alerting_node_action_failures = 'kibana_alerting_node_action_failures',
+  kibana_alerting_node_action_timeouts = 'kibana_alerting_node_action_timeouts',
+}
+
+export interface ClusterLevelMetricsType extends ValidMetricSet {
+  kibana_alerting_cluster_actions_overdue_count: number;
+  kibana_alerting_cluster_actions_overdue_delay_p50: number;
+  kibana_alerting_cluster_actions_overdue_delay_p99: number;
+}

x-pack/plugins/actions/server/action_type_registry.test.ts

@@ -13,10 +13,8 @@ import { actionsConfigMock } from './actions_config.mock';
 import { licenseStateMock } from './lib/license_state.mock';
 import { ActionsConfigurationUtilities } from './actions_config';
 import { licensingMock } from '@kbn/licensing-plugin/server/mocks';
-import { inMemoryMetricsMock } from './monitoring/in_memory_metrics.mock';
 
 const mockTaskManager = taskManagerMock.createSetup();
-const inMemoryMetrics = inMemoryMetricsMock.create();
 let mockedLicenseState: jest.Mocked<ILicenseState>;
 let mockedActionsConfig: jest.Mocked<ActionsConfigurationUtilities>;
 let actionTypeRegistryParams: ActionTypeRegistryOpts;
@@ -28,10 +26,7 @@ beforeEach(() => {
   actionTypeRegistryParams = {
     licensing: licensingMock.createSetup(),
     taskManager: mockTaskManager,
-    taskRunnerFactory: new TaskRunnerFactory(
-      new ActionExecutor({ isESOCanEncrypt: true }),
-      inMemoryMetrics
-    ),
+    taskRunnerFactory: new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true })),
     actionsConfigUtils: mockedActionsConfig,
     licenseState: mockedLicenseState,
     preconfiguredActions: [

x-pack/plugins/actions/server/actions_client.test.ts

@@ -36,7 +36,6 @@ import { ConnectorTokenClient } from './builtin_action_types/lib/connector_token
 import { encryptedSavedObjectsMock } from '@kbn/encrypted-saved-objects-plugin/server/mocks';
 import { Logger } from '@kbn/core/server';
 import { connectorTokenClientMock } from './builtin_action_types/lib/connector_token_client.mock';
-import { inMemoryMetricsMock } from './monitoring/in_memory_metrics.mock';
 
 jest.mock('@kbn/core/server/saved_objects/service/lib/utils', () => ({
   SavedObjectsUtils: {
@@ -83,18 +82,14 @@ const executor: ExecutorType<{}, {}, {}, void> = async (options) => {
 };
 
 const connectorTokenClient = connectorTokenClientMock.create();
-const inMemoryMetrics = inMemoryMetricsMock.create();
 
 beforeEach(() => {
   jest.resetAllMocks();
   mockedLicenseState = licenseStateMock.create();
   actionTypeRegistryParams = {
     licensing: licensingMock.createSetup(),
     taskManager: mockTaskManager,
-    taskRunnerFactory: new TaskRunnerFactory(
-      new ActionExecutor({ isESOCanEncrypt: true }),
-      inMemoryMetrics
-    ),
+    taskRunnerFactory: new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true })),
     actionsConfigUtils: actionsConfigMock.create(),
     licenseState: mockedLicenseState,
     preconfiguredActions: [],
@@ -503,10 +498,7 @@ describe('create()', () => {
     const localActionTypeRegistryParams = {
       licensing: licensingMock.createSetup(),
       taskManager: mockTaskManager,
-      taskRunnerFactory: new TaskRunnerFactory(
-        new ActionExecutor({ isESOCanEncrypt: true }),
-        inMemoryMetrics
-      ),
+      taskRunnerFactory: new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true })),
       actionsConfigUtils: localConfigUtils,
       licenseState: licenseStateMock.create(),
       preconfiguredActions: [],

x-pack/plugins/actions/server/builtin_action_types/index.test.ts

@@ -14,7 +14,6 @@ import { loggingSystemMock } from '@kbn/core/server/mocks';
 import { actionsConfigMock } from '../actions_config.mock';
 import { licenseStateMock } from '../lib/license_state.mock';
 import { licensingMock } from '@kbn/licensing-plugin/server/mocks';
-import { inMemoryMetricsMock } from '../monitoring/in_memory_metrics.mock';
 
 const ACTION_TYPE_IDS = [
   '.index',
@@ -33,14 +32,10 @@ export function createActionTypeRegistry(): {
   actionTypeRegistry: ActionTypeRegistry;
 } {
   const logger = loggingSystemMock.create().get() as jest.Mocked<Logger>;
-  const inMemoryMetrics = inMemoryMetricsMock.create();
   const actionTypeRegistry = new ActionTypeRegistry({
     taskManager: taskManagerMock.createSetup(),
     licensing: licensingMock.createSetup(),
-    taskRunnerFactory: new TaskRunnerFactory(
-      new ActionExecutor({ isESOCanEncrypt: true }),
-      inMemoryMetrics
-    ),
+    taskRunnerFactory: new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true })),
     actionsConfigUtils: actionsConfigMock.create(),
     licenseState: licenseStateMock.create(),
     preconfiguredActions: [],

x-pack/plugins/actions/server/lib/action_executor.test.ts

@@ -15,6 +15,7 @@ import { eventLoggerMock } from '@kbn/event-log-plugin/server/mocks';
 import { spacesServiceMock } from '@kbn/spaces-plugin/server/spaces_service/spaces_service.mock';
 import { ActionType } from '../types';
 import { actionsMock, actionsClientMock } from '../mocks';
+import { nodeLevelMetricsMock } from '../monitoring/node_level_metrics.mock';
 import { pick } from 'lodash';
 
 const actionExecutor = new ActionExecutor({ isESOCanEncrypt: true });
@@ -34,6 +35,7 @@ const executeParams = {
   request: {} as KibanaRequest,
 };
 
+const nodeLevelMetrics = nodeLevelMetricsMock.create();
 const spacesMock = spacesServiceMock.createStartContract();
 const loggerMock = loggingSystemMock.create().get();
 const getActionsClientWithRequest = jest.fn();
@@ -46,6 +48,7 @@ actionExecutor.initialize({
   encryptedSavedObjectsClient,
   eventLogger,
   preconfiguredActions: [],
+  nodeLevelMetrics,
 });
 
 beforeEach(() => {
@@ -734,6 +737,39 @@ test('writes to event log for execute and execute start when consumer and relate
   });
 });
 
+test('increments monitoring metrics after execution', async () => {
+  const executorMock = setupActionExecutorMock();
+  executorMock.mockResolvedValue({
+    actionId: '1',
+    status: 'ok',
+  });
+  await actionExecutor.execute(executeParams);
+
+  expect(nodeLevelMetrics.execution).toHaveBeenCalledTimes(1);
+});
+
+test('increments monitoring metrics after a failed execution', async () => {
+  const executorMock = setupActionExecutorMock();
+  executorMock.mockRejectedValue(new Error('this action execution is intended to fail'));
+  await actionExecutor.execute(executeParams);
+  expect(nodeLevelMetrics.execution).toHaveBeenCalledTimes(1);
+  expect(nodeLevelMetrics.failure).toHaveBeenCalledTimes(1);
+});
+
+test('increments monitoring metrics after a timeout', async () => {
+  setupActionExecutorMock();
+
+  await actionExecutor.logCancellation({
+    actionId: 'action1',
+    executionId: '123abc',
+    consumer: 'test-consumer',
+    relatedSavedObjects: [],
+    request: {} as KibanaRequest,
+  });
+
+  expect(nodeLevelMetrics.timeout).toHaveBeenCalledTimes(1);
+});
+
 function setupActionExecutorMock() {
   const actionType: jest.Mocked<ActionType> = {
     id: 'test',

x-pack/plugins/actions/server/lib/action_executor.ts

@@ -30,6 +30,7 @@ import { ActionsClient } from '../actions_client';
 import { ActionExecutionSource } from './action_execution_source';
 import { RelatedSavedObjects } from './related_saved_objects';
 import { createActionEventLogRecordObject } from './create_action_event_log_record_object';
+import { NodeLevelMetrics } from '../monitoring';
 
 // 1,000,000 nanoseconds in 1 millisecond
 const Millis2Nanos = 1000 * 1000;
@@ -46,6 +47,7 @@ export interface ActionExecutorContext {
   actionTypeRegistry: ActionTypeRegistryContract;
   eventLogger: IEventLogger;
   preconfiguredActions: PreConfiguredAction[];
+  nodeLevelMetrics?: NodeLevelMetrics;
 }
 
 export interface TaskInfo {
@@ -249,6 +251,12 @@ export class ActionExecutor {
 
         event.event = event.event || {};
 
+        this.actionExecutorContext?.nodeLevelMetrics?.execution(
+          actionId,
+          actionTypeId,
+          event.event?.duration ? event.event?.duration / Millis2Nanos : undefined
+        );
+
         if (result.status === 'ok') {
           span?.setOutcome('success');
           event.event.outcome = 'success';
@@ -260,6 +268,7 @@ export class ActionExecutor {
           event.error = event.error || {};
           event.error.message = actionErrorToMessage(result);
           logger.warn(`action execution failure: ${actionLabel}: ${event.error.message}`);
+          this.actionExecutorContext?.nodeLevelMetrics?.failure(actionId, actionTypeId);
         } else {
           span?.setOutcome('failure');
           event.event.outcome = 'failure';
@@ -269,6 +278,7 @@ export class ActionExecutor {
           logger.warn(
             `action execution failure: ${actionLabel}: returned unexpected result "${result.status}"`
           );
+          this.actionExecutorContext?.nodeLevelMetrics?.failure(actionId, actionTypeId);
         }
 
         eventLogger.logEvent(event);
@@ -345,6 +355,7 @@ export class ActionExecutor {
     });
 
     eventLogger.logEvent(event);
+    this.actionExecutorContext?.nodeLevelMetrics?.timeout(actionId, this.actionInfo.actionTypeId);
   }
 }
 

x-pack/plugins/actions/server/lib/task_runner_factory.test.ts

@@ -17,15 +17,14 @@ import { savedObjectsClientMock, loggingSystemMock, httpServiceMock } from '@kbn
 import { eventLoggerMock } from '@kbn/event-log-plugin/server/mocks';
 import { ActionTypeDisabledError } from './errors';
 import { actionsClientMock } from '../mocks';
-import { inMemoryMetricsMock } from '../monitoring/in_memory_metrics.mock';
-import { IN_MEMORY_METRICS } from '../monitoring';
+import { nodeLevelMetricsMock } from '../monitoring/node_level_metrics.mock';
 
 const spaceIdToNamespace = jest.fn();
 const actionTypeRegistry = actionTypeRegistryMock.create();
 const mockedEncryptedSavedObjectsClient = encryptedSavedObjectsMock.createClient();
 const mockedActionExecutor = actionExecutorMock.create();
 const eventLogger = eventLoggerMock.create();
-const inMemoryMetrics = inMemoryMetricsMock.create();
+const mockedNodeLevelMetrics = nodeLevelMetricsMock.create();
 
 let fakeTimer: sinon.SinonFakeTimers;
 let taskRunnerFactory: TaskRunnerFactory;
@@ -49,7 +48,7 @@ beforeAll(() => {
     },
     taskType: 'actions:1',
   };
-  taskRunnerFactory = new TaskRunnerFactory(mockedActionExecutor, inMemoryMetrics);
+  taskRunnerFactory = new TaskRunnerFactory(mockedActionExecutor);
   mockedActionExecutor.initialize(actionExecutorInitializerParams);
   taskRunnerFactory.initialize(taskRunnerFactoryInitializerParams);
 });
@@ -76,6 +75,7 @@ const taskRunnerFactoryInitializerParams = {
   encryptedSavedObjectsClient: mockedEncryptedSavedObjectsClient,
   basePathService: httpServiceMock.createBasePath(),
   getUnsecuredSavedObjectsClient: jest.fn().mockReturnValue(services.savedObjectsClient),
+  nodeLevelMetrics: mockedNodeLevelMetrics,
 };
 
 beforeEach(() => {
@@ -87,20 +87,14 @@ beforeEach(() => {
 });
 
 test(`throws an error if factory isn't initialized`, () => {
-  const factory = new TaskRunnerFactory(
-    new ActionExecutor({ isESOCanEncrypt: true }),
-    inMemoryMetrics
-  );
+  const factory = new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true }));
   expect(() =>
     factory.create({ taskInstance: mockedTaskInstance })
   ).toThrowErrorMatchingInlineSnapshot(`"TaskRunnerFactory not initialized"`);
 });
 
 test(`throws an error if factory is already initialized`, () => {
-  const factory = new TaskRunnerFactory(
-    new ActionExecutor({ isESOCanEncrypt: true }),
-    inMemoryMetrics
-  );
+  const factory = new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true }));
   factory.initialize(taskRunnerFactoryInitializerParams);
   expect(() =>
     factory.initialize(taskRunnerFactoryInitializerParams)
@@ -627,7 +621,7 @@ test('sanitizes invalid relatedSavedObjects when provided', async () => {
 });
 
 test(`doesn't use API key when not provided`, async () => {
-  const factory = new TaskRunnerFactory(mockedActionExecutor, inMemoryMetrics);
+  const factory = new TaskRunnerFactory(mockedActionExecutor);
   factory.initialize(taskRunnerFactoryInitializerParams);
   const taskRunner = factory.create({ taskInstance: mockedTaskInstance });
 
@@ -850,92 +844,3 @@ test('treats errors as errors if the error is thrown instead of returned', async
     `Action '2' failed and will retry: undefined`
   );
 });
-
-test('increments monitoring metrics after execution', async () => {
-  const taskRunner = taskRunnerFactory.create({
-    taskInstance: mockedTaskInstance,
-  });
-
-  mockedActionExecutor.execute.mockResolvedValueOnce({ status: 'ok', actionId: '2' });
-  spaceIdToNamespace.mockReturnValueOnce('namespace-test');
-  mockedEncryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({
-    id: '3',
-    type: 'action_task_params',
-    attributes: {
-      actionId: '2',
-      params: { baz: true },
-      executionId: '123abc',
-      apiKey: Buffer.from('123:abc').toString('base64'),
-    },
-    references: [],
-  });
-
-  await taskRunner.run();
-
-  expect(inMemoryMetrics.increment).toHaveBeenCalledTimes(1);
-  expect(inMemoryMetrics.increment.mock.calls[0][0]).toBe(IN_MEMORY_METRICS.ACTION_EXECUTIONS);
-});
-
-test('increments monitoring metrics after a failed execution', async () => {
-  const taskRunner = taskRunnerFactory.create({
-    taskInstance: mockedTaskInstance,
-  });
-
-  mockedActionExecutor.execute.mockResolvedValueOnce({
-    status: 'error',
-    actionId: '2',
-    message: 'Error message',
-    data: { foo: true },
-    retry: false,
-  });
-
-  spaceIdToNamespace.mockReturnValueOnce('namespace-test');
-  mockedEncryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({
-    id: '3',
-    type: 'action_task_params',
-    attributes: {
-      actionId: '2',
-      params: { baz: true },
-      executionId: '123abc',
-      apiKey: Buffer.from('123:abc').toString('base64'),
-    },
-    references: [],
-  });
-
-  let err;
-  try {
-    await taskRunner.run();
-  } catch (e) {
-    err = e;
-  }
-
-  expect(err).toBeDefined();
-  expect(inMemoryMetrics.increment).toHaveBeenCalledTimes(2);
-  expect(inMemoryMetrics.increment.mock.calls[0][0]).toBe(IN_MEMORY_METRICS.ACTION_EXECUTIONS);
-  expect(inMemoryMetrics.increment.mock.calls[1][0]).toBe(IN_MEMORY_METRICS.ACTION_FAILURES);
-});
-
-test('increments monitoring metrics after a timeout', async () => {
-  const taskRunner = taskRunnerFactory.create({
-    taskInstance: mockedTaskInstance,
-  });
-
-  mockedActionExecutor.execute.mockResolvedValueOnce({ status: 'ok', actionId: '2' });
-  spaceIdToNamespace.mockReturnValueOnce('namespace-test');
-  mockedEncryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({
-    id: '3',
-    type: 'action_task_params',
-    attributes: {
-      actionId: '2',
-      params: { baz: true },
-      executionId: '123abc',
-      apiKey: Buffer.from('123:abc').toString('base64'),
-    },
-    references: [],
-  });
-
-  await taskRunner.cancel();
-
-  expect(inMemoryMetrics.increment).toHaveBeenCalledTimes(1);
-  expect(inMemoryMetrics.increment.mock.calls[0][0]).toBe(IN_MEMORY_METRICS.ACTION_TIMEOUTS);
-});