Added support for Kibana visualisations (#350)

* Added support for Kibana visualisations * Fixed query map init path * Updated code comments * Merge branch 'main', moved Kibana endpoints from daemon * Updated tests --------- Co-authored-by: nick <[email protected]>
manticoresoftware · Dec 25, 2024 · 3179c59 · 3179c59
1 parent 856621a
commit 3179c59
Show file tree

Hide file tree

Showing 77 changed files with 7,430 additions and 69 deletions.
diff --git a/composer.lock b/composer.lock
diff --git a/src/Plugin/EmulateElastic/AddAliasHandler.php b/src/Plugin/EmulateElastic/AddAliasHandler.php
@@ -0,0 +1,134 @@
+<?php declare(strict_types=1);
+
+/*
+ Copyright (c) 2024, Manticore Software LTD (https://manticoresearch.com)
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License version 3 or any later
+ version. You should have received a copy of the GPL license along with this
+ program; if you did not, you can find it at http://www.gnu.org/
+ */
+
+namespace Manticoresearch\Buddy\Base\Plugin\EmulateElastic;
+
+use Manticoresearch\Buddy\Core\ManticoreSearch\Client as HTTPClient;
+use Manticoresearch\Buddy\Core\Task\Task;
+use Manticoresearch\Buddy\Core\Task\TaskResult;
+use RuntimeException;
+
+/**
+ * This is the parent class to handle erroneous Manticore queries
+ */
+class AddAliasHandler extends BaseEntityHandler {
+
+	use QueryMapLoaderTrait;
+
+	/**
+	 *  Initialize the executor
+	 *
+	 * @param Payload $payload
+	 * @return void
+	 */
+	public function __construct(public Payload $payload) {
+	}
+
+	/**
+	 * Process the request and return self for chaining
+	 *
+	 * @return Task
+	 * @throws RuntimeException
+	 */
+	public function run(): Task {
+		$taskFn = static function (Payload $payload, HTTPClient $manticoreClient): TaskResult {
+			/** @var array{actions:array<mixed>} */
+			$request = json_decode($payload->body, true);
+			if (!is_array($request)) {
+				throw new \Exception('Cannot parse request');
+			}
+			[$index, $alias] = self::extractIndexInfo($request['actions']);
+
+			$query = "SHOW TABLES LIKE '{$index}'";
+			/** @var array{0:array{data?:array<mixed>}} $queryResult */
+			$queryResult = $manticoreClient->sendRequest($query)->getResult();
+			if (!isset($queryResult[0]['data']) || !$queryResult[0]['data']) {
+				throw new \Exception("Index '{$index}' does not exist");
+			}
+
+			$query = "CREATE TABLE IF NOT EXISTS {$payload->table} (index string, alias string)";
+			$manticoreClient->sendRequest($query);
+
+			$query = "SELECT 1 FROM {$payload->table} WHERE index='{$index}'";
+			/** @var array{0:array{data:array<mixed>}} $queryResult */
+			$queryResult = $manticoreClient->sendRequest($query)->getResult();
+			$isExistingIndex = (bool)$queryResult[0]['data'];
+			$query = $isExistingIndex
+				? "UPDATE {$payload->table} SET alias='{$alias}' WHERE index='{$index}'"
+				: "INSERT INTO {$payload->table} (index, alias) VALUES('{$index}', '{$alias}')";
+			/** @var array{error?:string,0:array{data?:array<mixed>}} $queryResult */
+			$queryResult = $manticoreClient->sendRequest($query)->getResult();
+			if (isset($queryResult['error'])) {
+				throw new \Exception('Unknown error on template creation');
+			}
+			if (!$isExistingIndex) {
+				self::refreshAliasedEntities($index, $alias, $manticoreClient);
+			}
+
+			return TaskResult::raw(['acknowledged' => 'true']);
+		};
+
+		return Task::create(
+			$taskFn, [$this->payload, $this->manticoreClient]
+		)->run();
+	}
+
+	/**
+	 *
+	 * @param array<mixed> $actions
+	 * @return array{0:string,1:string}
+	 * @throws RuntimeException
+	 */
+	protected static function extractIndexInfo(array $actions): array {
+		$index = $alias = '';
+		foreach ($actions as $action) {
+			if (is_array($action) && array_key_exists('add', $action)) {
+				extract($action['add']);
+				return [$index, $alias];
+			}
+		}
+		throw new \Exception('Unknown error on alias creation');
+	}
+
+	/**
+	 *
+	 * @param string $index
+	 * @param string $alias
+	 * @param HTTPClient $manticoreClient
+	 * @return void
+	 * @throws RuntimeException
+	 */
+	protected static function refreshAliasedEntities(string $index, string $alias, HTTPClient $manticoreClient): void {
+		$query = 'CREATE TABLE IF NOT EXISTS ' . parent::ENTITY_TABLE
+			. ' (_id string, _index string, _index_alias string, _type string, _source json)';
+		$queryResult = $manticoreClient->sendRequest($query)->getResult();
+		if (isset($queryResult['error'])) {
+			throw new \Exception('Unknown error on entity table creation');
+		}
+		$queryMapName = ($alias === '.kibana') ? 'Settings' : 'ManagerSettings';
+		self::initQueryMap($queryMapName);
+		/** @var array{settings:array{index:array{created:string,uuid:string}}} $sourceObj */
+		$sourceObj = self::$queryMap[$queryMapName][$alias];
+		$created = time();
+		$uuid = substr(md5(microtime()), 0, 16);
+		$sourceObj['settings']['index']['created'] = $created;
+		$sourceObj['settings']['index']['uuid'] = $uuid;
+		$source = (string)json_encode($sourceObj);
+		$entityId = "settings:{$index}";
+		AddEntityHandler::add($source, $entityId, 'settings', $index, $alias, $manticoreClient);
+
+		$query = 'UPDATE ' . parent::ENTITY_TABLE . " SET _index='{$index}' WHERE _index=''";
+		$queryResult = $manticoreClient->sendRequest($query)->getResult();
+		if (isset($queryResult['error'])) {
+			throw new \Exception('Unknown error on entity alias update');
+		}
+	}
+}
diff --git a/src/Plugin/EmulateElastic/AddEntityHandler.php b/src/Plugin/EmulateElastic/AddEntityHandler.php
@@ -0,0 +1,86 @@
+<?php declare(strict_types=1);
+
+/*
+ Copyright (c) 2024, Manticore Software LTD (https://manticoresearch.com)
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License version 3 or any later
+ version. You should have received a copy of the GPL license along with this
+ program; if you did not, you can find it at http://www.gnu.org/
+ */
+
+namespace Manticoresearch\Buddy\Base\Plugin\EmulateElastic;
+
+use Manticoresearch\Buddy\Core\ManticoreSearch\Client as HTTPClient;
+use Manticoresearch\Buddy\Core\Task\Task;
+use Manticoresearch\Buddy\Core\Task\TaskResult;
+use RuntimeException;
+
+/**
+ * This is the class to update Kibana-related entities, like dashboards, visualizations, etc.
+ */
+class AddEntityHandler extends BaseEntityHandler {
+
+	/**
+	 * Process the request and return self for chaining
+	 *
+	 * @return Task
+	 * @throws RuntimeException
+	 */
+	public function run(): Task {
+		$taskFn = static function (Payload $payload, HTTPClient $manticoreClient): TaskResult {
+			[$entityId, $entityIndex, $entityIndexAlias] = self::getEntityInfo($payload->path, $manticoreClient);
+			$indexType = explode(':', $entityId)[0];
+			self::add($payload->body, $entityId, $indexType, $entityIndex, $entityIndexAlias, $manticoreClient);
+
+			return TaskResult::raw(
+				self::buildResponse($entityId, $entityIndex, 'created')
+			);
+		};
+
+		return Task::create(
+			$taskFn, [$this->payload, $this->manticoreClient]
+		)->run();
+	}
+
+	/**
+	 *
+	 * @param string $entitySource
+	 * @param string $entityId
+	 * @param string $entityType
+	 * @param string $entityIndex
+	 * @param string $entityIndexAlias
+	 * @param HttpClient $manticoreClient
+	 * @return void
+	 */
+	public static function add(
+		string $entitySource,
+		string $entityId,
+		string $entityType,
+		string $entityIndex,
+		string $entityIndexAlias,
+		HTTPClient $manticoreClient
+	): void {
+		$query = 'CREATE TABLE IF NOT EXISTS ' . parent::ENTITY_TABLE
+			. ' (_id string, _index string, _index_alias string, _type string, _source json)';
+		$manticoreClient->sendRequest($query);
+		$updatedSource = str_replace('\\"', '\\\\"', $entitySource);
+		$query = 'INSERT INTO ' . parent::ENTITY_TABLE . '(_id, _index, _index_alias,  _type, _source) '
+			. "VALUES('{$entityId}', '{$entityIndex}', '{$entityIndexAlias}', '{$entityType}', '{$updatedSource}')";
+		self::executeQuery($query, $manticoreClient, 'entity creation');
+
+		// Checking if there're previously imported entities related to this pattern and updating them
+		if ($entityType !== 'index-pattern') {
+			return;
+		}
+		$importedPatternId = ImportKibanaHandler::checkEntityPatterns(
+			$entitySource,
+			'import',
+			$manticoreClient
+		);
+		if (!$importedPatternId) {
+			return;
+		}
+		ImportKibanaHandler::updateEntityPatterns($importedPatternId, $entityId, $manticoreClient);
+	}
+}
diff --git a/src/Plugin/EmulateElastic/AddTemplateHandler.php b/src/Plugin/EmulateElastic/AddTemplateHandler.php
@@ -0,0 +1,90 @@
+<?php declare(strict_types=1);
+
+/*
+ Copyright (c) 2024, Manticore Software LTD (https://manticoresearch.com)
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License version 3 or any later
+ version. You should have received a copy of the GPL license along with this
+ program; if you did not, you can find it at http://www.gnu.org/
+ */
+
+namespace Manticoresearch\Buddy\Base\Plugin\EmulateElastic;
+
+use Manticoresearch\Buddy\Core\ManticoreSearch\Client as HTTPClient;
+use Manticoresearch\Buddy\Core\Plugin\BaseHandlerWithClient;
+use Manticoresearch\Buddy\Core\Task\Task;
+use Manticoresearch\Buddy\Core\Task\TaskResult;
+use RuntimeException;
+
+/**
+ * This is the parent class to handle erroneous Manticore queries
+ */
+class AddTemplateHandler extends BaseHandlerWithClient {
+
+	const TEMPLATE_TABLE = '_templates';
+
+	/**
+	 *  Initialize the executor
+	 *
+	 * @param Payload $payload
+	 * @return void
+	 */
+	public function __construct(public Payload $payload) {
+	}
+
+	/**
+	 * Process the request and return self for chaining
+	 *
+	 * @return Task
+	 * @throws RuntimeException
+	 */
+	public function run(): Task {
+		$taskFn = static function (Payload $payload, HTTPClient $manticoreClient): TaskResult {
+			/** @var array<string,mixed> */
+			$request = json_decode($payload->body, true);
+			if (!is_array($request) || !isset($request['index_patterns'])) {
+				throw new \Exception('Cannot parse request');
+			}
+			$patterns = json_encode($request['index_patterns']);
+			foreach (['index_patterns', 'settings', 'mappings'] as $removeProp) {
+				if (!isset($request[$removeProp])) {
+					continue;
+				}
+				unset($request[$removeProp]);
+			}
+			$content = json_encode($request);
+			$templateTable = self::TEMPLATE_TABLE;
+
+			// We may need to do a wildcard search by template name so we index it
+			$query = "CREATE TABLE IF NOT EXISTS {$templateTable} "
+				. "(name string indexed attribute, patterns json, content json) min_prefix_len='2'";
+			$manticoreClient->sendRequest($query);
+
+			$query = "SELECT 1 FROM {$templateTable} WHERE name='{$payload->table}'";
+			/** @var array{0:array{data:mixed}} $queryResult */
+			$queryResult = $manticoreClient->sendRequest($query)->getResult();
+			$query = $queryResult[0]['data']
+				? "UPDATE {$templateTable} SET patterns='{$patterns}', content='{$content}'"
+					. "WHERE name='{$payload->table}'"
+				: "INSERT INTO {$templateTable} (name, patterns, content) "
+					. "VALUES ('{$payload->table}', '{$patterns}', '{$content}')";
+			$queryResult = $manticoreClient->sendRequest($query)->getResult();
+			if (isset($queryResult['error'])) {
+				throw new \Exception('Unknown error on alias creation');
+			}
+
+			return TaskResult::raw(
+				[
+					'acknowledged' => 'true',
+					'index' => $payload->table,
+					'shards_acknowledged' => 'true',
+				]
+			);
+		};
+
+		return Task::create(
+			$taskFn, [$this->payload, $this->manticoreClient]
+		)->run();
+	}
+}