Updated the autoinsert of text fields in Elastic-like requests (#445)

* Updated the autoinsert of text fields in Elastic-like requests * Merged 'main' branch * Added support for json_secondary_indexes, updated tests * Added extra table options for elastic-like create requests --------- Co-authored-by: nick <[email protected]>
manticoresoftware · Jan 23, 2025 · e7ab5e9 · e7ab5e9
1 parent 3a661d8
commit e7ab5e9
Show file tree

Hide file tree

Showing 10 changed files with 91 additions and 24 deletions.
diff --git a/composer.lock b/composer.lock
diff --git a/src/Plugin/Insert/Payload.php b/src/Plugin/Insert/Payload.php
@@ -20,6 +20,9 @@
  * @phpstan-extends BasePayload<array>
  */
 final class Payload extends BasePayload {
+
+	const ELASTIC_LIKE_TABLE_OPTIONS = ["min_infix_len='2'"];
+
 	/** @var array<string> */
 	public array $queries = [];
 
@@ -53,7 +56,11 @@ public static function fromRequest(Request $request): static {
 			$self->path = '_bulk';
 			$request->payload = trim($request->payload) . "\n";
 		}
-		$self->queries[] = $self->buildCreateTableQuery(...$parser->parse($request->payload));
+		$createTableQuery = $self->buildCreateTableQuery(...$parser->parse($request->payload));
+		if (Loader::isElasticLikeRequest($request->path, $request->endpointBundle)) {
+			$createTableQuery .= ' ' . implode(' ', self::ELASTIC_LIKE_TABLE_OPTIONS);
+		}
+		$self->queries[] = $createTableQuery;
 		$self->queries[] = (str_contains($self->path, '_doc') || str_contains($self->path, '_create'))
 			? self::preprocessElasticLikeRequest($self->path, $request->payload)
 			: $request->payload;
@@ -92,14 +99,14 @@ protected static function buildCreateTableQuery(string $name, array $cols, array
 			',',
 			array_map(
 				function ($a, $b) {
-					return "$a $b";
+					return "`$a` $b";
 				},
 				$cols,
 				$colTypes
 			)
 		);
 		$repls = ['%NAME%' => $name, '%COL_EXPR%' => $colExpr];
-		return strtr('CREATE TABLE IF NOT EXISTS %NAME% (%COL_EXPR%)', $repls);
+		return strtr('CREATE TABLE IF NOT EXISTS `%NAME%` (%COL_EXPR%)', $repls);
 	}
 
 	/**

diff --git a/src/Plugin/Insert/QueryParser/CheckInsertDataTrait.php b/src/Plugin/Insert/QueryParser/CheckInsertDataTrait.php
@@ -102,9 +102,9 @@ protected static function checkTypeBundlesCompatibility(
 		string &$error
 	): void {
 		$typeBundles = [
-			[Datatype::Json, Datatype::Multi64, Datatype::Multi, Datatype::Null],
+			[Datatype::Indexedjson, Datatype::Json, Datatype::Multi64, Datatype::Multi, Datatype::Null],
 			[Datatype::Float, Datatype::Bigint, Datatype::Int, Datatype::Null],
-			[Datatype::Text, Datatype::String, Datatype::Null],
+			[Datatype::Text, Datatype::Indexedstring, Datatype::String, Datatype::Null],
 			[Datatype::Text, Datatype::Timestamp, Datatype::Null],
 		];
 		$isNewErrorCol = true;

diff --git a/src/Plugin/Insert/QueryParser/Datatype.php b/src/Plugin/Insert/QueryParser/Datatype.php
@@ -20,6 +20,8 @@ enum Datatype: string {
 	case Bigint = 'bigint';
 	case Text = 'text';
 	case String = 'string';
+	case Indexedjson = "json secondary_index='1'";
+	case Indexedstring = 'string attribute indexed';
 	case Timestamp = 'timestamp';
 	case Null = 'null';
 }
diff --git a/src/Plugin/Insert/QueryParser/ElasticJSONInsertParser.php b/src/Plugin/Insert/QueryParser/ElasticJSONInsertParser.php
@@ -32,6 +32,7 @@ class ElasticJSONInsertParser extends JSONInsertParser {
 	* @return void
 	*/
 	public function __construct(string $path) {
+		$this->isElasticQuery = true;
 		[$this->name, $this->id, $this->isBulkQuery] = $this->parseQueryPath($path);
 	}
 

diff --git a/src/Plugin/Insert/QueryParser/JSONInsertParser.php b/src/Plugin/Insert/QueryParser/JSONInsertParser.php
@@ -22,6 +22,18 @@ class JSONInsertParser extends JSONParser implements InsertQueryParserInterface
 	 */
 	protected ?int $id = null;
 
+	/**
+	 * @var bool $isElasticQuery
+	 */
+	protected bool $isElasticQuery;
+
+	/**
+	 * @return void
+	 */
+	public function __construct() {
+		$this->isElasticQuery = false;
+	}
+
 	/**
 	 * @param string $query
 	 * @return array{name:string,cols:array<string>,colTypes:array<string>}
@@ -122,13 +134,13 @@ protected function parseInsertValues(mixed $insertRow): array {
 	 * @param array<mixed> $val
 	 * @return Datatype
 	 */
-	protected static function detectArrayVal(array $val): Datatype {
+	protected function detectArrayVal(array $val): Datatype {
 		if (!array_is_list($val)) {
 			return Datatype::Json;
 		}
 		$returnType = Datatype::Multi;
 		foreach ($val as $subVal) {
-			$subValType = self::detectValType($subVal);
+			$subValType = $this->detectValType($subVal);
 			if ($returnType === Datatype::Multi && $subValType === Datatype::Bigint) {
 				$returnType = Datatype::Multi64;
 			} elseif ($subValType !== Datatype::Bigint && $subValType !== Datatype::Int) {
@@ -162,14 +174,24 @@ protected static function detectStringVal(string $val): Datatype {
 	 * @param mixed $val
 	 * @return Datatype
 	 */
-	protected static function detectValType(mixed $val): Datatype {
-		return match (true) {
+	protected function detectValType(mixed $val): Datatype {
+		$type = match (true) {
 			($val === null) => Datatype::Null,
 			is_float($val) => Datatype::Float,
 			is_int($val) => self::detectIntVal($val),
-			is_array($val) => self::detectArrayVal($val),
 			is_string($val) => self::detectStringVal($val),
+			is_array($val) => $this->detectArrayVal($val),
 			default => Datatype::Text,
 		};
+		if (!$this->isElasticQuery) {
+			return $type;
+		}
+		if ($type === Datatype::Text || $type === Datatype::String) {
+			$type = Datatype::Indexedstring;
+		} elseif ($type === Datatype::Json) {
+			$type = Datatype::Indexedjson;
+		}
+
+		return $type;
 	}
 }
diff --git a/src/Plugin/Insert/QueryParser/Loader.php b/src/Plugin/Insert/QueryParser/Loader.php
@@ -17,6 +17,16 @@
 
 class Loader {
 
+	/**
+	 *
+	 * @param string $requestPath
+	 * @param ManticoreEndpoint $requestEndpointBundle
+	 * @return bool
+	 */
+	public static function isElasticLikeRequest(string $requestPath, ManticoreEndpoint $requestEndpointBundle): bool {
+		return ($requestEndpointBundle->value !== $requestPath);
+	}
+
 	/**
 	 * @param string $reqPath
 	 * @param ManticoreEndpoint $reqEndpointBundle
@@ -34,9 +44,9 @@ public static function getInsertQueryParser(
 		};
 		$parserClass = match ($reqFormat) {
 			RequestFormat::SQL => 'SQLInsertParser',
-			RequestFormat::JSON => ($reqEndpointBundle->value === $reqPath)
-				? 'JSONInsertParser'
-				: 'ElasticJSONInsertParser',
+			RequestFormat::JSON => self::isElasticLikeRequest($reqPath, $reqEndpointBundle)
+				? 'ElasticJSONInsertParser'
+				: 'JSONInsertParser',
 		};
 		$parserClassFull = __NAMESPACE__ . '\\' . $parserClass;
 		$parser = ($parserClassFull === __NAMESPACE__ . '\ElasticJSONInsertParser')

diff --git a/test/Buddy/functional/InsertQueryTest.php b/test/Buddy/functional/InsertQueryTest.php
@@ -181,14 +181,14 @@ public function testAutoColumnAddOnInsert(): void {
 		$this->assertEquals(3, sizeof($outData['items']));
 		$out = static::runSqlQuery("describe {$this->testTable}");
 		$res = [
-			'+-----------+--------+----------------+',
-			'| Field     | Type   | Properties     |',
-			'+-----------+--------+----------------+',
-			'| id        | bigint |                |',
-			'| title     | text   | indexed stored |',
-			'| price     | uint   |                |',
-			'| new_price | float  |                |',
-			'+-----------+--------+----------------+',
+			'+-----------+--------+-------------------+',
+			'| Field     | Type   | Properties        |',
+			'+-----------+--------+-------------------+',
+			'| id        | bigint |                   |',
+			'| title     | string | indexed attribute |',
+			'| price     | uint   |                   |',
+			'| new_price | float  |                   |',
+			'+-----------+--------+-------------------+',
 		];
 		$this->assertEquals($res, $out);
 	}

diff --git a/test/Plugin/Insert/InsertQuery/InsertQueryPayloadTest.php b/test/Plugin/Insert/InsertQuery/InsertQueryPayloadTest.php
@@ -39,7 +39,8 @@ public function testCreationFromNetworkRequest(): void {
 		$this->assertEquals(2, sizeof($payload->queries));
 		$this->assertEquals(
 			[
-				'CREATE TABLE IF NOT EXISTS test (int_col int,string_col text,float_col float,@timestamp timestamp)',
+				'CREATE TABLE IF NOT EXISTS `test` (`int_col` int,`string_col` text,`float_col` float,'
+					. '`@timestamp` timestamp)',
 				'INSERT INTO test(int_col, string_col, float_col, @timestamp) VALUES(1, \'string\', 2.22,'
 					. ' \'2000-01-01T12:00:00Z\')',
 			], $payload->queries

diff --git a/test/Plugin/Insert/QueryParser/JSONInsertParserTest.php b/test/Plugin/Insert/QueryParser/JSONInsertParserTest.php
@@ -10,6 +10,7 @@
  */
 
 use Manticoresearch\Buddy\Base\Plugin\Insert\QueryParser\Datatype;
+use Manticoresearch\Buddy\Base\Plugin\Insert\QueryParser\ElasticJSONInsertParser;
 use Manticoresearch\Buddy\Base\Plugin\Insert\QueryParser\JSONInsertParser;
 use Manticoresearch\Buddy\Core\Error\QueryParseError;
 use Manticoresearch\Buddy\CoreTest\Trait\TestProtectedTrait;
@@ -37,6 +38,15 @@ public function testNdJSONParse(): void {
 
 	public function testInsertValTypeDetection(): void {
 		echo "\nTesting the detection of an Insert value datatype\n";
+
+		// Testing the specific treatment of text values in elastic-like requests
+		$parser = new ElasticJSONInsertParser('_bulk');
+		$this->assertEquals(Datatype::Indexedstring, self::invokeMethod($parser, 'detectValType', ['test text']));
+		$this->assertEquals(Datatype::Indexedstring, self::invokeMethod($parser, 'detectValType', ['0.1']));
+		$this->assertEquals(Datatype::Indexedstring, self::invokeMethod($parser, 'detectValType', ['11111111111']));
+		$this->assertEquals(Datatype::Indexedjson, self::invokeMethod($parser, 'detectValType', [['a' => 1]]));
+		$this->assertEquals(Datatype::Indexedjson, self::invokeMethod($parser, 'detectValType', [[2, 0.5]]));
+
 		$parser = new JSONInsertParser();
 		self::$parser = $parser;
 
@@ -153,4 +163,18 @@ public function testParseFail(): void {
 		$this->assertEquals(QueryParseError::class, $exCls);
 		$this->assertEquals("Operation name 'insert' is missing", $exMsg);
 	}
+
+	public function testElasticParseOk(): void {
+		// Testing the specific treatment of text values in elastic-like requests
+		echo "\nTesting the parsing of JSON insert elastic-like request\n";
+		$parser = new ElasticJSONInsertParser('test_elastic');
+		$query = '{"col1" : 10, "col2": "a", "col3": {"a":1} }';
+		$res = [
+			'name' => 'test_elastic',
+			'cols' => ['col1', 'col2', 'col3'],
+			'colTypes' => ['int', 'string attribute indexed', "json secondary_index='1'"],
+		];
+		$this->assertEquals($res, $parser->parse($query));
+	}
+
 }