From e7ab5e9ec08f32216617fac9376410f6fac1ec12 Mon Sep 17 00:00:00 2001 From: Nick Sergeev <45552280+Nick-S-2018@users.noreply.github.com> Date: Thu, 23 Jan 2025 23:50:33 +0700 Subject: [PATCH] Updated the autoinsert of text fields in Elastic-like requests (#445) * Updated the autoinsert of text fields in Elastic-like requests * Merged 'main' branch * Added support for json_secondary_indexes, updated tests * Added extra table options for elastic-like create requests --------- Co-authored-by: nick --- composer.lock | 4 +-- src/Plugin/Insert/Payload.php | 13 ++++++-- .../QueryParser/CheckInsertDataTrait.php | 4 +-- src/Plugin/Insert/QueryParser/Datatype.php | 2 ++ .../QueryParser/ElasticJSONInsertParser.php | 1 + .../Insert/QueryParser/JSONInsertParser.php | 32 ++++++++++++++++--- src/Plugin/Insert/QueryParser/Loader.php | 16 ++++++++-- test/Buddy/functional/InsertQueryTest.php | 16 +++++----- .../InsertQuery/InsertQueryPayloadTest.php | 3 +- .../QueryParser/JSONInsertParserTest.php | 24 ++++++++++++++ 10 files changed, 91 insertions(+), 24 deletions(-) diff --git a/composer.lock b/composer.lock index 22b3dc8f..2703987f 100644 --- a/composer.lock +++ b/composer.lock @@ -5052,8 +5052,8 @@ }, "prefer-stable": false, "prefer-lowest": false, - "platform": {}, - "platform-dev": {}, + "platform": [], + "platform-dev": [], "platform-overrides": { "php": "8.1.0" }, diff --git a/src/Plugin/Insert/Payload.php b/src/Plugin/Insert/Payload.php index 89049c64..50daafdf 100644 --- a/src/Plugin/Insert/Payload.php +++ b/src/Plugin/Insert/Payload.php @@ -20,6 +20,9 @@ * @phpstan-extends BasePayload */ final class Payload extends BasePayload { + + const ELASTIC_LIKE_TABLE_OPTIONS = ["min_infix_len='2'"]; + /** @var array */ public array $queries = []; @@ -53,7 +56,11 @@ public static function fromRequest(Request $request): static { $self->path = '_bulk'; $request->payload = trim($request->payload) . "\n"; } - $self->queries[] = $self->buildCreateTableQuery(...$parser->parse($request->payload)); + $createTableQuery = $self->buildCreateTableQuery(...$parser->parse($request->payload)); + if (Loader::isElasticLikeRequest($request->path, $request->endpointBundle)) { + $createTableQuery .= ' ' . implode(' ', self::ELASTIC_LIKE_TABLE_OPTIONS); + } + $self->queries[] = $createTableQuery; $self->queries[] = (str_contains($self->path, '_doc') || str_contains($self->path, '_create')) ? self::preprocessElasticLikeRequest($self->path, $request->payload) : $request->payload; @@ -92,14 +99,14 @@ protected static function buildCreateTableQuery(string $name, array $cols, array ',', array_map( function ($a, $b) { - return "$a $b"; + return "`$a` $b"; }, $cols, $colTypes ) ); $repls = ['%NAME%' => $name, '%COL_EXPR%' => $colExpr]; - return strtr('CREATE TABLE IF NOT EXISTS %NAME% (%COL_EXPR%)', $repls); + return strtr('CREATE TABLE IF NOT EXISTS `%NAME%` (%COL_EXPR%)', $repls); } /** diff --git a/src/Plugin/Insert/QueryParser/CheckInsertDataTrait.php b/src/Plugin/Insert/QueryParser/CheckInsertDataTrait.php index 9cdc23c3..cd7b3353 100644 --- a/src/Plugin/Insert/QueryParser/CheckInsertDataTrait.php +++ b/src/Plugin/Insert/QueryParser/CheckInsertDataTrait.php @@ -102,9 +102,9 @@ protected static function checkTypeBundlesCompatibility( string &$error ): void { $typeBundles = [ - [Datatype::Json, Datatype::Multi64, Datatype::Multi, Datatype::Null], + [Datatype::Indexedjson, Datatype::Json, Datatype::Multi64, Datatype::Multi, Datatype::Null], [Datatype::Float, Datatype::Bigint, Datatype::Int, Datatype::Null], - [Datatype::Text, Datatype::String, Datatype::Null], + [Datatype::Text, Datatype::Indexedstring, Datatype::String, Datatype::Null], [Datatype::Text, Datatype::Timestamp, Datatype::Null], ]; $isNewErrorCol = true; diff --git a/src/Plugin/Insert/QueryParser/Datatype.php b/src/Plugin/Insert/QueryParser/Datatype.php index 13b4ad67..17bf12b7 100644 --- a/src/Plugin/Insert/QueryParser/Datatype.php +++ b/src/Plugin/Insert/QueryParser/Datatype.php @@ -20,6 +20,8 @@ enum Datatype: string { case Bigint = 'bigint'; case Text = 'text'; case String = 'string'; + case Indexedjson = "json secondary_index='1'"; + case Indexedstring = 'string attribute indexed'; case Timestamp = 'timestamp'; case Null = 'null'; } diff --git a/src/Plugin/Insert/QueryParser/ElasticJSONInsertParser.php b/src/Plugin/Insert/QueryParser/ElasticJSONInsertParser.php index a036b4f4..966acc48 100644 --- a/src/Plugin/Insert/QueryParser/ElasticJSONInsertParser.php +++ b/src/Plugin/Insert/QueryParser/ElasticJSONInsertParser.php @@ -32,6 +32,7 @@ class ElasticJSONInsertParser extends JSONInsertParser { * @return void */ public function __construct(string $path) { + $this->isElasticQuery = true; [$this->name, $this->id, $this->isBulkQuery] = $this->parseQueryPath($path); } diff --git a/src/Plugin/Insert/QueryParser/JSONInsertParser.php b/src/Plugin/Insert/QueryParser/JSONInsertParser.php index 5a4c6cca..3c9a6f9d 100644 --- a/src/Plugin/Insert/QueryParser/JSONInsertParser.php +++ b/src/Plugin/Insert/QueryParser/JSONInsertParser.php @@ -22,6 +22,18 @@ class JSONInsertParser extends JSONParser implements InsertQueryParserInterface */ protected ?int $id = null; + /** + * @var bool $isElasticQuery + */ + protected bool $isElasticQuery; + + /** + * @return void + */ + public function __construct() { + $this->isElasticQuery = false; + } + /** * @param string $query * @return array{name:string,cols:array,colTypes:array} @@ -122,13 +134,13 @@ protected function parseInsertValues(mixed $insertRow): array { * @param array $val * @return Datatype */ - protected static function detectArrayVal(array $val): Datatype { + protected function detectArrayVal(array $val): Datatype { if (!array_is_list($val)) { return Datatype::Json; } $returnType = Datatype::Multi; foreach ($val as $subVal) { - $subValType = self::detectValType($subVal); + $subValType = $this->detectValType($subVal); if ($returnType === Datatype::Multi && $subValType === Datatype::Bigint) { $returnType = Datatype::Multi64; } elseif ($subValType !== Datatype::Bigint && $subValType !== Datatype::Int) { @@ -162,14 +174,24 @@ protected static function detectStringVal(string $val): Datatype { * @param mixed $val * @return Datatype */ - protected static function detectValType(mixed $val): Datatype { - return match (true) { + protected function detectValType(mixed $val): Datatype { + $type = match (true) { ($val === null) => Datatype::Null, is_float($val) => Datatype::Float, is_int($val) => self::detectIntVal($val), - is_array($val) => self::detectArrayVal($val), is_string($val) => self::detectStringVal($val), + is_array($val) => $this->detectArrayVal($val), default => Datatype::Text, }; + if (!$this->isElasticQuery) { + return $type; + } + if ($type === Datatype::Text || $type === Datatype::String) { + $type = Datatype::Indexedstring; + } elseif ($type === Datatype::Json) { + $type = Datatype::Indexedjson; + } + + return $type; } } diff --git a/src/Plugin/Insert/QueryParser/Loader.php b/src/Plugin/Insert/QueryParser/Loader.php index 4a28fbd5..0921b445 100644 --- a/src/Plugin/Insert/QueryParser/Loader.php +++ b/src/Plugin/Insert/QueryParser/Loader.php @@ -17,6 +17,16 @@ class Loader { + /** + * + * @param string $requestPath + * @param ManticoreEndpoint $requestEndpointBundle + * @return bool + */ + public static function isElasticLikeRequest(string $requestPath, ManticoreEndpoint $requestEndpointBundle): bool { + return ($requestEndpointBundle->value !== $requestPath); + } + /** * @param string $reqPath * @param ManticoreEndpoint $reqEndpointBundle @@ -34,9 +44,9 @@ public static function getInsertQueryParser( }; $parserClass = match ($reqFormat) { RequestFormat::SQL => 'SQLInsertParser', - RequestFormat::JSON => ($reqEndpointBundle->value === $reqPath) - ? 'JSONInsertParser' - : 'ElasticJSONInsertParser', + RequestFormat::JSON => self::isElasticLikeRequest($reqPath, $reqEndpointBundle) + ? 'ElasticJSONInsertParser' + : 'JSONInsertParser', }; $parserClassFull = __NAMESPACE__ . '\\' . $parserClass; $parser = ($parserClassFull === __NAMESPACE__ . '\ElasticJSONInsertParser') diff --git a/test/Buddy/functional/InsertQueryTest.php b/test/Buddy/functional/InsertQueryTest.php index 3100a86b..041fbe83 100644 --- a/test/Buddy/functional/InsertQueryTest.php +++ b/test/Buddy/functional/InsertQueryTest.php @@ -181,14 +181,14 @@ public function testAutoColumnAddOnInsert(): void { $this->assertEquals(3, sizeof($outData['items'])); $out = static::runSqlQuery("describe {$this->testTable}"); $res = [ - '+-----------+--------+----------------+', - '| Field | Type | Properties |', - '+-----------+--------+----------------+', - '| id | bigint | |', - '| title | text | indexed stored |', - '| price | uint | |', - '| new_price | float | |', - '+-----------+--------+----------------+', + '+-----------+--------+-------------------+', + '| Field | Type | Properties |', + '+-----------+--------+-------------------+', + '| id | bigint | |', + '| title | string | indexed attribute |', + '| price | uint | |', + '| new_price | float | |', + '+-----------+--------+-------------------+', ]; $this->assertEquals($res, $out); } diff --git a/test/Plugin/Insert/InsertQuery/InsertQueryPayloadTest.php b/test/Plugin/Insert/InsertQuery/InsertQueryPayloadTest.php index f8eb03df..fc28f3a8 100644 --- a/test/Plugin/Insert/InsertQuery/InsertQueryPayloadTest.php +++ b/test/Plugin/Insert/InsertQuery/InsertQueryPayloadTest.php @@ -39,7 +39,8 @@ public function testCreationFromNetworkRequest(): void { $this->assertEquals(2, sizeof($payload->queries)); $this->assertEquals( [ - 'CREATE TABLE IF NOT EXISTS test (int_col int,string_col text,float_col float,@timestamp timestamp)', + 'CREATE TABLE IF NOT EXISTS `test` (`int_col` int,`string_col` text,`float_col` float,' + . '`@timestamp` timestamp)', 'INSERT INTO test(int_col, string_col, float_col, @timestamp) VALUES(1, \'string\', 2.22,' . ' \'2000-01-01T12:00:00Z\')', ], $payload->queries diff --git a/test/Plugin/Insert/QueryParser/JSONInsertParserTest.php b/test/Plugin/Insert/QueryParser/JSONInsertParserTest.php index 93cde500..204c7429 100644 --- a/test/Plugin/Insert/QueryParser/JSONInsertParserTest.php +++ b/test/Plugin/Insert/QueryParser/JSONInsertParserTest.php @@ -10,6 +10,7 @@ */ use Manticoresearch\Buddy\Base\Plugin\Insert\QueryParser\Datatype; +use Manticoresearch\Buddy\Base\Plugin\Insert\QueryParser\ElasticJSONInsertParser; use Manticoresearch\Buddy\Base\Plugin\Insert\QueryParser\JSONInsertParser; use Manticoresearch\Buddy\Core\Error\QueryParseError; use Manticoresearch\Buddy\CoreTest\Trait\TestProtectedTrait; @@ -37,6 +38,15 @@ public function testNdJSONParse(): void { public function testInsertValTypeDetection(): void { echo "\nTesting the detection of an Insert value datatype\n"; + + // Testing the specific treatment of text values in elastic-like requests + $parser = new ElasticJSONInsertParser('_bulk'); + $this->assertEquals(Datatype::Indexedstring, self::invokeMethod($parser, 'detectValType', ['test text'])); + $this->assertEquals(Datatype::Indexedstring, self::invokeMethod($parser, 'detectValType', ['0.1'])); + $this->assertEquals(Datatype::Indexedstring, self::invokeMethod($parser, 'detectValType', ['11111111111'])); + $this->assertEquals(Datatype::Indexedjson, self::invokeMethod($parser, 'detectValType', [['a' => 1]])); + $this->assertEquals(Datatype::Indexedjson, self::invokeMethod($parser, 'detectValType', [[2, 0.5]])); + $parser = new JSONInsertParser(); self::$parser = $parser; @@ -153,4 +163,18 @@ public function testParseFail(): void { $this->assertEquals(QueryParseError::class, $exCls); $this->assertEquals("Operation name 'insert' is missing", $exMsg); } + + public function testElasticParseOk(): void { + // Testing the specific treatment of text values in elastic-like requests + echo "\nTesting the parsing of JSON insert elastic-like request\n"; + $parser = new ElasticJSONInsertParser('test_elastic'); + $query = '{"col1" : 10, "col2": "a", "col3": {"a":1} }'; + $res = [ + 'name' => 'test_elastic', + 'cols' => ['col1', 'col2', 'col3'], + 'colTypes' => ['int', 'string attribute indexed', "json secondary_index='1'"], + ]; + $this->assertEquals($res, $parser->parse($query)); + } + }