Support for OpenSearch alias type (opensearch-project#3246)

Signed-off-by: Heng Qian <[email protected]>
Bit-Quill · Jan 23, 2025 · 1284c12 · 1284c12
1 parent b738faf
commit 1284c12
Show file tree

Hide file tree

Showing 17 changed files with 312 additions and 11 deletions.
diff --git a/core/src/main/java/org/opensearch/sql/data/type/ExprType.java b/core/src/main/java/org/opensearch/sql/data/type/ExprType.java
@@ -9,6 +9,7 @@
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 import org.opensearch.sql.data.model.ExprValue;
 import org.opensearch.sql.expression.Expression;
 
@@ -54,4 +55,17 @@ default List<ExprType> getParent() {
   default String legacyTypeName() {
     return typeName();
   }
+
+  /** Get the original path. Types like alias type will set the actual path in field property. */
+  default Optional<String> getOriginalPath() {
+    return Optional.empty();
+  }
+
+  /**
+   * Get the original path. Types like alias type should be derived from the type of the original
+   * field.
+   */
+  default ExprType getOriginalExprType() {
+    return this;
+  }
 }
diff --git a/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java b/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java
@@ -23,6 +23,8 @@
 public class ReferenceExpression implements Expression {
   @Getter private final String attr;
 
+  @Getter private final String rawPath;
+
   @Getter private final List<String> paths;
 
   private final ExprType type;
@@ -36,8 +38,11 @@ public class ReferenceExpression implements Expression {
   public ReferenceExpression(String ref, ExprType type) {
     this.attr = ref;
     // Todo. the define of paths need to be redefined after adding multiple index/variable support.
-    this.paths = Arrays.asList(ref.split("\\."));
-    this.type = type;
+    // For AliasType, the actual path is set in the property of `path` and the type is derived
+    // from the type of field on that path; Otherwise, use ref itself as the path
+    this.rawPath = type.getOriginalPath().orElse(ref);
+    this.paths = Arrays.asList(rawPath.split("\\."));
+    this.type = type.getOriginalExprType();
   }
 
   @Override

diff --git a/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java b/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java
@@ -24,6 +24,7 @@
 import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED;
 import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN;
 
+import java.util.Optional;
 import org.hamcrest.Matchers;
 import org.junit.jupiter.api.Test;
 
@@ -85,4 +86,16 @@ void defaultLegacyTypeName() {
     final ExprType exprType = () -> "dummy";
     assertEquals("dummy", exprType.legacyTypeName());
   }
+
+  @Test
+  void getOriginalPath() {
+    final ExprType exprType = () -> "dummy";
+    assertEquals(Optional.empty(), exprType.getOriginalPath());
+  }
+
+  @Test
+  void getOriginalExprType() {
+    final ExprType exprType = () -> "dummy";
+    assertEquals(exprType, exprType.getOriginalExprType());
+  }
 }
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java
@@ -8,6 +8,7 @@
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static org.opensearch.sql.legacy.TestUtils.createIndexByRestClient;
 import static org.opensearch.sql.legacy.TestUtils.getAccountIndexMapping;
+import static org.opensearch.sql.legacy.TestUtils.getAliasIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getBankIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getDataTypeNonnumericIndexMapping;
@@ -751,7 +752,12 @@ public enum Index {
         TestsConstants.TEST_INDEX_JSON_TEST,
         "json",
         getJsonTestIndexMapping(),
-        "src/test/resources/json_test.json");
+        "src/test/resources/json_test.json"),
+    DATA_TYPE_ALIAS(
+        TestsConstants.TEST_INDEX_ALIAS,
+        "alias",
+        getAliasIndexMapping(),
+        "src/test/resources/alias.json");
 
     private final String name;
     private final String type;

diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java
@@ -255,6 +255,11 @@ public static String getJsonTestIndexMapping() {
     return getMappingFile(mappingFile);
   }
 
+  public static String getAliasIndexMapping() {
+    String mappingFile = "alias_index_mapping.json";
+    return getMappingFile(mappingFile);
+  }
+
   public static void loadBulk(Client client, String jsonPath, String defaultIndex)
       throws Exception {
     System.out.println(String.format("Loading file %s into opensearch cluster", jsonPath));

diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java
@@ -59,6 +59,7 @@ public class TestsConstants {
   public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls";
   public static final String TEST_INDEX_GEOPOINT = TEST_INDEX + "_geopoint";
   public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test";
+  public static final String TEST_INDEX_ALIAS = TEST_INDEX + "_alias";
   public static final String DATASOURCES = ".ql-datasources";
 
   public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";

diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java
@@ -5,8 +5,10 @@
 
 package org.opensearch.sql.ppl;
 
+import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_ALIAS;
 import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NONNUMERIC;
 import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ALIAS;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC;
 import static org.opensearch.sql.util.MatcherUtils.schema;
@@ -22,6 +24,7 @@ public class DataTypeIT extends PPLIntegTestCase {
   public void init() throws IOException {
     loadIndex(DATA_TYPE_NUMERIC);
     loadIndex(DATA_TYPE_NONNUMERIC);
+    loadIndex(DATA_TYPE_ALIAS);
   }
 
   @Test
@@ -75,4 +78,14 @@ public void test_long_integer_data_type() throws IOException {
         schema("long1", "long"),
         schema("long2", "long"));
   }
+
+  @Test
+  public void test_alias_data_type() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | where alias_col > 1 " + "| fields original_col, alias_col ",
+                TEST_INDEX_ALIAS));
+    verifySchema(result, schema("original_col", "integer"), schema("alias_col", "integer"));
+  }
 }
diff --git a/integ-test/src/test/resources/alias.json b/integ-test/src/test/resources/alias.json
@@ -0,0 +1,6 @@
+{"index":{"_id":"1"}}
+{"original_col" : 1}
+{"index":{"_id":"2"}}
+{"original_col" : 2}
+{"index":{"_id":"3"}}
+{"original_col" : 3}
diff --git a/integ-test/src/test/resources/indexDefinitions/alias_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/alias_index_mapping.json
@@ -0,0 +1,13 @@
+{
+  "mappings": {
+    "properties": {
+      "original_col": {
+        "type": "integer"
+      },
+      "alias_col": {
+        "type": "alias",
+        "path": "original_col"
+      }
+    }
+  }
+}
diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchAliasType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchAliasType.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.opensearch.data.type;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import org.opensearch.sql.data.type.ExprType;
+
+/**
+ * The type of alias. See <a
+ * href="https://opensearch.org/docs/latest/opensearch/supported-field-types/alias/">doc</a>
+ */
+public class OpenSearchAliasType extends OpenSearchDataType {
+
+  public static final String typeName = "alias";
+  public static final String pathPropertyName = "path";
+  public static final Set<MappingType> objectFieldTypes =
+      Set.of(MappingType.Object, MappingType.Nested);
+  private final String path;
+  private final OpenSearchDataType originalType;
+
+  public OpenSearchAliasType(String path, OpenSearchDataType type) {
+    super(type.getExprCoreType());
+    if (type instanceof OpenSearchAliasType) {
+      throw new IllegalStateException(
+          String.format("Alias field cannot refer to the path [%s] of alias type", path));
+    } else if (objectFieldTypes.contains(type.getMappingType())) {
+      throw new IllegalStateException(
+          String.format("Alias field cannot refer to the path [%s] of object type", path));
+    }
+    this.path = path;
+    this.originalType = type;
+  }
+
+  @Override
+  public Optional<String> getOriginalPath() {
+    return Optional.of(this.path);
+  }
+
+  @Override
+  public ExprType getOriginalExprType() {
+    return originalType.getExprType();
+  }
+
+  @Override
+  public ExprType getExprType() {
+    return this;
+  }
+
+  @Override
+  public OpenSearchDataType cloneEmpty() {
+    return new OpenSearchAliasType(this.path, originalType.cloneEmpty());
+  }
+
+  @Override
+  public boolean isCompatible(ExprType other) {
+    return originalType.isCompatible(other);
+  }
+
+  @Override
+  public List<ExprType> getParent() {
+    return originalType.getParent();
+  }
+
+  @Override
+  public String typeName() {
+    return originalType.typeName();
+  }
+
+  @Override
+  public String legacyTypeName() {
+    return originalType.legacyTypeName();
+  }
+
+  @Override
+  public boolean shouldCast(ExprType other) {
+    return originalType.shouldCast(other);
+  }
+}
diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java
@@ -109,14 +109,19 @@ public static Map<String, OpenSearchDataType> parseMapping(Map<String, Object> i
       return result;
     }
 
+    Map<String, String> aliasMapping = new LinkedHashMap<>();
     indexMapping.forEach(
         (k, v) -> {
           var innerMap = (Map<String, Object>) v;
           // by default, the type is treated as an Object if "type" is not provided
           var type = ((String) innerMap.getOrDefault("type", "object")).replace("_", "");
           if (!EnumUtils.isValidEnumIgnoreCase(OpenSearchDataType.MappingType.class, type)) {
             // unknown type, e.g. `alias`
-            // TODO resolve alias reference
+            // Record fields of the alias type and resolve them later in case their references have
+            // not been resolved.
+            if (OpenSearchAliasType.typeName.equals(type)) {
+              aliasMapping.put(k, (String) innerMap.get(OpenSearchAliasType.pathPropertyName));
+            }
             return;
           }
           // create OpenSearchDataType
@@ -126,6 +131,18 @@ public static Map<String, OpenSearchDataType> parseMapping(Map<String, Object> i
                   EnumUtils.getEnumIgnoreCase(OpenSearchDataType.MappingType.class, type),
                   innerMap));
         });
+
+    // Begin to parse alias type fields
+    aliasMapping.forEach(
+        (k, v) -> {
+          if (result.containsKey(v)) {
+            result.put(k, new OpenSearchAliasType(v, result.get(v)));
+          } else {
+            throw new IllegalStateException(
+                String.format("Cannot find the path [%s] for alias type field [%s]", v, k));
+          }
+        });
+
     return result;
   }
 

diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java
@@ -281,7 +281,7 @@ public void pushDownHighlight(String field, Map<String, Literal> arguments) {
   /** Push down project list to DSL requests. */
   public void pushDownProjects(Set<ReferenceExpression> projects) {
     sourceBuilder.fetchSource(
-        projects.stream().map(ReferenceExpression::getAttr).distinct().toArray(String[]::new),
+        projects.stream().map(ReferenceExpression::getRawPath).distinct().toArray(String[]::new),
         new String[0]);
   }
 

diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java
@@ -70,6 +70,7 @@
 import org.opensearch.sql.data.model.ExprIntegerValue;
 import org.opensearch.sql.data.model.ExprTupleValue;
 import org.opensearch.sql.data.model.ExprValue;
+import org.opensearch.sql.opensearch.data.type.OpenSearchAliasType;
 import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
 import org.opensearch.sql.opensearch.data.type.OpenSearchTextType;
 import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory;
@@ -159,9 +160,9 @@ void get_index_mappings() throws IOException {
     var parsedTypes = OpenSearchDataType.traverseAndFlatten(mapping);
     assertAll(
         () -> assertEquals(1, indexMappings.size()),
-        // 10 types extended to 17 after flattening
-        () -> assertEquals(10, mapping.size()),
-        () -> assertEquals(17, parsedTypes.size()),
+        // 11 types extended to 18 after flattening
+        () -> assertEquals(11, mapping.size()),
+        () -> assertEquals(18, parsedTypes.size()),
         () -> assertEquals("TEXT", mapping.get("address").legacyTypeName()),
         () -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("address")),
         () -> assertEquals("INTEGER", mapping.get("age").legacyTypeName()),
@@ -186,6 +187,11 @@ void get_index_mappings() throws IOException {
         () -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("employer")),
         // `employer` is a `text` with `fields`
         () -> assertTrue(((OpenSearchTextType) parsedTypes.get("employer")).getFields().size() > 0),
+        () -> assertEquals("TEXT", mapping.get("employer_alias").legacyTypeName()),
+        () ->
+            assertEquals(
+                new OpenSearchAliasType("employer", OpenSearchTextType.of(MappingType.Text)),
+                parsedTypes.get("employer_alias")),
         () -> assertEquals("NESTED", mapping.get("projects").legacyTypeName()),
         () -> assertEquals(OpenSearchTextType.of(MappingType.Nested), parsedTypes.get("projects")),
         () ->

diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java
@@ -68,6 +68,7 @@
 import org.opensearch.sql.data.model.ExprIntegerValue;
 import org.opensearch.sql.data.model.ExprTupleValue;
 import org.opensearch.sql.data.model.ExprValue;
+import org.opensearch.sql.opensearch.data.type.OpenSearchAliasType;
 import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
 import org.opensearch.sql.opensearch.data.type.OpenSearchTextType;
 import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory;
@@ -162,9 +163,9 @@ void get_index_mappings() throws IOException {
     var parsedTypes = OpenSearchDataType.traverseAndFlatten(mapping);
     assertAll(
         () -> assertEquals(1, indexMappings.size()),
-        // 10 types extended to 17 after flattening
-        () -> assertEquals(10, mapping.size()),
-        () -> assertEquals(17, parsedTypes.size()),
+        // 11 types extended to 18 after flattening
+        () -> assertEquals(11, mapping.size()),
+        () -> assertEquals(18, parsedTypes.size()),
         () -> assertEquals("TEXT", mapping.get("address").legacyTypeName()),
         () -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("address")),
         () -> assertEquals("INTEGER", mapping.get("age").legacyTypeName()),
@@ -189,6 +190,11 @@ void get_index_mappings() throws IOException {
         () -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("employer")),
         // `employer` is a `text` with `fields`
         () -> assertTrue(((OpenSearchTextType) parsedTypes.get("employer")).getFields().size() > 0),
+        () -> assertEquals("TEXT", mapping.get("employer_alias").legacyTypeName()),
+        () ->
+            assertEquals(
+                new OpenSearchAliasType("employer", OpenSearchTextType.of(MappingType.Text)),
+                parsedTypes.get("employer_alias")),
         () -> assertEquals("NESTED", mapping.get("projects").legacyTypeName()),
         () -> assertEquals(OpenSearchTextType.of(MappingType.Nested), parsedTypes.get("projects")),
         () ->