#1020: earliest/latest not supported with where command (#1021)

* Add earliest/latest to `conditionFunctionBase`. Signed-off-by: currantw <[email protected]> * Update doc and integ tests Signed-off-by: currantw <[email protected]> * Update/fix doc tests Signed-off-by: currantw <[email protected]> * Rename variable Signed-off-by: currantw <[email protected]> * scalafmtAll Signed-off-by: currantw <[email protected]> * Fix failing integ test Signed-off-by: currantw <[email protected]> * More scalafmt. Grrr... Signed-off-by: currantw <[email protected]> --------- Signed-off-by: currantw <[email protected]>
opensearch-project · Jan 27, 2025 · 3832906 · 3832906
1 parent 54248bb
commit 3832906
Show file tree

Hide file tree

Showing 6 changed files with 185 additions and 179 deletions.
diff --git a/docs/ppl-lang/functions/ppl-datetime.md b/docs/ppl-lang/functions/ppl-datetime.md
@@ -411,30 +411,15 @@ Return type: BOOLEAN
 
 Example:
 
-    os> source=people | eval earliest = earliest("-1s", now()) | fields earliest | head 1
-    fetched rows / total rows = 1/1
-    +----------+
-    | earliest |
-    |----------|
-    | True     |
-    +----------+
-
-    os> source=people | eval earliest = earliest("now", now()) | fields earliest | head 1
-    fetched rows / total rows = 1/1
-    +----------+
-    | earliest |
-    |----------|
-    | True     |
-    +----------+
-
-    os> source=people | eval earliest = earliest("+1s", now()) | fields earliest | head 1
-    fetched rows / total rows = 1/1
-    +----------+
-    | earliest |
-    |----------|
-    | False    |
-    +----------+
-
+    os> source=relative_datetime | eval timestamp = relative_timestamp(relative_string) | where earliest("now",timestamp) | sort timestamp | fields description, relative_string
+    fetched rows / total rows = 3/3
+    +--------------+-----------------+
+    | description  | relative_string |
+    +--------------+-----------------+
+    | Now          | NOW             |
+    | Tomorrow     | +D@D            |
+    | In one month | +month          |
+    +--------------+-----------------+
 
 ### `FROM_UNIXTIME`
 
@@ -560,29 +545,15 @@ Return type: BOOLEAN
 
 Example:
 
-    os> source=people | eval latest = latest("-1s", now()) | fields latest | head 1
-    fetched rows / total rows = 1/1
-    +--------+
-    | latest |
-    |--------|
-    | False  |
-    +--------+
-
-    os> source=people | eval latest = latest("now", now()) | fields latest | head 1
-    fetched rows / total rows = 1/1
-    +--------+
-    | latest |
-    |--------|
-    | True   |
-    +--------+
-
-    os> source=people | eval latest = latest("+1s", now()) | fields latest | head 1
-    fetched rows / total rows = 1/1
-    +--------+
-    | latest |
-    |--------|
-    | True   |
-    +--------+
+    os> source=relative_datetime | eval timestamp = relative_timestamp(relative_string) | where latest("now",timestamp) | sort timestamp | fields description, relative_string
+    fetched rows / total rows = 3/3
+    +---------------+-----------------+
+    | description   | relative_string |
+    +---------------+-----------------+
+    | Two weeks ago | -2wk            |
+    | Yesterday     | -1d@d           |
+    | Now           | NOW             |
+    +---------------+-----------------+
 
 
 ### `LOCALTIMESTAMP`
@@ -881,29 +852,17 @@ Return type: TIMESTAMP
 
 Example:
 
-    os> source=people | eval seconds_diff = timestampdiff(SECOND, now(), relative_timestamp("now")) | fields seconds_diff | head 1
-    fetched rows / total rows = 1/1
-    +--------------+
-    | seconds_diff |
-    |--------------+
-    | 0            |
-    +--------------+
-
-    os> source=people | eval hours_diff = timestampdiff(HOUR, now(), relative_timestamp("+1h")) | fields hours_diff | head 1
-    fetched rows / total rows = 1/1
-    +------------+
-    | hours_diff |
-    |------------+
-    | 1          |
-    +------------+
-
-    os> source=people | eval day = day_of_week(relative_timestamp("@w0")) | fields day | head 1
+    os> source=relative_datetime | eval relative = relative_timestamp(relative_string) | sort relative | fields description, relative_string
     fetched rows / total rows = 1/1
-    +-----+
-    | day |
-    |-----|
-    | 1   |
-    +-----+
+    +---------------+-----------------+
+    | description   | relative_string |
+    +---------------+-----------------+
+    | Two weeks ago | -2wk            |
+    | Yesterday     | -1d@d           |
+    | Now           | NOW             |
+    | Tomorrow      | +D@D            |
+    | In one month  | +month          |
+    +---------------+-----------------+
 
 ### `SECOND`
 
@@ -1227,7 +1186,7 @@ Examples::
 **Description:**
 
 Usage: TIMESTAMPDIFF(interval, start, end) returns the difference between the start and end date/times in interval units.
-Arguments will be automatically converted to a ]TIMESTAMP when appropriate.
+Arguments will be automatically converted to a TIMESTAMP when appropriate.
 Any argument that is a STRING must be formatted as a valid TIMESTAMP.
 
 Argument type: INTERVAL, DATE/TIMESTAMP/STRING, DATE/TIMESTAMP/STRING

diff --git a/integ-test/script/data/relative_datetime.json b/integ-test/script/data/relative_datetime.json
@@ -0,0 +1,10 @@
+{"index": {"_index": "relative_datetime"}}
+{"description": "Now", "relative_string": "NOW"}
+{"index": {"_index": "relative_datetime"}}
+{"description": "Tomorrow", "relative_string": "+D@D"}
+{"index": {"_index": "relative_datetime"}}
+{"description": "In one month", "relative_string": "+month"}
+{"index": {"_index": "relative_datetime"}}
+{"description": "Two weeks ago", "relative_string": "-2wk"}
+{"index": {"_index": "relative_datetime"}}
+{"description": "Yesterday", "relative_string": "-1d@d"}
diff --git a/integ-test/script/data/relative_datetime.mapping.json b/integ-test/script/data/relative_datetime.mapping.json
@@ -0,0 +1,12 @@
+{
+  "mappings": {
+    "properties": {
+      "description": {
+        "type": "text"
+      },
+      "relative_string": {
+        "type": "text"
+      }
+    }
+  }
+}
diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkSuite.scala
@@ -229,6 +229,26 @@ trait FlintSparkSuite extends QueryTest with FlintSuite with OpenSearchSuite wit
            | """.stripMargin)
   }
 
+  protected def createRelativeDateTimeTable(testTable: String): Unit = {
+    sql(s"""
+           | CREATE TABLE $testTable
+           | (
+           |   description STRING,
+           |   relative_string STRING
+           | )
+           | USING $tableType $tableOptions
+           |""".stripMargin)
+
+    sql(s"""
+           | INSERT INTO $testTable
+           | VALUES ('Now', 'NOW'),
+           |        ('Tomorrow', '+D@D'),
+           |        ('In one month', '+month'),
+           |        ('Two weeks ago', '-2wk'),
+           |        ('Yesterday', '-1d@d')
+           | """.stripMargin)
+  }
+
   protected def createNullableStateCountryTable(testTable: String): Unit = {
     sql(s"""
            | CREATE TABLE $testTable