From 1be7c606094fb245bb5fc92d112a23cbfc79b7ac Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Tue, 14 Jan 2025 14:29:44 +0700 Subject: [PATCH 1/6] added doc for polars --- .pre-commit-config.yaml | 10 +- .../data-integration/data-node-usage.md | 141 ++++++++++++++++++ 2 files changed, 146 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4cc0074e2..bcce870ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,8 +34,8 @@ repos: - id: black args: [--line-length=120] language_version: python3.9 -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - additional_dependencies: [flake8-typing-imports==1.10.0] +# - repo: https://gitlab.com/pycqa/flake8 +# rev: 3.9.2 +# hooks: +# - id: flake8 +# additional_dependencies: [flake8-typing-imports==1.10.0] diff --git a/docs/userman/scenario_features/data-integration/data-node-usage.md b/docs/userman/scenario_features/data-integration/data-node-usage.md index 133d1c985..86073f852 100644 --- a/docs/userman/scenario_features/data-integration/data-node-usage.md +++ b/docs/userman/scenario_features/data-integration/data-node-usage.md @@ -285,6 +285,7 @@ The following examples represent the results when reading from a CSV data node w ``` === "exposed_type = SaleRow" + ```python [ SaleRow("12/24/2018", 1550), @@ -293,6 +294,27 @@ The following examples represent the results when reading from a CSV data node w ] ``` + === "exposed_type = polars" + + ```python + shape: (3, 2) + ┌────────────┬──────────┐ + │ date ┆ nb_sales │ + │ --- ┆ --- │ + │ str ┆ i64 │ + ╞════════════╪══════════╡ + │ 12/24/2018 ┆ 1550 │ + │ 12/25/2018 ┆ 2315 │ + │ 12/26/2018 ┆ 1832 │ + └────────────┴──────────┘ + ``` + + !!! warning "Available in Taipy Enterprise edition" + + The Polars exposed type is only available in the Enterprise edition of Taipy. + [Contact us](https://taipy.io/book-a-call){: .tp-btn .tp-btn--accent target='blank' } + + When writing data to a CSV data node, the `CSVDataNode.write()^` method can take several datatype as the input: - list, numpy array @@ -439,6 +461,26 @@ The following examples represent the results when reading from an Excel data nod ] ``` + === "exposed_type = polars" + + ```python + shape: (3, 2) + ┌────────────┬──────────┐ + │ date ┆ nb_sales │ + │ --- ┆ --- │ + │ str ┆ i64 │ + ╞════════════╪══════════╡ + │ 12/24/2018 ┆ 1550 │ + │ 12/25/2018 ┆ 2315 │ + │ 12/26/2018 ┆ 1832 │ + └────────────┴──────────┘ + ``` + + !!! warning "Available in Taipy Enterprise edition" + + The Polars exposed type is only available in the Enterprise edition of Taipy. + [Contact us](https://taipy.io/book-a-call){: .tp-btn .tp-btn--accent target='blank' } + When writing data to an Excel data node, the `ExcelDataNode.write()^` method can take several datatype as the input: - list, numpy array @@ -587,6 +629,26 @@ node with different _exposed_type_: ] ``` + === "exposed_type = polars" + + ```python + shape: (3, 3) + ┌─────┬────────────┬──────────┐ + │ ID ┆ date ┆ nb_sales │ + │ --- ┆ --- ┆ --- │ + │ i64 ┆ str ┆ i64 │ + ╞═════╪════════════╪══════════╡ + │ 1 ┆ 12/24/2018 ┆ 1550 │ + │ 2 ┆ 12/25/2018 ┆ 2315 │ + │ 3 ┆ 12/26/2018 ┆ 1832 │ + └─────┴────────────┴──────────┘ + ``` + + !!! warning "Available in Taipy Enterprise edition" + + The Polars exposed type is only available in the Enterprise edition of Taipy. + [Contact us](https://taipy.io/book-a-call){: .tp-btn .tp-btn--accent target='blank' } + When writing data to a SQL Table data node, the `SQLTableDataNode.write()^` method can take several datatype as the input: @@ -957,12 +1019,33 @@ The following examples represent the results when read from Parquet data node wi ] ``` + === "exposed_type = polars" + + ```python + shape: (3, 2) + ┌────────────┬──────────┐ + │ date ┆ nb_sales │ + │ --- ┆ --- │ + │ str ┆ i64 │ + ╞════════════╪══════════╡ + │ 12/24/2018 ┆ 1550 │ + │ 12/25/2018 ┆ 2315 │ + │ 12/26/2018 ┆ 1832 │ + └────────────┴──────────┘ + ``` + + !!! warning "Available in Taipy Enterprise edition" + + The Polars exposed type is only available in the Enterprise edition of Taipy. + [Contact us](https://taipy.io/book-a-call){: .tp-btn .tp-btn--accent target='blank' } + When writing data to a Parquet data node, the `ParquetDataNode.write()^` method can take several datatype as the input depending on the _exposed type_: - pandas dataframes - numpy arrays - any object, which will be passed to the `pd.DataFrame` constructor (e.g., list of dictionaries) +- polars dataframes (Available in Taipy Enterprise edition only) The following examples will write to the path of the Parquet data node: @@ -1576,6 +1659,24 @@ filtered_data = data_node.filter(("nb_sales", 1550, Operator.EQUAL)) [SaleRow("12/24/2018", 1550)] ``` + === "exposed_type = polars" + + ```python + shape: (3, 2) + ┌────────────┬──────────┐ + │ date ┆ nb_sales │ + │ --- ┆ --- │ + │ str ┆ i64 │ + ╞════════════╪══════════╡ + │ 12/24/2018 ┆ 1550 │ + └────────────┴──────────┘ + ``` + + !!! warning "Available in Taipy Enterprise edition" + + The Polars exposed type is only available in the Enterprise edition of Taipy. + [Contact us](https://taipy.io/book-a-call){: .tp-btn .tp-btn--accent target='blank' } + If a list of operators is provided, it is necessary to provide a join operator that will be used to combine the filtered results from the operators. The default join operator is `JoinOperator.AND`. @@ -1621,6 +1722,26 @@ filtered_data = data_node.filter( ] ``` + === "exposed_type = polars" + + ```python + shape: (3, 2) + ┌────────────┬──────────┐ + │ date ┆ nb_sales │ + │ --- ┆ --- │ + │ str ┆ i64 │ + ╞════════════╪══════════╡ + │ 12/24/2018 ┆ 1550 │ + │ 12/25/2018 ┆ 2315 │ + │ 12/26/2018 ┆ 1832 │ + └────────────┴──────────┘ + ``` + + !!! warning "Available in Taipy Enterprise edition" + + The Polars exposed type is only available in the Enterprise edition of Taipy. + [Contact us](https://taipy.io/book-a-call){: .tp-btn .tp-btn--accent target='blank' } + In another example, the `DataNode.filter()^` method will return all the records from the data node where the value of the "nb_sales" field is equal to 1550 or greater than 2000. The following examples represent the results when read from a data node with different _exposed_type_: @@ -1664,6 +1785,26 @@ filtered_data = data_node.filter( ] ``` + === "exposed_type = polars" + + ```python + shape: (3, 2) + ┌────────────┬──────────┐ + │ date ┆ nb_sales │ + │ --- ┆ --- │ + │ str ┆ i64 │ + ╞════════════╪══════════╡ + │ 12/24/2018 ┆ 1550 │ + │ 12/25/2018 ┆ 2315 │ + │ 12/26/2018 ┆ 1832 │ + └────────────┴──────────┘ + ``` + + !!! warning "Available in Taipy Enterprise edition" + + The Polars exposed type is only available in the Enterprise edition of Taipy. + [Contact us](https://taipy.io/book-a-call){: .tp-btn .tp-btn--accent target='blank' } + With Pandas data frame as the exposed type, it is also possible to use pandas indexing and filtering style: From 0f8ece0b926800e0de434f8ef9ae24d3dbca6cf1 Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Tue, 14 Jan 2025 14:32:08 +0700 Subject: [PATCH 2/6] re-added flak8 --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bcce870ff..4cc0074e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,8 +34,8 @@ repos: - id: black args: [--line-length=120] language_version: python3.9 -# - repo: https://gitlab.com/pycqa/flake8 -# rev: 3.9.2 -# hooks: -# - id: flake8 -# additional_dependencies: [flake8-typing-imports==1.10.0] +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + additional_dependencies: [flake8-typing-imports==1.10.0] From 9b4d526cd168a7589ae3dffbd0e0fc351e25f1b5 Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Mon, 20 Jan 2025 15:41:32 +0700 Subject: [PATCH 3/6] replace polars representation with actual polars code --- .pre-commit-config.yaml | 10 +- .../data-integration/data-node-usage.md | 110 +++++++----------- 2 files changed, 47 insertions(+), 73 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4cc0074e2..bcce870ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,8 +34,8 @@ repos: - id: black args: [--line-length=120] language_version: python3.9 -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - additional_dependencies: [flake8-typing-imports==1.10.0] +# - repo: https://gitlab.com/pycqa/flake8 +# rev: 3.9.2 +# hooks: +# - id: flake8 +# additional_dependencies: [flake8-typing-imports==1.10.0] diff --git a/docs/userman/scenario_features/data-integration/data-node-usage.md b/docs/userman/scenario_features/data-integration/data-node-usage.md index 86073f852..f877b6765 100644 --- a/docs/userman/scenario_features/data-integration/data-node-usage.md +++ b/docs/userman/scenario_features/data-integration/data-node-usage.md @@ -297,16 +297,12 @@ The following examples represent the results when reading from a CSV data node w === "exposed_type = polars" ```python - shape: (3, 2) - ┌────────────┬──────────┐ - │ date ┆ nb_sales │ - │ --- ┆ --- │ - │ str ┆ i64 │ - ╞════════════╪══════════╡ - │ 12/24/2018 ┆ 1550 │ - │ 12/25/2018 ┆ 2315 │ - │ 12/26/2018 ┆ 1832 │ - └────────────┴──────────┘ + polars.DataFrame( + { + "date": ["12/24/2018", "12/25/2018", "12/26/2018"], + "nb_sales": [1550, 2315, 1832] + } + ) ``` !!! warning "Available in Taipy Enterprise edition" @@ -464,16 +460,12 @@ The following examples represent the results when reading from an Excel data nod === "exposed_type = polars" ```python - shape: (3, 2) - ┌────────────┬──────────┐ - │ date ┆ nb_sales │ - │ --- ┆ --- │ - │ str ┆ i64 │ - ╞════════════╪══════════╡ - │ 12/24/2018 ┆ 1550 │ - │ 12/25/2018 ┆ 2315 │ - │ 12/26/2018 ┆ 1832 │ - └────────────┴──────────┘ + polars.DataFrame( + { + "date": ["12/24/2018", "12/25/2018", "12/26/2018"], + "nb_sales": [1550, 2315, 1832] + } + ) ``` !!! warning "Available in Taipy Enterprise edition" @@ -632,16 +624,12 @@ node with different _exposed_type_: === "exposed_type = polars" ```python - shape: (3, 3) - ┌─────┬────────────┬──────────┐ - │ ID ┆ date ┆ nb_sales │ - │ --- ┆ --- ┆ --- │ - │ i64 ┆ str ┆ i64 │ - ╞═════╪════════════╪══════════╡ - │ 1 ┆ 12/24/2018 ┆ 1550 │ - │ 2 ┆ 12/25/2018 ┆ 2315 │ - │ 3 ┆ 12/26/2018 ┆ 1832 │ - └─────┴────────────┴──────────┘ + polars.DataFrame( + { + "date": ["12/24/2018", "12/25/2018", "12/26/2018"], + "nb_sales": [1550, 2315, 1832] + } + ) ``` !!! warning "Available in Taipy Enterprise edition" @@ -1022,16 +1010,12 @@ The following examples represent the results when read from Parquet data node wi === "exposed_type = polars" ```python - shape: (3, 2) - ┌────────────┬──────────┐ - │ date ┆ nb_sales │ - │ --- ┆ --- │ - │ str ┆ i64 │ - ╞════════════╪══════════╡ - │ 12/24/2018 ┆ 1550 │ - │ 12/25/2018 ┆ 2315 │ - │ 12/26/2018 ┆ 1832 │ - └────────────┴──────────┘ + polars.DataFrame( + { + "date": ["12/24/2018", "12/25/2018", "12/26/2018"], + "nb_sales": [1550, 2315, 1832] + } + ) ``` !!! warning "Available in Taipy Enterprise edition" @@ -1662,14 +1646,12 @@ filtered_data = data_node.filter(("nb_sales", 1550, Operator.EQUAL)) === "exposed_type = polars" ```python - shape: (3, 2) - ┌────────────┬──────────┐ - │ date ┆ nb_sales │ - │ --- ┆ --- │ - │ str ┆ i64 │ - ╞════════════╪══════════╡ - │ 12/24/2018 ┆ 1550 │ - └────────────┴──────────┘ + polars.DataFrame( + { + "date": ["12/24/2018"], + "nb_sales": [1550] + } + ) ``` !!! warning "Available in Taipy Enterprise edition" @@ -1725,16 +1707,12 @@ filtered_data = data_node.filter( === "exposed_type = polars" ```python - shape: (3, 2) - ┌────────────┬──────────┐ - │ date ┆ nb_sales │ - │ --- ┆ --- │ - │ str ┆ i64 │ - ╞════════════╪══════════╡ - │ 12/24/2018 ┆ 1550 │ - │ 12/25/2018 ┆ 2315 │ - │ 12/26/2018 ┆ 1832 │ - └────────────┴──────────┘ + polars.DataFrame( + { + "date": ["12/24/2018", "12/25/2018"], + "nb_sales": [1550, 2315] + } + ) ``` !!! warning "Available in Taipy Enterprise edition" @@ -1788,16 +1766,12 @@ filtered_data = data_node.filter( === "exposed_type = polars" ```python - shape: (3, 2) - ┌────────────┬──────────┐ - │ date ┆ nb_sales │ - │ --- ┆ --- │ - │ str ┆ i64 │ - ╞════════════╪══════════╡ - │ 12/24/2018 ┆ 1550 │ - │ 12/25/2018 ┆ 2315 │ - │ 12/26/2018 ┆ 1832 │ - └────────────┴──────────┘ + polars.DataFrame( + { + "date": ["12/24/2018", "12/25/2018"], + "nb_sales": [1550, 2315] + } + ) ``` !!! warning "Available in Taipy Enterprise edition" From 708707a6e58892df2813a06c2fa975e1a8df592d Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Mon, 20 Jan 2025 15:41:56 +0700 Subject: [PATCH 4/6] reset minor changes --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bcce870ff..4cc0074e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,8 +34,8 @@ repos: - id: black args: [--line-length=120] language_version: python3.9 -# - repo: https://gitlab.com/pycqa/flake8 -# rev: 3.9.2 -# hooks: -# - id: flake8 -# additional_dependencies: [flake8-typing-imports==1.10.0] +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + additional_dependencies: [flake8-typing-imports==1.10.0] From b88b1246c503357cc723c1ec9ef41493a2580132 Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Tue, 21 Jan 2025 14:01:17 +0700 Subject: [PATCH 5/6] align the pandas constructor Python --- .pre-commit-config.yaml | 10 +-- .../data-integration/data-node-usage.md | 72 +++++++++---------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4cc0074e2..bcce870ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,8 +34,8 @@ repos: - id: black args: [--line-length=120] language_version: python3.9 -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - additional_dependencies: [flake8-typing-imports==1.10.0] +# - repo: https://gitlab.com/pycqa/flake8 +# rev: 3.9.2 +# hooks: +# - id: flake8 +# additional_dependencies: [flake8-typing-imports==1.10.0] diff --git a/docs/userman/scenario_features/data-integration/data-node-usage.md b/docs/userman/scenario_features/data-integration/data-node-usage.md index f877b6765..6bdaba2fc 100644 --- a/docs/userman/scenario_features/data-integration/data-node-usage.md +++ b/docs/userman/scenario_features/data-integration/data-node-usage.md @@ -263,12 +263,11 @@ The following examples represent the results when reading from a CSV data node w === "exposed_type = "pandas"" ```python - pandas.DataFrame - ( - date nb_sales - 0 12/24/2018 1550 - 1 12/25/2018 2315 - 2 12/26/2018 1832 + pandas.DataFrame( + { + "date": ["12/24/2018", "12/25/2018", "12/26/2018"], + "nb_sales": [1550, 2315, 1832] + } ) ``` @@ -427,12 +426,11 @@ The following examples represent the results when reading from an Excel data nod === "exposed_type = "pandas"" ```python - pandas.DataFrame - ( - date nb_sales - 0 12/24/2018 1550 - 1 12/25/2018 2315 - 2 12/26/2018 1832 + pandas.DataFrame( + { + "date": ["12/24/2018", "12/25/2018", "12/26/2018"], + "nb_sales": [1550, 2315, 1832] + } ) ``` @@ -591,12 +589,12 @@ node with different _exposed_type_: === "exposed_type = "pandas"" ```python - pandas.DataFrame - ( - ID date nb_sales - 0 1 12/24/2018 1550 - 1 2 12/25/2018 2315 - 2 3 12/26/2018 1832 + pandas.DataFrame( + { + "ID": [1, 2, 3], + "date": ["12/24/2018", "12/25/2018", "12/26/2018"], + "nb_sales": [1550, 2315, 1832] + } ) ``` @@ -626,6 +624,7 @@ node with different _exposed_type_: ```python polars.DataFrame( { + "ID": [1, 2, 3], "date": ["12/24/2018", "12/25/2018", "12/26/2018"], "nb_sales": [1550, 2315, 1832] } @@ -977,12 +976,11 @@ The following examples represent the results when read from Parquet data node wi === "exposed_type = "pandas"" ```python - pandas.DataFrame - ( - date nb_sales - 0 12/24/2018 1550 - 1 12/25/2018 2315 - 2 12/26/2018 1832 + pandas.DataFrame( + { + "date": ["12/24/2018", "12/25/2018", "12/26/2018"], + "nb_sales": [1550, 2315, 1832] + } ) ``` @@ -1623,11 +1621,7 @@ filtered_data = data_node.filter(("nb_sales", 1550, Operator.EQUAL)) === "exposed_type = "pandas"" ```python - pandas.DataFrame - ( - date nb_sales - 0 12/24/2018 1550 - ) + pandas.DataFrame({"date": ["12/24/2018"], "nb_sales": [1550]}) ``` === "exposed_type = "numpy"" @@ -1683,6 +1677,12 @@ filtered_data = data_node.filter( 0 12/24/2018 1550 1 12/26/2018 1832 ) + pandas.DataFrame( + { + "date": ["12/24/2018", "12/26/2018"], + "nb_sales": [1550, 1832] + } + ) ``` === "exposed_type = "numpy"" @@ -1709,8 +1709,8 @@ filtered_data = data_node.filter( ```python polars.DataFrame( { - "date": ["12/24/2018", "12/25/2018"], - "nb_sales": [1550, 2315] + "date": ["12/24/2018", "12/26/2018"], + "nb_sales": [1550, 1832] } ) ``` @@ -1736,11 +1736,11 @@ filtered_data = data_node.filter( === "exposed_type = "pandas"" ```python - pandas.DataFrame - ( - date nb_sales - 0 12/24/2018 1550 - 1 12/25/2018 2315 + pandas.DataFrame( + { + "date": ["12/24/2018", "12/25/2018"], + "nb_sales": [1550, 2315] + } ) ``` From 30e13cc7b501de5786d46210fb33215cebc37772 Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Tue, 21 Jan 2025 14:01:38 +0700 Subject: [PATCH 6/6] return flake8 --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bcce870ff..4cc0074e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,8 +34,8 @@ repos: - id: black args: [--line-length=120] language_version: python3.9 -# - repo: https://gitlab.com/pycqa/flake8 -# rev: 3.9.2 -# hooks: -# - id: flake8 -# additional_dependencies: [flake8-typing-imports==1.10.0] +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + additional_dependencies: [flake8-typing-imports==1.10.0]