Skip to content

Commit

Permalink
Add PageFile format support for CTE Materialization
Browse files Browse the repository at this point in the history
  • Loading branch information
pdabre12 authored and Pratik Joseph Dabre committed Jan 30, 2025
1 parent c229714 commit ecc5bcc
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 5 deletions.
3 changes: 2 additions & 1 deletion .gitmodules
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[submodule "presto-native-execution/velox"]
path = presto-native-execution/velox
url = https://github.com/facebookincubator/velox.git
url = https://github.com/pdabre12/velox.git
branch = cte-page-reader
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ dwio::common::FileFormat toVeloxFileFormat(
return dwio::common::FileFormat::NIMBLE;
}
VELOX_UNSUPPORTED(
"Unsupported file format: {} {}", format.inputFormat, format.serDe);
"Unsupported file format123: {} {}", format.inputFormat, format.serDe);
}

dwio::common::FileFormat toVeloxFileFormat(
Expand All @@ -104,7 +104,7 @@ dwio::common::FileFormat toVeloxFileFormat(
} else if (format == protocol::iceberg::FileFormat::PARQUET) {
return dwio::common::FileFormat::PARQUET;
}
VELOX_UNSUPPORTED("Unsupported file format: {}", fmt::underlying(format));
VELOX_UNSUPPORTED("Unsupported file format123: {}", fmt::underlying(format));
}

template <typename T>
Expand Down Expand Up @@ -805,7 +805,7 @@ std::unique_ptr<connector::ConnectorTableHandle> toHiveTableHandle(
const protocol::Map<protocol::String, protocol::String>& tableParameters,
const VeloxExprConverter& exprConverter,
const TypeParser& typeParser) {
common::SubfieldFilters subfieldFilters;
velox::connector::hive::SubfieldFilters subfieldFilters;
auto domains = domainPredicate.domains;
for (const auto& domain : *domains) {
auto filter = domain.second;
Expand Down Expand Up @@ -905,6 +905,8 @@ dwio::common::FileFormat toFileFormat(
case protocol::hive::HiveStorageFormat::ALPHA:
// This has been renamed in Velox from ALPHA to NIMBLE.
return dwio::common::FileFormat::NIMBLE;
case protocol::hive::HiveStorageFormat::PAGEFILE:
return dwio::common::FileFormat::PAGEFILE;
default:
VELOX_UNSUPPORTED(
"Unsupported file format in {}: {}.",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.nativeworker;

import com.facebook.presto.Session;
import com.facebook.presto.testing.QueryRunner;

import static com.facebook.presto.SystemSessionProperties.CTE_FILTER_AND_PROJECTION_PUSHDOWN_ENABLED;
import static com.facebook.presto.SystemSessionProperties.CTE_MATERIALIZATION_STRATEGY;
import static com.facebook.presto.SystemSessionProperties.PARTITIONING_PROVIDER_CATALOG;
import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_SUBFIELDS_ENABLED;
import static com.facebook.presto.SystemSessionProperties.VERBOSE_OPTIMIZER_INFO_ENABLED;

public class TestPrestoNativeCteExecutionPageFile
extends AbstractTestNativeCteExecution
{
@Override
protected QueryRunner createQueryRunner()
throws Exception
{
return PrestoNativeQueryRunnerUtils.createNativeCteQueryRunner(true, "PAGEFILE");
}

@Override
protected QueryRunner createExpectedQueryRunner()
throws Exception
{
return PrestoNativeQueryRunnerUtils.createJavaQueryRunner("PAGEFILE");
}

@Override
protected Session getSession()
{
return Session.builder(super.getSession())
.setSystemProperty(PUSHDOWN_SUBFIELDS_ENABLED, "true")
.setSystemProperty(CTE_MATERIALIZATION_STRATEGY, "NONE")
.build();
}

@Override
protected Session getMaterializedSession()
{
return Session.builder(super.getSession())
.setSystemProperty(PUSHDOWN_SUBFIELDS_ENABLED, "true")
.setSystemProperty(VERBOSE_OPTIMIZER_INFO_ENABLED, "true")
.setSystemProperty(PARTITIONING_PROVIDER_CATALOG, "hive")
.setSystemProperty(CTE_MATERIALIZATION_STRATEGY, "ALL")
.setSystemProperty(CTE_FILTER_AND_PROJECTION_PUSHDOWN_ENABLED, "true")
.build();
}
}
2 changes: 1 addition & 1 deletion presto-native-execution/velox
Submodule velox updated 77 files
+2 −2 CMake/VeloxUtils.cmake
+0 −4 CMake/resolve_dependency_modules/simdjson.cmake
+0 −2 CMakeLists.txt
+0 −6 README.md
+1 −1 scripts/setup-centos9.sh
+1 −2 scripts/setup-helper-functions.sh
+1 −1 scripts/setup-macos.sh
+1 −1 scripts/setup-ubuntu.sh
+36 −10 velox/common/caching/CachedFactory.h
+4 −4 velox/common/caching/tests/AsyncDataCacheTest.cpp
+6 −3 velox/common/caching/tests/CachedFactoryTest.cpp
+0 −24 velox/common/config/GlobalConfig.h
+1 −1 velox/common/file/FileInputStream.cpp
+2 −2 velox/common/file/FileInputStream.h
+2 −1 velox/common/file/FileSystems.cpp
+3 −1 velox/common/file/FileSystems.h
+2 −1 velox/common/file/tests/FaultyFileSystem.cpp
+4 −3 velox/common/file/tests/FaultyFileSystem.h
+23 −0 velox/common/io/IoStatistics.cpp
+7 −0 velox/common/io/IoStatistics.h
+0 −122 velox/connectors/Connector.h
+3 −2 velox/connectors/hive/FileHandle.cpp
+4 −1 velox/connectors/hive/FileHandle.h
+3 −3 velox/connectors/hive/HiveConnectorUtil.cpp
+3 −5 velox/connectors/hive/HiveConnectorUtil.h
+5 −0 velox/connectors/hive/HiveDataSource.cpp
+4 −1 velox/connectors/hive/HiveDataSource.h
+2 −2 velox/connectors/hive/SplitReader.cpp
+2 −2 velox/connectors/hive/TableHandle.cpp
+6 −3 velox/connectors/hive/TableHandle.h
+3 −0 velox/connectors/hive/iceberg/PositionalDeleteFileReader.h
+2 −1 velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.cpp
+2 −1 velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h
+2 −1 velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.cpp
+2 −1 velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h
+2 −1 velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.cpp
+2 −1 velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h
+2 −1 velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.cpp
+2 −1 velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h
+2 −2 velox/connectors/hive/tests/HiveConnectorUtilTest.cpp
+3 −1 velox/core/PlanNode.h
+1 −0 velox/dwio/CMakeLists.txt
+4 −0 velox/dwio/common/Options.cpp
+1 −0 velox/dwio/common/Options.h
+1 −0 velox/dwio/common/Throttler.cpp
+1 −0 velox/dwio/common/Throttler.h
+2 −2 velox/dwio/common/tests/LoggedExceptionTest.cpp
+2 −0 velox/dwio/common/tests/utils/FilterGenerator.h
+20 −0 velox/dwio/cte/CMakeLists.txt
+25 −0 velox/dwio/cte/RegisterCtePageReader.h
+25 −0 velox/dwio/cte/RegisterCtePageWriter.h
+16 −0 velox/dwio/cte/reader/CMakeLists.txt
+127 −0 velox/dwio/cte/reader/CtePageReader.cpp
+135 −0 velox/dwio/cte/reader/CtePageReader.h
+15 −0 velox/dwio/cte/tests/CMakeLists.txt
+15 −0 velox/dwio/cte/tests/CtePageReaderTest.cpp
+16 −0 velox/dwio/cte/writer/CMakeLists.txt
+112 −0 velox/dwio/cte/writer/CtePageWriter.cpp
+69 −0 velox/dwio/cte/writer/CtePageWriter.h
+2 −1 velox/exec/SpillFile.cpp
+1 −6 velox/exec/fuzzer/CacheFuzzer.cpp
+7 −8 velox/exec/tests/TableScanTest.cpp
+1 −1 velox/exec/tests/utils/HiveConnectorTestBase.h
+1 −1 velox/exec/tests/utils/PlanBuilder.cpp
+2 −3 velox/experimental/wave/common/KernelCache.cpp
+27 −10 velox/expression/Expr.cpp
+13 −0 velox/expression/Expr.h
+1 −1 velox/expression/tests/ExprEncodingsTest.cpp
+6 −5 velox/expression/tests/ExprTest.cpp
+0 −26 velox/flag_definitions/flags.cpp
+2 −7 velox/functions/lib/Re2Functions.cpp
+0 −14 velox/functions/lib/tests/Re2FunctionsTest.cpp
+0 −2 velox/functions/prestosql/tests/BinaryFunctionsTest.cpp
+5 −5 velox/substrait/SubstraitToVeloxPlan.cpp
+1 −1 velox/substrait/SubstraitToVeloxPlan.h
+0 −3 velox/type/Filter.h
+3 −0 velox/type/tests/SubfieldFiltersBuilder.h

0 comments on commit ecc5bcc

Please sign in to comment.