Skip to content

Commit

Permalink
Adding Arrow Flight connector template (#24427)
Browse files Browse the repository at this point in the history 
This adds the Arrow Flight base module as a template to build connectors
that use a Flight service to transfer data with Presto in Arrow format.
A concrete Arrow Flight connector implementation will extend the
BaseArrowFlightClientHandler that can connect to a Flight service that
handles the required RPC calls from the client handler. An example
connector implementation is provided in the unit testing.

RFC: https://github.com/prestodb/rfcs/blob/main/RFC-0004-arrow-flight-connector.md

Co-authored-by: sai bhaskar reddy <[email protected]>
Co-authored-by: SthuthiGhosh9400 <[email protected]>
Co-authored-by: lithinwxd <[email protected]>
Co-authored-by: Steve Burnett <[email protected]>
Co-authored-by: elbinpallimalilibm <[email protected]>
Co-authored-by: Tim Meehan <[email protected]>
  • Loading branch information
@BryanCutler @sabbasani
@SthuthiGhosh9400 @lithinwxd @steveburnett @elbinpallimalilibm @tdcmeehan
7 people authored Jan 29, 2025
1 parent c229714 commit 4464129
Show file tree
Hide file tree
Showing 50 changed files with 6,343 additions and 1 deletion.
82 changes: 82 additions & 0 deletions .github/workflows/arrow-flight-tests.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
name: arrow flight tests

on:
pull_request:

env:
CONTINUOUS_INTEGRATION: true
MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError"
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true"
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end"
RETRY: .github/bin/retry

jobs:
changes:
runs-on: ubuntu-latest
permissions:
pull-requests: read
outputs:
codechange: ${{ steps.filter.outputs.codechange }}
steps:
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
codechange:
- '!presto-docs/**'
test:
runs-on: ubuntu-latest
needs: changes
strategy:
fail-fast: false
matrix:
modules:
- ":presto-base-arrow-flight" # Only run tests for the `presto-base-arrow-flight` module

timeout-minutes: 80
concurrency:
group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }}
cancel-in-progress: true

steps:
# Checkout the code only if there are changes in the relevant files
- uses: actions/checkout@v4
if: needs.changes.outputs.codechange == 'true'
with:
show-progress: false

# Set up Java for the build environment
- uses: actions/setup-java@v2
if: needs.changes.outputs.codechange == 'true'
with:
distribution: 'temurin'
java-version: 8

# Cache Maven dependencies to speed up the build
- name: Cache local Maven repository
if: needs.changes.outputs.codechange == 'true'
id: cache-maven
uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-2-
# Resolve Maven dependencies (if cache is not found)
- name: Populate Maven cache
if: steps.cache-maven.outputs.cache-hit != 'true' && needs.changes.outputs.codechange == 'true'
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs

# Install dependencies for the target module
- name: Maven Install
if: needs.changes.outputs.codechange == 'true'
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }}
# Run Maven tests for the target module
- name: Maven Tests
if: needs.changes.outputs.codechange == 'true'
run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }}
3 changes: 2 additions & 1 deletion .github/workflows/test-other-modules.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,4 +87,5 @@ jobs:
!presto-test-coverage,
!presto-iceberg,
!presto-singlestore,
!presto-native-sidecar-plugin'
!presto-native-sidecar-plugin,
!presto-base-arrow-flight'
26 changes: 26 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@
<dep.gson.version>2.11.0</dep.gson.version>
<dep.commons.lang3.version>3.14.0</dep.commons.lang3.version>
<dep.guice.version>5.1.0</dep.guice.version>
<dep.arrow.version>17.0.0</dep.arrow.version>

<!--
America/Bahia_Banderas has:
Expand Down Expand Up @@ -209,6 +210,7 @@
<module>presto-hana</module>
<module>presto-openapi</module>
<module>presto-native-sidecar-plugin</module>
<module>presto-base-arrow-flight</module>
</modules>

<dependencyManagement>
Expand Down Expand Up @@ -2442,6 +2444,30 @@
<version>2.12.0</version>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-core</artifactId>
<version>${dep.arrow.version}</version>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty</artifactId>
<version>${dep.arrow.version}</version>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<version>${dep.arrow.version}</version>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-core</artifactId>
<version>${dep.arrow.version}</version>
</dependency>

</dependencies>
</dependencyManagement>

Expand Down
256 changes: 256 additions & 0 deletions presto-base-arrow-flight/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
<?xml version="1.0"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-root</artifactId>
<version>0.292-SNAPSHOT</version>
</parent>

<artifactId>presto-base-arrow-flight</artifactId>
<name>presto-base-arrow-flight</name>
<description>Presto - Base Arrow Flight Connector</description>

<properties>
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty</artifactId>
<scope>runtime</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<!-- Excluding to resolve upper bound errors -->
<exclusion>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>bootstrap</artifactId>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>log</artifactId>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>

<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-spi</artifactId>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>slice</artifactId>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>log-manager</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-common</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>

<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<optional>true</optional>
</dependency>

<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>configuration</artifactId>
</dependency>

<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>

<dependency>
<groupId>org.jdbi</groupId>
<artifactId>jdbi3-core</artifactId>
</dependency>

<!-- for testing -->
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.airlift.tpch</groupId>
<artifactId>tpch</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-tpch</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>json</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-testng-services</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>testing</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-main</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-tests</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-jdbc</artifactId>
<version>${dep.arrow.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>-Xss10M</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.basepom.maven</groupId>
<artifactId>duplicate-finder-maven-plugin</artifactId>
<version>1.2.1</version>
<configuration>
<ignoredClassPatterns>
<ignoredClassPattern>module-info</ignoredClassPattern>
<ignoredClassPattern>META-INF.versions.9.module-info</ignoredClassPattern>
</ignoredClassPatterns>
<ignoredResourcePatterns>
<ignoredResourcePattern>arrow-git.properties</ignoredResourcePattern>
<ignoredResourcePattern>about.html</ignoredResourcePattern>
</ignoredResourcePatterns>
</configuration>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
Loading

0 comments on commit 4464129

Please sign in to comment.