From 1bed01b50f79661a2ad4d5d0d954a78e7ebdd39e Mon Sep 17 00:00:00 2001 From: Github Actions Date: Fri, 27 Jan 2023 07:47:09 +0100 Subject: [PATCH 1/5] Update SNFLK and Synapse drivers --- packages/php-table-backend-utils/Dockerfile | 6 +++--- .../php-table-backend-utils/docker/snowflake/generic.pol | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/php-table-backend-utils/Dockerfile b/packages/php-table-backend-utils/Dockerfile index 9f6805dd4..583707355 100644 --- a/packages/php-table-backend-utils/Dockerfile +++ b/packages/php-table-backend-utils/Dockerfile @@ -11,9 +11,9 @@ ARG DEBIAN_FRONTEND=noninteractive ENV COMPOSER_ALLOW_SUPERUSER 1 ENV COMPOSER_PROCESS_TIMEOUT 3600 -ARG SQLSRV_VERSION=5.9.0 -ARG SNOWFLAKE_ODBC_VERSION=2.21.1 -ARG SNOWFLAKE_GPG_KEY=EC218558EABB25A1 +ARG SQLSRV_VERSION=5.10.1 +ARG SNOWFLAKE_ODBC_VERSION=2.25.6 +ARG SNOWFLAKE_GPG_KEY=630D9F3CAB551AF3 WORKDIR /code/ diff --git a/packages/php-table-backend-utils/docker/snowflake/generic.pol b/packages/php-table-backend-utils/docker/snowflake/generic.pol index ca1cc1c5c..0c16bf39f 100644 --- a/packages/php-table-backend-utils/docker/snowflake/generic.pol +++ b/packages/php-table-backend-utils/docker/snowflake/generic.pol @@ -1,11 +1,11 @@ - + - + - + From 2d434eef9f9993315b32e022c1668df012bef7cf Mon Sep 17 00:00:00 2001 From: Github Actions Date: Fri, 27 Jan 2023 07:47:35 +0100 Subject: [PATCH 2/5] Use php 8.1 --- packages/php-table-backend-utils/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/php-table-backend-utils/Dockerfile b/packages/php-table-backend-utils/Dockerfile index 583707355..1dbae4584 100644 --- a/packages/php-table-backend-utils/Dockerfile +++ b/packages/php-table-backend-utils/Dockerfile @@ -4,7 +4,7 @@ ARG AWS_ACCESS_KEY_ID RUN /usr/bin/aws s3 cp s3://keboola-drivers/teradata/tdodbc1710-17.10.00.08-1.x86_64.deb /tmp/teradata/tdodbc.deb RUN /usr/bin/aws s3 cp s3://keboola-drivers/exasol/EXASOL_ODBC-7.1.10.tar.gz /tmp/exasol/odbc.tar.gz -FROM php:7.4-cli-buster +FROM php:8.1-cli-buster ARG COMPOSER_FLAGS="--prefer-dist --no-interaction" ARG DEBIAN_FRONTEND=noninteractive From 19b7d7f21c8ae3a85489d10c94da6635922a9d15 Mon Sep 17 00:00:00 2001 From: Github Actions Date: Fri, 27 Jan 2023 07:52:03 +0100 Subject: [PATCH 3/5] CS fix --- packages/php-table-backend-utils/phpcs.xml | 19 +++++++++++++++++++ .../Auth/SynapseGrantQueryBuilderTest.php | 11 ++++++----- .../SynapseDatabaseReflectionTest.php | 3 ++- .../Functional/Synapse/SynapseBaseCase.php | 6 ------ .../Functional/Teradata/TeradataBaseCase.php | 4 ---- .../Unit/Column/ColumnCollectionTest.php | 8 ++++---- 6 files changed, 31 insertions(+), 20 deletions(-) diff --git a/packages/php-table-backend-utils/phpcs.xml b/packages/php-table-backend-utils/phpcs.xml index 67e37cfc8..1075b1b2a 100644 --- a/packages/php-table-backend-utils/phpcs.xml +++ b/packages/php-table-backend-utils/phpcs.xml @@ -1,4 +1,23 @@ + + + + + + + + + + + + + + + + + + + diff --git a/packages/php-table-backend-utils/tests/Functional/Synapse/Auth/SynapseGrantQueryBuilderTest.php b/packages/php-table-backend-utils/tests/Functional/Synapse/Auth/SynapseGrantQueryBuilderTest.php index 2255c6546..f6e0f1b42 100644 --- a/packages/php-table-backend-utils/tests/Functional/Synapse/Auth/SynapseGrantQueryBuilderTest.php +++ b/packages/php-table-backend-utils/tests/Functional/Synapse/Auth/SynapseGrantQueryBuilderTest.php @@ -10,6 +10,7 @@ use Keboola\TableBackendUtils\Auth\Grant\Synapse\Permission; use Keboola\TableBackendUtils\Auth\Grant\Synapse\RevokeOptions; use Keboola\TableBackendUtils\Auth\SynapseGrantQueryBuilder; +use Keboola\TableBackendUtils\Escaping\SynapseQuote; class SynapseGrantQueryBuilderTest extends BaseAuthTestCase { @@ -80,20 +81,20 @@ protected function setUp(): void $this->connection->executeStatement(sprintf( 'CREATE ROLE %s', - $this->platform->quoteSingleIdentifier($this->currentLogin . '_ROLE') + SynapseQuote::quoteSingleIdentifier($this->currentLogin . '_ROLE') )); assert($this->currentLogin !== null); $this->connection->executeStatement(sprintf( 'CREATE SCHEMA %s AUTHORIZATION %s', - $this->platform->quoteSingleIdentifier(self::TEST_SCHEMA), - $this->platform->quoteSingleIdentifier($this->currentLogin) + SynapseQuote::quoteSingleIdentifier(self::TEST_SCHEMA), + SynapseQuote::quoteSingleIdentifier($this->currentLogin) )); $this->connection->executeStatement(sprintf( 'CREATE TABLE %s.%s ([col1] nvarchar(4000) NOT NULL DEFAULT \'\')', - $this->platform->quoteSingleIdentifier(self::TEST_SCHEMA), - $this->platform->quoteSingleIdentifier(self::TEST_TABLE) + SynapseQuote::quoteSingleIdentifier(self::TEST_SCHEMA), + SynapseQuote::quoteSingleIdentifier(self::TEST_TABLE) )); } diff --git a/packages/php-table-backend-utils/tests/Functional/Synapse/Database/SynapseDatabaseReflectionTest.php b/packages/php-table-backend-utils/tests/Functional/Synapse/Database/SynapseDatabaseReflectionTest.php index 64a236fd1..d8a6c4b10 100644 --- a/packages/php-table-backend-utils/tests/Functional/Synapse/Database/SynapseDatabaseReflectionTest.php +++ b/packages/php-table-backend-utils/tests/Functional/Synapse/Database/SynapseDatabaseReflectionTest.php @@ -5,6 +5,7 @@ namespace Tests\Keboola\TableBackendUtils\Functional\Synapse\Database; use Keboola\TableBackendUtils\Database\SynapseDatabaseReflection; +use Keboola\TableBackendUtils\Escaping\SynapseQuote; use Tests\Keboola\TableBackendUtils\Functional\Synapse\Auth\BaseAuthTestCase; class SynapseDatabaseReflectionTest extends BaseAuthTestCase @@ -24,7 +25,7 @@ public function testGetRolesNames(): void $this->connection->executeStatement(sprintf( 'CREATE ROLE %s', - $this->platform->quoteSingleIdentifier($this->currentLogin . '_ROLE') + SynapseQuote::quoteSingleIdentifier($this->currentLogin . '_ROLE') )); $ref = new SynapseDatabaseReflection($this->connection); $names = $ref->getRolesNames(self::LOGIN_PREFIX . '%'); diff --git a/packages/php-table-backend-utils/tests/Functional/Synapse/SynapseBaseCase.php b/packages/php-table-backend-utils/tests/Functional/Synapse/SynapseBaseCase.php index 468c7657b..9332f8154 100644 --- a/packages/php-table-backend-utils/tests/Functional/Synapse/SynapseBaseCase.php +++ b/packages/php-table-backend-utils/tests/Functional/Synapse/SynapseBaseCase.php @@ -6,8 +6,6 @@ use Doctrine\DBAL\Connection; use Doctrine\DBAL\DriverManager; -use Doctrine\DBAL\Platforms\AbstractPlatform; -use Doctrine\DBAL\Platforms\SQLServer2012Platform; use Keboola\TableBackendUtils\Connection\Synapse\SynapseDriver; use Keboola\TableBackendUtils\Schema\SynapseSchemaQueryBuilder; use Keboola\TableBackendUtils\Schema\SynapseSchemaReflection; @@ -20,9 +18,6 @@ class SynapseBaseCase extends TestCase protected Connection $connection; - /** @var SQLServer2012Platform|AbstractPlatform */ - protected $platform; - protected SynapseSchemaQueryBuilder $schemaQb; protected SynapseTableQueryBuilder $tableQb; @@ -64,7 +59,6 @@ protected function setUp(): void { parent::setUp(); $this->connection = $this->getSynapseConnection(); - $this->platform = $this->connection->getDatabasePlatform(); $this->schemaQb = new SynapseSchemaQueryBuilder(); $this->tableQb = new SynapseTableQueryBuilder(); } diff --git a/packages/php-table-backend-utils/tests/Functional/Teradata/TeradataBaseCase.php b/packages/php-table-backend-utils/tests/Functional/Teradata/TeradataBaseCase.php index 1cca2200c..9a681d435 100644 --- a/packages/php-table-backend-utils/tests/Functional/Teradata/TeradataBaseCase.php +++ b/packages/php-table-backend-utils/tests/Functional/Teradata/TeradataBaseCase.php @@ -23,14 +23,10 @@ class TeradataBaseCase extends TestCase protected Connection $connection; - /** @var TeradataPlatform|AbstractPlatform */ - protected $platform; - protected function setUp(): void { parent::setUp(); $this->connection = $this->getTeradataConnection(); - $this->platform = $this->connection->getDatabasePlatform(); } protected function initTable( diff --git a/packages/php-table-backend-utils/tests/Unit/Column/ColumnCollectionTest.php b/packages/php-table-backend-utils/tests/Unit/Column/ColumnCollectionTest.php index b6af47941..39fc4426d 100644 --- a/packages/php-table-backend-utils/tests/Unit/Column/ColumnCollectionTest.php +++ b/packages/php-table-backend-utils/tests/Unit/Column/ColumnCollectionTest.php @@ -27,9 +27,9 @@ public function testGetIterator(): void /** * @dataProvider tooMuchColumnsProviderWithLimits - * @param SynapseColumn|TeradataColumn $definitionClass + * @param class-string $definitionClass */ - public function testTooMuchColumns($definitionClass, int $limit): void + public function testTooMuchColumns(string $definitionClass, int $limit): void { $cols = []; for ($i = 0; $i < $limit + 2; $i++) { @@ -43,9 +43,9 @@ public function testTooMuchColumns($definitionClass, int $limit): void /** * @dataProvider tooMuchColumnsProviderWithNoLimits - * @param SnowflakeColumn|ExasolColumn $definitionClass + * @param class-string $definitionClass */ - public function testNoColumnsLimit($definitionClass, int $limit): void + public function testNoColumnsLimit(string $definitionClass, int $limit): void { $cols = []; for ($i = 0; $i < $limit + 2; $i++) { From 1220aabc6bd5d12758eb3b7ee7838c908c384d01 Mon Sep 17 00:00:00 2001 From: zajca Date: Wed, 1 Feb 2023 16:10:29 +0100 Subject: [PATCH 4/5] utils check php7.4 --- .../build-php-table-backend-utils.yml | 15 +++ .../php-table-backend-utils/Dockerfile.74 | 127 ++++++++++++++++++ .../docker-compose.yml | 8 ++ 3 files changed, 150 insertions(+) create mode 100644 packages/php-table-backend-utils/Dockerfile.74 diff --git a/.github/workflows/build-php-table-backend-utils.yml b/.github/workflows/build-php-table-backend-utils.yml index a28ebd670..7605f4bc0 100644 --- a/.github/workflows/build-php-table-backend-utils.yml +++ b/.github/workflows/build-php-table-backend-utils.yml @@ -57,6 +57,21 @@ defaults: working-directory: packages/php-table-backend-utils jobs: + build_image_php74_table_backend_utils: + if: ${{ inputs.hasCodeChanged || inputs.isTag }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Build docker image + env: + AWS_ACCESS_KEY_ID: ${{ inputs.BUILD_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.BUILD_AWS_SECRET_ACCESS_KEY }} + run: | + docker-compose build production74 + - name: Check + run: | + docker-compose run production74 composer check build_image_php_table_backend_utils: if: ${{ inputs.hasCodeChanged || inputs.isTag }} runs-on: ubuntu-latest diff --git a/packages/php-table-backend-utils/Dockerfile.74 b/packages/php-table-backend-utils/Dockerfile.74 new file mode 100644 index 000000000..9f6805dd4 --- /dev/null +++ b/packages/php-table-backend-utils/Dockerfile.74 @@ -0,0 +1,127 @@ +FROM quay.io/keboola/aws-cli +ARG AWS_SECRET_ACCESS_KEY +ARG AWS_ACCESS_KEY_ID +RUN /usr/bin/aws s3 cp s3://keboola-drivers/teradata/tdodbc1710-17.10.00.08-1.x86_64.deb /tmp/teradata/tdodbc.deb +RUN /usr/bin/aws s3 cp s3://keboola-drivers/exasol/EXASOL_ODBC-7.1.10.tar.gz /tmp/exasol/odbc.tar.gz + +FROM php:7.4-cli-buster + +ARG COMPOSER_FLAGS="--prefer-dist --no-interaction" +ARG DEBIAN_FRONTEND=noninteractive +ENV COMPOSER_ALLOW_SUPERUSER 1 +ENV COMPOSER_PROCESS_TIMEOUT 3600 + +ARG SQLSRV_VERSION=5.9.0 +ARG SNOWFLAKE_ODBC_VERSION=2.21.1 +ARG SNOWFLAKE_GPG_KEY=EC218558EABB25A1 + +WORKDIR /code/ + +COPY docker/php-prod.ini /usr/local/etc/php/php.ini +COPY docker/composer-install.sh /tmp/composer-install.sh + +RUN apt-get update -q \ + && apt-get install gnupg -y --no-install-recommends \ + && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ + && curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list \ + && apt-get update -q \ + && ACCEPT_EULA=Y apt-get install -y --no-install-recommends\ + git \ + locales \ + unzip \ + unixodbc \ + unixodbc-dev \ + libpq-dev \ + gpg \ + debsig-verify \ + dirmngr \ + gpg-agent \ + msodbcsql17 \ + libonig-dev \ + libxml2-dev \ + && rm -r /var/lib/apt/lists/* \ + && sed -i 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen \ + && locale-gen \ + && chmod +x /tmp/composer-install.sh \ + && /tmp/composer-install.sh + +ENV LANGUAGE=en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 + +# Snowflake ODBC +# https://github.com/docker-library/php/issues/103#issuecomment-353674490 +RUN set -ex; \ + docker-php-source extract; \ + { \ + echo '# https://github.com/docker-library/php/issues/103#issuecomment-353674490'; \ + echo 'AC_DEFUN([PHP_ALWAYS_SHARED],[])dnl'; \ + echo; \ + cat /usr/src/php/ext/odbc/config.m4; \ + } > temp.m4; \ + mv temp.m4 /usr/src/php/ext/odbc/config.m4; \ + docker-php-ext-configure odbc --with-unixODBC=shared,/usr; \ + docker-php-ext-install odbc; \ + docker-php-source delete + + + +#Synapse ODBC +RUN set -ex; \ + pecl install sqlsrv-$SQLSRV_VERSION pdo_sqlsrv-$SQLSRV_VERSION; \ + docker-php-ext-enable sqlsrv pdo_sqlsrv; \ + docker-php-source delete + +## Snowflake +COPY ./docker/snowflake/generic.pol /etc/debsig/policies/$SNOWFLAKE_GPG_KEY/generic.pol +COPY ./docker/snowflake/simba.snowflake.ini /usr/lib/snowflake/odbc/lib/simba.snowflake.ini + +RUN mkdir -p ~/.gnupg \ + && chmod 700 ~/.gnupg \ + && echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf \ + && mkdir -p /usr/share/debsig/keyrings/$SNOWFLAKE_GPG_KEY \ + && gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys $SNOWFLAKE_GPG_KEY \ + && gpg --export $SNOWFLAKE_GPG_KEY > /usr/share/debsig/keyrings/$SNOWFLAKE_GPG_KEY/debsig.gpg \ + && curl https://sfc-repo.snowflakecomputing.com/odbc/linux/$SNOWFLAKE_ODBC_VERSION/snowflake-odbc-$SNOWFLAKE_ODBC_VERSION.x86_64.deb --output /tmp/snowflake-odbc.deb \ + && debsig-verify /tmp/snowflake-odbc.deb \ + && gpg --batch --delete-key --yes $SNOWFLAKE_GPG_KEY \ + && dpkg -i /tmp/snowflake-odbc.deb + +# Teradata +COPY --from=0 /tmp/teradata/tdodbc.deb /tmp/teradata/tdodbc.deb +COPY docker/teradata/odbc.ini /tmp/teradata/odbc_td.ini +COPY docker/teradata/odbcinst.ini /tmp/teradata/odbcinst_td.ini + +RUN dpkg -i /tmp/teradata/tdodbc.deb \ + && cat /tmp/teradata/odbc_td.ini >> /etc/odbc.ini \ + && cat /tmp/teradata/odbcinst_td.ini >> /etc/odbcinst.ini \ + && rm -r /tmp/teradata \ + && docker-php-ext-configure pdo_odbc --with-pdo-odbc=unixODBC,/usr \ + && docker-php-ext-install pdo_odbc \ + && docker-php-source delete + +ENV ODBCHOME = /opt/teradata/client/ODBC_64/ +ENV ODBCINI = /opt/teradata/client/ODBC_64/odbc.ini +ENV ODBCINST = /opt/teradata/client/ODBC_64/odbcinst.ini +ENV LD_LIBRARY_PATH = /opt/teradata/client/ODBC_64/lib + +#Exasol +COPY --from=0 /tmp/exasol/odbc.tar.gz /tmp/exasol/odbc.tar.gz +RUN set -ex; \ + mkdir -p /tmp/exasol/odbc /opt/exasol ;\ + tar -xzf /tmp/exasol/odbc.tar.gz -C /tmp/exasol/odbc --strip-components 1; \ + cp /tmp/exasol/odbc/lib/linux/x86_64/libexaodbc-uo2214lv2.so /opt/exasol/;\ + echo "\n[exasol]\nDriver=/opt/exasol/libexaodbc-uo2214lv2.so\n" >> /etc/odbcinst.ini;\ + rm -rf /tmp/exasol; + +## Composer - deps always cached unless changed +# First copy only composer files +COPY composer.* /code/ +# Download dependencies, but don't run scripts or init autoloaders as the app is missing +RUN composer install $COMPOSER_FLAGS --no-scripts --no-autoloader +# copy rest of the app +COPY . /code/ +# run normal composer - all deps are cached already +RUN composer install $COMPOSER_FLAGS + +CMD ["php", "/code/src/run.php"] diff --git a/packages/php-table-backend-utils/docker-compose.yml b/packages/php-table-backend-utils/docker-compose.yml index 9e05465dc..85a1c5dbf 100644 --- a/packages/php-table-backend-utils/docker-compose.yml +++ b/packages/php-table-backend-utils/docker-compose.yml @@ -45,5 +45,13 @@ services: privileged: true volumes: - exa-volume:/exa + production74: + <<: *prod + build: + context: . + dockerfile: Dockerfile.74 + args: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY volumes: exa-volume: From ce725de8e43633c04184707a3e6ef985bec95e83 Mon Sep 17 00:00:00 2001 From: zajca Date: Wed, 1 Feb 2023 18:10:18 +0100 Subject: [PATCH 5/5] php7.4 fix snflk --- packages/php-table-backend-utils/Dockerfile.74 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/php-table-backend-utils/Dockerfile.74 b/packages/php-table-backend-utils/Dockerfile.74 index 9f6805dd4..583707355 100644 --- a/packages/php-table-backend-utils/Dockerfile.74 +++ b/packages/php-table-backend-utils/Dockerfile.74 @@ -11,9 +11,9 @@ ARG DEBIAN_FRONTEND=noninteractive ENV COMPOSER_ALLOW_SUPERUSER 1 ENV COMPOSER_PROCESS_TIMEOUT 3600 -ARG SQLSRV_VERSION=5.9.0 -ARG SNOWFLAKE_ODBC_VERSION=2.21.1 -ARG SNOWFLAKE_GPG_KEY=EC218558EABB25A1 +ARG SQLSRV_VERSION=5.10.1 +ARG SNOWFLAKE_ODBC_VERSION=2.25.6 +ARG SNOWFLAKE_GPG_KEY=630D9F3CAB551AF3 WORKDIR /code/