Backup to google storage

.github/workflows/push.yml

@@ -29,7 +29,8 @@ env:
   # GCP Credentials
   TEST_GCP_STORAGE_API_URL: "https://connection.europe-west3.gcp.keboola.com/"
   TEST_GCP_STORAGE_API_TOKEN: "${{ secrets.TEST_GCP_STORAGE_API_TOKEN }}"
-
+  TEST_GCP_BUCKET: "ci-php-kbc-project-backup"
+  TEST_GCP_SERVICE_ACCOUNT: "${{ secrets.TEST_GCP_SERVICE_ACCOUNT }}"
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -66,5 +67,7 @@ jobs:
           -e TEST_AZURE_CONTAINER_NAME
           -e TEST_GCP_STORAGE_API_URL
           -e TEST_GCP_STORAGE_API_TOKEN
+          -e TEST_GCP_BUCKET
+          -e TEST_GCP_SERVICE_ACCOUNT
           ${{env.APP_IMAGE}}
           composer ci

README.md

@@ -36,13 +36,20 @@ TEST_AZURE_STORAGE_API_TOKEN=
 TEST_AZURE_ACCOUNT_NAME=
 TEST_AZURE_ACCOUNT_KEY=
 TEST_AZURE_CONTAINER_NAME=
+TEST_GCP_STORAGE_API_URL=
+TEST_GCP_STORAGE_API_TOKEN=
+TEST_GCP_BUCKET=
+TEST_GCP_SERVICE_ACCOUNT=
 ```
 
-- `TEST_AWS_STORAGE_API_*` variables are from the project with AWS S3 backend whhich you want to backup
+- `TEST_AWS_STORAGE_API_*` variables are from the project with AWS S3 backend which you want to backup
 - `TEST_AWS_*` variables are from the S3 bucket the backup will be stored to _(Use [aws-cf-template.json](./aws-cf-template.json) CloudFormation stack template to create all required AWS resources)_
 - `TEST_AZURE_STORAGE_API_*` variables are from the project with Azure Blob storage backend which you want to backup
 - `TEST_AZURE_ACCOUNT_` create new Storage Account in your Azure Subscription
-- `TEST_AZURE_CONTAINER_NAME` container whhich will be created in your Storage Account
+- `TEST_AZURE_CONTAINER_NAME` container which will be created in your Storage Account
+- `TEST_GCP_STORAGE_API_*` variables are from the project with GCP Storage backend which you want to backup
+- `TEST_GCP_BUCKET` bucket in your Google Storage
+- `TEST_GCP_SERVICE_ACCOUNT` Service account with permissions to write to the bucket
 
 ```bash
 docker-compose run --rm tests

docker-compose.yml

@@ -28,3 +28,5 @@ services:
       - TEST_AZURE_CONTAINER_NAME
       - TEST_GCP_STORAGE_API_URL
       - TEST_GCP_STORAGE_API_TOKEN
+      - TEST_GCP_BUCKET
+      - TEST_GCP_SERVICE_ACCOUNT

src/GcsBackup.php

@@ -0,0 +1,69 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Keboola\ProjectBackup;
+
+use DateTimeImmutable;
+use Google\Cloud\Storage\StorageClient;
+use Google\Cloud\Storage\StorageObject;
+use Keboola\StorageApi\Client as StorageApi;
+use Psr\Log\LoggerInterface;
+
+class GcsBackup extends Backup
+{
+    private array $signedUrls = [];
+
+    public function __construct(
+        StorageApi $sapiClient,
+        readonly StorageClient $storageClient,
+        readonly string $bucketName,
+        readonly string $path,
+        readonly bool $generateSignedUrls = false,
+        ?LoggerInterface $logger = null,
+    ) {
+        parent::__construct($sapiClient, $logger);
+    }
+
+    /**
+     * @param resource|string $content
+     */
+    protected function putToStorage(string $name, $content): void
+    {
+        $bucket = $this->storageClient->bucket($this->bucketName);
+        $object = $bucket->upload($content, [
+            'name' => sprintf('%s%s', $this->path, $name),
+        ]);
+
+        if ($this->generateSignedUrls) {
+            $this->buildTreeFromPath($object, $name);
+        }
+    }
+
+    public function backupSignedUrls(): void
+    {
+        $name = 'signedUrls.json';
+        $bucket = $this->storageClient->bucket($this->bucketName);
+        $content = (string) json_encode($this->signedUrls, JSON_PRETTY_PRINT);
+
+        $bucket->upload($content, [
+            'name' => sprintf('%s%s', $this->path, $name),
+        ]);
+    }
+
+    private function buildTreeFromPath(StorageObject $object, string $path): void
+    {
+        $parts = explode('/', $path);
+        $filename = pathinfo(array_pop($parts), PATHINFO_BASENAME);
+
+        $current = &$this->signedUrls;
+        foreach ($parts as $part) {
+            if (!isset($current[$part])) {
+                $current[$part] = [];
+            }
+            $current = &$current[$part];
+        }
+
+        $current[$filename] = $object->signedUrl(new DateTimeImmutable('+2 days'));
+    }
+}

tests/FileClient/AbstractFileClientTestCase.php

@@ -56,7 +56,11 @@ protected function createTableInStorage(string $tableId): void
         $tableId = sprintf('%s.%s', $bucketId, $table);
 
         if (!$this->storageClient->bucketExists($bucketId)) {
-            $this->storageClient->createBucket($bucket, $stage);
+            $bucketName = $bucket;
+            if (str_starts_with($bucket, 'c-')) {
+                $bucketName = substr($bucket, 2);
+            }
+            $this->storageClient->createBucket($bucketName, $stage);
         }
         if ($this->storageClient->tableExists($tableId)) {
             $this->storageClient->dropTable($tableId);