[AWS S3] Improvements (magento#6262)

ajeetsinghcedcoss · Oct 28, 2020 · de8edcf · de8edcf
1 parent 00a170a
commit de8edcf
Show file tree

Hide file tree

Showing 60 changed files with 1,287 additions and 590 deletions.
diff --git a/.gitignore b/.gitignore
@@ -58,6 +58,8 @@ atlassian*
 /pub/media/tmp/*
 !/pub/media/tmp/.htaccess
 /pub/media/captcha/*
+/pub/media/sitemap/*
+!/pub/media/sitemap/.htaccess
 /pub/static/*
 !/pub/static/.htaccess
 

diff --git a/app/code/Magento/AwsS3/Driver/AwsS3.php b/app/code/Magento/AwsS3/Driver/AwsS3.php
@@ -7,24 +7,29 @@
 
 namespace Magento\AwsS3\Driver;
 
+use Exception;
 use League\Flysystem\AwsS3v3\AwsS3Adapter;
 use League\Flysystem\Config;
 use Magento\Framework\Exception\FileSystemException;
 use Magento\Framework\Filesystem\DriverInterface;
 use Magento\Framework\Phrase;
 use Psr\Log\LoggerInterface;
+use Magento\RemoteStorage\Driver\DriverException;
+use Magento\RemoteStorage\Driver\RemoteDriverInterface;
 
 /**
  * Driver for AWS S3 IO operations.
  *
  * @SuppressWarnings(PHPMD.ExcessiveClassComplexity)
  */
-class AwsS3 implements DriverInterface
+class AwsS3 implements RemoteDriverInterface
 {
     public const TYPE_DIR = 'dir';
     public const TYPE_FILE = 'file';
 
-    private const CONFIG = ['ACL' => 'public-read'];
+    private const TEST_FLAG = 'storage.flag';
+
+    private const CONFIG = ['ACL' => 'private'];
 
     /**
      * @var AwsS3Adapter
@@ -68,6 +73,18 @@ public function __destruct()
         }
     }
 
+    /**
+     * @inheritDoc
+     */
+    public function test(): void
+    {
+        try {
+            $this->adapter->write(self::TEST_FLAG, '', new Config(self::CONFIG));
+        } catch (Exception $exception) {
+            throw new DriverException(__($exception->getMessage()), $exception);
+        }
+    }
+
     /**
      * @inheritDoc
      */
@@ -177,7 +194,7 @@ public function deleteDirectory($path): bool
     /**
      * @inheritDoc
      */
-    public function filePutContents($path, $content, $mode = null, $context = null): int
+    public function filePutContents($path, $content, $mode = null): int
     {
         $path = $this->normalizeRelativePath($path);
 
@@ -345,6 +362,7 @@ public function isDirectory($path): bool
         if ($this->adapter->has($path) && ($meta = $this->adapter->getMetadata($path))) {
             return ($meta['type'] ?? null) === self::TYPE_DIR;
         }
+
         return false;
     }
 
@@ -416,10 +434,44 @@ public function stat($path): array
             'blksize' => 0,
             'blocks' => 0,
             'size' => $metaInfo['size'] ?? 0,
-            'type' => $metaInfo['type'] ?? 0,
+            'type' => $metaInfo['type'] ?? '',
             'mtime' => $metaInfo['timestamp'] ?? 0,
-            'disposition' => null,
-            'mimetype' => $metaInfo['mimetype'] ?? 0
+            'disposition' => null
+        ];
+    }
+
+    /**
+     * @inheritDoc
+     */
+    public function getMetadata(string $path): array
+    {
+        $path = $this->normalizeRelativePath($path);
+        $metaInfo = $this->adapter->getMetadata($path);
+
+        if (!$metaInfo) {
+            throw new FileSystemException(__('Cannot gather meta info! %1', [$this->getWarningMessage()]));
+        }
+
+        $extra = [
+            'image-width' => 0,
+            'image-height' => 0
+        ];
+
+        if (isset($metaInfo['image-width'], $metaInfo['image-height'])) {
+            $extra['image-width'] = $metaInfo['image-width'];
+            $extra['image-height'] = $metaInfo['image-height'];
+        }
+
+        return [
+            'path' => $metaInfo['path'],
+            'dirname' => $metaInfo['dirname'],
+            'basename' => $metaInfo['basename'],
+            'extension' => $metaInfo['extension'],
+            'filename' => $metaInfo['filename'],
+            'timestamp' => $metaInfo['timestamp'],
+            'size' => $metaInfo['size'],
+            'mimetype' => $metaInfo['mimetype'],
+            'extra' => $extra
         ];
     }
 
@@ -778,6 +830,7 @@ private function getSearchPattern(string $pattern, array $parentPattern, string
             '/\?/' => '.',
             '/\//' => '\/'
         ];
+
         return preg_replace(array_keys($replacement), array_values($replacement), $searchPattern);
     }
 
@@ -816,6 +869,7 @@ private function getDirectoryContent(
                 }
             }
         }
+
         return $directoryContent;
     }
 }
diff --git a/app/code/Magento/AwsS3/Driver/AwsS3Factory.php b/app/code/Magento/AwsS3/Driver/AwsS3Factory.php
@@ -9,9 +9,9 @@
 
 use Aws\S3\S3Client;
 use League\Flysystem\AwsS3v3\AwsS3Adapter;
-use Magento\Framework\Filesystem\DriverInterface;
 use Magento\Framework\ObjectManagerInterface;
 use Magento\RemoteStorage\Driver\DriverFactoryInterface;
+use Magento\RemoteStorage\Driver\RemoteDriverInterface;
 
 /**
  * Creates a pre-configured instance of AWS S3 driver.
@@ -36,13 +36,15 @@ public function __construct(ObjectManagerInterface $objectManager)
      *
      * @param array $config
      * @param string $prefix
-     * @return DriverInterface
+     * @return RemoteDriverInterface
      */
-    public function create(array $config, string $prefix): DriverInterface
+    public function create(array $config, string $prefix): RemoteDriverInterface
     {
-        $config += [
-            'version' => 'latest'
-        ];
+        $config['version'] = 'latest';
+
+        if (empty($config['credentials']['key']) || empty($config['credentials']['secret'])) {
+            unset($config['credentials']);
+        }
 
         return $this->objectManager->create(
             AwsS3::class,

diff --git a/app/code/Magento/AwsS3/Test/Mftf/Data/ConfigData.xml b/app/code/Magento/AwsS3/Test/Mftf/Data/ConfigData.xml
@@ -13,5 +13,7 @@
         <data key="bucket">{{_ENV.REMOTE_STORAGE_AWSS3_BUCKET}}</data>
         <data key="access_key">{{_ENV.REMOTE_STORAGE_AWSS3_ACCESS_KEY}}</data>
         <data key="secret_key">{{_ENV.REMOTE_STORAGE_AWSS3_SECRET_KEY}}</data>
+        <data key="enable_options">--remote-storage-driver={{_ENV.REMOTE_STORAGE_AWSS3_DRIVER}} --remote-storage-bucket={{_ENV.REMOTE_STORAGE_AWSS3_BUCKET}} --remote-storage-region={{_ENV.REMOTE_STORAGE_AWSS3_REGION}} --remote-storage-prefix={{_ENV.REMOTE_STORAGE_AWSS3_PREFIX}} --remote-storage-key={{_ENV.REMOTE_STORAGE_AWSS3_ACCESS_KEY}} --remote-storage-secret={{_ENV.REMOTE_STORAGE_AWSS3_SECRET_KEY}} -n</data>
+        <data key="disable_options">--remote-storage-driver=file -n</data>
     </entity>
 </entities>
diff --git a/app/code/Magento/AwsS3/Test/Mftf/Test/AwsS3AdminAddImageForCategoryTest.xml b/app/code/Magento/AwsS3/Test/Mftf/Test/AwsS3AdminAddImageForCategoryTest.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+-->
+
+<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
+    <test name="AwsS3AdminAddImageForCategoryTest" extends="AdminAddImageForCategoryTest">
+        <annotations>
+            <title value="AWS S3 Admin should be able to add image to a Category"/>
+            <stories value="Add/remove images and videos for all product types and category"/>
+            <description value="Admin should be able to add image to a Category"/>
+            <severity value="CRITICAL"/>
+            <testCaseId value="MC-38688"/>
+            <group value="remote_storage_aws_s3"/>
+        </annotations>
+        <before>
+            <magentoCLI command="setup:config:set {{RemoteStorageAwsS3ConfigData.enable_options}}" stepKey="enableRemoteStorage"/>
+        </before>
+        <after>
+            <magentoCLI command="setup:config:set {{RemoteStorageAwsS3ConfigData.disable_options}}" stepKey="disableRemoteStorage"/>
+        </after>
+    </test>
+</tests>
diff --git a/app/code/Magento/AwsS3/Test/Mftf/Test/AwsS3AdminAddImageToWYSIWYGBlockTest.xml b/app/code/Magento/AwsS3/Test/Mftf/Test/AwsS3AdminAddImageToWYSIWYGBlockTest.xml
@@ -7,74 +7,20 @@
 -->
 <tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
-    <test name="AwsS3AdminAddImageToWYSIWYGBlockTest">
+    <test name="AwsS3AdminAddImageToWYSIWYGBlockTest" extends="AdminAddImageToWYSIWYGBlockTest">
         <annotations>
-            <features value="Cms"/>
-            <stories value="MC-37460: Support by Magento CMS"/>
-            <group value="Cms"/>
-            <title value="Admin should be able to add image to WYSIWYG content of Block with remote filesystem enabled"/>
-            <description value="Admin should be able to add image to WYSIWYG content of Block with remote filesystem enabled"/>
+            <title value="AWS S3 Admin should be able to add image to WYSIWYG content of Block with remote filesystem enabled"/>
+            <stories value="Default WYSIWYG toolbar configuration with Magento Media Gallery"/>
+            <description value="Admin should be able to add image to WYSIWYG content of Block"/>
             <severity value="BLOCKER"/>
             <testCaseId value="MC-38302"/>
             <group value="remote_storage_aws_s3"/>
         </annotations>
         <before>
-            <magentoCLI command="remote-storage:enable {{RemoteStorageAwsS3ConfigData.driver}} {{RemoteStorageAwsS3ConfigData.bucket}} {{RemoteStorageAwsS3ConfigData.region}} {{RemoteStorageAwsS3ConfigData.prefix}} {{RemoteStorageAwsS3ConfigData.access_key}} {{RemoteStorageAwsS3ConfigData.secret_key}} --is-public true" stepKey="enableRemoteStorage"/>
-            <createData entity="_defaultCmsPage" stepKey="createCMSPage" />
-            <createData entity="_defaultBlock" stepKey="createPreReqBlock" />
-            <actionGroup ref="AdminLoginActionGroup" stepKey="login"/>
-            <actionGroup ref="AdminDisableWYSIWYGActionGroup" stepKey="disableWYSIWYGBeforeTest" />
-            <magentoCLI command='config:set cms/wysiwyg/enabled enabled' stepKey="enableWYSIWYGBeforeTest"/>
-            <actionGroup ref="SwitchToVersion4ActionGroup" stepKey="switchToTinyMCE4" />
+            <magentoCLI command="setup:config:set {{RemoteStorageAwsS3ConfigData.enable_options}}" stepKey="enableRemoteStorage"/>
         </before>
-        <actionGroup ref="AssignBlockToCMSPage" stepKey="assignBlockToCMSPage">
-            <argument name="Block" value="$$createPreReqBlock$$"/>
-            <argument name="CmsPage" value="$$createCMSPage$$"/>
-        </actionGroup>
-        <actionGroup ref="NavigateToCreatedCMSBlockPageActionGroup" stepKey="navigateToCreatedCMSBlockPage1">
-            <argument name="CMSBlockPage" value="$$createPreReqBlock$$"/>
-        </actionGroup>
-        <selectOption selector="{{BlockNewPageBasicFieldsSection.storeView}}" userInput="All Store View" stepKey="selectAllStoreView" />
-        <waitForElementVisible selector="{{TinyMCESection.TinyMCE4}}" stepKey="waitForTinyMCE" />
-        <click selector="{{TinyMCESection.InsertImageIcon}}" stepKey="clickInsertImageIcon" />
-        <waitForPageLoad stepKey="waitForPageLoad2" />
-        <actionGroup ref="ClickBrowseBtnOnUploadPopupActionGroup" stepKey="clickBrowserBtn"/>
-        <actionGroup ref="VerifyMediaGalleryStorageActionsActionGroup" stepKey="VerifyMediaGalleryStorageBtn"/>
-        <actionGroup ref="NavigateToMediaFolderActionGroup" stepKey="navigateToFolder">
-            <argument name="FolderName" value="Storage Root"/>
-        </actionGroup>
-        <actionGroup ref="CreateImageFolderActionGroup" stepKey="CreateImageFolder">
-            <argument name="ImageFolder" value="ImageFolder"/>
-        </actionGroup>
-        <actionGroup ref="AttachImageActionGroup" stepKey="attachImage1">
-            <argument name="Image" value="ImageUpload"/>
-        </actionGroup>
-        <actionGroup ref="DeleteImageActionGroup" stepKey="deleteImage"/>
-        <actionGroup ref="AttachImageActionGroup" stepKey="attachImage2">
-            <argument name="Image" value="ImageUpload"/>
-        </actionGroup>
-        <actionGroup ref="SaveImageActionGroup" stepKey="insertImage"/>
-        <actionGroup ref="FillOutUploadImagePopupActionGroup" stepKey="fillOutUploadImagePopup" />
-        <click selector="{{BlockNewPagePageActionsSection.saveBlock}}" stepKey="clickSaveBlock"/>
-        <amOnPage url="$$createCMSPage.identifier$$" stepKey="amOnPageTestPage"/>
-        <waitForPageLoad stepKey="waitForPageLoad11" />
-        <!--see image on Storefront-->
-        <seeElement selector="{{StorefrontBlockSection.mediaDescription}}" stepKey="assertMediaDescription"/>
-        <seeElementInDOM selector="{{StorefrontBlockSection.imageSource(ImageUpload.fileName)}}" stepKey="assertMediaSource"/>
         <after>
-            <actionGroup ref="NavigateToMediaGalleryActionGroup" stepKey="navigateToMediaGallery"/>
-            <actionGroup ref="DeleteFolderActionGroup" stepKey="DeleteCreatedFolder">
-                <argument name="ImageFolder" value="ImageFolder"/>
-            </actionGroup>
-            <amOnPage url="{{CmsPagesPage.url}}" stepKey="amOnEditPage"/>
-            <waitForPageLoad stepKey="waitForPageLoad"/>
-            <conditionalClick selector="{{CmsPagesPageActionsSection.clearAllButton}}" dependentSelector="{{CmsPagesPageActionsSection.activeFilters}}" stepKey="clickToResetFilter" visible="true"/>
-            <waitForPageLoad stepKey="waitForGridReload"/>
-            <deleteData createDataKey="createPreReqBlock" stepKey="deletePreReqBlock" />
-            <deleteData createDataKey="createCMSPage" stepKey="deletePreReqCMSPage" />
-            <actionGroup ref="AdminLogoutActionGroup" stepKey="logout"/>
-            <actionGroup ref="AdminEnableWYSIWYGActionGroup" stepKey="enableWYSIWYGAfterTest" />
-            <magentoCLI command="remote-storage:disable" stepKey="disableRemoteStorage"/>
+            <magentoCLI command="setup:config:set {{RemoteStorageAwsS3ConfigData.disable_options}}" stepKey="disableRemoteStorage"/>
         </after>
     </test>
 </tests>
diff --git a/app/code/Magento/AwsS3/Test/Mftf/Test/AwsS3AdminAddImageToWYSIWYGCMSTest.xml b/app/code/Magento/AwsS3/Test/Mftf/Test/AwsS3AdminAddImageToWYSIWYGCMSTest.xml
@@ -7,69 +7,20 @@
 -->
 <tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
-    <test name="AwsS3AdminAddImageToWYSIWYGCMSTest">
+    <test name="AwsS3AdminAddImageToWYSIWYGCMSTest" extends="AdminAddImageToWYSIWYGCMSTest">
         <annotations>
-            <features value="Cms"/>
-            <stories value="MC-37460: Support by Magento CMS"/>
-            <group value="Cms"/>
-            <title value="Admin should be able to add image to WYSIWYG content of CMS Page with remote filesystem enabled"/>
-            <description value="Admin should be able to add image to WYSIWYG content of CMS Page with remote filesystem enabled"/>
+            <title value="AWS S3 Admin should be able to add image to WYSIWYG content of CMS Page with remote filesystem enabled"/>
+            <stories value="Default WYSIWYG toolbar configuration with Magento Media Gallery"/>
+            <description value="Admin should be able to add image to WYSIWYG content of CMS Page"/>
             <severity value="BLOCKER"/>
             <testCaseId value="MC-38295"/>
             <group value="remote_storage_aws_s3"/>
         </annotations>
         <before>
-            <magentoCLI command="remote-storage:enable {{RemoteStorageAwsS3ConfigData.driver}} {{RemoteStorageAwsS3ConfigData.bucket}} {{RemoteStorageAwsS3ConfigData.region}} {{RemoteStorageAwsS3ConfigData.prefix}} {{RemoteStorageAwsS3ConfigData.access_key}} {{RemoteStorageAwsS3ConfigData.secret_key}} --is-public true" stepKey="enableRemoteStorage"/>
-            <createData entity="_defaultCmsPage" stepKey="createCMSPage" />
-            <actionGroup ref="AdminLoginActionGroup" stepKey="login"/>
-            <actionGroup ref="AdminDisableWYSIWYGActionGroup" stepKey="disableWYSIWYGBeforeTest" />
-            <magentoCLI command='config:set cms/wysiwyg/enabled enabled' stepKey="enableWYSIWYGBeforeTest"/>
-            <actionGroup ref="SwitchToVersion4ActionGroup" stepKey="switchToTinyMCE4" />
+            <magentoCLI command="setup:config:set {{RemoteStorageAwsS3ConfigData.enable_options}}" stepKey="enableRemoteStorage"/>
         </before>
         <after>
-            <actionGroup ref="NavigateToMediaGalleryActionGroup" stepKey="navigateToMediaGallery"/>
-            <actionGroup ref="DeleteFolderActionGroup" stepKey="DeleteCreatedFolder">
-                <argument name="ImageFolder" value="ImageFolder"/>
-            </actionGroup>
-            <deleteData createDataKey="createCMSPage" stepKey="deletePreReqCMSPage" />
-            <actionGroup ref="AdminEnableWYSIWYGActionGroup" stepKey="enableWYSIWYGAfterTest" />
-            <actionGroup ref="AdminLogoutActionGroup" stepKey="logout"/>
-            <magentoCLI command="remote-storage:disable" stepKey="disableRemoteStorage"/>
+            <magentoCLI command="setup:config:set {{RemoteStorageAwsS3ConfigData.disable_options}}" stepKey="disableRemoteStorage"/>
         </after>
-
-        <actionGroup ref="NavigateToCreatedCMSPageActionGroup" stepKey="navigateToCreatedCMSPage">
-            <argument name="CMSPage" value="$$createCMSPage$$"/>
-        </actionGroup>
-        <click selector="{{CmsNewPagePageContentSection.header}}" stepKey="clickExpandContent"/>
-        <waitForElementVisible selector="{{TinyMCESection.TinyMCE4}}" stepKey="waitForTinyMCE4" />
-        <click selector="{{TinyMCESection.InsertImageIcon}}" stepKey="clickInsertImageIcon" />
-        <waitForPageLoad stepKey="waitForPageLoad" />
-        <actionGroup ref="ClickBrowseBtnOnUploadPopupActionGroup" stepKey="clickBrowserBtn"/>
-        <actionGroup ref="VerifyMediaGalleryStorageActionsActionGroup" stepKey="VerifyMediaGalleryStorageBtn"/>
-        <actionGroup ref="NavigateToMediaFolderActionGroup" stepKey="navigateToFolder">
-             <argument name="FolderName" value="Storage Root"/>
-        </actionGroup>
-        <actionGroup ref="CreateImageFolderActionGroup" stepKey="CreateImageFolder">
-            <argument name="ImageFolder" value="ImageFolder"/>
-        </actionGroup>
-        <actionGroup ref="AttachImageActionGroup" stepKey="attachImage1">
-            <argument name="Image" value="ImageUpload3"/>
-        </actionGroup>
-        <actionGroup ref="DeleteImageActionGroup" stepKey="deleteImage"/>
-        <actionGroup ref="AttachImageActionGroup" stepKey="attachImage2">
-            <argument name="Image" value="ImageUpload3"/>
-        </actionGroup>
-        <actionGroup ref="SaveImageActionGroup" stepKey="insertImage"/>
-        <actionGroup ref="FillOutUploadImagePopupActionGroup" stepKey="fillOutUploadImagePopup" />
-        <click selector="{{CmsNewPagePageSeoSection.header}}" stepKey="clickExpandSearchEngineOptimisation"/>
-        <fillField selector="{{CmsNewPagePageSeoSection.urlKey}}" userInput="$$createCMSPage.identifier$$" stepKey="fillFieldUrlKey"/>
-        <click selector="{{CmsNewPagePageActionsSection.expandSplitButton}}" stepKey="expandButtonMenu"/>
-        <waitForElementVisible selector="{{CmsNewPagePageActionsSection.splitButtonMenu}}" stepKey="waitForSplitButtonMenuVisible"/>
-        <click selector="{{CmsNewPagePageActionsSection.savePage}}" stepKey="clickSavePage"/>
-        <see userInput="You saved the page." stepKey="seeSuccessMessage"/>
-        <amOnPage url="$$createCMSPage.identifier$$" stepKey="amOnPageTestPage"/>
-        <waitForPageLoad  stepKey="wait4"/>
-        <seeElement selector="{{StorefrontCMSPageSection.mediaDescription}}" stepKey="assertMediaDescription"/>
-        <seeElementInDOM selector="{{StorefrontCMSPageSection.imageSource(ImageUpload3.fileName)}}" stepKey="assertMediaSource"/>
     </test>
 </tests>
diff --git a/app/code/Magento/AwsS3/Test/Mftf/Test/AwsS3AdminAddImageToWYSIWYGNewsletterTest.xml b/app/code/Magento/AwsS3/Test/Mftf/Test/AwsS3AdminAddImageToWYSIWYGNewsletterTest.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
+    <test name="AwsS3AdminAddImageToWYSIWYGNewsletterTest" extends="AdminAddImageToWYSIWYGNewsletterTest">
+        <annotations>
+            <features value="Newsletter"/>
+            <stories value="Apply new WYSIWYG in Newsletter"/>
+            <group value="Newsletter"/>
+            <title value="AWS S3 Admin should be able to add image to WYSIWYG content of Newsletter"/>
+            <description value="Admin should be able to add image to WYSIWYG content Newsletter"/>
+            <severity value="CRITICAL"/>
+            <testCaseId value="MC-38716"/>
+            <group value="remote_storage_aws_s3"/>
+        </annotations>
+        <before>
+            <magentoCLI command="setup:config:set {{RemoteStorageAwsS3ConfigData.enable_options}}" stepKey="enableRemoteStorage"/>
+        </before>
+        <after>
+            <magentoCLI command="setup:config:set {{RemoteStorageAwsS3ConfigData.disable_options}}" stepKey="disableRemoteStorage"/>
+        </after>
+    </test>
+</tests>