support for optional category-path url-rewrites

bigbridge-nl · Sep 16, 2018 · 049132e · 049132e
1 parent 43c0598
commit 049132e
Show file tree

Hide file tree

Showing 11 changed files with 145 additions and 20 deletions.
diff --git a/Api/ImportConfig.php b/Api/ImportConfig.php
@@ -156,4 +156,14 @@ class ImportConfig
 
     const KEEP_REDIRECTS = "keep"; // keep existing redirects, create new ones if the Magento settings is thus set
     const DELETE_REDIRECTS = "delete"; // remove any existing redirects, and do not create new ones
+
+    /**
+     * How to handle product url_rewrites with category paths?
+     *
+     * @string
+     */
+    public $handleCategoryRewrites = self::KEEP_CATEGORY_REWRITES;
+
+    const KEEP_CATEGORY_REWRITES = "keep"; // keep url_rewrites with category paths, create new ones
+    const DELETE_CATEGORY_REWRITES = "delete"; // remove any existing redirects, and do not create new ones
 }
diff --git a/Api/UrlRewriteUpdater.php b/Api/UrlRewriteUpdater.php
@@ -41,14 +41,14 @@ public function __construct(
      * @param array $storeViewCodes
      * @throws \Exception
      */
-    public function updateUrlRewrites(array $storeViewCodes, UrlRewriteUpdateLogger $logger, bool $keepRedirects)
+    public function updateUrlRewrites(array $storeViewCodes, UrlRewriteUpdateLogger $logger, bool $keepRedirects, bool $keepCategories)
     {
         $storeViewIds = $this->metaData->getStoreViewIds($storeViewCodes);
         $productIds = $this->information->getProductIds();
 
         $i = 0;
         while ($chunkedIds = array_slice($productIds, $i, self::BUNCH_SIZE)) {
-            $this->urlRewriteStorage->updateRewritesByProductIds($chunkedIds, $storeViewIds, $keepRedirects);
+            $this->urlRewriteStorage->updateRewritesByProductIds($chunkedIds, $storeViewIds, $keepRedirects, $keepCategories);
             $i += self::BUNCH_SIZE;
 
             $logger->info($i);

diff --git a/Console/Command/ProductImportCommand.php b/Console/Command/ProductImportCommand.php
@@ -33,6 +33,8 @@ class ProductImportCommand extends Command
     const OPTION_EMPTY_TEXT = "empty-text";
     const OPTION_EMPTY_NON_TEXT = "empty-non-text";
     const OPTION_SKIP_XSD = "skip-xsd";
+    const OPTION_REDIRECTS = 'redirects';
+    const OPTION_CATEGORY_PATH_URLS = "category-path-urls";
 
     /** @var ObjectManagerInterface */
     protected $objectManager;
@@ -137,6 +139,20 @@ protected function configure()
                 'Handle empty non-textual values: ignore, remove',
                 ImportConfig::EMPTY_NONTEXTUAL_VALUE_STRATEGY_IGNORE
             ),
+            new InputOption(
+                self::OPTION_REDIRECTS,
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'url_rewrite: Handle 301 redirects (delete: delete all existing and new url-rewrite redirects)',
+                ImportConfig::KEEP_REDIRECTS
+            ),
+            new InputOption(
+                self::OPTION_CATEGORY_PATH_URLS,
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'url_rewrite: Handle category paths (delete: delete all existing and new category url-rewrites)',
+                ImportConfig::KEEP_CATEGORY_REWRITES
+            ),
             new InputOption(
                 self::OPTION_SKIP_XSD,
                 null,
@@ -173,6 +189,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $config->duplicateUrlKeyStrategy = $input->getOption(self::OPTION_URL_KEY_STRATEGY);
         $config->emptyTextValueStrategy = $input->getOption(self::OPTION_EMPTY_TEXT);
         $config->emptyNonTextValueStrategy = $input->getOption(self::OPTION_EMPTY_NON_TEXT);
+        $config->handleRedirects = $input->getOption(self::OPTION_REDIRECTS);
+        $config->handleCategoryRewrites = $input->getOption(self::OPTION_CATEGORY_PATH_URLS);
 
         $config->imageSourceDir = $this->guessImageSourceDir($fileName, $input->getOption(self::OPTION_IMAGE_SOURCE_DIR));
 

diff --git a/Console/Command/ProductUrlRewriteCommand.php b/Console/Command/ProductUrlRewriteCommand.php
@@ -19,7 +19,9 @@
 class ProductUrlRewriteCommand extends Command
 {
     const ARGUMENT_STOREVIEW_CODE = 'storeview';
-    const ARGUMENT_REDIRECTS = 'redirects';
+
+    const OPTION_REDIRECTS = 'redirects';
+    const OPTION_CATEGORY_PATH_URLS = "category-path-urls";
 
     /** @var ObjectManagerInterface */
     protected $objectManager;
@@ -47,12 +49,20 @@ protected function configure()
                 []
             ),
             new InputOption(
-                self::ARGUMENT_REDIRECTS,
+                self::OPTION_REDIRECTS,
                 'r',
                 InputOption::VALUE_OPTIONAL,
                 'Handle 301 redirects (delete: delete all existing and new url-rewrite redirects)',
                 ImportConfig::KEEP_REDIRECTS
             ),
+            new InputOption(
+                self::OPTION_CATEGORY_PATH_URLS,
+                'c',
+                InputOption::VALUE_OPTIONAL,
+                'Handle category paths (delete: delete all existing and new category url-rewrites)',
+                ImportConfig::KEEP_CATEGORY_REWRITES
+
+            ),
         ]);
     }
 
@@ -71,7 +81,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $information = $this->objectManager->create(Information::class);
 
         $storeViewCodes = $input->getOption(self::ARGUMENT_STOREVIEW_CODE);
-        $handleRedirects = $input->getOption(self::ARGUMENT_REDIRECTS);
+        $handleRedirects = $input->getOption(self::OPTION_REDIRECTS);
+        $handleCategories = $input->getOption(self::OPTION_CATEGORY_PATH_URLS);
 
         if (empty($storeViewCodes)) {
             $storeViewCodes =  $information->getNonGlobalStoreViewCodes();
@@ -85,7 +96,11 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $logger = new UrlRewriteUpdateCommandLogger($output);
 
         try {
-            $urlRewriteUpdater->updateUrlRewrites($storeViewCodes, $logger, $handleRedirects === ImportConfig::KEEP_REDIRECTS);
+            $urlRewriteUpdater->updateUrlRewrites(
+                $storeViewCodes,
+                $logger,
+                $handleRedirects === ImportConfig::KEEP_REDIRECTS,
+                $handleCategories === ImportConfig::KEEP_CATEGORY_REWRITES);
         } catch (Exception $e) {
             $output->writeln("<error>" . $e->getMessage() . "</error>");
             return \Magento\Framework\Console\Cli::RETURN_FAILURE;

diff --git a/Model/Resource/ProductStorage.php b/Model/Resource/ProductStorage.php
@@ -286,7 +286,9 @@ protected function saveProducts(array $validProducts, ImportConfig $config)
         $this->tierPriceStorage->updateTierPrices($validProducts);
 
         // url_rewrite (must be done after url_key and category_id)
-        $this->urlRewriteStorage->updateRewrites($validProducts, $config->handleRedirects === ImportConfig::KEEP_REDIRECTS);
+        $this->urlRewriteStorage->updateRewrites($validProducts,
+            $config->handleRedirects === ImportConfig::KEEP_REDIRECTS,
+            $config->handleCategoryRewrites === ImportConfig::KEEP_CATEGORY_REWRITES);
 
         $this->downloadableStorage->performTypeSpecificStorage($productsByType[DownloadableProduct::TYPE_DOWNLOADABLE]);
         $this->groupedStorage->performTypeSpecificStorage($productsByType[GroupedProduct::TYPE_GROUPED]);

diff --git a/Model/Resource/Storage/UrlRewriteStorage.php b/Model/Resource/Storage/UrlRewriteStorage.php
@@ -47,20 +47,21 @@ public function __construct(
         $this->categoryImporter = $categoryImporter;
     }
 
-    public function updateRewrites(array $products, bool $keepRedirects)
+    public function updateRewrites(array $products, bool $keepRedirects, bool $keepCategories)
     {
         $productIds = array_column($products, 'id');
         $nonGlobalStoreIds = $this->metaData->getNonGlobalStoreViewIds();
 
-        $this->updateRewritesByProductIds($productIds, $nonGlobalStoreIds, $keepRedirects);
+        $this->updateRewritesByProductIds($productIds, $nonGlobalStoreIds, $keepRedirects, $keepCategories);
     }
 
     /**
      * @param int[] $productIds
      * @param array $storeViewIds
      * @param bool $keepRedirects
+     * @param bool $keepCategories
      */
-    public function updateRewritesByProductIds(array $productIds, array $storeViewIds, bool $keepRedirects)
+    public function updateRewritesByProductIds(array $productIds, array $storeViewIds, bool $keepRedirects, bool $keepCategories)
     {
         $allExistingUrlRewrites = $this->getExistingUrlRewrites($productIds, $storeViewIds);
         $allProductCategoryIds = $this->getExistingProductCategoryIds($productIds);
@@ -89,7 +90,8 @@ public function updateRewritesByProductIds(array $productIds, array $storeViewId
 
                 $existingUrlRewrites = isset($allExistingUrlRewrites[$storeViewId][$productId]) ? $allExistingUrlRewrites[$storeViewId][$productId] : [];
 
-                list($newDeletes, $newInserts) = $this->updateRewriteGroup($productId, $storeViewId, $existingUrlRewrites, $productCategoryIds, $allProductUrlKeys, $keepRedirects);
+                list($newDeletes, $newInserts) =
+                    $this->updateRewriteGroup($productId, $storeViewId, $existingUrlRewrites, $productCategoryIds, $allProductUrlKeys, $keepRedirects, $keepCategories);
 
                 $inserts = array_merge($inserts, $newInserts);
                 $deletes = array_merge($deletes, $newDeletes);
@@ -221,16 +223,17 @@ protected function getExistingVisibilities(array $productIds)
      * @param array $categoryIds
      * @param array $allProductUrlKeys
      * @param bool $keepRedirects
+     * @param bool $keepCategories
      * @return array
      */
-    protected function updateRewriteGroup(int $productId, int $storeViewId, array $existingUrlRewrites, array $categoryIds, array $allProductUrlKeys, bool $keepRedirects)
+    protected function updateRewriteGroup(int $productId, int $storeViewId, array $existingUrlRewrites, array $categoryIds, array $allProductUrlKeys, bool $keepRedirects, bool $keepCategories)
     {
-        list($deletes, $inserts) = $this->updateRewrite($productId, $storeViewId, 0, $existingUrlRewrites, $allProductUrlKeys, $keepRedirects);
+        list($deletes, $inserts) = $this->updateRewrite($productId, $storeViewId, 0, $existingUrlRewrites, $allProductUrlKeys, $keepRedirects, $keepCategories);
 
         $productCategoryIds = $this->collectParentCategories($categoryIds);
 
         foreach ($productCategoryIds as $categoryId) {
-            list($newDeletes, $newInserts) = $this->updateRewrite($productId, $storeViewId, $categoryId, $existingUrlRewrites, $allProductUrlKeys, $keepRedirects);
+            list($newDeletes, $newInserts) = $this->updateRewrite($productId, $storeViewId, $categoryId, $existingUrlRewrites, $allProductUrlKeys, $keepRedirects, $keepCategories);
             $deletes = array_merge($deletes, $newDeletes);
             $inserts = array_merge($inserts, $newInserts);
         }
@@ -251,12 +254,13 @@ protected function updateRewriteGroup(int $productId, int $storeViewId, array $e
      * @param array $existingUrlRewrites
      * @param array $allProductUrlKeys
      * @param bool $keepRedirects
+     * @param bool $keepCategories
      * @return array
      */
-    protected function updateRewrite(int $productId, int $storeViewId, int $categoryId, array $existingUrlRewrites, array $allProductUrlKeys, bool $keepRedirects)
+    protected function updateRewrite(int $productId, int $storeViewId, int $categoryId, array $existingUrlRewrites, array $allProductUrlKeys, bool $keepRedirects, bool $keepCategories)
     {
         $oldRewrites = array_key_exists($categoryId, $existingUrlRewrites) ? $existingUrlRewrites[$categoryId] : [];
-        $newRewrites = $this->collectNewRewrites($productId, $storeViewId, $categoryId, $existingUrlRewrites, $allProductUrlKeys, $keepRedirects);
+        $newRewrites = $this->collectNewRewrites($productId, $storeViewId, $categoryId, $existingUrlRewrites, $allProductUrlKeys, $keepRedirects, $keepCategories);
 
         if ($newRewrites === false) {
             return [[], []];
@@ -278,10 +282,16 @@ protected function updateRewrite(int $productId, int $storeViewId, int $category
      * @param array $existingUrlRewrites
      * @param array $allProductUrlKeys
      * @param bool $keepRedirects
+     * @param bool $keepCategories
      * @return array|bool
      */
-    protected function collectNewRewrites(int $productId, int $storeViewId, int $categoryId, array $existingUrlRewrites, array $allProductUrlKeys, bool $keepRedirects)
+    protected function collectNewRewrites(int $productId, int $storeViewId, int $categoryId, array $existingUrlRewrites, array $allProductUrlKeys, bool $keepRedirects, bool $keepCategories)
     {
+        // category path urls unwanted? remove them and their rewrites
+        if ($categoryId !== 0 && !$keepCategories) {
+            return [];
+        }
+
         $targetPath = self::TARGET_PATH_BASE . $productId . ($categoryId === 0 ? '' : self::TARGET_PATH_EXT . $categoryId);
         $requestPath = $this->createRequestPath($productId, $storeViewId, $categoryId, $allProductUrlKeys);
         if ($requestPath === null) {
@@ -327,7 +337,7 @@ protected function collectNewRewrites(int $productId, int $storeViewId, int $cat
             }
         }
 
-        // always add the active non-redirect
+        // add the active non-redirect
         $newRewrites[$currentZeroRewrite->getKey()] = $currentZeroRewrite;
 
         return $newRewrites;

diff --git a/Test/Benchmark/MemoryTest.php b/Test/Benchmark/MemoryTest.php
@@ -100,7 +100,7 @@ public function testImport()
         echo "Updates: " . sprintf('%.1f', $time) . " seconds; " . $memory . " kB \n";
 
         $this->assertSame([], $lastErrors);
-        $this->assertLessThan(66, $memory);
+        $this->assertLessThan(67, $memory);
 
         $afterPeakMemory = memory_get_peak_usage();
 

diff --git a/Test/Integration/UrlRewriteTest.php b/Test/Integration/UrlRewriteTest.php
@@ -398,6 +398,29 @@ public function testUrlRewritesGeneration()
 
         $this->doAsserts($expectedRewrites, $expectedIndexes, $product1, $product3);
 
+        // do not keep categories (but do keep redirects)
+
+        $config = new ImportConfig();
+        $config->handleCategoryRewrites = ImportConfig::DELETE_CATEGORY_REWRITES;
+
+        $importer = self::$factory->createImporter($config);
+        $importer->importSimpleProduct($product1);
+        $importer->importSimpleProduct($product3);
+        $importer->flush();
+
+        $expectedRewrites = [
+            ["product", "grote-turquoise-doos-product-import.html", "catalog/product/view/id/{$product1->id}", "0", "1", "1", null],
+
+            ["product", "big-grass-green-box-product-import.html", "a-big-grass-green-box-product-import.html", "301", "1", "0", serialize([])],
+
+            ["product", "a-big-grass-green-box-product-import.html", "catalog/product/view/id/{$product3->id}", "0", "1", "1", null],
+        ];
+
+        $expectedIndexes = [
+        ];
+
+        $this->doAsserts($expectedRewrites, $expectedIndexes, $product1, $product3);
+
         // do not keep redirects
 
         $config = new ImportConfig();

diff --git a/doc/changelog.md b/doc/changelog.md
@@ -0,0 +1,13 @@
+# Changelog
+
+## 1.1.0 : Category path url-rewrites - 16-09-2018
+
+Added the option to remove category path url-rewrites.
+
+If a shop does not use category paths in its urls, url_rewrite generation can be simplified a lot. This saves time and reduces the size of the url_rewrite table.
+
+Based on the ideas from [fisheyehq/module-url-rewrite-optimiser](https://github.com/fisheyehq/module-url-rewrite-optimiser)
+
+## 1.0.0 : Public release - 10-09-2018
+
+Publication on Github, into the public domain.
diff --git a/doc/importer.md b/doc/importer.md
@@ -583,6 +583,26 @@ Entries in the table url_rewrite are automatically generated, but only if the at
 
 This is default Magento behaviour. Invisible products (like most configurable variants) need no urls.
 
+The following config options are applicable; use with care.
+
+When the url of a product page changes, by default Magento keeps the old url and creates an HTTP response 301 REDIRECT when this url is requested. If this Magento option is turned off, no redirects will be stored.
+
+However, the number of redirects is a known source of database bloat. When a shop is set up, and many products move from one category to the next, the url_rewrite table may be filled with hundreds of thousands of senseless redirects. In this case, the "delete" option is handy. It removes all 301's from the database and creates no new 301's in this run.
+
+Use in a production shop is inadvisable. It is a SEO killer: products will no longer be accessible via old urls that may exist on the internet.
+
+    $config->handleRedirects = "delete";
+
+If your shop has "Use Categories Path for Product URLs" (Configuration / Catalog / Search Engine Optimization) turned off, there is no sense creating url_rewrites with category paths (i.e. gear/bags/joust-duffie-bag.html), they are not used. But Magento and this library will create them anyway.
+
+This takes a lot of time in url_rewrite creation and it takes up most of the records in the url_rewrite table.
+
+This is how to get rid of these rewrites:
+
+    $config->handleCategoryRewrites = "delete";
+
+Note that this will also remove existing category path product url redirects.
+
 ## Custom options
 
 Magento allows you to specify unique "attributes" to a product that are applicable to that single product alone. These are called custom options.

diff --git a/doc/url-rewrite-tool.md b/doc/url-rewrite-tool.md
@@ -24,4 +24,18 @@ When the url of a product page changes, by default Magento keeps the old url and
 
 However, the number of redirects is a known source of database bloat. When a shop is set up, and many products move from one category to the next, the url_rewrite table may be filled with hundreds of thousands of senseless redirects. In this case, the "delete" option is handy. It removes all 301's from the database and creates no new 301's in this run.
 
-Use in a production shop is inadvisable. It is a SEO killer: products will no longer be accessible via old urls that may exist on the internet.
+Use in a production shop is inadvisable. It is a SEO killer: products will no longer be accessible via old urls that may exist on the internet.
+
+## Delete category path urls
+
+If your shop has "Use Categories Path for Product URLs" (Configuration / Catalog / Search Engine Optimization) turned off, there is no sense creating url_rewrites with category paths (i.e. gear/bags/joust-duffie-bag.html), they are not used. But Magento and this library will create them anyway.
+
+This takes a lot of time in url_rewrite creation and it takes up most of the records in the url_rewrite table.
+
+This is how to get rid of these rewrites:
+
+    bin/magento bigbridge:product:urlrewrite --categories delete
+
+It makes sure these are not created, and will be removed if they are exist.
+
+Note that this will also remove existing category path product url redirects.