Improve perf. Check modified since and use lazy loading

- allow to disable composer API v2 - workaround to avoid a lot of 304 requests, see composer/composer#11323
vtsykun · Feb 21, 2023 · 3ec55fc · 3ec55fc
1 parent 8775f0a
commit 3ec55fc
Show file tree

Hide file tree

Showing 34 changed files with 422 additions and 149 deletions.
diff --git a/composer.lock b/composer.lock
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
@@ -51,7 +51,7 @@ if [ -f /var/tmp/data/handler.sh ]; then
     bash /var/tmp/data/handler.sh
 fi
 
-mkdir var/cache var/log
+mkdir -p var/cache var/log
 rm -rf var/cache/*
 
 app cache:clear

diff --git a/public/packeton/css/main.css b/public/packeton/css/main.css
@@ -1747,3 +1747,9 @@ pre.github {
     border-radius: 3px;
     cursor: pointer;
 }
+
+@media (min-width: 1200px) {
+    .modal-lg {
+        width:1150px
+    }
+}
diff --git a/src/Composer/IO/BufferIO.php b/src/Composer/IO/BufferIO.php
@@ -26,9 +26,9 @@ class BufferIO extends ConsoleIO
     protected $verbosityMatrixCapacity = 3;
 
     protected $verbosityMatrix = [
-        IOInterface::VERBOSE => 3,
-        IOInterface::VERY_VERBOSE => 2,
-        IOInterface::DEBUG => 2,
+        IOInterface::VERBOSE => 10,
+        IOInterface::VERY_VERBOSE => 6,
+        IOInterface::DEBUG => 6,
     ];
 
     public function __construct(string $input = '', int $verbosity = StreamOutput::VERBOSITY_NORMAL, ?OutputFormatterInterface $formatter = null)
@@ -117,13 +117,14 @@ protected function dynamicVerbosity($verbosity)
         if ($verbosity === self::NORMAL && $this->verbosityMatrixCapacity > 0) {
             foreach ($this->verbosityMatrix as $verb => $value) {
                 if ($value <= 0) {
-                    $this->verbosityMatrix[$verb] = 2;
+                    $this->verbosityMatrix[$verb] = 6;
                     $this->verbosityMatrixCapacity--;
                 }
             }
         }
 
         if (isset($this->verbosityMatrix[$verbosity]) && $this->verbosityMatrix[$verbosity] > 0) {
+            $this->verbosityMatrix[$verbosity]--;
             return self::NORMAL;
         }
 

diff --git a/src/Composer/MetadataMinifier.php b/src/Composer/MetadataMinifier.php
@@ -14,7 +14,7 @@ class MetadataMinifier
     /**
      * Convert metadata v1 to metadata v2
      */
-    public function minify(array $metadata, bool $isDev = true, &$lastModified = null): array
+    public function minify(array $metadata, ?bool $isDev = true, &$lastModified = null): array
     {
         $packages = $metadata['packages'] ?? [];
         $metadata['minified'] = 'composer/2.0';
@@ -23,7 +23,7 @@ public function minify(array $metadata, bool $isDev = true, &$lastModified = nul
         $obj->time = '1970-01-01T00:00:00+00:00';
 
         foreach ($packages as $packName => $versions) {
-            $versions = \array_filter($versions, fn($v) => $this->isValidStability($v, $isDev));
+            $versions = \array_filter($versions, fn($v) => $isDev === null || $this->isValidStability($v, $isDev));
 
             \usort($versions, fn($v1, $v2) => -1 * version_compare($v1['version_normalized'], $v2['version_normalized']));
             \array_map(fn($v) => $obj->time < ($v['time'] ?? 0) ? $obj->time = ($v['time'] ?? 0) : null, $versions);

diff --git a/src/Composer/PackagistFactory.php b/src/Composer/PackagistFactory.php
@@ -14,7 +14,7 @@
 use Packeton\Composer\Repository\VcsRepository;
 use Packeton\Composer\Util\ConfigFactory;
 use Packeton\Composer\Util\ProcessExecutor;
-use Packeton\Entity\SshCredentials;
+use Packeton\Model\CredentialsInterface;
 
 class PackagistFactory
 {
@@ -71,10 +71,10 @@ public function createArchiveManager(IOInterface $io, RepositoryInterface $repos
     }
 
     /**
-     * @param SshCredentials|null $credentials
+     * @param CredentialsInterface|null $credentials
      * @return \Composer\Config
      */
-    public function createConfig(SshCredentials $credentials = null)
+    public function createConfig(CredentialsInterface $credentials = null)
     {
         $config = ConfigFactory::createConfig();
 
@@ -97,6 +97,9 @@ public function createConfig(SshCredentials $credentials = null)
                 ProcessExecutor::inheritEnv(['GIT_SSH_COMMAND']);
 
                 $config->merge(['config' => ['ssh-key-file' => $credentialsFile]]);
+            } else if ($credentials->getPrivkeyFile()) {
+                putenv("GIT_SSH_COMMAND=ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {$credentials->getPrivkeyFile()}");
+                ProcessExecutor::inheritEnv(['GIT_SSH_COMMAND']);
             }
         }
 
@@ -107,12 +110,12 @@ public function createConfig(SshCredentials $credentials = null)
      * @param string $url
      * @param IOInterface|null $io
      * @param Config|null $config
-     * @param SshCredentials|null $credentials
+     * @param CredentialsInterface|null $credentials
      * @param array $repoConfig
      *
      * @return Repository\VcsRepository
      */
-    public function createRepository(string $url, IOInterface $io = null, Config $config = null, SshCredentials $credentials = null, array $repoConfig = [])
+    public function createRepository(string $url, IOInterface $io = null, Config $config = null, CredentialsInterface $credentials = null, array $repoConfig = [])
     {
         $io = $io ?: new NullIO();
         if (null === $config) {

diff --git a/src/Controller/MirrorController.php b/src/Controller/MirrorController.php
@@ -61,9 +61,14 @@ public function metadataV2Action(string $package, string $alias, Request $reques
         $devStability = \str_ends_with($package, '~dev');
         $package = \preg_replace('/~dev$/', '', $package);
 
-        $metadata = $this->wrap404Error($alias, fn (PRI $repo) => $repo->findPackageMetadata($package));
+        $modifiedSince = $request->headers->get('If-Modified-Since');
+        $modifiedSince = $modifiedSince ? (\strtotime($modifiedSince) ?: null) : null;
+
+        $metadata = $this->wrap404Error($alias, fn (PRI $repo) => $repo->findPackageMetadata($package, $modifiedSince));
 
-        $metadata = $metadata->withContent(fn ($package) => $this->minifier->minify($package, $devStability));
+        if (false === $metadata->isNotModified()) {
+            $metadata = $metadata->withContent(fn ($package) => $this->minifier->minify($package, $devStability));
+        }
 
         return $this->renderMetadata($metadata, $request);
     }
@@ -114,6 +119,9 @@ protected function renderMetadata(JsonMetadata $metadata, Request $request, call
         $response = new Response($metadata->getContent(), 200, ['Content-Type' => 'application/json']);
         $response->setLastModified($metadata->lastModified());
         $notModified = $response->isNotModified($request);
+        if ($metadata->isNotModified()) {
+            $response->setNotModified();
+        }
 
         if (null !== $lazyLoad && false === $notModified) {
             $metadata = $lazyLoad($metadata);

diff --git a/src/Controller/ProxiesController.php b/src/Controller/ProxiesController.php
@@ -107,7 +107,7 @@ public function metadata(HtmlJsonHuman $jsonHuman, string $alias, string $packag
         }
 
         $json = $meta->decodeJson();
-        $metadata = $this->metadataMinifier->minify($json)['packages'][$package] ?? [];
+        $metadata = $this->metadataMinifier->minify($json, null)['packages'][$package] ?? [];
 
         return new Response($jsonHuman->buildToHtml($metadata));
     }

diff --git a/src/Cron/MirrorCronLoader.php b/src/Cron/MirrorCronLoader.php
@@ -28,6 +28,10 @@ public function getSchedules(array $options = []): iterable
 
         foreach ($this->registry->getAllRepos() as $name => $repo) {
             if ($repo instanceof RemoteProxyRepository) {
+                if (!$repo->getPackageManager()->isAutoSync()) {
+                    continue;
+                }
+
                 $repo->resetProxyOptions();
                 $config = $repo->getConfig();
                 $expr = '@random ' . $this->getSyncInterval($config);

diff --git a/src/Entity/SshCredentials.php b/src/Entity/SshCredentials.php
@@ -3,14 +3,15 @@
 namespace Packeton\Entity;
 
 use Doctrine\ORM\Mapping as ORM;
+use Packeton\Model\CredentialsInterface;
 
 /**
  * SshCredentials
  *
  * @ORM\Table(name="ssh_credentials")
  * @ORM\Entity()
  */
-class SshCredentials implements OwnerAwareInterface
+class SshCredentials implements OwnerAwareInterface, CredentialsInterface
 {
     /**
      * @var int
@@ -122,7 +123,7 @@ public function setKey($key)
      *
      * @return string
      */
-    public function getKey()
+    public function getKey(): ?string
     {
         return $this->key;
     }
@@ -180,7 +181,7 @@ public function getFingerprint()
      *
      * @return mixed|null
      */
-    public function getComposerConfigOption(string $name)
+    public function getComposerConfigOption(string $name): mixed
     {
         return $this->composerConfig[$name] ?? null;
     }
@@ -228,4 +229,9 @@ public function getVisibility(): ?string
     {
         return OwnerAwareInterface::STRICT_VISIBLE;
     }
+
+    public function getPrivkeyFile(): ?string
+    {
+        return null;
+    }
 }
diff --git a/src/Form/Type/ProxySettingsType.php b/src/Form/Type/ProxySettingsType.php
@@ -25,7 +25,15 @@ public function buildForm(FormBuilderInterface $builder, array $options): void
                     'All (new packages are automatically added when requested by composer)' => false,
                 ]
             ])
+            ->add('disable_v2', ChoiceType::class, [
+                'label' => 'Disable Composer API v2',
+                'choices' => [
+                    'No' => false,
+                    'Yes' => true,
+                ]
+            ])
             ->add('enabled_sync', CheckboxType::class, [
+                'required' => false,
                 'label' => 'Enable automatically synchronization',
             ]);
     }

diff --git a/src/Mirror/AbstractProxyRepository.php b/src/Mirror/AbstractProxyRepository.php
@@ -21,31 +21,31 @@ public function getConfig(): ProxyOptions
     {
         return $this->proxyOptions ??= new ProxyOptions(
             $this->repoConfig
-            + ['root' => $this->rootMetadata()?->decodeJson()]
+            + ['root' => $this->getRootMetadataInfo()]
             + ['stats' => $this->getStats()]
         );
     }
 
     /**
      * {@inheritdoc}
      */
-    public function findPackageMetadata(string $name): ?JsonMetadata
+    public function findPackageMetadata(string $name, int $modifiedSince = null): ?JsonMetadata
     {
         return null;
     }
 
     /**
      * {@inheritdoc}
      */
-    public function findProviderMetadata(string $name): ?JsonMetadata
+    public function findProviderMetadata(string $name, int $modifiedSince = null): ?JsonMetadata
     {
         return null;
     }
 
     /**
      * {@inheritdoc}
      */
-    public function rootMetadata(): ?JsonMetadata
+    public function rootMetadata(int $modifiedSince = null): ?JsonMetadata
     {
         return null;
     }
@@ -63,4 +63,15 @@ public function resetProxyOptions(): void
     {
         $this->proxyOptions = null;
     }
+
+    protected function getRootMetadataInfo(): array
+    {
+        if ($meta = $this->rootMetadata()) {
+            $data = $meta->decodeJson();
+            $data['modified_since'] = $meta->lastModified()->getTimestamp();
+            return $data;
+        }
+
+        return [];
+    }
 }
diff --git a/src/Mirror/Decorator/AbstractProxyRepositoryDecorator.php b/src/Mirror/Decorator/AbstractProxyRepositoryDecorator.php
@@ -10,17 +10,17 @@
 
 abstract class AbstractProxyRepositoryDecorator implements StrictProxyRepositoryInterface
 {
-    public function findPackageMetadata(string $nameOrUri): JsonMetadata
+    public function findPackageMetadata(string $nameOrUri, int $modifiedSince = null): JsonMetadata
     {
         throw new MetadataNotFoundException('Not found');
     }
 
-    public function findProviderMetadata(string $nameOrUri): JsonMetadata
+    public function findProviderMetadata(string $nameOrUri, int $modifiedSince = null): JsonMetadata
     {
         throw new MetadataNotFoundException('Not found');
     }
 
-    public function rootMetadata(): JsonMetadata
+    public function rootMetadata(int $modifiedSince = null): JsonMetadata
     {
         throw new MetadataNotFoundException('Not found');
     }

diff --git a/src/Mirror/Decorator/ProxyRepositoryACLDecorator.php b/src/Mirror/Decorator/ProxyRepositoryACLDecorator.php
@@ -5,7 +5,6 @@
 namespace Packeton\Mirror\Decorator;
 
 use Packeton\Mirror\Exception\ApproveRestrictException;
-use Packeton\Mirror\Exception\MetadataNotFoundException;
 use Packeton\Mirror\Model\ApprovalRepoInterface;
 use Packeton\Mirror\Model\JsonMetadata;
 use Packeton\Mirror\Model\StrictProxyRepositoryInterface as RPI;
@@ -30,16 +29,18 @@ public function __construct(
     /**
      * {@inheritdoc}
      */
-    public function rootMetadata(): JsonMetadata
+    public function rootMetadata(int $modifiedSince = null): JsonMetadata
     {
-        $metadata = $this->repository->rootMetadata();
+        $metadata = $this->repository->rootMetadata($modifiedSince);
+        $metadata->setOptions($this->approval->getSettings());
+
         if ($this->approval->requireApprove()) {
             $approved = $this->approval->getApproved();
             $metadata->setOption('available_packages', $approved);
 
             if (null !== $this->remote) {
                 $metadata->setOption('includes', function () use ($approved) {
-                    [$includes] = IncludeV1ApiMetadata::buildInclude($approved, $this->remote);
+                    [$includes] = IncludeV1ApiMetadata::buildIncludes($approved, $this->remote);
                     return $includes;
                 });
             }
@@ -51,25 +52,25 @@ public function rootMetadata(): JsonMetadata
     /**
      * {@inheritdoc}
      */
-    public function findProviderMetadata(string $nameOrUri): JsonMetadata
+    public function findProviderMetadata(string $nameOrUri, int $modifiedSince = null): JsonMetadata
     {
         if (\str_starts_with($nameOrUri, 'include-packeton/all$') && null !== $this->remote) {
             $approved = $this->approval->getApproved();
-            [$includes, $content] = IncludeV1ApiMetadata::buildInclude($approved, $this->remote);
+            [$includes, $content] = IncludeV1ApiMetadata::buildIncludes($approved, $this->remote);
             if (isset($includes[$nameOrUri])) {
                 return new JsonMetadata($content);
             }
         }
 
-        return $this->repository->findProviderMetadata($nameOrUri);
+        return $this->repository->findProviderMetadata($nameOrUri, $modifiedSince);
     }
 
     /**
      * {@inheritdoc}
      */
-    public function findPackageMetadata(string $nameOrUri): JsonMetadata
+    public function findPackageMetadata(string $nameOrUri, int $modifiedSince = null): JsonMetadata
     {
-        $metadata = $this->repository->findPackageMetadata($nameOrUri);
+        $metadata = $this->repository->findPackageMetadata($nameOrUri, $modifiedSince);
 
         [$package, ] = \explode('$', $nameOrUri);
         if ($this->approval->requireApprove()) {