Composer private proxies

config/packages/security.yaml

@@ -60,7 +60,7 @@ security:
         # Packagist
         - { path: (^(/change-password|/profile|/logout))+, roles: ROLE_USER }
         - { path: (^(/search|/packages/|/versions/))+, roles: ROLE_USER, allow_if: "is_granted('PACKETON_PUBLIC')" }
-        - { path: (^(/packages.json$|/p/|/p2/|/downloads/))+, roles: ROLE_USER, allow_if: "is_granted('PACKETON_PUBLIC')" }
+        - { path: (^(/packages.json$|/p/|/p2/|/mirror/|/downloads/))+, roles: ROLE_USER, allow_if: "is_granted('PACKETON_PUBLIC')" }
         - { path: (^(/zipball/))+, roles: ROLE_USER, allow_if: "is_granted('PACKETON_ARCHIVE_PUBLIC')" }
         - { path: (^(/api/webhook-invoke/))+, roles: ROLE_USER }
         - { path: (^(/api/(create-package|update-package|github|bitbucket)|/apidoc|/about))$, roles: ROLE_MAINTAINER }

config/services.yaml

@@ -53,6 +53,7 @@ services:
             $jwtTokenConfig: '%packeton_jws_config%'
             $jwtSignAlgorithm: '%packeton_jws_algo%'
             $mirrorRepoMetaDir: '%mirror_repos_meta_dir%'
+            $mirrorDistDir: '%mirror_repos_dist_dir%'
             'Symfony\Component\Security\Core\User\UserCheckerInterface': '@Packeton\Security\UserChecker'
 
     Packeton\:

src/Command/SyncMirrorsCommand.php

@@ -63,7 +63,8 @@ private function syncRepo(RemoteProxyRepository $repo, InputInterface $input, Ou
     {
         $io = new ConsoleIO($input, $output, new HelperSet());
         $flags = $input->getOption('force') ? 1 : 0;
-        $this->syncFacade->sync($repo, $io, $flags);
+        $stats = $this->syncFacade->sync($repo, $io, $flags);
+        $repo->setStats($stats);
 
         return 0;
     }

src/Controller/MirrorController.php

@@ -10,10 +10,13 @@
 use Packeton\Mirror\Model\ProxyRepositoryInterface as PRI;
 use Packeton\Mirror\RootMetadataMerger;
 use Packeton\Mirror\Service\ComposeProxyRegistry;
+use Packeton\Security\Acl\ObjectIdentity;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
+use Symfony\Component\HttpFoundation\BinaryFileResponse;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\Routing\Annotation\Route;
+use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
 
 #[Route('/mirror', defaults:['_format' => 'json'])]
 class MirrorController extends AbstractController
@@ -25,6 +28,21 @@ public function __construct(
     ) {
     }
 
+    #[Route('/{alias}', name: 'mirror_index', defaults: ['_format' => 'html'], methods: ['GET'])]
+    public function index(string $alias): Response
+    {
+        try {
+            $this->checkAccess($alias);
+            $this->proxyRegistry->createRepository($alias);
+        } catch (MetadataNotFoundException $e) {
+            throw $this->createNotFoundException($e->getMessage(), $e);
+        }
+
+        $repo = $this->generateUrl('mirror_index', ['alias' => $alias], UrlGeneratorInterface::ABSOLUTE_URL);
+
+        return $this->render('proxies/mirror.html.twig', ['alias' => $alias, 'repoUrl' => $repo]);
+    }
+
     #[Route('/{alias}/packages.json', name: 'mirror_root', methods: ['GET'])]
     public function rootAction(Request $request, string $alias): Response
     {
@@ -56,6 +74,30 @@ public function packageAction(string $package, string $alias, Request $request):
         return $this->renderMetadata($metadata, $request);
     }
 
+    #[Route(
+        '/{alias}/zipball/{package}/{version}/{ref}.{type}',
+        name: 'mirror_zipball',
+        requirements: ['package' => '%package_name_regex%'],
+        methods: ['GET']
+    )]
+    public function zipball(string $alias, string $package, string $version, string $ref): Response
+    {
+        try {
+            $this->checkAccess($alias);
+            $dm = $this->proxyRegistry->getProxyDownloadManager($alias);
+
+            $path = $dm->distPath($package, $version, $ref);
+        } catch (MetadataNotFoundException $e) {
+            throw $this->createNotFoundException($e->getMessage(), $e);
+        }
+
+        $response = new BinaryFileResponse($path);
+        $response->setAutoEtag();
+        $response->setPublic();
+
+        return $response;
+    }
+
     // provider - proxy full url name, include providers is not changed for root
     #[Route('/{alias}/{provider}', name: 'mirror_provider_includes', requirements: ['provider' => '.+'], methods: ['GET'])]
     public function providerAction(Request $request, $alias, $provider): Response
@@ -77,10 +119,20 @@ protected function renderMetadata(JsonMetadata $metadata, Request $request): Res
     protected function wrap404Error(string $alias, callable $callback): JsonMetadata
     {
         try {
+            $this->checkAccess($alias);
             $repo = $this->proxyRegistry->createRepository($alias);
+
             return $callback($repo);
         } catch (MetadataNotFoundException $e) {
             throw $this->createNotFoundException($e->getMessage(), $e);
         }
     }
+
+    protected function checkAccess(string $alias)
+    {
+        // ROLE_ADMIN have access to all proxies views
+        if (!$this->isGranted('ROLE_ADMIN') && !$this->isGranted('VIEW', new ObjectIdentity($alias, PRI::class))) {
+            throw $this->createAccessDeniedException();
+        }
+    }
 }

src/Controller/ProxiesController.php

@@ -4,13 +4,16 @@
 
 namespace Packeton\Controller;
 
+use Packeton\Form\Type\ProxySettingsType;
 use Packeton\Mirror\Model\ProxyInfoInterface;
 use Packeton\Mirror\Model\ProxyOptions;
 use Packeton\Mirror\ProxyRepositoryRegistry;
+use Packeton\Mirror\RemoteProxyRepository;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\Routing\Annotation\Route;
+use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
 
 #[Route('/proxies')]
 class ProxiesController extends AbstractController
@@ -21,7 +24,7 @@ public function __construct(
     }
 
     #[Route('', name: 'proxies_list')]
-    public function listAction(Request $request): Response
+    public function listAction(): Response
     {
         /** @var ProxyOptions[] $proxies */
         $proxies = [];
@@ -35,4 +38,63 @@ public function listAction(Request $request): Response
             'proxies' => $proxies
         ]);
     }
+
+    #[Route('/{alias}', name: 'proxy_view')]
+    public function viewAction(string $alias): Response
+    {
+        $repo = $this->getRemoteRepository($alias);
+        $data = $this->getProxyData($repo);
+
+        return $this->render('proxies/view.html.twig', $data + ['proxy' => $repo->getConfig()]);
+    }
+
+    #[Route('/{alias}/settings', name: 'proxy_settings')]
+    public function settings(Request $request, string $alias)
+    {
+        $repo = $this->getRemoteRepository($alias);
+        $settings = $repo->getPackageManager()->getSettings();
+
+        $form = $this->createForm(ProxySettingsType::class, $settings);
+        $form->handleRequest($request);
+
+        if ($form->isSubmitted() && $form->isValid()) {
+            $repo->getPackageManager()->setSettings($form->getData());
+            $this->addFlash('success', 'The proxy settings has been updated.');
+            return $this->redirect($this->generateUrl('proxy_view', ['alias' => $alias]));
+        }
+
+        return $this->render('proxies/settings.html.twig', [
+            'proxy' => $repo->getConfig(),
+            'form'    => $form->createView()
+        ]);
+    }
+
+    protected function getProxyData(RemoteProxyRepository $repo): array
+    {
+        $config = $repo->getConfig();
+        $repoUrl = $this->generateUrl('mirror_index', ['alias' => $config->getAlias()], UrlGeneratorInterface::ABSOLUTE_URL);
+
+        return [
+            'repoUrl' => $repoUrl,
+            'tooltips' => [
+                'API_V2' => 'Support metadata-url for Composer v2',
+                'API_V1' => 'Support Composer v1 API',
+                'API_META_CHANGE' => 'Support Metadata changes API',
+            ]
+        ];
+    }
+
+    protected function getRemoteRepository(string $alias): RemoteProxyRepository
+    {
+        try {
+            $repo = $this->proxyRepositoryRegistry->getRepository($alias);
+            if (!$repo instanceof RemoteProxyRepository) {
+                throw $this->createNotFoundException();
+            }
+        } catch (\Exception) {
+            throw $this->createNotFoundException();
+        }
+
+        return $repo;
+    }
 }

src/Cron/Handler/CleanupJobStorage.php

@@ -53,7 +53,7 @@ protected function selectKeepPeriod(&$count = null): int
             ->getQuery()
             ->getSingleScalarResult();
 
-        return match ($count) {
+        return match (true) {
             $count > 60000 => 2,
             $count > 40000 => 5,
             $count > 25000 => 10,

src/Cron/MirrorCronLoader.php

@@ -5,14 +5,50 @@
 namespace Packeton\Cron;
 
 use Okvpn\Bundle\CronBundle\Loader\ScheduleLoaderInterface;
+use Okvpn\Bundle\CronBundle\Model\ScheduleEnvelope;
+use Packeton\Mirror\Model\ProxyOptions;
+use Packeton\Mirror\ProxyRepositoryRegistry;
+use Packeton\Mirror\RemoteProxyRepository;
+use Okvpn\Bundle\CronBundle\Model;
 
 class MirrorCronLoader implements ScheduleLoaderInterface
 {
+    public function __construct(private readonly ProxyRepositoryRegistry $registry)
+    {
+    }
+
     /**
      * {@inheritdoc}
      */
     public function getSchedules(array $options = []): iterable
     {
-        return [];
+        if (!\in_array($options['groups'] ?? 'default', ['default', 'mirror'])) {
+            return;
+        }
+
+        foreach ($this->registry->getAllRepos() as $name => $repo) {
+            if ($repo instanceof RemoteProxyRepository) {
+                $repo->resetProxyOptions();
+                $config = $repo->getConfig();
+                $expr = '@random ' . $this->getSyncInterval($config);
+
+                yield new ScheduleEnvelope(
+                    'sync:mirrors',
+                    new Model\ScheduleStamp($expr),
+                    new WorkerStamp(true),
+                    new Model\ArgumentsStamp(['mirror' => $name,])
+                );
+            }
+        }
+    }
+
+    private function getSyncInterval(ProxyOptions $config): int
+    {
+        return $config->getSyncInterval() ?? match (true) {
+            $config->isLazy() && $config->getV2SyncApi() => 900,
+            $config->isLazy() && $config->hasV2Api() => 1800,
+            $config->isLazy() => 7200,
+            default => 86400,
+        };
     }
 }

src/Cron/WebhookCronLoader.php

@@ -26,7 +26,7 @@ public function __construct(ManagerRegistry $registry)
      */
     public function getSchedules(array $options = []): iterable
     {
-        if ('default' !== ($options['group'] ?? 'default')) {
+        if (!\in_array($options['groups'] ?? 'default', ['default', 'webhook'])) {
             return;
         }
 

src/Cron/WorkerMiddleware.php

@@ -6,6 +6,7 @@
 
 use Okvpn\Bundle\CronBundle\Middleware\MiddlewareEngineInterface;
 use Okvpn\Bundle\CronBundle\Middleware\StackInterface;
+use Okvpn\Bundle\CronBundle\Model\ArgumentsStamp;
 use Okvpn\Bundle\CronBundle\Model\ScheduleEnvelope;
 use Packeton\Service\JobScheduler;
 
@@ -27,8 +28,17 @@ public function handle(ScheduleEnvelope $envelope, StackInterface $stack): Sched
             return $stack->next()->handle($envelope, $stack);
         }
 
+        /** @var WorkerStamp $stamp */
+        $stamp = $envelope->get(WorkerStamp::class);
+        if (true === $stamp->asJob) {
+            $args = $envelope->get(ArgumentsStamp::class) ? $envelope->get(ArgumentsStamp::class)->getArguments() : [];
+            $this->jobScheduler->publish($envelope->getCommand(), $args, $stamp->hash);
+            return $stack->end()->handle($envelope, $stack);
+        }
+
         $envelopeData = \serialize($envelope->without(WorkerStamp::class));
-        $this->jobScheduler->publish(WorkerStamp::JOB_NAME, [
+
+        $this->jobScheduler->publish(WorkerStamp::DEFAULT_JOB_NAME, [
             'envelope' => $envelopeData
         ]);
 

src/Cron/WorkerStamp.php

@@ -8,5 +8,11 @@
 
 class WorkerStamp implements CommandStamp
 {
-    public const JOB_NAME = 'cron:execute';
+    public const DEFAULT_JOB_NAME = 'cron:execute';
+
+    public function __construct(
+        public readonly bool $asJob = false,
+        public readonly ?int $hash = null,
+    ) {
+    }
 }

src/DependencyInjection/CompilerPass/MirrorsConfigCompilerPass.php

@@ -7,6 +7,7 @@
 use Packeton\Mirror\RemoteProxyRepository;
 use Packeton\Mirror\ProxyRepositoryRegistry;
 use Packeton\Mirror\Service\RemotePackagesManager;
+use Packeton\Mirror\Service\ZipballDownloadManager;
 use Symfony\Component\DependencyInjection\ChildDefinition;
 use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
 use Symfony\Component\DependencyInjection\Compiler\ServiceLocatorTagPass;
@@ -39,9 +40,13 @@ private function createMirrorRepo(ContainerBuilder $container, array $repoConfig
         $rmp->setArgument('$repo', $name);
         $container->setDefinition($rmpId = 'packeton.mirror_rmp.' . $name, $rmp);
 
+        $container->setDefinition($dmId = 'packeton.mirror_dm.' . $name, new ChildDefinition(ZipballDownloadManager::class));
+
         $service = new ChildDefinition(RemoteProxyRepository::class);
+
         $service->setArgument('$repoConfig', ['name' => $name, 'type' => 'composer'] + $repoConfig)
-            ->setArgument('$rpm', new Reference($rmpId));
+            ->setArgument('$rpm', new Reference($rmpId))
+            ->setArgument('$zipballManager', new Reference($dmId));
 
         $container
             ->setDefinition($serviceId = $this->getMirrorServiceId($name), $service);

src/DependencyInjection/Configuration.php

@@ -100,11 +100,11 @@ private function addMirrorsRepositoriesConfiguration(ArrayNodeDefinition|NodeDef
                 ->end()
                 ->arrayNode('http_basic')
                     ->children()
-                        ->scalarNode('username')->cannotBeEmpty()->end()
-                        ->scalarNode('password')->cannotBeEmpty()->end()
+                        ->scalarNode('username')->isRequired()->end()
+                        ->scalarNode('password')->isRequired()->end()
                     ->end()
                 ->end()
-                ->scalarNode('sync_interval')->defaultValue(86400)->end()
+                ->scalarNode('sync_interval')->end()
                 ->booleanNode('sync_lazy')->end()
                 ->booleanNode('enable_dist_mirror')->defaultTrue()->end()
                 ->booleanNode('parent_notify')->end()
@@ -141,7 +141,6 @@ private function addMirrorsRepositoriesConfiguration(ArrayNodeDefinition|NodeDef
                     if (!isset($provider['url'])) {
                         return $provider;
                     }
-
                     $host = \parse_url($provider['url'], \PHP_URL_HOST);
 
                     $provider['url'] = \rtrim($provider['url'], '/');