From 259a9ba86cdf264928abdf711d972e2971057d1e Mon Sep 17 00:00:00 2001 From: Uladzimir Tsykun Date: Mon, 30 Jan 2023 05:21:50 +0100 Subject: [PATCH] Composer private proxies --- config/packages/security.yaml | 2 +- config/services.yaml | 1 + src/Command/SyncMirrorsCommand.php | 3 +- src/Controller/MirrorController.php | 52 +++++++++ src/Controller/ProxiesController.php | 64 ++++++++++- src/Cron/Handler/CleanupJobStorage.php | 2 +- src/Cron/MirrorCronLoader.php | 38 ++++++- src/Cron/WebhookCronLoader.php | 2 +- src/Cron/WorkerMiddleware.php | 12 +- src/Cron/WorkerStamp.php | 8 +- .../MirrorsConfigCompilerPass.php | 7 +- src/DependencyInjection/Configuration.php | 7 +- src/Entity/Group.php | 27 ++++- src/Entity/GroupAclPermission.php | 2 +- src/Form/Type/GroupType.php | 22 +++- src/Form/Type/ProxySettingsType.php | 32 ++++++ src/Menu/MenuBuilder.php | 1 + src/Mirror/AbstractProxyRepository.php | 7 +- src/Mirror/Model/JsonMetadata.php | 3 +- src/Mirror/Model/MetadataOptions.php | 7 +- src/Mirror/Model/ProxyOptions.php | 30 +++++ src/Mirror/RemoteProxyRepository.php | 20 +++- src/Mirror/RootMetadataMerger.php | 21 +++- src/Mirror/Service/ComposeProxyRegistry.php | 13 ++- src/Mirror/Service/RemotePackagesManager.php | 30 ++++- .../Service/RemoteSyncProxiesFacade.php | 18 +-- src/Mirror/Service/SyncMirrorWorker.php | 21 ++++ src/Mirror/Service/SyncProviderService.php | 3 +- src/Mirror/Service/ZipballDownloadManager.php | 107 ++++++++++++++++++ src/Package/InMemoryDumper.php | 2 +- src/Repository/GroupRepository.php | 20 ++++ src/Resolver/ControllerArgumentResolver.php | 32 ++++-- src/Security/Acl/ObjectIdentity.php | 28 +++++ src/Security/Acl/ObjectIdentityVoter.php | 39 +++++++ src/Service/JobPersister.php | 19 +++- src/Service/JobScheduler.php | 10 +- src/Service/QueueWorker.php | 1 + src/Twig/PackagistExtension.php | 36 ++++++ templates/base.html.twig | 4 +- templates/group/update.html.twig | 28 ++++- templates/proxies/index.html.twig | 39 ++++++- templates/proxies/info_caps.html.twig | 15 +++ templates/proxies/info_repo.html.twig | 18 +++ templates/proxies/mirror.html.twig | 10 ++ templates/proxies/settings.html.twig | 34 ++++++ templates/proxies/view.html.twig | 41 +++++++ translations/messages.en.yml | 1 + 47 files changed, 870 insertions(+), 69 deletions(-) create mode 100644 src/Form/Type/ProxySettingsType.php create mode 100644 src/Mirror/Service/ZipballDownloadManager.php create mode 100644 src/Security/Acl/ObjectIdentity.php create mode 100644 src/Security/Acl/ObjectIdentityVoter.php create mode 100644 templates/proxies/info_caps.html.twig create mode 100644 templates/proxies/info_repo.html.twig create mode 100644 templates/proxies/mirror.html.twig create mode 100644 templates/proxies/settings.html.twig create mode 100644 templates/proxies/view.html.twig diff --git a/config/packages/security.yaml b/config/packages/security.yaml index 0a8450c7..aa3f4f3c 100644 --- a/config/packages/security.yaml +++ b/config/packages/security.yaml @@ -60,7 +60,7 @@ security: # Packagist - { path: (^(/change-password|/profile|/logout))+, roles: ROLE_USER } - { path: (^(/search|/packages/|/versions/))+, roles: ROLE_USER, allow_if: "is_granted('PACKETON_PUBLIC')" } - - { path: (^(/packages.json$|/p/|/p2/|/downloads/))+, roles: ROLE_USER, allow_if: "is_granted('PACKETON_PUBLIC')" } + - { path: (^(/packages.json$|/p/|/p2/|/mirror/|/downloads/))+, roles: ROLE_USER, allow_if: "is_granted('PACKETON_PUBLIC')" } - { path: (^(/zipball/))+, roles: ROLE_USER, allow_if: "is_granted('PACKETON_ARCHIVE_PUBLIC')" } - { path: (^(/api/webhook-invoke/))+, roles: ROLE_USER } - { path: (^(/api/(create-package|update-package|github|bitbucket)|/apidoc|/about))$, roles: ROLE_MAINTAINER } diff --git a/config/services.yaml b/config/services.yaml index 858e5ebb..31c14aee 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -53,6 +53,7 @@ services: $jwtTokenConfig: '%packeton_jws_config%' $jwtSignAlgorithm: '%packeton_jws_algo%' $mirrorRepoMetaDir: '%mirror_repos_meta_dir%' + $mirrorDistDir: '%mirror_repos_dist_dir%' 'Symfony\Component\Security\Core\User\UserCheckerInterface': '@Packeton\Security\UserChecker' Packeton\: diff --git a/src/Command/SyncMirrorsCommand.php b/src/Command/SyncMirrorsCommand.php index 513ffa83..79507a18 100644 --- a/src/Command/SyncMirrorsCommand.php +++ b/src/Command/SyncMirrorsCommand.php @@ -63,7 +63,8 @@ private function syncRepo(RemoteProxyRepository $repo, InputInterface $input, Ou { $io = new ConsoleIO($input, $output, new HelperSet()); $flags = $input->getOption('force') ? 1 : 0; - $this->syncFacade->sync($repo, $io, $flags); + $stats = $this->syncFacade->sync($repo, $io, $flags); + $repo->setStats($stats); return 0; } diff --git a/src/Controller/MirrorController.php b/src/Controller/MirrorController.php index a4d48570..fa90947a 100644 --- a/src/Controller/MirrorController.php +++ b/src/Controller/MirrorController.php @@ -10,10 +10,13 @@ use Packeton\Mirror\Model\ProxyRepositoryInterface as PRI; use Packeton\Mirror\RootMetadataMerger; use Packeton\Mirror\Service\ComposeProxyRegistry; +use Packeton\Security\Acl\ObjectIdentity; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; +use Symfony\Component\HttpFoundation\BinaryFileResponse; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Symfony\Component\Routing\Annotation\Route; +use Symfony\Component\Routing\Generator\UrlGeneratorInterface; #[Route('/mirror', defaults:['_format' => 'json'])] class MirrorController extends AbstractController @@ -25,6 +28,21 @@ public function __construct( ) { } + #[Route('/{alias}', name: 'mirror_index', defaults: ['_format' => 'html'], methods: ['GET'])] + public function index(string $alias): Response + { + try { + $this->checkAccess($alias); + $this->proxyRegistry->createRepository($alias); + } catch (MetadataNotFoundException $e) { + throw $this->createNotFoundException($e->getMessage(), $e); + } + + $repo = $this->generateUrl('mirror_index', ['alias' => $alias], UrlGeneratorInterface::ABSOLUTE_URL); + + return $this->render('proxies/mirror.html.twig', ['alias' => $alias, 'repoUrl' => $repo]); + } + #[Route('/{alias}/packages.json', name: 'mirror_root', methods: ['GET'])] public function rootAction(Request $request, string $alias): Response { @@ -56,6 +74,30 @@ public function packageAction(string $package, string $alias, Request $request): return $this->renderMetadata($metadata, $request); } + #[Route( + '/{alias}/zipball/{package}/{version}/{ref}.{type}', + name: 'mirror_zipball', + requirements: ['package' => '%package_name_regex%'], + methods: ['GET'] + )] + public function zipball(string $alias, string $package, string $version, string $ref): Response + { + try { + $this->checkAccess($alias); + $dm = $this->proxyRegistry->getProxyDownloadManager($alias); + + $path = $dm->distPath($package, $version, $ref); + } catch (MetadataNotFoundException $e) { + throw $this->createNotFoundException($e->getMessage(), $e); + } + + $response = new BinaryFileResponse($path); + $response->setAutoEtag(); + $response->setPublic(); + + return $response; + } + // provider - proxy full url name, include providers is not changed for root #[Route('/{alias}/{provider}', name: 'mirror_provider_includes', requirements: ['provider' => '.+'], methods: ['GET'])] public function providerAction(Request $request, $alias, $provider): Response @@ -77,10 +119,20 @@ protected function renderMetadata(JsonMetadata $metadata, Request $request): Res protected function wrap404Error(string $alias, callable $callback): JsonMetadata { try { + $this->checkAccess($alias); $repo = $this->proxyRegistry->createRepository($alias); + return $callback($repo); } catch (MetadataNotFoundException $e) { throw $this->createNotFoundException($e->getMessage(), $e); } } + + protected function checkAccess(string $alias) + { + // ROLE_ADMIN have access to all proxies views + if (!$this->isGranted('ROLE_ADMIN') && !$this->isGranted('VIEW', new ObjectIdentity($alias, PRI::class))) { + throw $this->createAccessDeniedException(); + } + } } diff --git a/src/Controller/ProxiesController.php b/src/Controller/ProxiesController.php index b1bf96f0..0b84806f 100644 --- a/src/Controller/ProxiesController.php +++ b/src/Controller/ProxiesController.php @@ -4,13 +4,16 @@ namespace Packeton\Controller; +use Packeton\Form\Type\ProxySettingsType; use Packeton\Mirror\Model\ProxyInfoInterface; use Packeton\Mirror\Model\ProxyOptions; use Packeton\Mirror\ProxyRepositoryRegistry; +use Packeton\Mirror\RemoteProxyRepository; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Symfony\Component\Routing\Annotation\Route; +use Symfony\Component\Routing\Generator\UrlGeneratorInterface; #[Route('/proxies')] class ProxiesController extends AbstractController @@ -21,7 +24,7 @@ public function __construct( } #[Route('', name: 'proxies_list')] - public function listAction(Request $request): Response + public function listAction(): Response { /** @var ProxyOptions[] $proxies */ $proxies = []; @@ -35,4 +38,63 @@ public function listAction(Request $request): Response 'proxies' => $proxies ]); } + + #[Route('/{alias}', name: 'proxy_view')] + public function viewAction(string $alias): Response + { + $repo = $this->getRemoteRepository($alias); + $data = $this->getProxyData($repo); + + return $this->render('proxies/view.html.twig', $data + ['proxy' => $repo->getConfig()]); + } + + #[Route('/{alias}/settings', name: 'proxy_settings')] + public function settings(Request $request, string $alias) + { + $repo = $this->getRemoteRepository($alias); + $settings = $repo->getPackageManager()->getSettings(); + + $form = $this->createForm(ProxySettingsType::class, $settings); + $form->handleRequest($request); + + if ($form->isSubmitted() && $form->isValid()) { + $repo->getPackageManager()->setSettings($form->getData()); + $this->addFlash('success', 'The proxy settings has been updated.'); + return $this->redirect($this->generateUrl('proxy_view', ['alias' => $alias])); + } + + return $this->render('proxies/settings.html.twig', [ + 'proxy' => $repo->getConfig(), + 'form' => $form->createView() + ]); + } + + protected function getProxyData(RemoteProxyRepository $repo): array + { + $config = $repo->getConfig(); + $repoUrl = $this->generateUrl('mirror_index', ['alias' => $config->getAlias()], UrlGeneratorInterface::ABSOLUTE_URL); + + return [ + 'repoUrl' => $repoUrl, + 'tooltips' => [ + 'API_V2' => 'Support metadata-url for Composer v2', + 'API_V1' => 'Support Composer v1 API', + 'API_META_CHANGE' => 'Support Metadata changes API', + ] + ]; + } + + protected function getRemoteRepository(string $alias): RemoteProxyRepository + { + try { + $repo = $this->proxyRepositoryRegistry->getRepository($alias); + if (!$repo instanceof RemoteProxyRepository) { + throw $this->createNotFoundException(); + } + } catch (\Exception) { + throw $this->createNotFoundException(); + } + + return $repo; + } } diff --git a/src/Cron/Handler/CleanupJobStorage.php b/src/Cron/Handler/CleanupJobStorage.php index 1713dcf5..d04dc25d 100644 --- a/src/Cron/Handler/CleanupJobStorage.php +++ b/src/Cron/Handler/CleanupJobStorage.php @@ -53,7 +53,7 @@ protected function selectKeepPeriod(&$count = null): int ->getQuery() ->getSingleScalarResult(); - return match ($count) { + return match (true) { $count > 60000 => 2, $count > 40000 => 5, $count > 25000 => 10, diff --git a/src/Cron/MirrorCronLoader.php b/src/Cron/MirrorCronLoader.php index 99cf96b9..9ca3f246 100644 --- a/src/Cron/MirrorCronLoader.php +++ b/src/Cron/MirrorCronLoader.php @@ -5,14 +5,50 @@ namespace Packeton\Cron; use Okvpn\Bundle\CronBundle\Loader\ScheduleLoaderInterface; +use Okvpn\Bundle\CronBundle\Model\ScheduleEnvelope; +use Packeton\Mirror\Model\ProxyOptions; +use Packeton\Mirror\ProxyRepositoryRegistry; +use Packeton\Mirror\RemoteProxyRepository; +use Okvpn\Bundle\CronBundle\Model; class MirrorCronLoader implements ScheduleLoaderInterface { + public function __construct(private readonly ProxyRepositoryRegistry $registry) + { + } + /** * {@inheritdoc} */ public function getSchedules(array $options = []): iterable { - return []; + if (!\in_array($options['groups'] ?? 'default', ['default', 'mirror'])) { + return; + } + + foreach ($this->registry->getAllRepos() as $name => $repo) { + if ($repo instanceof RemoteProxyRepository) { + $repo->resetProxyOptions(); + $config = $repo->getConfig(); + $expr = '@random ' . $this->getSyncInterval($config); + + yield new ScheduleEnvelope( + 'sync:mirrors', + new Model\ScheduleStamp($expr), + new WorkerStamp(true), + new Model\ArgumentsStamp(['mirror' => $name,]) + ); + } + } + } + + private function getSyncInterval(ProxyOptions $config): int + { + return $config->getSyncInterval() ?? match (true) { + $config->isLazy() && $config->getV2SyncApi() => 900, + $config->isLazy() && $config->hasV2Api() => 1800, + $config->isLazy() => 7200, + default => 86400, + }; } } diff --git a/src/Cron/WebhookCronLoader.php b/src/Cron/WebhookCronLoader.php index 9e11408e..8d79ca11 100644 --- a/src/Cron/WebhookCronLoader.php +++ b/src/Cron/WebhookCronLoader.php @@ -26,7 +26,7 @@ public function __construct(ManagerRegistry $registry) */ public function getSchedules(array $options = []): iterable { - if ('default' !== ($options['group'] ?? 'default')) { + if (!\in_array($options['groups'] ?? 'default', ['default', 'webhook'])) { return; } diff --git a/src/Cron/WorkerMiddleware.php b/src/Cron/WorkerMiddleware.php index 94bc5587..ff2e79d3 100644 --- a/src/Cron/WorkerMiddleware.php +++ b/src/Cron/WorkerMiddleware.php @@ -6,6 +6,7 @@ use Okvpn\Bundle\CronBundle\Middleware\MiddlewareEngineInterface; use Okvpn\Bundle\CronBundle\Middleware\StackInterface; +use Okvpn\Bundle\CronBundle\Model\ArgumentsStamp; use Okvpn\Bundle\CronBundle\Model\ScheduleEnvelope; use Packeton\Service\JobScheduler; @@ -27,8 +28,17 @@ public function handle(ScheduleEnvelope $envelope, StackInterface $stack): Sched return $stack->next()->handle($envelope, $stack); } + /** @var WorkerStamp $stamp */ + $stamp = $envelope->get(WorkerStamp::class); + if (true === $stamp->asJob) { + $args = $envelope->get(ArgumentsStamp::class) ? $envelope->get(ArgumentsStamp::class)->getArguments() : []; + $this->jobScheduler->publish($envelope->getCommand(), $args, $stamp->hash); + return $stack->end()->handle($envelope, $stack); + } + $envelopeData = \serialize($envelope->without(WorkerStamp::class)); - $this->jobScheduler->publish(WorkerStamp::JOB_NAME, [ + + $this->jobScheduler->publish(WorkerStamp::DEFAULT_JOB_NAME, [ 'envelope' => $envelopeData ]); diff --git a/src/Cron/WorkerStamp.php b/src/Cron/WorkerStamp.php index babd985f..bf35f1e5 100644 --- a/src/Cron/WorkerStamp.php +++ b/src/Cron/WorkerStamp.php @@ -8,5 +8,11 @@ class WorkerStamp implements CommandStamp { - public const JOB_NAME = 'cron:execute'; + public const DEFAULT_JOB_NAME = 'cron:execute'; + + public function __construct( + public readonly bool $asJob = false, + public readonly ?int $hash = null, + ) { + } } diff --git a/src/DependencyInjection/CompilerPass/MirrorsConfigCompilerPass.php b/src/DependencyInjection/CompilerPass/MirrorsConfigCompilerPass.php index dd453689..b5980f47 100644 --- a/src/DependencyInjection/CompilerPass/MirrorsConfigCompilerPass.php +++ b/src/DependencyInjection/CompilerPass/MirrorsConfigCompilerPass.php @@ -7,6 +7,7 @@ use Packeton\Mirror\RemoteProxyRepository; use Packeton\Mirror\ProxyRepositoryRegistry; use Packeton\Mirror\Service\RemotePackagesManager; +use Packeton\Mirror\Service\ZipballDownloadManager; use Symfony\Component\DependencyInjection\ChildDefinition; use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; use Symfony\Component\DependencyInjection\Compiler\ServiceLocatorTagPass; @@ -39,9 +40,13 @@ private function createMirrorRepo(ContainerBuilder $container, array $repoConfig $rmp->setArgument('$repo', $name); $container->setDefinition($rmpId = 'packeton.mirror_rmp.' . $name, $rmp); + $container->setDefinition($dmId = 'packeton.mirror_dm.' . $name, new ChildDefinition(ZipballDownloadManager::class)); + $service = new ChildDefinition(RemoteProxyRepository::class); + $service->setArgument('$repoConfig', ['name' => $name, 'type' => 'composer'] + $repoConfig) - ->setArgument('$rpm', new Reference($rmpId)); + ->setArgument('$rpm', new Reference($rmpId)) + ->setArgument('$zipballManager', new Reference($dmId)); $container ->setDefinition($serviceId = $this->getMirrorServiceId($name), $service); diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index 80e2d59b..75b7660f 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -100,11 +100,11 @@ private function addMirrorsRepositoriesConfiguration(ArrayNodeDefinition|NodeDef ->end() ->arrayNode('http_basic') ->children() - ->scalarNode('username')->cannotBeEmpty()->end() - ->scalarNode('password')->cannotBeEmpty()->end() + ->scalarNode('username')->isRequired()->end() + ->scalarNode('password')->isRequired()->end() ->end() ->end() - ->scalarNode('sync_interval')->defaultValue(86400)->end() + ->scalarNode('sync_interval')->end() ->booleanNode('sync_lazy')->end() ->booleanNode('enable_dist_mirror')->defaultTrue()->end() ->booleanNode('parent_notify')->end() @@ -141,7 +141,6 @@ private function addMirrorsRepositoriesConfiguration(ArrayNodeDefinition|NodeDef if (!isset($provider['url'])) { return $provider; } - $host = \parse_url($provider['url'], \PHP_URL_HOST); $provider['url'] = \rtrim($provider['url'], '/'); diff --git a/src/Entity/Group.php b/src/Entity/Group.php index a20fc018..26e02afc 100644 --- a/src/Entity/Group.php +++ b/src/Entity/Group.php @@ -28,6 +28,13 @@ class Group */ private $name; + /** + * @var array + * + * @ORM\Column(name="proxies", type="simple_array", nullable=true) + */ + private $proxies; + /** * @var GroupAclPermission[]|Collection * @@ -75,6 +82,24 @@ public function getName() return $this->name; } + /** + * @return array + */ + public function getProxies() + { + return $this->proxies; + } + + /** + * @param array $proxies + * @return Group + */ + public function setProxies(?array $proxies) + { + $this->proxies = $proxies; + return $this; + } + /** * @return GroupAclPermission[]|Collection */ @@ -126,7 +151,7 @@ public function removeAclPermissions(GroupAclPermission $permission) { if ($this->aclPermissions->contains($permission)) { $this->aclPermissions->removeElement($permission); - $permission->setGroup(null); + $permission->setGroup(); } return $this; diff --git a/src/Entity/GroupAclPermission.php b/src/Entity/GroupAclPermission.php index 74e1052e..e0502d3a 100644 --- a/src/Entity/GroupAclPermission.php +++ b/src/Entity/GroupAclPermission.php @@ -88,7 +88,7 @@ public function getGroup() * @param Group $group * @return GroupAclPermission */ - public function setGroup(Group $group) + public function setGroup(Group $group = null) { $this->group = $group; return $this; diff --git a/src/Form/Type/GroupType.php b/src/Form/Type/GroupType.php index 2e38aaec..c5f667c5 100644 --- a/src/Form/Type/GroupType.php +++ b/src/Form/Type/GroupType.php @@ -3,20 +3,40 @@ namespace Packeton\Form\Type; use Packeton\Entity\Group; +use Packeton\Mirror\ProxyRepositoryRegistry; use Symfony\Component\Form\AbstractType; +use Symfony\Component\Form\Extension\Core\Type\ChoiceType; use Symfony\Component\Form\Extension\Core\Type\TextType; use Symfony\Component\Form\FormBuilderInterface; use Symfony\Component\OptionsResolver\OptionsResolver; class GroupType extends AbstractType { + public function __construct(private readonly ProxyRepositoryRegistry $registry) + { + } + /** * {@inheritdoc} */ public function buildForm(FormBuilderInterface $builder, array $options) { + $proxyChoice = $this->registry->getAllNames(); + $proxyChoice = \array_combine($proxyChoice, $proxyChoice); + + $builder + ->add('name', TextType::class, ['label' => 'Name']); + + if ($proxyChoice) { + $builder + ->add('proxies', ChoiceType::class, [ + 'choices' => $proxyChoice, + 'multiple' => true, + 'label' => 'Allowed Proxies' + ]); + } + $builder - ->add('name', TextType::class, ['label' => 'Name']) ->add('aclPermissions', GroupAclPermissionCollectionType::class); } diff --git a/src/Form/Type/ProxySettingsType.php b/src/Form/Type/ProxySettingsType.php new file mode 100644 index 00000000..6c93011e --- /dev/null +++ b/src/Form/Type/ProxySettingsType.php @@ -0,0 +1,32 @@ +add('strict_mirror', ChoiceType::class, [ + 'expanded' => true, + 'label' => 'Dependencies Usage Policy', + 'choices' => [ + 'Strict (new packages must be manually approved)' => true, + 'All (new packages are automatically added when requested by composer)' => false, + ] + ]) + ->add('enabled_sync', CheckboxType::class, [ + 'label' => 'Enable automatically synchronization', + ]); + } +} diff --git a/src/Menu/MenuBuilder.php b/src/Menu/MenuBuilder.php index 5dcea068..5c6b3052 100644 --- a/src/Menu/MenuBuilder.php +++ b/src/Menu/MenuBuilder.php @@ -49,6 +49,7 @@ public function createAdminMenu() $menu->addChild($this->translator->trans('menu.my_groups'), ['label' => '' . $this->translator->trans('menu.my_groups'), 'route' => 'groups_index', 'extras' => ['safe_label' => true]]); $menu->addChild($this->translator->trans('menu.ssh_keys'), ['label' => '' . $this->translator->trans('menu.ssh_keys'), 'route' => 'user_add_sshkey', 'extras' => ['safe_label' => true]]); $menu->addChild($this->translator->trans('menu.webhooks'), ['label' => '' . $this->translator->trans('menu.webhooks'), 'route' => 'webhook_index', 'extras' => ['safe_label' => true]]); + $menu->addChild($this->translator->trans('menu.proxies'), ['label' => '' . $this->translator->trans('menu.proxies'), 'route' => 'proxies_list', 'extras' => ['safe_label' => true]]); return $menu; } diff --git a/src/Mirror/AbstractProxyRepository.php b/src/Mirror/AbstractProxyRepository.php index d876d100..8013bc85 100644 --- a/src/Mirror/AbstractProxyRepository.php +++ b/src/Mirror/AbstractProxyRepository.php @@ -50,7 +50,7 @@ public function rootMetadata(): ?JsonMetadata return null; } - public function getStats(): ?array + public function getStats(): array { return []; } @@ -58,4 +58,9 @@ public function getStats(): ?array public function setStats(array $stats = []): void { } + + public function resetProxyOptions(): void + { + $this->proxyOptions = null; + } } diff --git a/src/Mirror/Model/JsonMetadata.php b/src/Mirror/Model/JsonMetadata.php index a903bed5..3b3d2bbe 100644 --- a/src/Mirror/Model/JsonMetadata.php +++ b/src/Mirror/Model/JsonMetadata.php @@ -31,7 +31,8 @@ public function getContent(): string public function decodeJson(): array { - return \json_decode($this->getContent(), true); + $data = \json_decode($this->getContent(), true); + return \is_array($data) ? $data : []; } public function hash(): ?string diff --git a/src/Mirror/Model/MetadataOptions.php b/src/Mirror/Model/MetadataOptions.php index b805f19d..4fa241ef 100644 --- a/src/Mirror/Model/MetadataOptions.php +++ b/src/Mirror/Model/MetadataOptions.php @@ -16,12 +16,7 @@ public function __construct(protected array $config) public function isLazy(): bool { - return ($this->config['lazy'] ?? false); - } - - public function isLazyProviders(): bool - { - return $this->repoConfig['skip_fetch_providers'] ?? false; + return ($this->config['sync_lazy'] ?? false); } public function parentNotify(): bool diff --git a/src/Mirror/Model/ProxyOptions.php b/src/Mirror/Model/ProxyOptions.php index 2910fb21..1c3c2331 100644 --- a/src/Mirror/Model/ProxyOptions.php +++ b/src/Mirror/Model/ProxyOptions.php @@ -74,6 +74,30 @@ public function getProviderIncludes($withHash = false): array return $providerIncludes; } + public function getSyncInterval(): ?int + { + return $this->config['sync_interval'] ?? null; + } + + public function capabilities(): array + { + $flags = [ + isset($this->config['root']['metadata-url']) ? 'API_V2' : null, + isset($this->config['root']['providers-url']) || isset($this->config['root']['providers-lazy-url']) ? 'API_V1' : null, + isset($this->config['root']['metadata-changes-url']) ? 'API_META_CHANGE' : null, + !isset($this->config['root']['providers-url']) && isset($this->config['root']['providers-lazy-url']) ? 'API_V1_LAZY' : null, + $this->config['packages'] ?? [] ? 'API_V1_PACKAGES' : null, + ]; + + return \array_values(\array_filter($flags)); + } + + public function isPackagist() + { + $hostname = \parse_url($this->getUrl(), \PHP_URL_HOST); + return \in_array($hostname, ['repo.packagist.org', 'packagist.org']); + } + public function logo(): ?string { return $this->config['logo'] ?? null; @@ -104,4 +128,10 @@ public function getComposerAuth(): ?array { return $this->config['composer_auth'] ?? null; } + + public function getStats(string $name = null, mixed $default = null): mixed + { + $stats = $this->config['stats'] ?? []; + return $name ? $stats[$name] ?? $default : $stats; + } } diff --git a/src/Mirror/RemoteProxyRepository.php b/src/Mirror/RemoteProxyRepository.php index 5e437f54..397227be 100644 --- a/src/Mirror/RemoteProxyRepository.php +++ b/src/Mirror/RemoteProxyRepository.php @@ -9,6 +9,7 @@ use Packeton\Mirror\Model\JsonMetadata; use Packeton\Mirror\Service\Filesystem; use Packeton\Mirror\Service\RemotePackagesManager; +use Packeton\Mirror\Service\ZipballDownloadManager; /** * Filesystem mirror proxy repo for metadata. @@ -28,7 +29,8 @@ public function __construct( protected ?string $mirrorRepoMetaDir, protected Filesystem $filesystem, protected \Redis $redis, - protected RemotePackagesManager $rpm + protected RemotePackagesManager $rpm, + protected ZipballDownloadManager $zipballManager ) { if (null === $mirrorRepoMetaDir) { $mirrorRepoMetaDir = ConfigFactory::getHomeDir(); @@ -38,6 +40,8 @@ public function __construct( $this->rootFilename = $this->mirrorRepoMetaDir . '/' . self::ROOT_PACKAGE; $this->providersDir = $this->mirrorRepoMetaDir . '/p/'; $this->packageDir = $this->mirrorRepoMetaDir . '/package/'; + + $this->zipballManager->setRepository($this); } /** @@ -175,7 +179,7 @@ public function getRootDir(): string return $this->mirrorRepoMetaDir; } - public function getStats(): ?array + public function getStats(): array { $stats = $this->redis->get("proxy-info-{$this->repoConfig['name']}"); $stats = $stats ? json_decode($stats, true) : []; @@ -183,8 +187,15 @@ public function getStats(): ?array return is_array($stats) ? $stats : []; } + public function clearStats(array $stats = []): void + { + $this->redis->set("proxy-info-{$this->repoConfig['name']}", json_encode($stats)); + } + public function setStats(array $stats = []): void { + $stats = \array_merge($this->getStats(), $stats); + $this->redis->set("proxy-info-{$this->repoConfig['name']}", json_encode($stats)); } @@ -193,6 +204,11 @@ public function getPackageManager(): RemotePackagesManager return $this->rpm; } + public function getDownloadManager(): ZipballDownloadManager + { + return $this->zipballManager; + } + protected function packageShort(string $package, string $hash = null): string { return $package . ($hash ? '__' . $hash : '') . '.json.gz'; diff --git a/src/Mirror/RootMetadataMerger.php b/src/Mirror/RootMetadataMerger.php index 0baa517c..e539b99e 100644 --- a/src/Mirror/RootMetadataMerger.php +++ b/src/Mirror/RootMetadataMerger.php @@ -16,7 +16,7 @@ public function __construct( public function merge(JsonMetadata ...$stamps): JsonMetadata { - if (count($stamps) > 1) { + if (\count($stamps) > 1) { throw new \LogicException('Todo, not implements'); } @@ -36,21 +36,21 @@ public function merge(JsonMetadata ...$stamps): JsonMetadata $newFile = []; $url = $this->router->generate('mirror_metadata_v2', ['package' => 'VND/PKG', 'alias' => $opt->getAlias()]); - $newFile['metadata-url'] = str_replace('VND/PKG', '%package%', $url); + $newFile['metadata-url'] = \str_replace('VND/PKG', '%package%', $url); if ($providerIncludes = $rootFile['provider-includes'] ?? []) { foreach ($providerIncludes as $name => $value) { unset($providerIncludes[$name]); - $providerIncludes[ltrim($name, '/')] = $value; + $providerIncludes[\ltrim($name, '/')] = $value; } $rootFile['provider-includes'] = $providerIncludes; } if ($providerUrl = $rootFile['providers-url'] ?? null) { - $hasHash = str_contains($providerUrl, '%hash%'); + $hasHash = \str_contains($providerUrl, '%hash%'); $url = $this->router->generate('mirror_metadata_v1', ['package' => 'VND/PKG', 'alias' => $opt->getAlias()]); - $rootFile['providers-url'] = str_replace('VND/PKG', $hasHash ? '%package%$%hash%' : '%package%', $url); + $rootFile['providers-url'] = \str_replace('VND/PKG', $hasHash ? '%package%$%hash%' : '%package%', $url); } if ($opt->getAvailablePatterns()) { @@ -63,6 +63,15 @@ public function merge(JsonMetadata ...$stamps): JsonMetadata unset($rootFile['packages']); } - return $stamps->withContent(array_merge($newFile, $rootFile)); + $zipball = $this->router->generate( + 'mirror_zipball', + ['package' => 'VND/PKG', 'alias' => $opt->getAlias(), 'version' => '__VER', 'ref' => '__REF', 'type' => '__TP'] + ); + + $rootFile['mirrors'] = [ + ['dist-url' => \str_replace(['VND/PKG', '__VER', '__REF', '__TP'], ['%package%', '%version%', '%reference%', '%type%'], $zipball), 'preferred' => true] + ]; + + return $stamps->withContent(\array_merge($newFile, $rootFile)); } } diff --git a/src/Mirror/Service/ComposeProxyRegistry.php b/src/Mirror/Service/ComposeProxyRegistry.php index 802dff6a..4d1a2b3b 100644 --- a/src/Mirror/Service/ComposeProxyRegistry.php +++ b/src/Mirror/Service/ComposeProxyRegistry.php @@ -7,6 +7,7 @@ use Packeton\Mirror\Exception\MetadataNotFoundException; use Packeton\Mirror\ProxyRepositoryFacade; use Packeton\Mirror\ProxyRepositoryRegistry; +use Packeton\Mirror\RemoteProxyRepository; class ComposeProxyRegistry { @@ -30,7 +31,17 @@ public function createRepository(string $name): ProxyRepositoryFacade return new ProxyRepositoryFacade($repo, ...$this->factoryArgs); } - public function createProxyDownloadManager(string $name) + public function getProxyDownloadManager(string $name): ZipballDownloadManager { + try { + $repo = $this->proxyRegistry->getRepository($name); + if (!$repo instanceof RemoteProxyRepository) { + throw new MetadataNotFoundException('Provider does not exists'); + } + } catch (\InvalidArgumentException $e) { + throw new MetadataNotFoundException('Provider does not exists', 0, $e); + } + + return $repo->getDownloadManager(); } } diff --git a/src/Mirror/Service/RemotePackagesManager.php b/src/Mirror/Service/RemotePackagesManager.php index 1fde94a6..b050a0b7 100644 --- a/src/Mirror/Service/RemotePackagesManager.php +++ b/src/Mirror/Service/RemotePackagesManager.php @@ -12,17 +12,43 @@ public function __construct( ) { } + public function getSettings(): array + { + $settings = $this->redis->get("repo:{$this->repo}:settings"); + $settings = $settings ? \json_decode($settings, true) : []; + + return [ + 'strict_mirror' => (bool)($settings['strict_mirror'] ?? false), + 'enabled_sync' => (bool)($settings['enabled_sync'] ?? true), + ]; + } + + public function setSettings(array $settings): void + { + $this->redis->set("repo:{$this->repo}:settings", \json_encode($settings)); + } + + public function isMinoring(): bool + { + return $this->getSettings()['strict_mirror']; + } + + public function isAutoSync(): bool + { + return $this->getSettings()['enabled_sync']; + } + public function markEnable(string $name): void { $this->redis->zadd("repo:{$this->repo}:enabled", time(), $name); } - public function allEnabled(): array + public function getEnabled(): array { return $this->redis->zRange("repo:{$this->repo}:enabled", 0, -1) ?: []; } - public function allApproved(): array + public function getApproved(): array { return $this->redis->zRange("repo:{$this->repo}:approve", 0, -1) ?: []; } diff --git a/src/Mirror/Service/RemoteSyncProxiesFacade.php b/src/Mirror/Service/RemoteSyncProxiesFacade.php index 7115fd36..c8b88fe7 100644 --- a/src/Mirror/Service/RemoteSyncProxiesFacade.php +++ b/src/Mirror/Service/RemoteSyncProxiesFacade.php @@ -24,6 +24,7 @@ public function sync(RemoteProxyRepository $repo, IOInterface $io, int $flags = { $this->syncProvider->setIO($io); + $stats = ['last_sync' => \date('Y-m-d H:i:s')]; if (self::FULL_RESET & $flags) { $io->notice('Remove all sync data and execute full resync!'); @@ -31,7 +32,7 @@ public function sync(RemoteProxyRepository $repo, IOInterface $io, int $flags = $cfs = new \Composer\Util\Filesystem(); $cfs->remove($repo->getRootDir()); - $repo->setStats([]); + $repo->clearStats(); $io->notice('All data removed!'); } @@ -47,7 +48,7 @@ public function sync(RemoteProxyRepository $repo, IOInterface $io, int $flags = if ($repo->isRootFresh($root)) { $io->info('Root is not changes, skip update providers'); - return []; + return $stats; } $providersForUpdate = []; @@ -63,14 +64,14 @@ public function sync(RemoteProxyRepository $repo, IOInterface $io, int $flags = if (empty($providerUrl) || $config->isLazy()) { $io->notice('Skipping sync packages, lazy sync.'); $repo->dumpRootMeta($root); - return []; + return $stats; } - $updated = 0; + $updated = $success = 0; $maxErrors = 3; foreach ($this->getAllProviders($providersForUpdate, $repo, $config) as $provName => $providersChunk) { $io->info("Loading chunk #$provName"); - $packages = $skipped = []; + try { [$packages, $skipped] = $this->syncProvider->loadPackages($repo, $providersChunk, $providerUrl); } catch (TransportException $e) { @@ -86,13 +87,14 @@ public function sync(RemoteProxyRepository $repo, IOInterface $io, int $flags = } $updated += \count($packages); - $success = \count($packages) + \count($skipped); + $success += \count($packages) + \count($skipped); $io->info("Total updated $updated packages."); } + $stats += ['pkg_updated' => $updated, 'pkg_total' => $success]; $repo->dumpRootMeta($root); - return ['pkg_updated' => $updated]; + return $stats; } private function getAllProviders($providersForUpdate, RemoteProxyRepository $repo, ProxyOptions $config): iterable @@ -113,7 +115,7 @@ private function getAllProviders($providersForUpdate, RemoteProxyRepository $rep private function syncLazy(RemoteProxyRepository $repo, IOInterface $io, ProxyOptions $config): array { $stats = []; - $packages = $repo->getPackageManager()->allEnabled(); + $packages = $repo->getPackageManager()->getEnabled(); $http = $this->syncProvider->initHttpDownloader($config); if ($apiUrl = $config->getV2SyncApi()) { diff --git a/src/Mirror/Service/SyncMirrorWorker.php b/src/Mirror/Service/SyncMirrorWorker.php index caecf38e..04b6dca7 100644 --- a/src/Mirror/Service/SyncMirrorWorker.php +++ b/src/Mirror/Service/SyncMirrorWorker.php @@ -4,9 +4,30 @@ namespace Packeton\Mirror\Service; +use Packeton\Attribute\AsWorker; +use Packeton\Mirror\ProxyRepositoryRegistry; +use Packeton\Mirror\RemoteProxyRepository; + +#[AsWorker('sync:mirrors')] class SyncMirrorWorker { + public function __construct( + private readonly ProxyRepositoryRegistry $registry, + private readonly RemoteSyncProxiesFacade $syncFacade + ) { + } + public function __invoke(array $arguments = []) { + if (!isset($arguments['mirror']) || !$this->registry->hasRepository($arguments['mirror'])) { + return ['message' => 'Repo not found']; + } + + /** @var RemoteProxyRepository $repo */ + $repo = $this->registry->getRepository($arguments['mirror']); + $stats = $this->syncFacade->sync($repo); + + $repo->setStats($stats); + return []; } } diff --git a/src/Mirror/Service/SyncProviderService.php b/src/Mirror/Service/SyncProviderService.php index 43947fa5..14c4bd7d 100644 --- a/src/Mirror/Service/SyncProviderService.php +++ b/src/Mirror/Service/SyncProviderService.php @@ -18,7 +18,8 @@ class SyncProviderService public function __construct( private readonly ProxyHttpDownloader $downloader, - ) {} + ) { + } public function loadRootComposer(RemoteProxyRepository $repo): array { diff --git a/src/Mirror/Service/ZipballDownloadManager.php b/src/Mirror/Service/ZipballDownloadManager.php new file mode 100644 index 00000000..a05fb5a7 --- /dev/null +++ b/src/Mirror/Service/ZipballDownloadManager.php @@ -0,0 +1,107 @@ +repository = $repository; + } + + public function distPath(string $package, string $version, string $ref): string + { + $etag = \preg_replace('#[^a-z0-9-]#i', '-', $version) . '-' . \sha1($package.$version.$ref) . '.zip'; + $distDir = $this->generatePackageDir($package); + $filename = $distDir . '/' . $etag; + if ($this->filesystem->exists($filename)) { + return $filename; + } + + $packages = $this->repository->findPackageMetadata($package)?->decodeJson(); + $packages = $packages['packages'][$package] ?? []; + $accessor = PropertyAccess::createPropertyAccessor(); + + $search = static function ($packages, $preference) use (&$search, $accessor) { + $candidate = null; + [$ref, $propertyPath] = \array_shift($preference); + + foreach ($packages as $package) { + try { + $expectedRef = $accessor->getValue($package, $propertyPath); + } catch (\Exception) { + continue; + } + + if (!empty($expectedRef) && $expectedRef === $ref) { + $candidate = $package; + break; + } + } + + if (null === $candidate && $preference) { + return $search($packages, $preference); + } + return $candidate; + }; + + $preference = [ + [$ref, '[dist][reference]'], + [$ref, '[source][reference]'], + [$version, '[version_normalized]'], + ]; + + if (!$candidate = $search($packages, $preference)) { + throw new MetadataNotFoundException('Not found reference in metadata'); + } + + $http = $this->service->initHttpDownloader($this->repository->getConfig()); + + $loader = new ArrayLoader(); + [$package] = $loader->loadPackages([$candidate + ['name' => $package, 'version' => $version]]); + $urls = $package->getDistUrls(); + + $hasFile = false; + $targetDir = \dirname($filename); + $this->filesystem->mkdir($targetDir); + foreach ($urls as $url) { + try { + $http->copy($url, $filename); + } catch (\Exception $e) { + continue; + } + + if ($hasFile = $this->filesystem->exists($filename)) { + break; + } + } + + if (false === $hasFile) { + throw new MetadataNotFoundException('Unable to download dist from source'); + } + + return $filename; + } + + public function generatePackageDir(string $packageName): string + { + $intermediatePath = \preg_replace('#[^a-z0-9-_/]#i', '-', $packageName); + return \sprintf('%s/%s', \rtrim($this->mirrorDistDir, '/'), $intermediatePath); + } +} diff --git a/src/Package/InMemoryDumper.php b/src/Package/InMemoryDumper.php index e6497f9e..53785b94 100644 --- a/src/Package/InMemoryDumper.php +++ b/src/Package/InMemoryDumper.php @@ -85,7 +85,6 @@ private function dumpRootPackages(UserInterface $user = null) $rootFile['providers-url'] = '/p/%package%$%hash%.json'; $rootFile['metadata-url'] = '/p2/%package%.json'; - $rootFile['available-packages'] = $availablePackages; $userHash = \hash('sha256', \json_encode($providers)); $rootFile['provider-includes'] = [ @@ -93,6 +92,7 @@ private function dumpRootPackages(UserInterface $user = null) 'sha256' => $userHash ] ]; + $rootFile['available-packages'] = $availablePackages; return [$rootFile, $providers, $packagesData]; } diff --git a/src/Repository/GroupRepository.php b/src/Repository/GroupRepository.php index efacb557..d4c2dfb0 100644 --- a/src/Repository/GroupRepository.php +++ b/src/Repository/GroupRepository.php @@ -101,4 +101,24 @@ public function getGroupsData(Group|int $group): array return ['packages' => $packages]; } + + public function getAllowedProxies(?UserInterface $user) + { + if (!$user instanceof User) { + return []; + } + + $proxies = $this->getEntityManager()->createQueryBuilder() + ->select('g.proxies') + ->from(User::class, 'u') + ->innerJoin('u.groups', 'g') + ->where('u.id = :uid') + ->setParameter('uid', $user->getId()) + ->getQuery() + ->getArrayResult(); + + $proxies = array_column($proxies, 'proxies'); + + return $proxies ? \array_unique(\array_merge(...$proxies)) : []; + } } diff --git a/src/Resolver/ControllerArgumentResolver.php b/src/Resolver/ControllerArgumentResolver.php index 9860d890..735ee38f 100644 --- a/src/Resolver/ControllerArgumentResolver.php +++ b/src/Resolver/ControllerArgumentResolver.php @@ -15,31 +15,32 @@ class ControllerArgumentResolver implements ArgumentValueResolverInterface { public function __construct( private readonly ManagerRegistry $registry - ){} + ){ + } /** * {@inheritdoc} */ - public function supports(Request $request, ArgumentMetadata $argument) + public function supports(Request $request, ArgumentMetadata $argument): bool { - return count($argument->getAttributes(Vars::class, ArgumentMetadata::IS_INSTANCEOF)) > 0 && - count($request->attributes->get('_route_params', [])) > 0; + return \count($argument->getAttributes(Vars::class, ArgumentMetadata::IS_INSTANCEOF)) > 0 && + \count($this->getRequestParams($request)) > 0; } /** * {@inheritdoc} */ - public function resolve(Request $request, ArgumentMetadata $argument) + public function resolve(Request $request, ArgumentMetadata $argument): iterable { /** @var Vars $attr */ $attr = $argument->getAttributes(Vars::class, ArgumentMetadata::IS_INSTANCEOF)[0]; $mapping = []; - if (is_string($attr->map)) { + if (\is_string($attr->map)) { $mapping[$attr->map] = $request->attributes->has($argument->getName()) ? $request->attributes->get($argument->getName()) : $request->attributes->get($attr->map); - } elseif (is_array($attr->map)) { + } elseif (\is_array($attr->map)) { foreach ($attr->map as $name => $field) { $mapping[$field] = $request->attributes->get($name); } @@ -49,8 +50,8 @@ public function resolve(Request $request, ArgumentMetadata $argument) if ($request->attributes->has($identifier)) { $mapping[$identifier] = $request->attributes->get($identifier); - } elseif (count($params = $request->attributes->get('_route_params', [])) === 1) { - $identifier = array_key_first($params); + } elseif (\count($params = $this->getRequestParams($request)) === 1) { + $identifier = \array_key_first($params); if ($metadata->hasField($identifier)) { $mapping[$identifier] = $params[$identifier]; } @@ -58,6 +59,9 @@ public function resolve(Request $request, ArgumentMetadata $argument) } if (empty($mapping)) { + if ($argument->hasDefaultValue()) { + return [$argument->getDefaultValue()]; + } throw new \UnexpectedValueException('Cannot resolve $'.$argument->getName() . ' argument'); } @@ -71,6 +75,14 @@ public function resolve(Request $request, ArgumentMetadata $argument) throw new NotFoundHttpException('Object not found'); } - yield $object; + return [$object]; + } + + private function getRequestParams(Request $request): array + { + $params = $request->attributes->get('_route_params', []); + unset($params['_format']); + + return $params; } } diff --git a/src/Security/Acl/ObjectIdentity.php b/src/Security/Acl/ObjectIdentity.php new file mode 100644 index 00000000..c1747861 --- /dev/null +++ b/src/Security/Acl/ObjectIdentity.php @@ -0,0 +1,28 @@ +identifier; + } + + /** + * @return string + */ + public function getType(): string + { + return $this->type; + } +} diff --git a/src/Security/Acl/ObjectIdentityVoter.php b/src/Security/Acl/ObjectIdentityVoter.php new file mode 100644 index 00000000..b7ce7ba2 --- /dev/null +++ b/src/Security/Acl/ObjectIdentityVoter.php @@ -0,0 +1,39 @@ +getType()) { + PRI::class => $this->checkProxyAccess($token, $subject->getIdentifier()), + default => self::ACCESS_ABSTAIN + }; + } + + private function checkProxyAccess(TokenInterface $token, $identifier): int + { + $user = $token->getUser(); + + $allowed = $this->registry->getRepository(Group::class)->getAllowedProxies($user); + + return \in_array($identifier, $allowed, true) ? self::ACCESS_GRANTED : self::ACCESS_DENIED; + } +} diff --git a/src/Service/JobPersister.php b/src/Service/JobPersister.php index e19b05c3..2c3dd328 100644 --- a/src/Service/JobPersister.php +++ b/src/Service/JobPersister.php @@ -29,7 +29,7 @@ public function persist(Job $job): void ->getQuery() ->getResult(); } else { - $job->setId(bin2hex(random_bytes(20))); + $job->setId(\bin2hex(\random_bytes(20))); } $data = [ @@ -60,6 +60,23 @@ public function persist(Job $job): void } } + public function getPendingJob(string $type, int $hash): ?string + { + $result = $this->getConn()->fetchAssociative( + 'SELECT id FROM job WHERE packageId = :package AND status = :status AND type = :type LIMIT 1', + [ + 'package' => $hash, + 'type' => $type, + 'status' => Job::STATUS_QUEUED, + ] + ); + + if ($result) { + return $result['id']; + } + + return null; + } /** * @return \Doctrine\DBAL\Connection diff --git a/src/Service/JobScheduler.php b/src/Service/JobScheduler.php index 765b232c..a178d461 100644 --- a/src/Service/JobScheduler.php +++ b/src/Service/JobScheduler.php @@ -22,9 +22,11 @@ public function __construct(\Redis $redis, JobPersister $persister) * * @param string $type * @param array|Job|null $job + * @param int|null $hash + * * @return Job */ - public function publish(string $type, $job = null): Job + public function publish(string $type, array|Job|null $job = null, int $hash = null): Job { if ($job instanceof Job) { $job->setCreatedAt(new \DateTime('now', new \DateTimeZone('UTC'))); @@ -37,6 +39,12 @@ public function publish(string $type, $job = null): Job $job->setCreatedAt(new \DateTime('now', new \DateTimeZone('UTC'))); } + $job->setPackageId($hash); + if (null !== $hash && null !== $this->persister->getPendingJob($type, $hash)) { + $job->setStatus(Job::STATUS_COMPLETED); + return $job; + } + $this->persister->persist($job); // trigger immediately if not scheduled for later if (!$job->getExecuteAfter()) { diff --git a/src/Service/QueueWorker.php b/src/Service/QueueWorker.php index 1a0cef01..266f335e 100644 --- a/src/Service/QueueWorker.php +++ b/src/Service/QueueWorker.php @@ -129,6 +129,7 @@ private function process(string $jobId, SignalHandler $signal): bool try { $result = $processor($job, $signal); + $result = $result + ['status' => Job::STATUS_COMPLETED, 'message' => 'Success']; } catch (JobException $e) { $result = [ 'status' => Job::STATUS_ERRORED, diff --git a/src/Twig/PackagistExtension.php b/src/Twig/PackagistExtension.php index 995850fc..d09645dc 100644 --- a/src/Twig/PackagistExtension.php +++ b/src/Twig/PackagistExtension.php @@ -7,8 +7,10 @@ use Packeton\Entity\Job; use Packeton\Entity\Package; use Packeton\Entity\User; +use Packeton\Form\Model\PackagePermission; use Packeton\Model\ProviderManager; use Packeton\Security\JWTUserManager; +use Symfony\Component\Form\FormView; use Symfony\Component\Security\Core\User\UserInterface; use Twig\Extension\AbstractExtension; use Twig\TwigFilter; @@ -38,6 +40,7 @@ public function getFunctions() return [ new TwigFunction('package_job_result', [$this, 'getLatestJobResult']), new TwigFunction('get_group_data', [$this, 'getGroupData']), + new TwigFunction('get_group_acl_form_data', [$this, 'getGroupAclForm']), new TwigFunction('get_api_token', [$this, 'getApiToken']), ]; } @@ -51,6 +54,39 @@ public function getFilters() ]; } + public function getGroupAclForm(FormView $items) + { + $byVendors = []; + foreach ($items->children as $item) { + $data = $item->vars['data'] ?? null; + if ($data instanceof PackagePermission) { + [$vendor] = \explode('/', $data->getName()); + $byVendors[$vendor][] = $item; + } + } + + $grouped = $otherVendors = []; + \uasort($byVendors, fn($a, $b) => -1 * (\count($a) <=> \count($b))); + foreach ($byVendors as $vendorName => $children) { + if (\count($grouped) < 4 && \count($children) > 1) { + $grouped[$vendorName] = $children; + } else { + $otherVendors = \array_merge($otherVendors, $children); + } + } + + if ($otherVendors) { + $grouped['other'] = $otherVendors; + } + foreach ($grouped as $vendor => $children) { + \usort($children, fn($a, $b) => $a->vars['data']->getName() <=> $b->vars['data']->getName()); + $selected = \count(\array_filter($children, fn($a) => $a->vars['data']->getSelected())); + $grouped[$vendor] = ['items' => $children, 'selected' => $selected]; + } + + return $grouped; + } + public function getGroupData(Group|int $group): array { return $this->registry->getRepository(Group::class)->getGroupsData($group); diff --git a/templates/base.html.twig b/templates/base.html.twig index 60f45e91..a48af44b 100644 --- a/templates/base.html.twig +++ b/templates/base.html.twig @@ -26,8 +26,8 @@ {% block stylesheets %}{% endblock %} {% block head_feeds %} - - + + {% endblock %} {% block head_additions %}{% endblock %} diff --git a/templates/group/update.html.twig b/templates/group/update.html.twig index 973475fa..5ac46cf2 100644 --- a/templates/group/update.html.twig +++ b/templates/group/update.html.twig @@ -13,12 +13,32 @@ {{ form_start(form, { attr: { class: 'col-md-6' } }) }} {{ form_errors(form) }} {{ form_row(form.name) }} + {% if form.proxies is defined %} + {{ form_row(form.proxies) }} + {% endif %}

Access matrix

-
- {% for permission in form.aclPermissions %} -
{{ form_row(permission) }}
- {% endfor %} + {% set aclPermissions = get_group_acl_form_data(form.aclPermissions) %} + {% if aclPermissions|length > 0 %} + + +
+ {% for vendor, items in aclPermissions %} + {% set vid = 'ven-' ~ vendor %} +
+ {% for permission in items['items'] %} +
{{ form_row(permission) }}
+ {% endfor %} +
+ {% endfor %}
+ {% endif %} + {{ form_end(form) }}
diff --git a/templates/proxies/index.html.twig b/templates/proxies/index.html.twig index 5cdc9b98..f65d4b70 100644 --- a/templates/proxies/index.html.twig +++ b/templates/proxies/index.html.twig @@ -1,15 +1,17 @@ {% extends "layout.html.twig" %} +{% block title %}Proxies - {{ parent() }}{% endblock %} + {% block content %}
+

Proxies and Mirroring

+ {% if proxies|length > 0 %}
    {% for proxy in proxies %} - - {% set viewPath = '' %} + {% set viewPath = path('proxy_view', {'alias': proxy.alias }) %}
  • -
    {% if proxy.logo %} @@ -20,16 +22,41 @@ {{ proxy.url }}
    -
    + {% set caps = proxy.capabilities %} + {% if caps|length > 0 %}

    - Active: Yes
    - Expires at: 1212 + Capabilities: {{ caps|map(value => "#{value}")|join(' ')|raw }} +

    + {% endif %} +

    + Lazy: {{ proxy.lazy ? 'Yes' : 'No' }}
    + Last sync: {{ proxy.getStats('last_sync', 'N/A') }}
    + Packages: {{ proxy.getStats('pkg_total', 'N/A') }}

{% endfor %} + {% else %} +
+

To enable mirroring and proxies, you need to update the application configuration, see docs

+ 
+
+# config/packages/example.yaml
+packeton:
+  mirrors:
+#   packagist:
+#     url: https://repo.packagist.org
+
+    magentocom:
+      url: https://packages.example.com
+      http_basic:
+        username: ****
+        password: ****
+
+
+ {% endif %} {% endblock %} diff --git a/templates/proxies/info_caps.html.twig b/templates/proxies/info_caps.html.twig new file mode 100644 index 00000000..f3ac24b1 --- /dev/null +++ b/templates/proxies/info_caps.html.twig @@ -0,0 +1,15 @@ +

+ {% if proxy.logo %} + {{ proxy.url }} + {% endif %} + {{ proxy.alias }} + {{ proxy.url }} +

+

+ Last updated: {{ proxy.getStats('last_sync', 'N/A') }} +

+

+ Repo Capabilities: {{ proxy.capabilities|map(value => "#{value}")|join(' ')|raw }}
+ Disk usage: {{ proxy.getStats('used_storage', 'N/A') }}
+ Lazy sync: {{ proxy.lazy ? 'Yes' : 'No' }}
+

diff --git a/templates/proxies/info_repo.html.twig b/templates/proxies/info_repo.html.twig new file mode 100644 index 00000000..261f36cf --- /dev/null +++ b/templates/proxies/info_repo.html.twig @@ -0,0 +1,18 @@ +

To enable proxy, please add these lines to your composer.json file:

+{% if proxy is defined and proxy.packagist %} + 
{
+  "repositories": [
+    { "type": "composer", "url": "{{ repoUrl }}"},
+    { "packagist": false }
+  ]
+}
+{% else %} + 
{
+  "repositories": [
+    {
+      "type": "composer",
+      "url": "{{ repoUrl }}"
+    }
+  ]
+}
+{% endif %} diff --git a/templates/proxies/mirror.html.twig b/templates/proxies/mirror.html.twig new file mode 100644 index 00000000..31fc6c09 --- /dev/null +++ b/templates/proxies/mirror.html.twig @@ -0,0 +1,10 @@ +{% extends "layout.html.twig" %} + +{% block content %} +

Mirror {{ alias }}

+
+
+ {% include 'proxies/info_repo.html.twig' %} +
+
+{% endblock %} diff --git a/templates/proxies/settings.html.twig b/templates/proxies/settings.html.twig new file mode 100644 index 00000000..61edd8ac --- /dev/null +++ b/templates/proxies/settings.html.twig @@ -0,0 +1,34 @@ +{% extends "layout.html.twig" %} + +{% block title %}{{ proxy.alias }} - Settings Proxy - {{ parent() }}{% endblock %} + +{% block content %} +

+ {% include 'proxies/info_caps.html.twig' %} +

+
+ + +
+ {{ form_start(form, { attr: { class: 'col-sm-6', id: 'settings-form' } }) }} + {{ form_widget(form) }} + + << Go Back + + {{ form_end(form) }} + +
+ {{ 'abandon.warning'|trans|raw }} +
+
+{% endblock %} diff --git a/templates/proxies/view.html.twig b/templates/proxies/view.html.twig new file mode 100644 index 00000000..eb3c3037 --- /dev/null +++ b/templates/proxies/view.html.twig @@ -0,0 +1,41 @@ +{% extends "layout.html.twig" %} + +{% block title %}{{ proxy.alias }} - Proxy - {{ parent() }}{% endblock %} + +{% block scripts %} +{% endblock %} + +{% block content %} +
+
+
+
+ {% include 'proxies/info_caps.html.twig' %} +
+
+ {% include 'proxies/info_repo.html.twig' %} +
+
+
+ +
+
+
+ +
+ +
+ + +
+ +
+ + +
+ +
+
+
+
+{% endblock %} diff --git a/translations/messages.en.yml b/translations/messages.en.yml index 6c52fb19..c544b786 100644 --- a/translations/messages.en.yml +++ b/translations/messages.en.yml @@ -26,6 +26,7 @@ menu: my_groups: My groups ssh_keys: Credentials webhooks: Webhooks + proxies: Composer Proxies security: login: