From 98b9f3ca425e27f7e98536b1ff2ae8a0dc6aa4f5 Mon Sep 17 00:00:00 2001 From: Uladzimir Tsykun Date: Sat, 25 Feb 2023 21:28:55 +0100 Subject: [PATCH] Performance improvement when the number of packets is large Fixes #75 --- README.md | 89 +++++++++++++++++-- config/services.yaml | 4 + src/Composer/Cache/MetadataCache.php | 7 +- src/Composer/MetadataFormat.php | 37 ++++++++ src/Controller/PackageController.php | 1 + src/Controller/ProviderController.php | 36 +++++--- src/Controller/ProxiesController.php | 5 +- src/DependencyInjection/Configuration.php | 7 ++ src/DependencyInjection/PacketonExtension.php | 3 + src/EventListener/DoctrineListener.php | 6 +- src/Mirror/RemoteProxyRepository.php | 1 + src/Mirror/Service/ComposeProxyRegistry.php | 6 +- src/Mirror/Utils/MirrorPackagesValidate.php | 6 +- src/Model/PackageManager.php | 60 +++++-------- src/Model/ProviderManager.php | 89 +++++++++++++++++-- src/Package/InMemoryDumper.php | 65 +++++++++++--- src/Package/Updater.php | 26 +++++- 17 files changed, 351 insertions(+), 97 deletions(-) create mode 100644 src/Composer/MetadataFormat.php diff --git a/README.md b/README.md index 3b4beb6d..9bb04c6d 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ Table of content - [JIRA issue fix version](/docs/webhook.md#jira-create-a-new-release-and-set-fix-version) - [Gitlab setup auto webhook](/docs/webhook.md#gitlab-auto-webhook) - [Ssh key access](#ssh-key-access-and-composer-oauth-token) +- [Configuration](#configuration) +- [LPAD Authenticating](/docs/authentication-ldap.md) - [Update Webhooks](#update-webhooks) - [Github](#github-webhooks) - [GitLab](#gitlab-service) @@ -56,7 +58,7 @@ Table of content - [Custom webhook format](#custom-webhook-format-transformer) - [Mirroring Composer repos](docs/usage/mirroring.md) - [Usage](#usage-and-authentication) - - [Create admin user](#create-admin-user) + - [Create admin user](#create-admin-and-maintainer-users) Demo ---- @@ -124,7 +126,7 @@ Installation 1. Clone the repository 2. Install dependencies: `composer install` 3. Create .env.local and copy needed environment variables into it, see docker Environment variables section -4. Run `bin/console doctrine:schema:create` to setup the DB +4. Run `bin/console doctrine:schema:update --force --complete` to set up the DB 5. Create admin user via console. ``` @@ -132,10 +134,15 @@ php bin/console packagist:user:manager username --email=admin@example.com --pass ``` 6. Enable cron tabs and background jobs. -Enable crontab `crontab -e -u www-data` +Enable crontab `crontab -e -u www-data` or use Docker friendly build-in cron demand runner. ``` -* * * * * /var/www/packagist/bin/console --env=prod okvpn:cron >> /dev/null +* * * * * /var/www/packagist/bin/console okvpn:cron >> /dev/null +``` + +Example, run cron as background process without crontab. Can use with supervisor. +``` +bin/console okvpn:cron --demand ``` Setup Supervisor to run worker. @@ -246,6 +253,72 @@ We disable usage GitHub API by default to force use ssh key or clone the reposit it would with any other git repository. You can enable it again with env option `GITHUB_NO_API` [see here](https://getcomposer.org/doc/06-config.md#use-github-api). +Configuration +------------- + +In order to add a configuration add a file with any name to the folder `config/packages/*`. +The config will merge with default values in priority sorted by filename. + +The configuration for Docker installation is available at `/data/config.yaml`. +Also, you can use docker volume to add config directly at path `config/packages/ldap.yaml`. + +```yaml +... + volumes: + - .docker:/data + - ${PWD}/ldap.yaml:/var/www/packagist/config/packages/ldap.yaml +``` + +Where `/var/www/packagist/` default ROOT for docker installation. + +Full example of configuration. + +```yaml +packeton: + github_no_api: '%env(bool:GITHUB_NO_API)%' # default true + rss_max_items: 30 + archive: true + + # default false + anonymous_access: '%env(bool:PUBLIC_ACCESS)%' + + anonymous_archive_access: '%env(bool:PUBLIC_ACCESS)%' # default false + + archive_options: + format: zip + basedir: '%env(resolve:PACKAGIST_DIST_PATH)%' + endpoint: '%env(PACKAGIST_DIST_HOST)%' # default auto detect by host headers + include_archive_checksum: false + + # disable by default + jwt_authentication: + algo: EdDSA + private_key: '%kernel.project_dir%/var/jwt/eddsa-key.pem' + public_key: '%kernel.project_dir%/var/jwt/eddsa-public.pem' + passphrase: ~ + + # See mirrors section + mirrors: ~ + + metadata: + format: auto # Default, see about metadata. + info_cmd_message: ~ # Bash logo, example - \u001b[37;44m#StandWith\u001b[30;43mUkraine\u001b[0m +``` + +### Metadata format. + +Packeton support metadata for Composer 1 and 2. For performance reasons, for Composer 1 uses metadata +depending on the user-agent header: `providers-lazy-url` if ua != 1; `provider-includes` if ua == 1; + +| Format strategy | UA 1 | UA 2 | UA is NULL | +|-----------------|--------------------------------|---------------------------------|---------------------------------| +| auto | provider-includes metadata-url | providers-lazy-url metadata-url | providers-lazy-url metadata-url | +| only_v1 | provider-includes | provider-includes | provider-includes | +| only_v2 | metadata-url | metadata-url | metadata-url | +| full | provider-includes metadata-url | provider-includes metadata-url | provider-includes metadata-url | + +Where `UA 1` - Composer User-Agent = 1. `UA 2` - Composer User-Agent = 2. + Update Webhooks --------------- You can use GitLab, Gitea, GitHub, and Bitbucket project post-receive hook to keep your packages up to date @@ -415,10 +488,10 @@ Configure this private repository in your `composer.json`. **Application Roles** -- ROLE_USER - minimal access level, these users only can read metadata only for selected packages. -- ROLE_FULL_CUSTOMER - Can read all packages metadata. -- ROLE_MAINTAINER - Can submit a new package and read all metadata. -- ROLE_ADMIN - Can create a new customer users, management webhooks and credentials. +- `ROLE_USER` - minimal access level, these users only can read metadata only for selected packages. +- `ROLE_FULL_CUSTOMER` - Can read all packages metadata. +- `ROLE_MAINTAINER` - Can submit a new package and read all metadata. +- `ROLE_ADMIN` - Can create a new customer users, management webhooks and credentials. You can create a user and then promote to admin or maintainer via console using fos user bundle commands. diff --git a/config/services.yaml b/config/services.yaml index eb2e5626..c2e5def8 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -104,6 +104,10 @@ services: arguments: $config: '%packeton_archive_opts%' + Packeton\Package\InMemoryDumper: + arguments: + $config: '%packeton_dumper_opts%' + Packeton\DBAL\OpensslCrypter: public: true arguments: diff --git a/src/Composer/Cache/MetadataCache.php b/src/Composer/Cache/MetadataCache.php index 709aef01..4d2e328b 100644 --- a/src/Composer/Cache/MetadataCache.php +++ b/src/Composer/Cache/MetadataCache.php @@ -16,7 +16,7 @@ public function __construct( ) { } - public function get(string $key, callable $callback, int $lastModify = null) + public function get(string $key, callable $callback, int $lastModify = null, callable $needClearCache = null) { // Use host key to prevent Cache Poisoning attack, if dist URL generated dynamic. // But for will protection must be used trusted_hosts @@ -27,9 +27,12 @@ public function get(string $key, callable $callback, int $lastModify = null) @[$ctime, $data] = $item->get(); $needRefresh = false; - if ($lastModify !== null) { + if (null !== $lastModify) { $needRefresh = $ctime < $lastModify || $ctime + $this->maxTtl < time(); } + if (null !== $needClearCache) { + $needRefresh = $needRefresh || $needClearCache($data); + } if (!$item->isHit() || $needRefresh || empty($data)) { $data = $callback($item); diff --git a/src/Composer/MetadataFormat.php b/src/Composer/MetadataFormat.php new file mode 100644 index 00000000..daed4900 --- /dev/null +++ b/src/Composer/MetadataFormat.php @@ -0,0 +1,37 @@ + true, + default => false + }; + } + + public function lazyProviders(int $version = null): bool + { + return match(true) { + $this === MetadataFormat::AUTO && $version !== 1 => true, + default => false + }; + } + + public function metadataUrl(int $version = null): bool + { + return match(true) { + $this === MetadataFormat::ONLY_V1 => false, + default => true + }; + } +} diff --git a/src/Controller/PackageController.php b/src/Controller/PackageController.php index 7047170b..b7fcb458 100644 --- a/src/Controller/PackageController.php +++ b/src/Controller/PackageController.php @@ -501,6 +501,7 @@ public function deletePackageVersionAction(Request $req, $versionId): Response throw new AccessDeniedException; } + $this->providerManager->setLastModify($package->getName()); $repo->remove($version); $this->registry->getManager()->flush(); $this->registry->getManager()->clear(); diff --git a/src/Controller/ProviderController.php b/src/Controller/ProviderController.php index 444de731..9472970a 100644 --- a/src/Controller/ProviderController.php +++ b/src/Controller/ProviderController.php @@ -10,7 +10,9 @@ use Packeton\Entity\Package; use Packeton\Entity\Version; use Packeton\Model\PackageManager; +use Packeton\Model\ProviderManager; use Packeton\Service\DistManager; +use Packeton\Util\UserAgentParser; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; use Symfony\Component\HttpFoundation\BinaryFileResponse; use Symfony\Component\HttpFoundation\Request; @@ -24,21 +26,27 @@ class ProviderController extends AbstractController public function __construct( private readonly PackageManager $packageManager, + private readonly ProviderManager $providerManager, private readonly ManagerRegistry $registry, - ){ + ) { } #[Route('/packages.json', name: 'root_packages', defaults: ['_format' => 'json'], methods: ['GET'])] public function packagesAction(Request $request): Response { - $rootPackages = $this->packageManager->getRootPackagesJson($this->getUser()); + $response = new JsonResponse([]); + $response->setLastModified($this->providerManager->getRootLastModify()); + if ($response->isNotModified($request)) { + return $response; + } + + $ua = new UserAgentParser($request->headers->get('User-Agent')); + $apiVersion = $request->query->get('ua') ? (int) $request->query->get('ua') : $ua->getComposerMajorVersion(); - $response = new JsonResponse($rootPackages); + $rootPackages = $this->packageManager->getRootPackagesJson($this->getUser(), $apiVersion); + + $response->setData($rootPackages); $response->setEncodingOptions(JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT); - if ($lastModify = $this->packageManager->getLastModify()) { - $response->setLastModified($lastModify); - $response->isNotModified($request); - } return $response; } @@ -120,20 +128,22 @@ public function packageAction(string $package): Response )] public function packageV2Action(Request $request, string $package): Response { + $response = new JsonResponse([]); + $response->setLastModified($this->providerManager->getLastModify($package)); + if ($response->isNotModified($request)) { + return $response; + } + $isDev = str_ends_with($package, '~dev'); $package = preg_replace('/~dev$/', '', $package); - $package = $this->packageManager->getPackageV2Json($this->getUser(), $package, $isDev, $lastModified); + $package = $this->packageManager->getPackageV2Json($this->getUser(), $package, $isDev); if (!$package) { return $this->createNotFound(); } - $response = new JsonResponse($package); $response->setEncodingOptions(\JSON_UNESCAPED_SLASHES); - if ($lastModified !== null) { - $response->setLastModified(new \DateTime($lastModified)); - $response->isNotModified($request); - } + $response->setData($package); return $response; } diff --git a/src/Controller/ProxiesController.php b/src/Controller/ProxiesController.php index ea0a1ca9..ed03c0b0 100644 --- a/src/Controller/ProxiesController.php +++ b/src/Controller/ProxiesController.php @@ -18,6 +18,7 @@ use Packeton\Mirror\Utils\MirrorTextareaParser; use Packeton\Mirror\Utils\MirrorUIFormatter; use Packeton\Model\PackageManager; +use Packeton\Model\ProviderManager; use Packeton\Service\JobScheduler; use Packeton\Util\HtmlJsonHuman; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; @@ -38,7 +39,7 @@ public function __construct( private readonly JobScheduler $jobScheduler, private readonly MirrorPackagesValidate $mirrorValidate, private readonly MetadataMinifier $metadataMinifier, - private readonly PackageManager $packageManager, + private readonly ProviderManager $providerManager, ) { } @@ -204,7 +205,7 @@ protected function getProxyData(RemoteProxyRepository $repo): array $repoUrl = $this->generateUrl('mirror_index', ['alias' => $config->getAlias()], UrlGeneratorInterface::ABSOLUTE_URL); $rpm = $repo->getPackageManager(); - $privatePackages = $this->packageManager->getPackageNames(); + $privatePackages = $this->providerManager->getPackageNames(); $packages = MirrorUIFormatter::getGridPackagesData($rpm->getApproved(), $rpm->getEnabled(), $privatePackages); diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index 41e1968f..5152172a 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -3,6 +3,7 @@ namespace Packeton\DependencyInjection; use Firebase\JWT\JWT; +use Packeton\Composer\MetadataFormat; use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition; use Symfony\Component\Config\Definition\Builder\NodeDefinition; use Symfony\Component\Config\Definition\Builder\TreeBuilder; @@ -27,6 +28,12 @@ public function getConfigTreeBuilder() ->children() ->booleanNode('github_no_api')->end() ->scalarNode('rss_max_items')->defaultValue(40)->end() + ->arrayNode('metadata') + ->children() + ->enumNode('format')->values(array_map(fn($o) => $o->value, MetadataFormat::cases()))->end() + ->scalarNode('info_cmd_message')->end() + ->end() + ->end() ->booleanNode('anonymous_access')->defaultFalse()->end() ->booleanNode('anonymous_archive_access')->defaultFalse()->end() ->booleanNode('archive') diff --git a/src/DependencyInjection/PacketonExtension.php b/src/DependencyInjection/PacketonExtension.php index 758fd01a..877602e5 100644 --- a/src/DependencyInjection/PacketonExtension.php +++ b/src/DependencyInjection/PacketonExtension.php @@ -5,6 +5,7 @@ namespace Packeton\DependencyInjection; use Packeton\Attribute\AsWorker; +use Packeton\Package\InMemoryDumper; use Symfony\Component\DependencyInjection\ChildDefinition; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Exception\LogicException; @@ -25,6 +26,8 @@ public function load(array $configs, ContainerBuilder $container) $container->setParameter('packeton_archive_opts', $config['archive_options'] ?? []); } + $container->setParameter('packeton_dumper_opts', $config['metadata'] ?? []); + $hasPublicMirror = array_filter($config['mirrors'] ?? [] , fn ($i) => $i['public_access'] ?? false); $container->setParameter('anonymous_mirror_access', (bool) $hasPublicMirror); diff --git a/src/EventListener/DoctrineListener.php b/src/EventListener/DoctrineListener.php index 55b23f04..f7171abc 100644 --- a/src/EventListener/DoctrineListener.php +++ b/src/EventListener/DoctrineListener.php @@ -12,7 +12,7 @@ use Packeton\Entity\Package; use Packeton\Entity\User; use Packeton\Entity\Version; -use Packeton\Model\PackageManager; +use Packeton\Model\ProviderManager; use Packeton\Service\DistConfig; use Symfony\Component\HttpFoundation\RequestStack; @@ -28,7 +28,7 @@ class DoctrineListener public function __construct( private readonly RequestStack $requestStack, - private readonly PackageManager $packageManager, + private readonly ProviderManager $providerManager, ){ } @@ -65,7 +65,7 @@ public function onFlush(OnFlushEventArgs $args): void foreach ($changes as $object) { $class = ClassUtils::getClass($object); if (isset(self::$trackLastModifyClasses[$class])) { - $this->packageManager->setLastModify(); + $this->providerManager->setRootLastModify(); return; } } diff --git a/src/Mirror/RemoteProxyRepository.php b/src/Mirror/RemoteProxyRepository.php index c4885251..79d62fd7 100644 --- a/src/Mirror/RemoteProxyRepository.php +++ b/src/Mirror/RemoteProxyRepository.php @@ -341,6 +341,7 @@ public function packageKey(string $package, string $hash = null): string $vendor = $this->safeName($vendor); $pkg = $this->safeName($pkg) ?: '_null_'; + $hash = $hash ? $this->safeName($hash) : null; return $vendor . $this->ds . $pkg . ($hash ? self::HASH_SEPARATOR . $hash : '') . '.json.gz'; } diff --git a/src/Mirror/Service/ComposeProxyRegistry.php b/src/Mirror/Service/ComposeProxyRegistry.php index 73b7dd3a..e5b4b0fd 100644 --- a/src/Mirror/Service/ComposeProxyRegistry.php +++ b/src/Mirror/Service/ComposeProxyRegistry.php @@ -12,7 +12,7 @@ use Packeton\Mirror\Model\StrictProxyRepositoryInterface as PRI; use Packeton\Mirror\ProxyRepositoryRegistry; use Packeton\Mirror\RemoteProxyRepository; -use Packeton\Model\PackageManager; +use Packeton\Model\ProviderManager; class ComposeProxyRegistry { @@ -20,7 +20,7 @@ public function __construct( protected ProxyRepositoryRegistry $proxyRegistry, protected SyncProviderService $syncService, protected MetadataMinifier $metadataMinifier, - protected PackageManager $packageManager, + protected ProviderManager $providerManager, ) { } @@ -46,7 +46,7 @@ public function createACLAwareRepository(string $name): PRI $repo, $repo->getConfig()->getAvailablePackages(), $repo->getConfig()->getAvailablePatterns(), - $this->packageManager->getPackageNames(), + $this->providerManager->getPackageNames(), ); } diff --git a/src/Mirror/Utils/MirrorPackagesValidate.php b/src/Mirror/Utils/MirrorPackagesValidate.php index e14536bb..b9b4406d 100644 --- a/src/Mirror/Utils/MirrorPackagesValidate.php +++ b/src/Mirror/Utils/MirrorPackagesValidate.php @@ -6,20 +6,20 @@ use Packeton\Mirror\RemoteProxyRepository; use Packeton\Mirror\Service\FetchPackageMetadataService; -use Packeton\Model\PackageManager; +use Packeton\Model\ProviderManager; class MirrorPackagesValidate { public function __construct( private readonly FetchPackageMetadataService $fetchMetadataService, - private readonly PackageManager $packageManager + private readonly ProviderManager $providerManager ) { } public function checkPackages(RemoteProxyRepository $repo, array $packages, array $enabled): array { $waiting = $valid = []; - $excluded = $this->packageManager->getPackageNames(); + $excluded = $this->providerManager->getPackageNames(); foreach ($packages as $package) { if ($meta = $this->getData($repo->findPackageMetadata($package)?->decodeJson(), $package)) { $valid[$package] = $meta; diff --git a/src/Model/PackageManager.php b/src/Model/PackageManager.php index 07241ae0..840b70e1 100644 --- a/src/Model/PackageManager.php +++ b/src/Model/PackageManager.php @@ -37,7 +37,8 @@ public function __construct( protected MetadataCache $cache, protected MetadataMinifier $metadataMinifier, protected \Redis $redis, - ) {} + ) { + } public function deletePackage(Package $package) { @@ -131,20 +132,24 @@ public function notifyUpdateFailure(Package $package, \Exception $e, $details = return true; } - public function getRootPackagesJson(UserInterface $user = null) + public function getRootPackagesJson(UserInterface $user = null, int $apiVersion = null) { - $packagesData = $this->dumpInMemory($user, false); + $packagesData = $this->dumpInMemory($user, false, $apiVersion); return $packagesData[0]; } /** * @param User|null|object $user * @param string $hash - * @return bool + * @return bool|array */ public function getProvidersJson(?UserInterface $user, $hash) { - list($root, $providers) = $this->dumpInMemory($user); + [$root, $providers] = $this->dumpInMemory($user); + if (null === $providers || !isset($root['provider-includes'])) { + return false; + } + $rootHash = \reset($root['provider-includes']); if ($hash && $rootHash['sha256'] !== $hash) { return false; @@ -165,19 +170,18 @@ public function getPackageJson(?UserInterface $user, string $package) $user = null; } - return $this->dumper->dumpPackage($user, $package); + $meta = $this->dumper->dumpPackage($user, $package); + return $meta ? ['packages' => [$package => $meta]] : []; } - public function getPackageV2Json(?UserInterface $user, string $package, bool $isDev = true, &$lastModified = null): array + public function getPackageV2Json(?UserInterface $user, string $package, bool $isDev = true): array { - $metadata = $this->getCachedPackageJson($user, $package) ?: - $this->getPackageJson($user, $package); - + $metadata = $this->getPackageJson($user, $package); if (empty($metadata)) { return []; } - return $this->metadataMinifier->minify($metadata, $isDev, $lastModified); + return $this->metadataMinifier->minify($metadata, $isDev); } /** @@ -200,37 +204,19 @@ public function getCachedPackageJson(?UserInterface $user, string $package, stri return $packages[$package]; } - private function dumpInMemory(UserInterface $user = null, bool $ignoreLastModify = true) + private function dumpInMemory(UserInterface $user = null, bool $ignoreLastModify = true, int $apiVersion = null) { if ($user && $this->authorizationChecker->isGranted('ROLE_FULL_CUSTOMER')) { $user = null; } - $cacheKey = 'pkg_user_cache_' . ($user ? $user->getUserIdentifier() : 0); - - $lastModify = false === $ignoreLastModify ? ($this->getLastModify()?->getTimestamp() ?: 0) : null; - return $this->cache->get($cacheKey, fn () => $this->dumper->dump($user), $lastModify); - } - - public function getPackageNames(): array - { - $cacheKey = 'all_packages_'; - $lastModify = $this->getLastModify()?->getTimestamp(); - $data = $this->cache->get($cacheKey, fn () => $this->doctrine->getRepository(Package::class)->getPackageNames(), $lastModify); - - return is_array($data) ? $data : []; - } - - public function setLastModify(): void - { - try { - $this->redis->set('packages-last-modify', time()); - } catch (\Exception) {} - } + $cacheKey = 'pkg_user_cache_' . $this->dumper->getFormat()->value . '_' . ($user ? $user->getUserIdentifier() : 0); - public function getLastModify(): ?\DateTimeInterface - { - $unix = $this->redis->get('packages-last-modify'); - return $unix ? \DateTime::createFromFormat('U', $unix) : null; + return $this->cache->get( + $cacheKey, + fn () => [...$this->dumper->dump($user, $apiVersion), $apiVersion], + false === $ignoreLastModify ? $this->providerManager->getRootLastModify()->getTimestamp() : null, + fn ($meta) => $ignoreLastModify === false && (($meta[4] ?? null) !== $apiVersion) + ); } } diff --git a/src/Model/ProviderManager.php b/src/Model/ProviderManager.php index 3b3520a1..d031739c 100644 --- a/src/Model/ProviderManager.php +++ b/src/Model/ProviderManager.php @@ -18,14 +18,18 @@ class ProviderManager { - protected $redis; - protected $registry; + public const DEV_UPDATED = 1; + public const STAB_UPDATED = 2; + protected $initializedProviders = false; - public function __construct(\Redis $redis, ManagerRegistry $registry) - { - $this->redis = $redis; - $this->registry = $registry; + protected $initializedPackages = []; + protected $initializedPackagesUnix = false; + + public function __construct( + protected \Redis $redis, + protected ManagerRegistry $registry + ) { } public function packageExists($name) @@ -45,20 +49,85 @@ public function packageIsProvided($name) return (bool) $this->redis->sismember('set:providers', strtolower($name)); } - public function getPackageNames() + public function getPackageNames(bool $reload = false): array { - if (!$this->redis->scard('set:packages')) { + $cacheKey = 'set:packages:last-modify'; + $lastModify = $this->getRootLastModify()->getTimestamp(); + if (false === $reload && $this->initializedPackagesUnix === $lastModify) { + return $this->initializedPackages; + } + + if (true === $reload || $lastModify !== (int)$this->redis->get($cacheKey)) { $names = $this->getRepo()->getPackageNames(); while ($names) { $nameSlice = array_splice($names, 0, 1000); $this->redis->sadd('set:packages', $nameSlice); } + + $this->redis->setOption($cacheKey, $lastModify); } $names = $this->redis->smembers('set:packages'); sort($names, SORT_STRING); + $this->initializedPackagesUnix = $lastModify; + + return $this->initializedPackages = $names; + } + + public function setRootLastModify(int $unix = null): void + { + try { + $this->redis->set('packages-last-modify', $unix ?: time()); + } catch (\Exception) {} + } + + public function getRootLastModify(): \DateTimeInterface + { + $unix = $this->redis->get('packages-last-modify'); + if (empty($unix) || !is_numeric($unix)) { + $this->setRootLastModify($unix = time()); + } + + return \DateTime::createFromFormat('U', (int)$unix); + } + + public function setLastModify(string $package, int $flags = null, int $unix = null): void + { + try { + $flags ??= 3; + $unix ??= time(); + $keys = []; + if ($flags & self::DEV_UPDATED) { + $keys[] = $package . '~dev'; + } + if ($flags & self::STAB_UPDATED) { + $keys[] = $package; + } + + foreach ($keys as $key) { + $this->redis->set('lm:'.$key, $unix); + } + } catch (\Exception) {} + } + + public function getLastModify(string $package, bool $isDev = null): \DateTimeInterface + { + $key = match ($isDev) { + $isDev === true => $package . '~dev', + default => $package, + }; + + $key = 'lm:'.$key; + $unix = $this->redis->get($key); + if (empty($unix) || !is_numeric($unix)) { + if (!$this->packageExists(preg_replace('/~dev$/', '', $package))) { + $unix = time(); + } else { + $this->setLastModify($package, null, $unix = time()); + } + } - return $names; + return \DateTime::createFromFormat('U', (int)$unix); } public function insertPackage(Package $package) @@ -69,6 +138,8 @@ public function insertPackage(Package $package) public function deletePackage(Package $package) { $this->redis->srem('set:packages', strtolower($package->getName())); + $this->redis->del('lm:'.$package->getName()); + $this->redis->del('lm:'.$package->getName().'~dev'); } private function populateProviders() diff --git a/src/Package/InMemoryDumper.php b/src/Package/InMemoryDumper.php index 53785b94..cff99897 100644 --- a/src/Package/InMemoryDumper.php +++ b/src/Package/InMemoryDumper.php @@ -5,6 +5,7 @@ namespace Packeton\Package; use Doctrine\Persistence\ManagerRegistry; +use Packeton\Composer\MetadataFormat; use Packeton\Entity\Group; use Packeton\Entity\Package; use Packeton\Entity\User; @@ -15,19 +16,32 @@ class InMemoryDumper { + private MetadataFormat $metadataFormat; + private ?string $infoMessage; + public function __construct( private readonly ManagerRegistry $registry, private readonly PackagesAclChecker $checker, - private readonly RouterInterface $router - ) {} + private readonly RouterInterface $router, + array $config = null, + ) { + $this->infoMessage = $config['info_cmd_message'] ?? null; + $this->metadataFormat = MetadataFormat::tryFrom((string) ($config['format'] ?? null)) ?: MetadataFormat::AUTO; + } /** * @param UserInterface|null $user + * @param int|null $apiVersion * @return array */ - public function dump(UserInterface $user = null): array + public function dump(UserInterface $user = null, int $apiVersion = null): array + { + return $this->dumpRootPackages($user, $apiVersion); + } + + public function getFormat(): MetadataFormat { - return $this->dumpRootPackages($user); + return $this->metadataFormat; } /** @@ -73,9 +87,9 @@ public function dumpPackage(?UserInterface $user, $package, array $versionData = return $packageData; } - private function dumpRootPackages(UserInterface $user = null) + private function dumpRootPackages(UserInterface $user = null, int $apiVersion = null) { - [$providers, $packagesData, $availablePackages] = $this->dumpUserPackages($user); + [$providers, $packagesData, $availablePackages] = $this->dumpUserPackages($user, $apiVersion); $rootFile = ['packages' => []]; $url = $this->router->generate('track_download', ['name' => 'VND/PKG']); @@ -86,19 +100,44 @@ private function dumpRootPackages(UserInterface $user = null) $rootFile['metadata-url'] = '/p2/%package%.json'; - $userHash = \hash('sha256', \json_encode($providers)); - $rootFile['provider-includes'] = [ - 'p/providers$%hash%.json' => [ - 'sha256' => $userHash - ] - ]; + if (null !== $providers) { + $userHash = \hash('sha256', \json_encode($providers)); + $rootFile['provider-includes'] = [ + 'p/providers$%hash%.json' => [ + 'sha256' => $userHash + ] + ]; + } + $rootFile['available-packages'] = $availablePackages; + if ($this->metadataFormat->lazyProviders($apiVersion)) { + unset($rootFile['provider-includes'], $rootFile['providers-url']); + $rootFile['providers-lazy-url'] = '/p/%package%.json'; + } + + if (false === $this->metadataFormat->metadataUrl($apiVersion)) { + unset($rootFile['metadata-url'], $rootFile['available-packages']); + } + + if ($this->infoMessage) { + $rootFile['info'] = $this->infoMessage; + } + return [$rootFile, $providers, $packagesData]; } - private function dumpUserPackages(UserInterface $user = null): array + private function dumpUserPackages(UserInterface $user = null, int $apiVersion = null): array { + if (false === $this->metadataFormat->providerIncludes($apiVersion)) { + $allowed = $user ? $this->registry->getRepository(Group::class) + ->getAllowedPackagesForUser($user, false) : null; + + $availablePackages = $this->registry->getRepository(Package::class)->getPackageNames($allowed); + + return [null, [], $availablePackages]; + } + $packages = $user ? $this->registry->getRepository(Group::class) ->getAllowedPackagesForUser($user) : diff --git a/src/Package/Updater.php b/src/Package/Updater.php index bb574b41..4b8d2afc 100644 --- a/src/Package/Updater.php +++ b/src/Package/Updater.php @@ -36,6 +36,7 @@ use Packeton\Entity\Version; use Packeton\Entity\SuggestLink; use Packeton\Event\UpdaterEvent; +use Packeton\Model\ProviderManager; use Packeton\Service\DistConfig; use Doctrine\DBAL\Connection; use Symfony\Component\EventDispatcher\EventDispatcherInterface; @@ -84,6 +85,7 @@ public function __construct( protected ManagerRegistry $doctrine, protected DistConfig $distConfig, protected PackagistFactory $packagistFactory, + protected ProviderManager $providerManager, protected EventDispatcherInterface $dispatcher, ) { ErrorHandler::register(); @@ -109,6 +111,7 @@ public function update(IOInterface $io, Config $config, Package $package, Reposi $start = new \DateTime(); } + $stabilityVersionUpdated = 0; $deleteDate = clone $start; $deleteDate->modify('-1day'); @@ -158,6 +161,7 @@ public function update(IOInterface $io, Config $config, Package $package, Reposi $versionRepository->remove($version); } + $stabilityVersionUpdated = 3; $em->flush(); $em->refresh($package); } @@ -196,6 +200,8 @@ public function update(IOInterface $io, Config $config, Package $package, Reposi if (!isset($result['id']) && $version instanceof Version) { $result['id'] = $version->getId(); } + + $stabilityVersionUpdated |= $lastProcessed->isDev() ? ProviderManager::DEV_UPDATED : ProviderManager::STAB_UPDATED; } else { $idsToMarkUpdated[] = $result['id']; } @@ -226,6 +232,7 @@ public function update(IOInterface $io, Config $config, Package $package, Reposi } else { // set it to be soft-deleted so next update that occurs after deleteDate (1day) if the // version is still missing it will be really removed + $stabilityVersionUpdated = 3; $em->getConnection()->executeStatement( 'UPDATE package_version SET softDeletedAt = :now WHERE id = :id', ['now' => date('Y-m-d H:i:s'), 'id' => $version['id']] @@ -244,7 +251,9 @@ public function update(IOInterface $io, Config $config, Package $package, Reposi } foreach ($deletedVersions as $versionId) { - $versionRepository->remove($versionRepository->findOneById($versionId)); + $stabilityVersionUpdated = 3; + $version = $versionRepository->find($versionId); + $versionRepository->remove($version); } if (preg_match('{^(?:git://|git@|https?://)github.com[:/]([^/]+)/(.+?)(?:\.git|/)?$}i', $package->getRepository(), $match) && $repository instanceof VcsRepository) { @@ -253,17 +262,26 @@ public function update(IOInterface $io, Config $config, Package $package, Reposi $this->updateReadme($io, $package, $repository); } + if ($stabilityVersionUpdated !== 0) { + $this->providerManager->setLastModify($package->getName(), $stabilityVersionUpdated); + } + $package->setUpdatedAt(new \DateTime('now', new \DateTimeZone('UTC'))); $package->setCrawledAt(new \DateTime('now', new \DateTimeZone('UTC'))); $em->flush(); - if ($repository->hadInvalidBranches()) { - throw new InvalidRepositoryException('Some branches contained invalid data and were discarded, it is advised to review the log and fix any issues present in branches'); - } if (true === $isNewPackage) { $this->dispatcher->dispatch(new UpdaterEvent($package, $flags), UpdaterEvent::PACKAGE_PERSIST); } + if (!$this->providerManager->packageExists($package->getName())) { + $this->providerManager->getPackageNames(true); + } + + if ($repository->hadInvalidBranches()) { + throw new InvalidRepositoryException('Some branches contained invalid data and were discarded, it is advised to review the log and fix any issues present in branches'); + } + return $package; }