diff --git a/composer.lock b/composer.lock index f2921aa5..6f127177 100644 --- a/composer.lock +++ b/composer.lock @@ -3116,16 +3116,16 @@ }, { "name": "okvpn/cron-bundle", - "version": "0.2.1", + "version": "0.2.3", "source": { "type": "git", "url": "https://github.com/vtsykun/cron-bundle.git", - "reference": "835df9afd2fcaf300de3246b5582e6f167b99a2d" + "reference": "9185eab70ff4e1443bcb211733284db40a53095c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/vtsykun/cron-bundle/zipball/835df9afd2fcaf300de3246b5582e6f167b99a2d", - "reference": "835df9afd2fcaf300de3246b5582e6f167b99a2d", + "url": "https://api.github.com/repos/vtsykun/cron-bundle/zipball/9185eab70ff4e1443bcb211733284db40a53095c", + "reference": "9185eab70ff4e1443bcb211733284db40a53095c", "shasum": "" }, "require": { @@ -3179,7 +3179,7 @@ "issues": "https://github.com/vtsykun/cron-bundle/issues", "source": "https://github.com/vtsykun/cron-bundle/releases" }, - "time": "2023-01-28T21:29:36+00:00" + "time": "2023-02-20T14:29:12+00:00" }, { "name": "oro/doctrine-extensions", diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 263b1881..ebca4220 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -51,7 +51,7 @@ if [ -f /var/tmp/data/handler.sh ]; then bash /var/tmp/data/handler.sh fi -mkdir var/cache var/log +mkdir -p var/cache var/log rm -rf var/cache/* app cache:clear diff --git a/public/packeton/css/main.css b/public/packeton/css/main.css index cca48766..88b77363 100644 --- a/public/packeton/css/main.css +++ b/public/packeton/css/main.css @@ -1747,3 +1747,9 @@ pre.github { border-radius: 3px; cursor: pointer; } + +@media (min-width: 1200px) { + .modal-lg { + width:1150px + } +} diff --git a/src/Composer/IO/BufferIO.php b/src/Composer/IO/BufferIO.php index 4612c043..9bb9a1fa 100644 --- a/src/Composer/IO/BufferIO.php +++ b/src/Composer/IO/BufferIO.php @@ -26,9 +26,9 @@ class BufferIO extends ConsoleIO protected $verbosityMatrixCapacity = 3; protected $verbosityMatrix = [ - IOInterface::VERBOSE => 3, - IOInterface::VERY_VERBOSE => 2, - IOInterface::DEBUG => 2, + IOInterface::VERBOSE => 10, + IOInterface::VERY_VERBOSE => 6, + IOInterface::DEBUG => 6, ]; public function __construct(string $input = '', int $verbosity = StreamOutput::VERBOSITY_NORMAL, ?OutputFormatterInterface $formatter = null) @@ -117,13 +117,14 @@ protected function dynamicVerbosity($verbosity) if ($verbosity === self::NORMAL && $this->verbosityMatrixCapacity > 0) { foreach ($this->verbosityMatrix as $verb => $value) { if ($value <= 0) { - $this->verbosityMatrix[$verb] = 2; + $this->verbosityMatrix[$verb] = 6; $this->verbosityMatrixCapacity--; } } } if (isset($this->verbosityMatrix[$verbosity]) && $this->verbosityMatrix[$verbosity] > 0) { + $this->verbosityMatrix[$verbosity]--; return self::NORMAL; } diff --git a/src/Composer/MetadataMinifier.php b/src/Composer/MetadataMinifier.php index 7ad7de81..2503d86a 100644 --- a/src/Composer/MetadataMinifier.php +++ b/src/Composer/MetadataMinifier.php @@ -14,7 +14,7 @@ class MetadataMinifier /** * Convert metadata v1 to metadata v2 */ - public function minify(array $metadata, bool $isDev = true, &$lastModified = null): array + public function minify(array $metadata, ?bool $isDev = true, &$lastModified = null): array { $packages = $metadata['packages'] ?? []; $metadata['minified'] = 'composer/2.0'; @@ -23,7 +23,7 @@ public function minify(array $metadata, bool $isDev = true, &$lastModified = nul $obj->time = '1970-01-01T00:00:00+00:00'; foreach ($packages as $packName => $versions) { - $versions = \array_filter($versions, fn($v) => $this->isValidStability($v, $isDev)); + $versions = \array_filter($versions, fn($v) => $isDev === null || $this->isValidStability($v, $isDev)); \usort($versions, fn($v1, $v2) => -1 * version_compare($v1['version_normalized'], $v2['version_normalized'])); \array_map(fn($v) => $obj->time < ($v['time'] ?? 0) ? $obj->time = ($v['time'] ?? 0) : null, $versions); diff --git a/src/Composer/PackagistFactory.php b/src/Composer/PackagistFactory.php index ffcc6339..d1b60c3b 100644 --- a/src/Composer/PackagistFactory.php +++ b/src/Composer/PackagistFactory.php @@ -14,7 +14,7 @@ use Packeton\Composer\Repository\VcsRepository; use Packeton\Composer\Util\ConfigFactory; use Packeton\Composer\Util\ProcessExecutor; -use Packeton\Entity\SshCredentials; +use Packeton\Model\CredentialsInterface; class PackagistFactory { @@ -71,10 +71,10 @@ public function createArchiveManager(IOInterface $io, RepositoryInterface $repos } /** - * @param SshCredentials|null $credentials + * @param CredentialsInterface|null $credentials * @return \Composer\Config */ - public function createConfig(SshCredentials $credentials = null) + public function createConfig(CredentialsInterface $credentials = null) { $config = ConfigFactory::createConfig(); @@ -97,6 +97,9 @@ public function createConfig(SshCredentials $credentials = null) ProcessExecutor::inheritEnv(['GIT_SSH_COMMAND']); $config->merge(['config' => ['ssh-key-file' => $credentialsFile]]); + } else if ($credentials->getPrivkeyFile()) { + putenv("GIT_SSH_COMMAND=ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {$credentials->getPrivkeyFile()}"); + ProcessExecutor::inheritEnv(['GIT_SSH_COMMAND']); } } @@ -107,12 +110,12 @@ public function createConfig(SshCredentials $credentials = null) * @param string $url * @param IOInterface|null $io * @param Config|null $config - * @param SshCredentials|null $credentials + * @param CredentialsInterface|null $credentials * @param array $repoConfig * * @return Repository\VcsRepository */ - public function createRepository(string $url, IOInterface $io = null, Config $config = null, SshCredentials $credentials = null, array $repoConfig = []) + public function createRepository(string $url, IOInterface $io = null, Config $config = null, CredentialsInterface $credentials = null, array $repoConfig = []) { $io = $io ?: new NullIO(); if (null === $config) { diff --git a/src/Controller/MirrorController.php b/src/Controller/MirrorController.php index 9b7107f4..6353f96f 100644 --- a/src/Controller/MirrorController.php +++ b/src/Controller/MirrorController.php @@ -61,9 +61,14 @@ public function metadataV2Action(string $package, string $alias, Request $reques $devStability = \str_ends_with($package, '~dev'); $package = \preg_replace('/~dev$/', '', $package); - $metadata = $this->wrap404Error($alias, fn (PRI $repo) => $repo->findPackageMetadata($package)); + $modifiedSince = $request->headers->get('If-Modified-Since'); + $modifiedSince = $modifiedSince ? (\strtotime($modifiedSince) ?: null) : null; + + $metadata = $this->wrap404Error($alias, fn (PRI $repo) => $repo->findPackageMetadata($package, $modifiedSince)); - $metadata = $metadata->withContent(fn ($package) => $this->minifier->minify($package, $devStability)); + if (false === $metadata->isNotModified()) { + $metadata = $metadata->withContent(fn ($package) => $this->minifier->minify($package, $devStability)); + } return $this->renderMetadata($metadata, $request); } @@ -114,6 +119,9 @@ protected function renderMetadata(JsonMetadata $metadata, Request $request, call $response = new Response($metadata->getContent(), 200, ['Content-Type' => 'application/json']); $response->setLastModified($metadata->lastModified()); $notModified = $response->isNotModified($request); + if ($metadata->isNotModified()) { + $response->setNotModified(); + } if (null !== $lazyLoad && false === $notModified) { $metadata = $lazyLoad($metadata); diff --git a/src/Controller/ProxiesController.php b/src/Controller/ProxiesController.php index 3669efbd..ea0a1ca9 100644 --- a/src/Controller/ProxiesController.php +++ b/src/Controller/ProxiesController.php @@ -107,7 +107,7 @@ public function metadata(HtmlJsonHuman $jsonHuman, string $alias, string $packag } $json = $meta->decodeJson(); - $metadata = $this->metadataMinifier->minify($json)['packages'][$package] ?? []; + $metadata = $this->metadataMinifier->minify($json, null)['packages'][$package] ?? []; return new Response($jsonHuman->buildToHtml($metadata)); } diff --git a/src/Cron/MirrorCronLoader.php b/src/Cron/MirrorCronLoader.php index ad24c474..012ee366 100644 --- a/src/Cron/MirrorCronLoader.php +++ b/src/Cron/MirrorCronLoader.php @@ -28,6 +28,10 @@ public function getSchedules(array $options = []): iterable foreach ($this->registry->getAllRepos() as $name => $repo) { if ($repo instanceof RemoteProxyRepository) { + if (!$repo->getPackageManager()->isAutoSync()) { + continue; + } + $repo->resetProxyOptions(); $config = $repo->getConfig(); $expr = '@random ' . $this->getSyncInterval($config); diff --git a/src/Entity/SshCredentials.php b/src/Entity/SshCredentials.php index 4d697912..32f1f54e 100644 --- a/src/Entity/SshCredentials.php +++ b/src/Entity/SshCredentials.php @@ -3,6 +3,7 @@ namespace Packeton\Entity; use Doctrine\ORM\Mapping as ORM; +use Packeton\Model\CredentialsInterface; /** * SshCredentials @@ -10,7 +11,7 @@ * @ORM\Table(name="ssh_credentials") * @ORM\Entity() */ -class SshCredentials implements OwnerAwareInterface +class SshCredentials implements OwnerAwareInterface, CredentialsInterface { /** * @var int @@ -122,7 +123,7 @@ public function setKey($key) * * @return string */ - public function getKey() + public function getKey(): ?string { return $this->key; } @@ -180,7 +181,7 @@ public function getFingerprint() * * @return mixed|null */ - public function getComposerConfigOption(string $name) + public function getComposerConfigOption(string $name): mixed { return $this->composerConfig[$name] ?? null; } @@ -228,4 +229,9 @@ public function getVisibility(): ?string { return OwnerAwareInterface::STRICT_VISIBLE; } + + public function getPrivkeyFile(): ?string + { + return null; + } } diff --git a/src/Form/Type/ProxySettingsType.php b/src/Form/Type/ProxySettingsType.php index 6c93011e..0bb67faa 100644 --- a/src/Form/Type/ProxySettingsType.php +++ b/src/Form/Type/ProxySettingsType.php @@ -25,7 +25,15 @@ public function buildForm(FormBuilderInterface $builder, array $options): void 'All (new packages are automatically added when requested by composer)' => false, ] ]) + ->add('disable_v2', ChoiceType::class, [ + 'label' => 'Disable Composer API v2', + 'choices' => [ + 'No' => false, + 'Yes' => true, + ] + ]) ->add('enabled_sync', CheckboxType::class, [ + 'required' => false, 'label' => 'Enable automatically synchronization', ]); } diff --git a/src/Mirror/AbstractProxyRepository.php b/src/Mirror/AbstractProxyRepository.php index 8013bc85..03d9b942 100644 --- a/src/Mirror/AbstractProxyRepository.php +++ b/src/Mirror/AbstractProxyRepository.php @@ -21,7 +21,7 @@ public function getConfig(): ProxyOptions { return $this->proxyOptions ??= new ProxyOptions( $this->repoConfig - + ['root' => $this->rootMetadata()?->decodeJson()] + + ['root' => $this->getRootMetadataInfo()] + ['stats' => $this->getStats()] ); } @@ -29,7 +29,7 @@ public function getConfig(): ProxyOptions /** * {@inheritdoc} */ - public function findPackageMetadata(string $name): ?JsonMetadata + public function findPackageMetadata(string $name, int $modifiedSince = null): ?JsonMetadata { return null; } @@ -37,7 +37,7 @@ public function findPackageMetadata(string $name): ?JsonMetadata /** * {@inheritdoc} */ - public function findProviderMetadata(string $name): ?JsonMetadata + public function findProviderMetadata(string $name, int $modifiedSince = null): ?JsonMetadata { return null; } @@ -45,7 +45,7 @@ public function findProviderMetadata(string $name): ?JsonMetadata /** * {@inheritdoc} */ - public function rootMetadata(): ?JsonMetadata + public function rootMetadata(int $modifiedSince = null): ?JsonMetadata { return null; } @@ -63,4 +63,15 @@ public function resetProxyOptions(): void { $this->proxyOptions = null; } + + protected function getRootMetadataInfo(): array + { + if ($meta = $this->rootMetadata()) { + $data = $meta->decodeJson(); + $data['modified_since'] = $meta->lastModified()->getTimestamp(); + return $data; + } + + return []; + } } diff --git a/src/Mirror/Decorator/AbstractProxyRepositoryDecorator.php b/src/Mirror/Decorator/AbstractProxyRepositoryDecorator.php index f235abbc..35fb671b 100644 --- a/src/Mirror/Decorator/AbstractProxyRepositoryDecorator.php +++ b/src/Mirror/Decorator/AbstractProxyRepositoryDecorator.php @@ -10,17 +10,17 @@ abstract class AbstractProxyRepositoryDecorator implements StrictProxyRepositoryInterface { - public function findPackageMetadata(string $nameOrUri): JsonMetadata + public function findPackageMetadata(string $nameOrUri, int $modifiedSince = null): JsonMetadata { throw new MetadataNotFoundException('Not found'); } - public function findProviderMetadata(string $nameOrUri): JsonMetadata + public function findProviderMetadata(string $nameOrUri, int $modifiedSince = null): JsonMetadata { throw new MetadataNotFoundException('Not found'); } - public function rootMetadata(): JsonMetadata + public function rootMetadata(int $modifiedSince = null): JsonMetadata { throw new MetadataNotFoundException('Not found'); } diff --git a/src/Mirror/Decorator/ProxyRepositoryACLDecorator.php b/src/Mirror/Decorator/ProxyRepositoryACLDecorator.php index 6c602bdc..30c71b27 100644 --- a/src/Mirror/Decorator/ProxyRepositoryACLDecorator.php +++ b/src/Mirror/Decorator/ProxyRepositoryACLDecorator.php @@ -5,7 +5,6 @@ namespace Packeton\Mirror\Decorator; use Packeton\Mirror\Exception\ApproveRestrictException; -use Packeton\Mirror\Exception\MetadataNotFoundException; use Packeton\Mirror\Model\ApprovalRepoInterface; use Packeton\Mirror\Model\JsonMetadata; use Packeton\Mirror\Model\StrictProxyRepositoryInterface as RPI; @@ -30,16 +29,18 @@ public function __construct( /** * {@inheritdoc} */ - public function rootMetadata(): JsonMetadata + public function rootMetadata(int $modifiedSince = null): JsonMetadata { - $metadata = $this->repository->rootMetadata(); + $metadata = $this->repository->rootMetadata($modifiedSince); + $metadata->setOptions($this->approval->getSettings()); + if ($this->approval->requireApprove()) { $approved = $this->approval->getApproved(); $metadata->setOption('available_packages', $approved); if (null !== $this->remote) { $metadata->setOption('includes', function () use ($approved) { - [$includes] = IncludeV1ApiMetadata::buildInclude($approved, $this->remote); + [$includes] = IncludeV1ApiMetadata::buildIncludes($approved, $this->remote); return $includes; }); } @@ -51,25 +52,25 @@ public function rootMetadata(): JsonMetadata /** * {@inheritdoc} */ - public function findProviderMetadata(string $nameOrUri): JsonMetadata + public function findProviderMetadata(string $nameOrUri, int $modifiedSince = null): JsonMetadata { if (\str_starts_with($nameOrUri, 'include-packeton/all$') && null !== $this->remote) { $approved = $this->approval->getApproved(); - [$includes, $content] = IncludeV1ApiMetadata::buildInclude($approved, $this->remote); + [$includes, $content] = IncludeV1ApiMetadata::buildIncludes($approved, $this->remote); if (isset($includes[$nameOrUri])) { return new JsonMetadata($content); } } - return $this->repository->findProviderMetadata($nameOrUri); + return $this->repository->findProviderMetadata($nameOrUri, $modifiedSince); } /** * {@inheritdoc} */ - public function findPackageMetadata(string $nameOrUri): JsonMetadata + public function findPackageMetadata(string $nameOrUri, int $modifiedSince = null): JsonMetadata { - $metadata = $this->repository->findPackageMetadata($nameOrUri); + $metadata = $this->repository->findPackageMetadata($nameOrUri, $modifiedSince); [$package, ] = \explode('$', $nameOrUri); if ($this->approval->requireApprove()) { diff --git a/src/Mirror/Decorator/ProxyRepositoryFacade.php b/src/Mirror/Decorator/ProxyRepositoryFacade.php index 192caeee..70869107 100644 --- a/src/Mirror/Decorator/ProxyRepositoryFacade.php +++ b/src/Mirror/Decorator/ProxyRepositoryFacade.php @@ -33,7 +33,7 @@ public function __construct( /** * {@inheritdoc} */ - public function findPackageMetadata(string $nameOrUri): JsonMetadata + public function findPackageMetadata(string $nameOrUri, int $modifiedSince = null): JsonMetadata { [$package, ] = \explode('$', $nameOrUri); @@ -53,7 +53,7 @@ public function findPackageMetadata(string $nameOrUri): JsonMetadata /** * {@inheritdoc} */ - public function findProviderMetadata(string $nameOrUri): JsonMetadata + public function findProviderMetadata(string $nameOrUri, int $modifiedSince = null): JsonMetadata { return $this->fetch(__FUNCTION__, func_get_args(), function () { throw new MetadataNotFoundException('Not found'); @@ -63,7 +63,7 @@ public function findProviderMetadata(string $nameOrUri): JsonMetadata /** * {@inheritdoc} */ - public function rootMetadata(): JsonMetadata + public function rootMetadata(int $modifiedSince = null): JsonMetadata { $metadata = $this->fetch(__FUNCTION__, [], function () { try { diff --git a/src/Mirror/Manager/RootMetadataMerger.php b/src/Mirror/Manager/RootMetadataMerger.php index e6270c38..25281e4b 100644 --- a/src/Mirror/Manager/RootMetadataMerger.php +++ b/src/Mirror/Manager/RootMetadataMerger.php @@ -21,7 +21,7 @@ public function merge(JsonMetadata $stamps, int $composerApi = null): JsonMetada $config = $stamps->getOptions(); // To avoid call parent host - unset($rootFile['providers-api']); + unset($rootFile['providers-api'], $rootFile['search'], $rootFile['list']); if (!$config->parentNotify()) { unset($rootFile['notify-batch']); @@ -69,6 +69,11 @@ public function merge(JsonMetadata $stamps, int $composerApi = null): JsonMetada $rootFile['providers-lazy-url'] = \str_replace('VND/PKG','%package%', $url); } + if ($config->disableV2Format()) { + $composerApi = 1; + unset($newFile['metadata-url'], $rootFile['metadata-url']); + } + // generate lazy load includes if enabled composer strict approve mode. if ($config->disableV1Format() === false && ComposerApi::API_V1 === $composerApi && ($includes = $config->getIncludes())) { unset($rootFile['provider-includes'], $rootFile['providers-url'], $rootFile['providers-lazy-url']); diff --git a/src/Mirror/Model/ApprovalRepoInterface.php b/src/Mirror/Model/ApprovalRepoInterface.php index 351b2974..6f37efbb 100644 --- a/src/Mirror/Model/ApprovalRepoInterface.php +++ b/src/Mirror/Model/ApprovalRepoInterface.php @@ -13,6 +13,11 @@ interface ApprovalRepoInterface */ public function getApproved(): array; + /** + * @return array + */ + public function getSettings(): array; + /** * Remove approve for package. * diff --git a/src/Mirror/Model/HttpMetadataTrait.php b/src/Mirror/Model/HttpMetadataTrait.php index ba72880d..bac75e8f 100644 --- a/src/Mirror/Model/HttpMetadataTrait.php +++ b/src/Mirror/Model/HttpMetadataTrait.php @@ -5,6 +5,7 @@ namespace Packeton\Mirror\Model; use Composer\Downloader\TransportException; +use Composer\IO\IOInterface; use Composer\Util\Http\Response; use Composer\Util\HttpDownloader; use Packeton\Composer\Util\SignalLoop; @@ -14,10 +15,12 @@ trait HttpMetadataTrait { protected ?SignalHandler $signal = null; + private $errorMap = []; + /** * @throws TransportException */ - private function requestMetadataVia2(HttpDownloader $downloader, iterable $packages, string $url, callable $onFulfilled, callable $onReject = null): void + private function requestMetadataVia2(HttpDownloader $downloader, iterable $packages, string $url, callable $onFulfilled, callable $onReject = null, callable $lastModifyLoader = null): void { $queue = new \ArrayObject(); $loop = new SignalLoop($downloader, $this->signal); @@ -33,13 +36,24 @@ private function requestMetadataVia2(HttpDownloader $downloader, iterable $packa foreach ($packages as $package) { $requester = function (Response $response) use ($package, &$queue, &$onFulfilled) { + if ($response->getStatusCode() > 299) { + return; + } + + try { + $body = $response->decodeJson(); + } catch (\Throwable $e) { + $this->handleUnexpectedException($e); + return; + } + if (isset($queue[$package])) { - $metadata = $this->metadataMinifier->expand($queue[$package], $response->decodeJson()); + $metadata = $this->metadataMinifier->expand($queue[$package], $body); unset($queue[$package]); $onFulfilled($package, $metadata); } else { - $queue[$package] = $response->decodeJson(); + $queue[$package] = $body; } }; @@ -47,8 +61,23 @@ private function requestMetadataVia2(HttpDownloader $downloader, iterable $packa return $onReject($e, $package); }; - $promise[] = $downloader->add(\str_replace('%package%', $package, $url))->then($requester, $reject); - $promise[] = $downloader->add(\str_replace('%package%', $package . '~dev', $url))->then($requester, $reject); + $options = []; + if ($lastModified = $lastModifyLoader ? $lastModifyLoader($package) : null) { + $headers = ['If-Modified-Since: '.$lastModified]; + $options = ['http' => ['header' => $headers]]; + } + + $promise[] = $downloader->add(\str_replace('%package%', $package, $url), $options)->then($requester, $reject); + $promise[] = $downloader->add(\str_replace('%package%', $package . '~dev', $url), $options)->then($requester, $reject); + } + + $loop->wait($promise); + + $promise = []; + // Try without last modify only uncompleted. + if ($lastModifyLoader && $queue->count() > 0) { + $packages = \array_keys($queue); + $this->requestMetadataVia2($downloader, $packages, $url, $onFulfilled, $onReject); } $loop->wait($promise); @@ -99,4 +128,32 @@ function ($e) use ($package, $onReject) { $loop->wait($promise); } + + private function onErrorIgnore(IOInterface $io): callable + { + return function ($e, $package) use ($io) { + if ($e instanceof TransportException) { + $code = $e->getStatusCode() === 404; + if (\in_array($code, [401, 403])) { + throw $e; + } + + $this->errorMap[$code] = ($this->errorMap[$code] ?? 0) + 1; + if ($this->errorMap[$code] < 12) { + $io->error("[$code] [$package] " . $e->getMessage()); + } else if ($this->errorMap[$code] === 12) { + $io->critical("A lot of exceptions [$code]. Stop logging."); + } + + return false; + } + + $this->handleUnexpectedException($e); + return false; + }; + } + + private function handleUnexpectedException($e): void + { + } } diff --git a/src/Mirror/Model/JsonMetadata.php b/src/Mirror/Model/JsonMetadata.php index e0a0ca8e..021f0363 100644 --- a/src/Mirror/Model/JsonMetadata.php +++ b/src/Mirror/Model/JsonMetadata.php @@ -8,6 +8,8 @@ class JsonMetadata { use GZipTrait; + protected bool $notModified = false; + public function __construct( private string $content, private ?int $unix = null, @@ -24,6 +26,11 @@ public function lastModified(): \DateTimeInterface return (new \DateTime('now', new \DateTimeZone('UTC')))->setTimestamp($this->unix); } + public function isNotModified(): bool + { + return $this->notModified; + } + public function getContent(): string { return $this->decode($this->content); @@ -40,6 +47,11 @@ public function hash(): ?string return $this->hash; } + public function setOptions(array $options): void + { + $this->options = \array_merge($this->options, $options); + } + public function setOption(string $name, mixed $value): void { $this->options[$name] = $value; @@ -64,4 +76,12 @@ public function withContent(string|array|callable $content, int $flags = \JSON_U return $clone; } + + public static function createNotModified(int $unix): static + { + $object = new static('', $unix); + $object->notModified = true; + + return $object; + } } diff --git a/src/Mirror/Model/MetadataOptions.php b/src/Mirror/Model/MetadataOptions.php index 94c56079..3cb88c81 100644 --- a/src/Mirror/Model/MetadataOptions.php +++ b/src/Mirror/Model/MetadataOptions.php @@ -34,6 +34,11 @@ public function disableV1Format(): bool return $this->config['disable_v1'] ?? false; } + public function disableV2Format(): bool + { + return $this->config['disable_v2'] ?? false; + } + public function isDistMirror(): bool { return $this->config['enable_dist_mirror'] ?? true; diff --git a/src/Mirror/Model/ProxyOptions.php b/src/Mirror/Model/ProxyOptions.php index dc20bd17..f978c42f 100644 --- a/src/Mirror/Model/ProxyOptions.php +++ b/src/Mirror/Model/ProxyOptions.php @@ -60,11 +60,6 @@ public function getMetadataV1Url(string $package = null, string $hash = null): ? return \str_replace(['%package%', '%hash%'], [$package ?? '%package%', $hash ?? '%hash%'], $url); } - public function getRoot(): array - { - return $this->config['root'] ?? []; - } - public function getRootProviders(): array { return $this->config['root']['providers'] ?? []; @@ -93,15 +88,38 @@ public function getSyncInterval(): ?int return $this->config['sync_interval'] ?? null; } + public function lastModifiedUnix(): int + { + return (int)($this->config['root']['modified_since'] ?? \time()); + } + + public function lastModified(): \DateTimeInterface + { + return new \DateTime('@'. $this->lastModifiedUnix()); + } + + public function matchCaps(RepoCaps|array $match): bool + { + $caps = $this->capabilities(); + $match = \is_array($match) ? $match : [$match]; + foreach ($match as $cap) { + if (\in_array($cap, $caps, true)) { + return true; + } + } + + return false; + } + public function capabilities(): array { $flags = [ - isset($this->config['root']['metadata-url']) ? 'API_V2' : null, - isset($this->config['root']['providers-url']) || isset($this->config['root']['providers-lazy-url']) ? 'API_V1' : null, - isset($this->config['root']['metadata-changes-url']) ? 'API_META_CHANGE' : null, - !isset($this->config['root']['providers-url']) && isset($this->config['root']['providers-lazy-url']) ? 'API_V1_LAZY' : null, - ($this->config['root']['packages'] ?? []) ? 'API_V1_PACKAGES' : null, - ($this->config['root']['includes'] ?? []) ? 'API_V1_INCLUDES' : null, + isset($this->config['root']['metadata-url']) ? RepoCaps::V2 : null, + isset($this->config['root']['providers-url']) || isset($this->config['root']['providers-lazy-url']) ? RepoCaps::V1 : null, + isset($this->config['root']['metadata-changes-url']) ? RepoCaps::META_CHANGE : null, + !isset($this->config['root']['providers-url']) && isset($this->config['root']['providers-lazy-url']) ? RepoCaps::LAZY : null, + ($this->config['root']['packages'] ?? ($this->config['root']['__packages'] ?? null)) ? RepoCaps::PACKAGES : null, + ($this->config['root']['includes'] ?? []) ? RepoCaps::INCLUDES : null, ]; return \array_values(\array_filter($flags)); diff --git a/src/Mirror/Model/ProxyRepositoryInterface.php b/src/Mirror/Model/ProxyRepositoryInterface.php index 55cd8908..618edb8a 100644 --- a/src/Mirror/Model/ProxyRepositoryInterface.php +++ b/src/Mirror/Model/ProxyRepositoryInterface.php @@ -10,22 +10,26 @@ interface ProxyRepositoryInterface * Base package metadata. * * @param string $nameOrUri + * @param int $modifiedSince * @return JsonMetadata|null */ - public function findPackageMetadata(string $nameOrUri): ?JsonMetadata; + public function findPackageMetadata(string $nameOrUri, int $modifiedSince = null): ?JsonMetadata; /** * Provider include metadata * * @param string $nameOrUri + * @param int $modifiedSince + * * @return JsonMetadata|null */ - public function findProviderMetadata(string $nameOrUri): ?JsonMetadata; + public function findProviderMetadata(string $nameOrUri, int $modifiedSince = null): ?JsonMetadata; /** * Get composer root * + * @param int $modifiedSince * @return JsonMetadata|null */ - public function rootMetadata(): ?JsonMetadata; + public function rootMetadata(int $modifiedSince = null): ?JsonMetadata; } diff --git a/src/Mirror/Model/RepoCaps.php b/src/Mirror/Model/RepoCaps.php new file mode 100644 index 00000000..c0b82079 --- /dev/null +++ b/src/Mirror/Model/RepoCaps.php @@ -0,0 +1,15 @@ +providersDir = $this->mirrorRepoMetaDir . $this->ds . 'p' . $this->ds; $this->packageDir = $this->mirrorRepoMetaDir . $this->ds . 'package' . $this->ds; + $this->redisRootKey = "proxy-root-{$this->repoConfig['name']}"; + $this->redisStatKey = "proxy-info-{$this->repoConfig['name']}"; + $this->zipballManager->setRepository($this); } @@ -60,7 +69,7 @@ public function getUrl(string $path = null): string /** * {@inheritdoc} */ - public function rootMetadata(): ?JsonMetadata + public function rootMetadata(int $modifiedSince = null): ?JsonMetadata { if ($this->filesystem->exists($this->rootFilename)) { return $this->createMetadataFromFile($this->rootFilename); @@ -72,7 +81,7 @@ public function rootMetadata(): ?JsonMetadata /** * {@inheritdoc} */ - public function findProviderMetadata(string $providerName): ?JsonMetadata + public function findProviderMetadata(string $providerName, int $modifiedSince = null): ?JsonMetadata { $filename = $this->providersDir . $this->providerKey($providerName); @@ -86,27 +95,31 @@ public function findProviderMetadata(string $providerName): ?JsonMetadata /** * {@inheritdoc} */ - public function findPackageMetadata(string $name): ?JsonMetadata + public function findPackageMetadata(string $name, int $modifiedSince = null): ?JsonMetadata { @[$package, $hash] = \explode('$', $name); - // Load packages data from root. - if ($packages = $this->lookIncludePackageMetadata($this->getConfig()->getRoot(), $package)) { - $content = \json_encode(['packages' => [$package => $packages]], \JSON_UNESCAPED_SLASHES); - $unix = @\filemtime($this->rootFilename) ?: null; - return new JsonMetadata($content, $unix, null, $this->repoConfig); + $config = $this->getConfig(); + // Satis API. Check includes without loading big data + if ($config->matchCaps([RepoCaps::INCLUDES, RepoCaps::PACKAGES])) { + if ($modifiedSince && $config->lastModifiedUnix() <= $modifiedSince) { + return JsonMetadata::createNotModified($config->lastModifiedUnix()); + } + + $root = $this->rootMetadata()?->decodeJson() ?: []; + if ($packages = $this->lookIncludePackageMetadata($root, $package)) { + $content = \json_encode(['packages' => [$package => $packages]], \JSON_UNESCAPED_SLASHES); + $unix = $config->lastModifiedUnix(); + return new JsonMetadata($content, $unix, null, $this->repoConfig); + } } - $packageName = \explode('/', $package)[1]; $filename = $this->packageDir . $this->packageKey($package, $hash); - if ($this->filesystem->exists($filename)) { - return $this->createMetadataFromFile($filename, $hash); + return $this->createMetadataFromFile($filename, $hash, $modifiedSince); } - $dir = \rtrim(\dirname($filename), $this->ds) . $this->ds; - $last = $this->filesystem->globLast($dir . $packageName . self::HASH_SEPARATOR . '*'); - return $last ? $this->createMetadataFromFile($last) : null; + return null; } // See loadIncludes in the ComposerRepository @@ -144,26 +157,62 @@ protected function lookIncludePackageMetadata(array $data, string $package): arr return $packages; } - protected function createMetadataFromFile(string $filename, string $hash = null): JsonMetadata + protected function createMetadataFromFile(string $filename, string $hash = null, int $modifiedSince = null): JsonMetadata { - $content = \file_get_contents($filename); - $unix = \filemtime($filename) ?: null; + $unix = @\filemtime($filename) ?: null; + if ($modifiedSince && $unix && $unix <= $modifiedSince) { + return JsonMetadata::createNotModified($modifiedSince); + } + $content = \file_get_contents($filename); return new JsonMetadata($content, $unix, $hash, $this->repoConfig); } public function dumpRootMeta(array $root): void { - $rootJson = \json_encode($root, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + $rootJson = \json_encode($root, \JSON_UNESCAPED_SLASHES); $this->filesystem->dumpFile($this->rootFilename, $rootJson); + $this->updateRootStats($root); + $this->proxyOptions = null; } + protected function updateRootStats(array $root = null): array + { + $root ??= $this->rootMetadata()?->decodeJson() ?: []; + if ($root) { + $modifiedSince = @\filemtime($this->rootFilename) ?: \time(); + $root['__packages'] = (bool)($root['packages'] ?? ($root['__packages'] ?? false)); + $root['modified_since'] = $modifiedSince; + unset($root['packages']); + + $this->redis->set($this->redisRootKey, \json_encode($root)); + return $root; + } + + return []; + } + + /** + * {@inheritdoc} + */ + protected function getRootMetadataInfo(): array + { + $root = $this->redis->get($this->redisRootKey); + $root = $root ? \json_decode($root, true) : []; + + if (empty($root) && ($root = parent::getRootMetadataInfo())) { + $root = $this->updateRootStats($root); + } + + return $root; + } + public function isRootFresh(array $root): bool { if ($this->filesystem->exists($this->rootFilename)) { - $rootJson = \json_encode($root, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); - return \sha1_file($this->rootFilename) === sha1($rootJson); + $rootJson = \json_encode($root, \JSON_UNESCAPED_SLASHES); + return \sha1_file($this->rootFilename) === \sha1($rootJson); } return false; } @@ -172,6 +221,13 @@ public function touchRoot(): void { if ($this->filesystem->exists($this->rootFilename)) { $this->filesystem->touch($this->rootFilename); + + $root = $this->redis->get($this->redisRootKey); + if ($root = $root ? \json_decode($root, true) : null) { + $modifiedSince = @\filemtime($this->rootFilename) ?: \time(); + $root['modified_since'] = $modifiedSince; + $this->redis->set($this->redisRootKey, \json_encode($root)); + } } } @@ -181,6 +237,12 @@ public function hasPackage(string $package, string $hash = null): bool return $this->filesystem->exists($filename); } + public function packageModifiedSince(string $package): ?int + { + $filename = $this->packageDir . $this->packageKey($package); + return $this->filesystem->exists($filename) ? (@\filemtime($filename) ?: null) : null; + } + public function dumpPackage(string $package, string|array|null $content, ?string $hash = null): void { $content = \is_array($content) ? \json_encode($content, \JSON_UNESCAPED_SLASHES) : $content; @@ -191,6 +253,11 @@ public function dumpPackage(string $package, string|array|null $content, ?string $filename = $this->packageDir . $this->packageKey($package, $hash); $this->filesystem->dumpFile($filename, $content); + + if ($hash !== null) { + $filename = $this->packageDir . $this->packageKey($package); + $this->filesystem->dumpFile($filename, $content); + } } public function hasProvider(string $uri): bool @@ -209,9 +276,9 @@ public function dumpProvider(string $uri, string|array $content): void $this->filesystem->dumpFile($filename, $content); } - public function lookupAllProviders(): iterable + public function lookupAllProviders(ProxyOptions $config = null): iterable { - $config = $this->getConfig(); + $config ??= $this->getConfig(); if ($config->getRootProviders()) { yield $config->getRootProviders(); } @@ -234,22 +301,28 @@ public function getRootDir(): string public function getStats(): array { - $stats = $this->redis->get("proxy-info-{$this->repoConfig['name']}"); + $stats = $this->redis->get($this->redisStatKey); $stats = $stats ? \json_decode($stats, true) : []; return \is_array($stats) ? $stats : []; } + public function clearAll(): void + { + $this->clearStats(); + $this->redis->del($this->redisRootKey); + } + public function clearStats(array $stats = []): void { - $this->redis->set("proxy-info-{$this->repoConfig['name']}", \json_encode($stats)); + $this->redis->set($this->redisStatKey, \json_encode($stats)); } public function setStats(array $stats = []): void { $stats = \array_merge($this->getStats(), $stats); - $this->redis->set("proxy-info-{$this->repoConfig['name']}", \json_encode($stats, \JSON_UNESCAPED_SLASHES)); + $this->redis->set($this->redisStatKey, \json_encode($stats, \JSON_UNESCAPED_SLASHES)); } public function getPackageManager(): RemotePackagesManager diff --git a/src/Mirror/Service/Filesystem.php b/src/Mirror/Service/Filesystem.php deleted file mode 100644 index 908de4ad..00000000 --- a/src/Mirror/Service/Filesystem.php +++ /dev/null @@ -1,35 +0,0 @@ -glob($pattern); - - $old = 0; - $selected = null; - foreach ($list as $file) { - $filename = \str_contains($file, '/') ? $file : $dir . $file; - $unix = \filemtime($filename); - if ($unix > $old) { - $selected = $filename; - $old = $unix; - } - } - - return $selected; - } -} diff --git a/src/Mirror/Service/RemotePackagesManager.php b/src/Mirror/Service/RemotePackagesManager.php index bb366622..d7202328 100644 --- a/src/Mirror/Service/RemotePackagesManager.php +++ b/src/Mirror/Service/RemotePackagesManager.php @@ -22,6 +22,7 @@ public function getSettings(): array return [ 'strict_mirror' => (bool)($settings['strict_mirror'] ?? false), 'enabled_sync' => (bool)($settings['enabled_sync'] ?? true), + 'disable_v2' => (bool)($settings['disable_v2'] ?? false), ]; } diff --git a/src/Mirror/Service/RemoteSyncProxiesFacade.php b/src/Mirror/Service/RemoteSyncProxiesFacade.php index 0d11e6d0..8f4536d2 100644 --- a/src/Mirror/Service/RemoteSyncProxiesFacade.php +++ b/src/Mirror/Service/RemoteSyncProxiesFacade.php @@ -39,6 +39,7 @@ public function sync(RPR $repo, IOInterface $io, int $flags = 0, SignalHandler $ } finally { $this->signal = null; $this->syncProvider->reset(); + $this->errorMap = []; } } @@ -54,7 +55,7 @@ private function doSync(RPR $repo, IOInterface $io, int $flags = 0): array $cfs = new \Composer\Util\Filesystem(); $cfs->remove($repo->getRootDir()); - $repo->clearStats(); + $repo->clearAll(); $io->notice('All data removed!'); } @@ -67,8 +68,11 @@ private function doSync(RPR $repo, IOInterface $io, int $flags = 0): array } if ($config->isLazy() && $config->hasV2Api()) { - $stats += $this->syncLazyV2($repo, $io, $config); - $repo->dumpRootMeta($root); + $stats += $this->syncLazyV2($repo, $io, $config, $updated); + if ($updated > 0) { + $repo->dumpRootMeta($root); + } + return $stats; } @@ -96,7 +100,7 @@ private function doSync(RPR $repo, IOInterface $io, int $flags = 0): array $availablePackages = \array_merge( $this->loadRootPackagesNames($root, $repo), - $this->loadProviderPackagesNames($repo, $config->maxCountOfAvailablePackages()) + $this->loadProviderPackagesNames($repo, $config->maxCountOfAvailablePackages(), $config) ); $stats['available_packages'] = \count($availablePackages) < $config->maxCountOfAvailablePackages() ? $availablePackages : []; @@ -143,10 +147,10 @@ private function doSync(RPR $repo, IOInterface $io, int $flags = 0): array return $stats; } - private function loadProviderPackagesNames(RPR $repo, int $limit): array + private function loadProviderPackagesNames(RPR $repo, int $limit, ProxyOptions $config): array { $packages = []; - foreach ($repo->lookupAllProviders() as $providerInclude) { + foreach ($repo->lookupAllProviders($config) as $providerInclude) { $packages = \array_merge($packages, \array_keys($providerInclude)); if (\count($packages) > $limit) { break; @@ -222,12 +226,12 @@ private function syncLazyV1(RPR $repo, IOInterface $io, ProxyOptions $config): a return $stats; } - $this->requestMetadataVia1($http, $packages, $config->getMetadataV1Url(), $onFulfilled, null, $repo->lookupAllProviders()); + $this->requestMetadataVia1($http, $packages, $config->getMetadataV1Url(), $onFulfilled, null, $repo->lookupAllProviders($config)); return $stats; } - private function syncLazyV2(RPR $repo, IOInterface $io, ProxyOptions $config): array + private function syncLazyV2(RPR $repo, IOInterface $io, ProxyOptions $config, &$updated = null): array { $stats = []; $rmp = $repo->getPackageManager(); @@ -270,10 +274,14 @@ private function syncLazyV2(RPR $repo, IOInterface $io, ProxyOptions $config): a $updated = 0; $onFulfilled = static function (string $name, $meta) use ($repo, &$updated) { $updated++; - $repo->dumpPackage($name, $meta); + // $repo->dumpPackage($name, $meta); + }; + + $modifiedSinceLoader = static function($package) use ($repo) { + return $repo->packageModifiedSince($package); }; - $this->requestMetadataVia2($http, $packages, $config->getMetadataV2Url(), $onFulfilled); + $this->requestMetadataVia2($http, $packages, $config->getMetadataV2Url(), $onFulfilled, $this->onErrorIgnore($io), $modifiedSinceLoader); $io->info('Updated packages: ' . $updated); diff --git a/src/Mirror/Service/ZipballDownloadManager.php b/src/Mirror/Service/ZipballDownloadManager.php index 3fb1ce51..1df16530 100644 --- a/src/Mirror/Service/ZipballDownloadManager.php +++ b/src/Mirror/Service/ZipballDownloadManager.php @@ -7,6 +7,7 @@ use Composer\Package\Loader\ArrayLoader; use Packeton\Mirror\Exception\MetadataNotFoundException; use Packeton\Mirror\RemoteProxyRepository; +use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\PropertyAccess\PropertyAccess; class ZipballDownloadManager diff --git a/src/Mirror/Utils/IncludeV1ApiMetadata.php b/src/Mirror/Utils/IncludeV1ApiMetadata.php index 08a9d031..03cddfb5 100644 --- a/src/Mirror/Utils/IncludeV1ApiMetadata.php +++ b/src/Mirror/Utils/IncludeV1ApiMetadata.php @@ -8,14 +8,24 @@ class IncludeV1ApiMetadata { - public static function buildInclude(array $packages, RemoteProxyRepository $repository): array + public static function buildIncludes(array $packages, RemoteProxyRepository $repository): array + { + return static::buildIncludesLazy( + $packages, + fn ($p) => $repository->findPackageMetadata($p)?->decodeJson()['packages'][$p] ?? null + ); + } + + public static function buildIncludesLazy(array $packages, callable $packageProvider): array { if (empty($packages)) { $metadataString = \json_encode(['packages' => []]); } else { $metadataString = '{"packages": {'; foreach ($packages as $i => $package) { - $item = $repository->findPackageMetadata($package)?->decodeJson()['packages'][$package] ?? []; + if (!$item = $packageProvider($package)) { + continue; + } $metadataString .= \sprintf('%s: %s', \json_encode($package, \JSON_UNESCAPED_SLASHES), \json_encode($item, \JSON_UNESCAPED_SLASHES)); if (\count($packages) !== $i+1) { $metadataString .= ", "; diff --git a/src/Model/CredentialsInterface.php b/src/Model/CredentialsInterface.php new file mode 100644 index 00000000..7207eaa0 --- /dev/null +++ b/src/Model/CredentialsInterface.php @@ -0,0 +1,34 @@ + 0 %} {% endif %} diff --git a/templates/proxies/settings.html.twig b/templates/proxies/settings.html.twig index 61edd8ac..0968e2ef 100644 --- a/templates/proxies/settings.html.twig +++ b/templates/proxies/settings.html.twig @@ -28,7 +28,16 @@ {{ form_end(form) }}
- {{ 'abandon.warning'|trans|raw }} +

New releases update will be frozen if auto sync is disabled. Metadata frozen as permanent snapshot

+ +

If strict mode is enabled, you will be able to use those packages that you have manually approved from the user interface.

+ +

+ You can disable Composer v2 API to avoid too many 304 requests. + Instead, the Packeton takes a snapshot of all packages and uses includes directive. + This may be useful if missing HTTPS 2 or your network is slower. +

+
{% endblock %}