From 8c8a35fa7836c67631b398644d3df5abc7c53f01 Mon Sep 17 00:00:00 2001 From: Nyholm Date: Fri, 20 Dec 2019 11:48:23 +0100 Subject: [PATCH] Create a small CLI application --- .gitignore | 3 +- .phpcs.xml.dist | 147 +++++++++++++++++++++++++++++++++ .prettyci.composer.json | 5 ++ .travis.yml | 20 +++++ Makefile | 1 - Readme.md | 2 +- bref-extra | 12 +++ checksums.json | 1 + composer.json | 9 +- export/.gitignore | 1 - export/checksums | 6 -- layers.json | 1 + src/Application.php | 45 ++++++++++ src/Aws/LayerProvider.php | 66 +++++++++++++++ src/Aws/LayerPublisher.php | 135 ++++++++++++++++++++++++++++++ src/Command/HelpCommand.php | 20 +++++ src/Command/ListCommand.php | 54 ++++++++++++ src/Command/PublishCommand.php | 74 +++++++++++++++++ src/Service/RegionProvider.php | 20 +++++ src/layer-list.php | 66 --------------- src/publish.php | 147 --------------------------------- 21 files changed, 611 insertions(+), 224 deletions(-) create mode 100644 .phpcs.xml.dist create mode 100644 .prettyci.composer.json create mode 100644 .travis.yml create mode 100755 bref-extra create mode 100644 checksums.json delete mode 100644 export/checksums create mode 100644 layers.json create mode 100644 src/Application.php create mode 100644 src/Aws/LayerProvider.php create mode 100644 src/Aws/LayerPublisher.php create mode 100644 src/Command/HelpCommand.php create mode 100644 src/Command/ListCommand.php create mode 100644 src/Command/PublishCommand.php create mode 100644 src/Service/RegionProvider.php delete mode 100644 src/layer-list.php delete mode 100644 src/publish.php diff --git a/.gitignore b/.gitignore index 55940e57..f8602e8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /vendor/ -composer.lock \ No newline at end of file +composer.lock +.php_cs.cache \ No newline at end of file diff --git a/.phpcs.xml.dist b/.phpcs.xml.dist new file mode 100644 index 00000000..c80af217 --- /dev/null +++ b/.phpcs.xml.dist @@ -0,0 +1,147 @@ + + + + + + + + + + bref + src + tests + tests/Bridge/Symfony/var + tests/Bridge/Symfony/cache + tests/Bridge/Symfony/logs + tests/Bridge/Laravel/bootstrap/cache + + + + + + + 0 + + + + + 0 + + + + + + + + + + + + + + + + + + + + 0 + + + + + 0 + + + + + + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + 0 + + + + + 0 + + + + + 0 + + + 0 + + + 0 + + + + + + + + + tests/* + + + + + tests/* + + + diff --git a/.prettyci.composer.json b/.prettyci.composer.json new file mode 100644 index 00000000..5ae6e7a5 --- /dev/null +++ b/.prettyci.composer.json @@ -0,0 +1,5 @@ +{ + "require": { + "doctrine/coding-standard": "^5.0" + } +} diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..358e76ab --- /dev/null +++ b/.travis.yml @@ -0,0 +1,20 @@ +language: php + +notifications: + email: + on_success: never + +services: + - docker + +env: + - LAYER=amqp PHP=72 + - LAYER=amqp PHP=73 + - LAYER=amqp PHP=74 + - LAYER=mecached PHP=72 + - LAYER=mecached PHP=73 + - LAYER=mecached PHP=74 + +script: + - cd layer/$LAYER + - docker build --build-arg PHP_VERSION=$PHP . diff --git a/Makefile b/Makefile index 17b1d019..67ee939f 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ SHELL := /bin/bash php_versions = 72 73 74 -php_versions = 73 docker-images: PWD=pwd diff --git a/Readme.md b/Readme.md index dc6a0fcf..3509c07c 100644 --- a/Readme.md +++ b/Readme.md @@ -1,4 +1,4 @@ -# Bref layers +# Bref extra layers Create small layers with specific PHP extensions. This is useful when you want something "off the shelf". If you ever need more than one layer you should consider creating your own layer. That is because AWS has diff --git a/bref-extra b/bref-extra new file mode 100755 index 00000000..dbc6cf0a --- /dev/null +++ b/bref-extra @@ -0,0 +1,12 @@ +#!/usr/bin/env php +command('publish', \Bref\Extra\Command\PublishCommand::class)->descriptions('Publish new layers. Create checksums.json'); +$app->command('list', \Bref\Extra\Command\ListCommand::class)->descriptions('Create layer.json'); +$app->command('help', \Bref\Extra\Command\HelpCommand::class)->descriptions('Prints some help texts.'); + +$app->setDefaultCommand('help'); + +$app->run(); diff --git a/checksums.json b/checksums.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/checksums.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/composer.json b/composer.json index b751b5a4..65f751dc 100644 --- a/composer.json +++ b/composer.json @@ -4,7 +4,14 @@ "require-dev": { "symfony/process": "^5.0", "symfony/finder": "^5.0", - "aws/aws-sdk-php": "^3.130" + "aws/aws-sdk-php": "^3.130", + "mnapoli/silly": "^1.7", + "mnapoli/silly-php-di": "^1.2" + }, + "autoload-dev": { + "psr-4": { + "Bref\\Extra\\": "src/" + } }, "license": "MIT", "authors": [ diff --git a/export/.gitignore b/export/.gitignore index 1cd26acc..d6b7ef32 100644 --- a/export/.gitignore +++ b/export/.gitignore @@ -1,3 +1,2 @@ * !.gitignore -!checksums \ No newline at end of file diff --git a/export/checksums b/export/checksums deleted file mode 100644 index 7523f658..00000000 --- a/export/checksums +++ /dev/null @@ -1,6 +0,0 @@ -b63635b18295f039aee4a46d82669ef4 -577e4e2bcb7d5bac9888129501cdfd61 -0a4b17da009296aaea8e1b1b023b6667 -e28ef694afd4d5fc14d2d7f120554270 -f9f52e3c11929eb677f9aee8a2018906 -4e1039a62f9500ad9e057c5915900362 \ No newline at end of file diff --git a/layers.json b/layers.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/layers.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/src/Application.php b/src/Application.php new file mode 100644 index 00000000..d307dcde --- /dev/null +++ b/src/Application.php @@ -0,0 +1,45 @@ +addDefinitions([ + 'project_dir' => $projectDir, + 'aws_id' => $awsId, + 'layer_names' => $localLayers, + LayerProvider::class => function (ContainerInterface $c) { + return new LayerProvider($c->get('layer_names'), $c->get('aws_id')); + }, + ListCommand::class => function (ContainerInterface $c) { + return new ListCommand($c->get(LayerProvider::class), $c->get(RegionProvider::class), $c->get('project_dir')); + }, + PublishCommand::class => function (ContainerInterface $c) { + return new PublishCommand($c->get(LayerPublisher::class), $c->get(RegionProvider::class), $c->get('project_dir')); + }, + ]); + + return $builder->build(); + } +} \ No newline at end of file diff --git a/src/Aws/LayerProvider.php b/src/Aws/LayerProvider.php new file mode 100644 index 00000000..7101c628 --- /dev/null +++ b/src/Aws/LayerProvider.php @@ -0,0 +1,66 @@ + + */ +class LayerProvider +{ + private $awsId; + + /** + * @var array + */ + private $layerNames; + + /** + * @param array $layerNames the name of the layers to list. + * @param string $awsId The account id + */ + public function __construct(array $layerNames, string $awsId) + { + $this->awsId = $awsId; + $this->layerNames = $layerNames; + } + + + public function listLayers(string $selectedRegion): array + { + $lambda = new LambdaClient([ + 'version' => 'latest', + 'region' => $selectedRegion, + ]); + + $accountId = $this->awsId; + // Run the API calls in parallel (thanks to async) + $promises = array_combine($this->layerNames, array_map(function (string $layerName) use ($lambda, $selectedRegion, $accountId) { + return $lambda->listLayerVersionsAsync([ + 'LayerName' => "arn:aws:lambda:$selectedRegion:$accountId:layer:$layerName", + 'MaxItems' => 1, + ]); + }, $this->layerNames)); + + // Wait on all of the requests to complete. Throws a ConnectException + // if any of the requests fail + $results = \GuzzleHttp\Promise\unwrap($promises); + + $layers = []; + foreach ($results as $layerName => $result) { + $versions = $result['LayerVersions']; + $latestVersion = end($versions); + $layers[$layerName] = $latestVersion['Version']; + } + + return $layers; + } + + +} \ No newline at end of file diff --git a/src/Aws/LayerPublisher.php b/src/Aws/LayerPublisher.php new file mode 100644 index 00000000..e8ab350a --- /dev/null +++ b/src/Aws/LayerPublisher.php @@ -0,0 +1,135 @@ +publishSingleLayer($region, $layer); + } + } + $this->finishProcesses($publishingProcesses); + + + // Add public permissions on the layers + /** @var Process[] $permissionProcesses */ + $permissionProcesses = []; + foreach ($regions as $region) { + foreach ($layers as $layer) { + $publishLayer = $publishingProcesses[$region . $layer]; + $layerVersion = trim($publishLayer->getOutput()); + + $permissionProcesses[] = $this->addPublicLayerPermissions($region, $layer, $layerVersion); + } + } + $this->finishProcesses($permissionProcesses); + } + + /** + * @param string $region The AWS region to publish the layer to + * @param string $layer The file path to the layer relative ??? + * @return Process + */ + private function publishSingleLayer(string $region, string $layer): Process + { + $file = __DIR__ . "/export/$layer.zip"; + + $process = new Process([ + 'aws', + 'lambda', + 'publish-layer-version', + '--region', + $region, + '--layer-name', + $layer, + '--description', + $layer, + '--license-info', + 'MIT', + '--zip-file', + 'fileb://' . $file, + '--compatible-runtimes', + 'provided', + // Output the version so that we can fetch it and use it + '--output', + 'text', + '--query', + 'Version', + ]); + $process->setTimeout(null); + + return $process; + } + + /** + * @param Process[] $processes + */ + private function finishProcesses(array $processes): void + { + // Run the processes in batches to parallelize them without overloading the machine and the network + foreach (array_chunk($processes, 4) as $batch) { + // Start all the processes + array_map(function (Process $process): void { + $process->start(); + }, $batch); + // Wait for them to finish + array_map(function (Process $process): void { + $status = $process->wait(); + echo '.'; + // Make sure the process ran successfully + if ($status !== 0) { + echo 'Process ' . $process->getCommandLine() . ' failed:' . PHP_EOL; + echo $process->getErrorOutput(); + echo $process->getOutput(); + exit(1); + } + }, $batch); + } + } + + /** + * @param string $region + * @param string $layer name + * @param string $layerVersion + * @return Process + */ + private function addPublicLayerPermissions(string $region, string $layer, string $layerVersion): Process + { + $process = new Process([ + 'aws', + 'lambda', + 'add-layer-version-permission', + '--region', + $region, + '--layer-name', + $layer, + '--version-number', + $layerVersion, + '--statement-id', + 'public', + '--action', + 'lambda:GetLayerVersion', + '--principal', + '*', + ]); + $process->setTimeout(null); + + return $process; + } + + + +} \ No newline at end of file diff --git a/src/Command/HelpCommand.php b/src/Command/HelpCommand.php new file mode 100644 index 00000000..11328396 --- /dev/null +++ b/src/Command/HelpCommand.php @@ -0,0 +1,20 @@ +writeln('With this small application you may publish new layers and list existing ones in layer.json'); + + return 0; + } +} \ No newline at end of file diff --git a/src/Command/ListCommand.php b/src/Command/ListCommand.php new file mode 100644 index 00000000..2cdbda47 --- /dev/null +++ b/src/Command/ListCommand.php @@ -0,0 +1,54 @@ +provider = $provider; + $this->projectDir = $projectDir; + $this->regionProvider = $regionProvider; + } + + public function __invoke(OutputInterface $output) + { + $export = []; + foreach ($this->regionProvider->getAll() as $region) { + $layers = $this->provider->listLayers($region); + + //$export[$layerName][$region] = $layers[$layerName]; + + $output->writeln($region); + } + file_put_contents($this->projectDir . '/layers.json', json_encode($export, JSON_PRETTY_PRINT)); + + return 0; + } +} \ No newline at end of file diff --git a/src/Command/PublishCommand.php b/src/Command/PublishCommand.php new file mode 100644 index 00000000..427817b1 --- /dev/null +++ b/src/Command/PublishCommand.php @@ -0,0 +1,74 @@ +publisher = $publisher; + $this->projectDir = $projectDir; + $this->regionProvider = $regionProvider; + } + + public function __invoke(OutputInterface $output) + { + $checksums = file_get_contents($this->projectDir.'/checksums.json'); + $discoveredChecksums = []; + + $layers = []; + $finder = new Finder(); + $finder->in(__DIR__.'/../export') + ->name('layer-*'); + foreach ($finder->files() as $file) { + /** @var \SplFileInfo $file */ + $layerFile = $file->getFilename(); + $layerName = substr($file->getFilenameWithoutExtension(), 6); + $md5 = md5_file($file->getRealPath()); + $discoveredChecksums[$layerName] = $md5; + if (false === strstr($checksums, $md5)) { + // This layer is new. + $layers[$layerName] = $layerFile; + } + } + $output->writeln(sprintf('Found %d new layers', count($layers))); + + try { + $this->publisher->publishLayers(array_values($layers), $this->regionProvider->getAll()); + }catch(\Exception $e) { + // TODO write output. + exit(1); + } + + // Dump checksums + file_put_contents($this->projectDir.'/checksums.json', json_encode($discoveredChecksums)); + + $output->writeln('Done'); + $output->writeln('Remember to commit and push changes to ./checksums.json'); + + return 0; + } +} \ No newline at end of file diff --git a/src/Service/RegionProvider.php b/src/Service/RegionProvider.php new file mode 100644 index 00000000..583f9659 --- /dev/null +++ b/src/Service/RegionProvider.php @@ -0,0 +1,20 @@ + + */ +class RegionProvider +{ + public function getAll(): array + { + return json_decode(file_get_contents('https://raw.githubusercontent.com/brefphp/bref/master/runtime/layers/regions.json'), true); + } +} diff --git a/src/layer-list.php b/src/layer-list.php deleted file mode 100644 index 815c87dd..00000000 --- a/src/layer-list.php +++ /dev/null @@ -1,66 +0,0 @@ - 'latest', - 'region' => $selectedRegion, - ]); - - // Run the API calls in parallel (thanks to async) - $promises = array_combine(LAYER_NAMES, array_map(function (string $layerName) use ($lambda, $selectedRegion) { - return $lambda->listLayerVersionsAsync([ - 'LayerName' => "arn:aws:lambda:$selectedRegion:209497400698:layer:$layerName", - 'MaxItems' => 1, - ]); - }, LAYER_NAMES)); - - // Wait on all of the requests to complete. Throws a ConnectException - // if any of the requests fail - $results = unwrap($promises); - - $layers = []; - foreach ($results as $layerName => $result) { - $versions = $result['LayerVersions']; - $latestVersion = end($versions); - $layers[$layerName] = $latestVersion['Version']; - } - - return $layers; -} - diff --git a/src/publish.php b/src/publish.php deleted file mode 100644 index 220795e4..00000000 --- a/src/publish.php +++ /dev/null @@ -1,147 +0,0 @@ -in(__DIR__.'/../export') - ->name('layer-*'); -foreach ($finder->files() as $file) { - /** @var SplFileInfo $file */ - $layer = $file->getFilenameWithoutExtension(); - $md5 = md5_file($file->getRealPath()); - $discoveredChecksums[] = $md5; - if (false === strstr($checksums, $md5)) { - // This layer is new. - $layers[] = substr($layer, 6); - } -} -echo sprintf("\n%d new layers\n", count($layers)); - -file_put_contents(__DIR__.'/../export/checksums', implode("\n", $discoveredChecksums)); -return; - -// Publish the layers -/** @var Process[] $publishingProcesses */ -$publishingProcesses = []; -foreach ($regions as $region) { - foreach ($layers as $layer) { - $publishingProcesses[$region.$layer] = publishLayer($region, $layer); - } -} -runProcessesInParallel($publishingProcesses); -echo sprintf("\n%d layers are published, adding permissions now\n", count($layers)); - -// Add public permissions on the layers -/** @var Process[] $permissionProcesses */ -$permissionProcesses = []; -foreach ($regions as $region) { - foreach ($layers as $layer) { - $publishLayer = $publishingProcesses[$region . $layer]; - $layerVersion = trim($publishLayer->getOutput()); - - $permissionProcesses[] = addPublicLayerPermissions($region, $layer, $layerVersion); - } -} -runProcessesInParallel($permissionProcesses); - -// Dump checksums -file_put_contents(__DIR__.'/../export/checksums', implode("\n", $discoveredChecksums)); -echo "\nDone\n"; -echo "Remember to commit and push changes to export/checksums\n"; - -function publishLayer(string $region, string $layer): Process -{ - $file = __DIR__ . "/export/$layer.zip"; - - $process = new Process([ - 'aws', - 'lambda', - 'publish-layer-version', - '--region', - $region, - '--layer-name', - $layer, - '--description', - $layer, - '--license-info', - 'MIT', - '--zip-file', - 'fileb://' . $file, - '--compatible-runtimes', - 'provided', - // Output the version so that we can fetch it and use it - '--output', - 'text', - '--query', - 'Version', - ]); - $process->setTimeout(null); - - return $process; -} - -/** - * @param Process[] $processes - */ -function runProcessesInParallel(array $processes): void -{ - // Run the processes in batches to parallelize them without overloading the machine and the network - foreach (array_chunk($processes, 4) as $batch) { - // Start all the processes - array_map(function (Process $process): void { - $process->start(); - }, $batch); - // Wait for them to finish - array_map(function (Process $process): void { - $status = $process->wait(); - echo '.'; - // Make sure the process ran successfully - if ($status !== 0) { - echo 'Process ' . $process->getCommandLine() . ' failed:' . PHP_EOL; - echo $process->getErrorOutput(); - echo $process->getOutput(); - exit(1); - } - }, $batch); - } -} - -function addPublicLayerPermissions(string $region, string $layer, string $layerVersion): Process -{ - $process = new Process([ - 'aws', - 'lambda', - 'add-layer-version-permission', - '--region', - $region, - '--layer-name', - $layer, - '--version-number', - $layerVersion, - '--statement-id', - 'public', - '--action', - 'lambda:GetLayerVersion', - '--principal', - '*', - ]); - $process->setTimeout(null); - - return $process; -}