chore: new artifact version and naming rules

*	Get rid of Terraform / Atlas Artifact Registry. Close #78
*	Close #49
*	Separate OS name and version with dash. Fix #22
*	Include version in artifact names and tags
*	Synchronize versions of Amazon and VirtualBox base images

See doc/NAMING.md for complete guidelines.

BREAKING CHANGE

build.gradle

@@ -30,6 +30,7 @@ buildscript {
   }
   dependencies {
     classpath 'gradle.plugin.fidata.org:gradle-packer-plugin:2.4.0'
+    classpath 'com.amazonaws:aws-java-sdk-ec2:1.11.289'
     classpath 'org.apache.commons:commons-exec:1.3'
     classpath 'com.jcraft:jsch:0.1.54'
   }
@@ -65,6 +66,12 @@ import groovy.json.JsonSlurper
 import groovy.json.JsonOutput
 import org.apache.commons.exec.*
 import com.jcraft.jsch.*
+import org.apache.commons.io.filefilter.DirectoryFileFilter
+import com.amazonaws.services.ec2.AmazonEC2ClientBuilder
+import com.amazonaws.auth.AWSStaticCredentialsProvider
+import com.amazonaws.auth.BasicAWSCredentials
+import com.amazonaws.services.ec2.model.DescribeImagesRequest
+import com.amazonaws.services.ec2.model.Image
 
 task wrapper(type: Wrapper) {
   gradleVersion = '4.6'
@@ -168,10 +175,7 @@ ext.with {
   rubocopHTMLOutput = file("$htmlReportsDir/rubocop.html")
   cookstyleHTMLOutput = file("$htmlReportsDir/cookstyle.html")
 
-  terraformDirs = ["$libDir/get_atlas_artifact_metadata_full"]
-  terraformDir.eachDir { d ->
-    terraformDirs << d
-  }
+  terraformDirs = Arrays.asList(terraformDir.listFiles((FileFilter)DirectoryFileFilter.DIRECTORY))
 }
 
 visteg {
@@ -405,7 +409,6 @@ ext.with {
     'CHEF_VERSION': chefVersion,
     'AWS_ACCESS_KEY_ID': awsAccessKey,
     'AWS_SECRET_ACCESS_KEY': awsSecretKey,
-    'ATLAS_TOKEN': atlasToken,
     'KITCHEN_SSH_KEY': kitchenSSHPrivateKey.absolutePath,
     'ENCRYPTED_DATA_BAG_SECRET': encryptedDataBagSecret.absolutePath,
   ]
@@ -415,7 +418,6 @@ ext.with {
     'keys_dir': keysDir.absolutePath,
     'aws_access_key': awsAccessKey,
     'aws_secret_key': awsSecretKey,
-    'atlas_token': atlasToken,
     'cloudflare_email': cloudflareEmail,
     'cloudflare_token': cloudflareToken,
   ]
@@ -431,7 +433,7 @@ ext.with {
   ]
 
   jenkinsSlaves = [
-    'UbuntuServer16.04.4+amd64': [
+    'UbuntuServer-16.04.4+amd64': [
       'version': '0.3.2',
       'security_groups': ['SSH'] as Set<String>,
       'type_data': [
@@ -455,28 +457,30 @@ ext.with {
     'JenkinsEC2Cloud': [:],
     'ReleaseCredentials': [:],
   ]
-}
 
-Map getAtlasArtifactMetadataFull(name, type, version) {
-  new ByteArrayOutputStream().withStream { os ->
-    exec {
-      environment([
-        'ATLAS_TOKEN': atlasToken
-      ])
-      commandLine([
-        bundleExecutable, 'exec',
-        'ruby'
-      ] +
-      rubyLogLevelArgs +
-      [
-        '--',
-        "$libDir/get_atlas_artifact_metadata_full/get_atlas_artifact_metadata_full.rb",
-        'get',
-        name, type, version
-      ])
-      standardOutput = os
-    }
-    new JsonSlurper().parseText(os.toString())
+  ec2 = null
+}
+
+String findAMI(name, version) {
+  ec2 = ec2 ?: AmazonEC2ClientBuilder.standard()
+    .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(awsAccessKey, awsSecretKey)))
+    .withRegion('eu-west-1')
+    .build()
+  List<Image> result = ec2.describeImages(new DescribeImagesRequest()
+    .withFilters([
+      name: name,
+      'tag:version': version
+    ])
+    .withOwners('880042334380')
+  ).images
+  if (result.size() == 0) {
+    logger.error('findAMI: Can\'t find image')
+  }
+  else if (result.size() > 1) {
+    logger.error('findAMI: Too many images found')
+  }
+  else {
+    return result[0].imageId
   }
 }
 
@@ -672,30 +676,29 @@ packer {
     'manifest_output_dir': manifestOutputDir,
     'aws_access_key': awsAccessKey,
     'aws_secret_key': awsSecretKey,
-    'atlas_token': atlasToken,
     'vagrant_cloud_token': vagrantCloudToken,
     'encrypted_data_bag_secret': encryptedDataBagSecret,
   ]
   // Kernel images
-  template 'src/packer/kernel_images/kernel-UbuntuServer16.04.4_amd64.json'
+  template 'src/packer/kernel_images/kernel-UbuntuServer-16.04.4+amd64.json'
 
   // Base images
-  template 'src/packer/base_images/UbuntuServer16.04.4_amd64.json'
+  template 'src/packer/base_images/UbuntuServer-16.04.4+amd64.json'
 
   // Instances
   template 'src/packer/instances/JenkinsMaster.json'
   template 'src/packer/instances/JenkinsSlaves.json'
 }
 
-tasks['build-kernel-UbuntuServer16.04.4_amd64-vbox'].dependsOn 'createVideocaptureDir', 'createManifestOutputDir'
-tasks['build-UbuntuServer16.04.4_amd64-vbox'].dependsOn 'createManifestOutputDir'
-tasks['build-UbuntuServer16.04.4_amd64-vbox'].mustRunAfter 'build-kernel-UbuntuServer16.04.4_amd64-vbox'
-tasks['build-UbuntuServer16.04.4_amd64-amazon'].dependsOn 'createManifestOutputDir'
-tasks['build-UbuntuServer16.04.4_amd64'].dependsOn 'createManifestOutputDir'
+tasks['build-kernel-UbuntuServer-16.04.4+amd64-vbox'].dependsOn 'createVideocaptureDir', 'createManifestOutputDir'
+tasks['build-UbuntuServer-16.04.4+amd64-vbox'].dependsOn 'createManifestOutputDir'
+tasks['build-UbuntuServer-16.04.4+amd64-vbox'].mustRunAfter 'build-kernel-UbuntuServer-16.04.4+amd64-vbox'
+tasks['build-UbuntuServer-16.04.4+amd64-amazon'].dependsOn 'createManifestOutputDir'
+tasks['build-UbuntuServer-16.04.4+amd64'].dependsOn 'createManifestOutputDir'
 
 task('clean-base') {
   group 'Build'
-  dependsOn 'clean-UbuntuServer16.04.4_amd64'
+  dependsOn 'clean-UbuntuServer-16.04.4+amd64'
 }
 
 task('clean') {
@@ -704,17 +707,17 @@ task('clean') {
 
 task('build-base-vbox') {
   group 'Build'
-  dependsOn 'build-UbuntuServer16.04.4_amd64-vbox'
+  dependsOn 'build-UbuntuServer-16.04.4+amd64-vbox'
 }
 
 task('build-base-amazon') {
   group 'Build'
-  dependsOn 'build-UbuntuServer16.04.4_amd64-amazon'
+  dependsOn 'build-UbuntuServer-16.04.4+amd64-amazon'
 }
 
 task('build-base') {
   group 'Build'
-  dependsOn 'build-UbuntuServer16.04.4_amd64'
+  dependsOn 'build-UbuntuServer-16.04.4+amd64'
 }
 
 task('berksUpdate') {
@@ -940,7 +943,7 @@ task('generateRuntimeDataBag') {
     }
     property 'JenkinsEC2CloudDataBag', {
       for (s in jenkinsSlaves) {
-        s.value['ami'] = getAtlasArtifactMetadataFull("fidata/JenkinsSlave-${s.key.replace('+', '_')}", 'amazon.image', s.value['version'])['ami_id']
+        s.value['ami'] = findAMI("JenkinsSlave-${s.key.replace('+', '_')}-*", s.value['version'])
       }
       dataBags['JenkinsEC2Cloud'] << [
         'region': 'eu-west-1',
@@ -1004,7 +1007,7 @@ task('generateRuntimeDataBag') {
 task('kitchenTest-JenkinsMaster-vbox', type: CrossPlatformExec) {
   group 'Test'
   dependsOn 'terraformOutput-common', 'generateRuntimeDataBag', 'berksUpdate'
-  mustRunAfter 'build-UbuntuServer16.04.4_amd64-vbox'
+  mustRunAfter 'build-UbuntuServer-16.04.4+amd64-vbox'
   workingDir chefDir
   doFirst {
     environment << kitchenEnvironment
@@ -1026,7 +1029,7 @@ task('kitchenTest-JenkinsMaster-vbox', type: CrossPlatformExec) {
 task('kitchenTest-JenkinsMaster-amazon', type: CrossPlatformExec) {
   group 'Test'
   dependsOn 'terraformApply-common', 'terraformOutput-common', 'generateRuntimeDataBag', 'berksUpdate'
-  mustRunAfter 'build-UbuntuServer16.04.4_amd64-amazon', 'build-UbuntuServer16.04.4_amd64'
+  mustRunAfter 'build-UbuntuServer-16.04.4+amd64-amazon', 'build-UbuntuServer-16.04.4+amd64'
   workingDir chefDir
   doFirst {
     environment << kitchenEnvironment
@@ -1048,7 +1051,7 @@ task('kitchenTest-JenkinsMaster-amazon', type: CrossPlatformExec) {
 task('kitchenConverge-JenkinsMaster-vbox', type: CrossPlatformExec) {
   group 'Test'
   dependsOn 'terraformOutput-common', 'generateRuntimeDataBag'
-  mustRunAfter 'berksUpdate', 'build-UbuntuServer16.04.4_amd64-vbox'
+  mustRunAfter 'berksUpdate', 'build-UbuntuServer-16.04.4+amd64-vbox'
   workingDir chefDir
   doFirst {
     environment << kitchenEnvironment
@@ -1070,7 +1073,7 @@ task('kitchenConverge-JenkinsMaster-vbox', type: CrossPlatformExec) {
 task('kitchenConverge-JenkinsMaster-amazon', type: CrossPlatformExec) {
   group 'Test'
   dependsOn 'terraformApply-common', 'terraformOutput-common', 'generateRuntimeDataBag'
-  mustRunAfter 'berksUpdate', 'build-UbuntuServer16.04.4_amd64-amazon', 'build-UbuntuServer16.04.4_amd64'
+  mustRunAfter 'berksUpdate', 'build-UbuntuServer-16.04.4+amd64-amazon', 'build-UbuntuServer-16.04.4+amd64'
   workingDir chefDir
   doFirst {
     environment << kitchenEnvironment
@@ -1183,7 +1186,7 @@ task('kitchenDestroy-JenkinsMaster-amazon', type: CrossPlatformExec) {
 
 tasks['build-JenkinsMaster'].with {
   dependsOn 'createManifestOutputDir', 'terraformApply-common', 'terraformOutput-common', 'generateRuntimeDataBag', 'berksVendor'
-  mustRunAfter 'build-UbuntuServer16.04.4_amd64-amazon', 'build-UbuntuServer16.04.4_amd64', 'kitchenTest-JenkinsMaster-vbox', 'kitchenConverge-JenkinsMaster-vbox', 'kitchenTest-JenkinsMaster-amazon', 'kitchenConverge-JenkinsMaster-amazon'
+  mustRunAfter 'build-UbuntuServer-16.04.4+amd64-amazon', 'build-UbuntuServer-16.04.4+amd64', 'kitchenTest-JenkinsMaster-vbox', 'kitchenConverge-JenkinsMaster-vbox', 'kitchenTest-JenkinsMaster-amazon', 'kitchenConverge-JenkinsMaster-amazon'
 }
 
 task('terraformValidate-instances', type: Exec) {

doc/NAMING.md

@@ -0,0 +1,57 @@
+Artifact Naming Guidelines
+--------------------------
+
+The following artifacts are named with version:
+*	AMIs
+*	VirtualBox VMs
+*	VirtualBox images (files)
+*	Vagrant box files
+*	Packer output directories for VirtualBox builders
+
+The following artifacts are named without version:
+*	Vagrant boxes
+
+The following artifacts are named with version and timestamp:
+*	Packer manifests
+
+Each obtainment of artifact is made with explicit version:
+*	AMIs - with version tag
+*	VagrantBox images - with version specified
+	in names of directory and file
+*	Vagrant boxes - with built-in version property
+
+Use of `most_recent` filter is prohibited.
+
+Labels in names are named according to Semantic Versioning
+with the following interpretations and deviations:
+
+1.	Environment name is considered as the part of the name. It is
+separated from the base name with hyphen `-` and precedes a version.
+
+2.	Architecture and builder (provider, format) are considered as build
+metadata.
+
+3.	If timestamp is present it is appended in the end with hyphen `-`
+separator.
+
+4.	In the names of the following artifacts plus `+` separator is
+replaced with low line (ground) `_`:
+
+	*	VirtualBox VMs
+	*	VirtualBox images
+	*	AMIs
+	*	Vagrant box files
+	*	Vagrant boxes
+
+5.	The version can contain pre-release label as usual.
+
+
+------------------------------------------------------------------------
+Copyright © 2018  Basil Peace
+
+This is part of FIDATA Infrastructure.
+
+Copying and distribution of this file, with or without modification,
+are permitted in any medium without royalty provided the copyright
+notice and this notice are preserved.  This file is offered as-is,
+without any warranty.

src/chef/.kitchen.yml

@@ -15,13 +15,15 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# <% require_relative '../lib/get_atlas_artifact_metadata_full/get_atlas_artifact_metadata_full' %>
 platforms:
-  - name: UbuntuServer16.04.4+amd64-amazon
+  - name: UbuntuServer-16.04.4+amd64-amazon
     driver:
       name: ec2
-      image_id: <%= get_atlas_artifact_metadata_full('fidata/UbuntuServer16.04.4_amd64', 'amazon.image', '0.1.0')['ami_id'] %>
+      image_search:
+        name: UbuntuServer-16.04.4-*
+        "tag:version": "0.1.0"
+        "tag:architecture": amd64
+        owner-id: "880042334380"
       region: eu-west-1
       subnet_id: <%= ENV['AWS_SUBNET_ID'] %>
       aws_ssh_key_id: kitchen
@@ -31,12 +33,12 @@ platforms:
       no-host-key-verify: true
       connection_timeout: 30
       keepalive: false
-  - name: UbuntuServer16.04.4+amd64-vbox
+  - name: UbuntuServer-16.04.4+amd64-vbox
     driver:
       name: vagrant
       cachier: machine
-      box: fidata/UbuntuServer16.04.4_amd64
-      box_url: fidata/UbuntuServer16.04.4_amd64
+      box: fidata/UbuntuServer-16.04.4_amd64
+      box_url: fidata/UbuntuServer-16.04.4_amd64
       box_version: "0.1.0"
       box_check_update: true
       username: ubuntu
@@ -59,7 +61,7 @@ verifier:
 suites:
   - name: BuildToolset
     includes:
-      - UbuntuServer16.04.4+amd64-amazon
+      - UbuntuServer-16.04.4+amd64-amazon
     driver:
       instance_type: t2.medium
       block_device_mappings:
@@ -81,7 +83,7 @@ suites:
           tag: v1.4.0
   - name: JenkinsMaster-vbox
     includes:
-      - UbuntuServer16.04.4+amd64-vbox
+      - UbuntuServer-16.04.4+amd64-vbox
     driver:
       vm_hostname: JenkinsMaster
       network:
@@ -99,7 +101,7 @@ suites:
       - recipe[fidata-jenkins-master::default]
   - name: JenkinsMaster-amazon
     includes:
-      - UbuntuServer16.04.4+amd64-amazon
+      - UbuntuServer-16.04.4+amd64-amazon
     driver:
       instance_type: t2.small
       block_device_mappings: