diff --git a/ansible/roles/vector/tasks/configure.yml b/ansible/roles/vector/tasks/configure.yml index 78641dc..9731d34 100644 --- a/ansible/roles/vector/tasks/configure.yml +++ b/ansible/roles/vector/tasks/configure.yml @@ -1,6 +1,6 @@ --- - name: Install Vector configuration - template: + copy: src: "{{ configuration_file }}" dest: /etc/vector/vector.toml mode: 0644 diff --git a/ansible/roles/verifiable-logger/tasks/install.yml b/ansible/roles/verifiable-logger/tasks/install.yml new file mode 100644 index 0000000..7cde03b --- /dev/null +++ b/ansible/roles/verifiable-logger/tasks/install.yml @@ -0,0 +1,21 @@ +--- +- name: Create verifiable-logger directory + file: + path: /var/lib/verifiable-logger + state: directory + +- name: Copy verifiable-logger binary + get_url: + url: "https://verifiable-logger-builds.s3.us-east-2.amazonaws.com/verifiable-logger" + dest: /usr/local/bin/verifiable-logger + mode: 755 + +- name: Copy generate-logs unit file + template: + src: generate-logs.service + dest: /etc/systemd/system/generate-logs.service + +- name: Copy verify-logs unit file + template: + src: verify-logs.service + dest: /etc/systemd/system/verify-logs.service diff --git a/ansible/roles/verifiable-logger/tasks/main.yml b/ansible/roles/verifiable-logger/tasks/main.yml new file mode 100644 index 0000000..b017e8b --- /dev/null +++ b/ansible/roles/verifiable-logger/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include: "{{ action }}.yml" diff --git a/ansible/roles/verifiable-logger/tasks/start-generating.yml b/ansible/roles/verifiable-logger/tasks/start-generating.yml new file mode 100644 index 0000000..e59b184 --- /dev/null +++ b/ansible/roles/verifiable-logger/tasks/start-generating.yml @@ -0,0 +1,6 @@ +--- +- name: Start generating logs + systemd: + name: generate-logs + state: started + daemon-reload: yes diff --git a/ansible/roles/verifiable-logger/tasks/start-verifying.yml b/ansible/roles/verifiable-logger/tasks/start-verifying.yml new file mode 100644 index 0000000..e29c256 --- /dev/null +++ b/ansible/roles/verifiable-logger/tasks/start-verifying.yml @@ -0,0 +1,6 @@ +--- +- name: Start verifying logs + systemd: + name: verify-logs + state: started + daemon-reload: yes diff --git a/ansible/roles/verifiable-logger/templates/generate-logs.service b/ansible/roles/verifiable-logger/templates/generate-logs.service new file mode 100644 index 0000000..0184536 --- /dev/null +++ b/ansible/roles/verifiable-logger/templates/generate-logs.service @@ -0,0 +1,6 @@ +[Unit] +Description=generate-logs + +[Service] +ExecStart=/usr/local/bin/verifiable-logger generate --rate 10 --output output.log +WorkingDirectory=/var/lib/verifiable-logger/ diff --git a/ansible/roles/verifiable-logger/templates/verify-logs.service b/ansible/roles/verifiable-logger/templates/verify-logs.service new file mode 100644 index 0000000..f554a33 --- /dev/null +++ b/ansible/roles/verifiable-logger/templates/verify-logs.service @@ -0,0 +1,6 @@ +[Unit] +Description=verify-logs + +[Service] +ExecStart=/usr/local/bin/verifiable-logger verify file-to-s3-reliability-test-data us-east-1 --prefix "host=%H" --tail +WorkingDirectory=/var/lib/verifiable-logger/ diff --git a/cases/file_to_s3_reliability/README.md b/cases/file_to_s3_reliability/README.md new file mode 100644 index 0000000..632b79a --- /dev/null +++ b/cases/file_to_s3_reliability/README.md @@ -0,0 +1,25 @@ +# File to S3 Reliability Test + +This is a long-running test of Vector tailing a file and sending the data to S3. +It uses the [`verifiable-logger`][0] project both to generate the log data and +verify that it all reaches S3. + +## Try it + +You can run this test via: + +``` +test -t file_to_s3_reliability +``` + +## Resources + +* [Setup][setup] +* [Development][development] +* [How it works][how_it_works] +* [Vector docs][docs] +* [Vector repo][repo] +* [Vector website][website] + + +[0]: https://github.com/timberio/verifiable-logger diff --git a/cases/file_to_s3_reliability/ansible/bootstrap.yml b/cases/file_to_s3_reliability/ansible/bootstrap.yml new file mode 100644 index 0000000..dc310fc --- /dev/null +++ b/cases/file_to_s3_reliability/ansible/bootstrap.yml @@ -0,0 +1,21 @@ +--- +- hosts: '{{ test_namespace }}' + tasks: + - meta: refresh_inventory + - name: Wait 600 seconds for target connection to become reachable/usable + wait_for_connection: + + - debug: + var: playbook_dir + +- hosts: '{{ test_namespace }}:&tag_TestRole_subject' + become: true + roles: + - role: vector + action: install + + - role: vector + action: configure + + - role: verifiable-logger + action: install diff --git a/cases/file_to_s3_reliability/ansible/config_files/vector.toml b/cases/file_to_s3_reliability/ansible/config_files/vector.toml new file mode 100644 index 0000000..4027f3d --- /dev/null +++ b/cases/file_to_s3_reliability/ansible/config_files/vector.toml @@ -0,0 +1,16 @@ +data_dir = "/var/lib/vector" + +[sources.file] +type = "file" +include = ["/var/lib/verifiable-logger/output.log"] +start_at_beginning = true + +[sinks.s3] +inputs = ["file"] +type = "aws_s3" +region = "us-east-1" +bucket = "file-to-s3-reliability-test-data" +key_prefix = "host={{ host }}/date=%F/" +encoding = "text" +compression = "none" +batch.timeout_secs = 30 diff --git a/cases/file_to_s3_reliability/ansible/run.yml b/cases/file_to_s3_reliability/ansible/run.yml new file mode 100644 index 0000000..5015262 --- /dev/null +++ b/cases/file_to_s3_reliability/ansible/run.yml @@ -0,0 +1,12 @@ +--- +- hosts: '{{ test_namespace }}:&tag_TestRole_subject' + become: true + roles: + - role: verifiable-logger + action: start-generating + + - role: verifiable-logger + action: start-verifying + + - role: vector + action: start diff --git a/cases/file_to_s3_reliability/configurations/default/ansible.yml b/cases/file_to_s3_reliability/configurations/default/ansible.yml new file mode 100644 index 0000000..c4dd98e --- /dev/null +++ b/cases/file_to_s3_reliability/configurations/default/ansible.yml @@ -0,0 +1,2 @@ +--- +foo: "bar" diff --git a/cases/file_to_s3_reliability/configurations/default/terraform.tfvars b/cases/file_to_s3_reliability/configurations/default/terraform.tfvars new file mode 100644 index 0000000..f9170c6 --- /dev/null +++ b/cases/file_to_s3_reliability/configurations/default/terraform.tfvars @@ -0,0 +1,6 @@ +# WARNING! +# +# Do not modify the parameters of this file since historical test results are +# based on these parameters. Please create a new configuration and specify taht +# configuration instead. +subject_instance_type = "c5.large" diff --git a/cases/file_to_s3_reliability/terraform/main.tf b/cases/file_to_s3_reliability/terraform/main.tf new file mode 100644 index 0000000..02218ee --- /dev/null +++ b/cases/file_to_s3_reliability/terraform/main.tf @@ -0,0 +1,70 @@ +provider "aws" { + region = "us-east-1" + version = "~> 2.53" +} + +terraform { + required_version = ">= 0.12" + backend "s3" {} +} + +module "topology" { + source = "../../../terraform/aws_uni_topology" + + providers = { + aws = aws + } + + pub_key = var.pub_key + subject_instance_type = var.subject_instance_type + test_configuration = var.test_configuration + test_name = var.test_name + user_id = var.user_id + results_s3_bucket_name = var.results_s3_bucket_name +} + +resource "aws_s3_bucket" "logs-bucket" { + # data is namespaced by host within the bucket + bucket = "file-to-s3-reliability-test-data" + + lifecycle_rule { + enabled = true + + expiration { + days = 14 + } + } +} + +data "aws_iam_policy_document" "logs-bucket-policy" { + statement { + sid = "AllowTestHarnessListBucket" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + aws_s3_bucket.logs-bucket.arn, + ] + } + + statement { + sid = "AllowTestHarnessEverythingElse" + + actions = [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + ] + + resources = [ + "${aws_s3_bucket.logs-bucket.arn}/*", + ] + } +} + +resource "aws_iam_role_policy" "default" { + role = module.topology.instance_profile_name + policy = data.aws_iam_policy_document.logs-bucket-policy.json +} diff --git a/cases/file_to_s3_reliability/terraform/variables.tf b/cases/file_to_s3_reliability/terraform/variables.tf new file mode 100644 index 0000000..c220a8a --- /dev/null +++ b/cases/file_to_s3_reliability/terraform/variables.tf @@ -0,0 +1,24 @@ +variable "pub_key" { + type = string +} + +variable "subject_instance_type" { + type = string +} + +variable "test_configuration" { + type = string +} + +variable "test_name" { + type = string +} + +variable "user_id" { + type = string +} + +// don't actually need this, but need to accept it +variable "results_s3_bucket_name" { + type = string +} diff --git a/cases/file_to_s3_reliability/terraform/versions.tf b/cases/file_to_s3_reliability/terraform/versions.tf new file mode 100644 index 0000000..d9b6f79 --- /dev/null +++ b/cases/file_to_s3_reliability/terraform/versions.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 0.12" +} diff --git a/terraform/aws_uni_topology/main.tf b/terraform/aws_uni_topology/main.tf index 81c187d..912d1b2 100644 --- a/terraform/aws_uni_topology/main.tf +++ b/terraform/aws_uni_topology/main.tf @@ -11,7 +11,7 @@ locals { } module "vpc" { - source = "../../../terraform/aws_vpc" + source = "../aws_vpc" providers = { aws = aws @@ -24,7 +24,7 @@ module "vpc" { } module "aws_instance_profile" { - source = "../../../terraform/aws_instance_profile" + source = "../aws_instance_profile" providers = { aws = aws @@ -36,7 +36,7 @@ module "aws_instance_profile" { } module "aws_instance_subject" { - source = "../../../terraform/aws_instance" + source = "../aws_instance" providers = { aws = aws diff --git a/terraform/aws_uni_topology/outputs.tf b/terraform/aws_uni_topology/outputs.tf new file mode 100644 index 0000000..37c8afd --- /dev/null +++ b/terraform/aws_uni_topology/outputs.tf @@ -0,0 +1,3 @@ +output "instance_profile_name" { + value = module.aws_instance_profile.name +}