Skip to content

Commit

Permalink
Switch to "$defs" to collect our definitions
Browse files Browse the repository at this point in the history
  • Loading branch information
@thomas-vogels
thomas-vogels committed Oct 31, 2021
1 parent 889cc57 commit ff28a0b
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 83 deletions.
108 changes: 54 additions & 54 deletions python/etl/config/settings.schema
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Settings",
"description": "Schema for settings of data warehouse, upstream sources, and ETL",
"definitions": {
"$defs": {
"identifier": {
"description": "Standard identifier (either SQL or AWS)",
"type": "string",
Expand All @@ -12,17 +12,17 @@
},
"identifier_list": {
"type": "array",
"items": { "$ref": "#/definitions/identifier" },
"items": { "$ref": "#/$defs/identifier" },
"uniqueItems": true,
"minItems": 1
},
"user_info": {
"type": "object",
"properties": {
"name": { "$ref": "#/definitions/identifier" },
"name": { "$ref": "#/$defs/identifier" },
"description": { "type": "string" },
"group": { "$ref": "#/definitions/identifier" },
"schema": { "$ref": "#/definitions/identifier" }
"group": { "$ref": "#/$defs/identifier" },
"schema": { "$ref": "#/$defs/identifier" }
},
"required": [ "name", "group" ],
"additionalProperties": false
Expand Down Expand Up @@ -51,13 +51,13 @@
"description": "Description of an upstream database with access, tables and permissions for the copy",
"type": "object",
"properties": {
"name": { "$ref": "#/definitions/identifier" },
"name": { "$ref": "#/$defs/identifier" },
"description": { "type": "string" },
"read_access": { "$ref": "#/definitions/identifier" },
"include_tables": { "$ref": "#/definitions/glob_pattern_list" },
"exclude_tables": { "$ref": "#/definitions/glob_pattern_list" },
"readers": { "$ref": "#/definitions/identifier_list" },
"writers": { "$ref": "#/definitions/identifier_list" }
"read_access": { "$ref": "#/$defs/identifier" },
"include_tables": { "$ref": "#/$defs/glob_pattern_list" },
"exclude_tables": { "$ref": "#/$defs/glob_pattern_list" },
"readers": { "$ref": "#/$defs/identifier_list" },
"writers": { "$ref": "#/$defs/identifier_list" }
},
"required": [ "name", "read_access", "include_tables" ],
"additionalProperties": false
Expand All @@ -66,15 +66,15 @@
"description": "Description of an upstream source consisting of files in S3, may also be an unload target",
"type": "object",
"properties": {
"name": { "$ref": "#/definitions/identifier" },
"name": { "$ref": "#/$defs/identifier" },
"description": { "type": "string" },
"s3_bucket": { "$ref": "#/definitions/bucket_template" },
"s3_path_template": { "$ref": "#/definitions/path_template" },
"s3_data_format": { "$ref": "#/definitions/s3_data_format" },
"s3_unload_path_template": { "$ref": "#/definitions/path_template" },
"include_tables": { "$ref": "#/definitions/glob_pattern_list" },
"readers": { "$ref": "#/definitions/identifier_list" },
"writers": { "$ref": "#/definitions/identifier_list" }
"s3_bucket": { "$ref": "#/$defs/bucket_template" },
"s3_path_template": { "$ref": "#/$defs/path_template" },
"s3_data_format": { "$ref": "#/$defs/s3_data_format" },
"s3_unload_path_template": { "$ref": "#/$defs/path_template" },
"include_tables": { "$ref": "#/$defs/glob_pattern_list" },
"readers": { "$ref": "#/$defs/identifier_list" },
"writers": { "$ref": "#/$defs/identifier_list" }
},
"anyOf": [
{ "required": [ "name", "s3_bucket", "s3_path_template" ] },
Expand All @@ -89,12 +89,12 @@
"description": "Description of an external schema in AWS Redshift (listed in an AWS Glue Data Catalog)",
"type": "object",
"properties": {
"name": { "$ref": "#/definitions/identifier" },
"name": { "$ref": "#/$defs/identifier" },
"description": { "type": "string" },
"external": { "enum": [ true ] },
"groups": { "$ref": "#/definitions/identifier_list" },
"database": { "$ref": "#/definitions/identifier" },
"iam_role": { "$ref": "#/definitions/aws_arn" }
"groups": { "$ref": "#/$defs/identifier_list" },
"database": { "$ref": "#/$defs/identifier" },
"iam_role": { "$ref": "#/$defs/aws_arn" }
},
"required": [ "name", "external" ],
"additionalProperties": false
Expand Down Expand Up @@ -231,12 +231,12 @@
"data_warehouse": {
"type": "object",
"properties": {
"admin_access": { "$ref": "#/definitions/identifier" },
"etl_access": { "$ref": "#/definitions/identifier" },
"owner": { "$ref": "#/definitions/user_info" },
"admin_access": { "$ref": "#/$defs/identifier" },
"etl_access": { "$ref": "#/$defs/identifier" },
"owner": { "$ref": "#/$defs/user_info" },
"users": {
"type": "array",
"items": { "$ref": "#/definitions/user_info" },
"items": { "$ref": "#/$defs/user_info" },
"uniqueItems": true,
"minItems": 1
},
Expand All @@ -246,9 +246,9 @@
"items": {
"type": "object",
"properties": {
"name": { "$ref": "#/definitions/identifier" },
"name": { "$ref": "#/$defs/identifier" },
"description": { "type": "string" },
"groups": { "$ref": "#/definitions/identifier_list" }
"groups": { "$ref": "#/$defs/identifier_list" }
},
"required": [ "name" ],
"additionalProperties": false
Expand All @@ -258,7 +258,7 @@
"required_for_success": {
"description": "List of patterns to identify 'required' relations whose fault will stop the ETL",
"oneOf": [
{ "$ref": "#/definitions/glob_pattern_list" },
{ "$ref": "#/$defs/glob_pattern_list" },
{ "type": "boolean" }
]
}
Expand All @@ -271,9 +271,9 @@
"description": "List of upstream sources (databases or files in S3)",
"items": {
"oneOf": [
{ "$ref": "#/definitions/upstream_database" },
{ "$ref": "#/definitions/upstream_static" },
{ "$ref": "#/definitions/upstream_external" }
{ "$ref": "#/$defs/upstream_database" },
{ "$ref": "#/$defs/upstream_static" },
{ "$ref": "#/$defs/upstream_external" }
]
},
"minItems": 1,
Expand Down Expand Up @@ -311,13 +311,13 @@
"properties": {
"iam_role": {
"description": "DEPRECATED Use the more specific IAM role for S3",
"$ref": "#/definitions/aws_arn"
"$ref": "#/$defs/aws_arn"
},
"s3": {
"type": "object",
"properties": {
"bucket_name": { "$ref": "#/definitions/identifier" },
"iam_role": { "$ref": "#/definitions/aws_arn" }
"bucket_name": { "$ref": "#/$defs/identifier" },
"iam_role": { "$ref": "#/$defs/aws_arn" }
},
"required": [ "bucket_name" ],
"additionalProperties": false
Expand All @@ -330,13 +330,13 @@
"description": "Location and attributes for (temporary) storage for code and extracted data",
"type": "object",
"properties": {
"iam_role": { "$ref": "#/definitions/aws_arn" },
"iam_role": { "$ref": "#/$defs/aws_arn" },
"s3": {
"type": "object",
"properties": {
"bucket_name": { "$ref": "#/definitions/identifier" },
"iam_role": { "$ref": "#/definitions/aws_arn" },
"upload_acl": { "$ref": "#/definitions/s3_acl" }
"bucket_name": { "$ref": "#/$defs/identifier" },
"iam_role": { "$ref": "#/$defs/aws_arn" },
"upload_acl": { "$ref": "#/$defs/s3_acl" }
},
"required": [ "bucket_name" ],
"additionalProperties": false
Expand All @@ -348,19 +348,19 @@
"resources": {
"type": "object",
"properties": {
"key_name": { "$ref": "#/definitions/identifier" },
"key_name": { "$ref": "#/$defs/identifier" },
"VPC": {
"type": "object",
"properties": {
"region": { "$ref": "#/definitions/identifier" },
"region": { "$ref": "#/$defs/identifier" },
"account": {
"type": "string",
"pattern": "^[0-9]+$"
},
"name": { "$ref": "#/definitions/identifier" },
"public_subnet": { "$ref": "#/definitions/identifier" },
"private_subnet": { "$ref": "#/definitions/identifier" },
"whitelist_security_group": { "$ref": "#/definitions/identifier" }
"name": { "$ref": "#/$defs/identifier" },
"public_subnet": { "$ref": "#/$defs/identifier" },
"private_subnet": { "$ref": "#/$defs/identifier" },
"whitelist_security_group": { "$ref": "#/$defs/identifier" }
},
"required": [ "region", "account", "name", "public_subnet", "whitelist_security_group" ],
"additionalProperties": false
Expand All @@ -370,8 +370,8 @@
"properties": {
"instance_type": { "type": "string" },
"image_id": { "type": "string" },
"public_security_group": { "$ref": "#/definitions/identifier" },
"iam_instance_profile": { "$ref": "#/definitions/identifier" }
"public_security_group": { "$ref": "#/$defs/identifier" },
"iam_instance_profile": { "$ref": "#/$defs/identifier" }
},
"required": [ "instance_type", "image_id", "public_security_group", "iam_instance_profile" ],
"additionalProperties": false
Expand All @@ -380,8 +380,8 @@
"type": "object",
"properties": {
"release_label": { "type": "string" },
"master": { "$ref": "#/definitions/cluster_group" },
"core": { "$ref": "#/definitions/cluster_group" },
"master": { "$ref": "#/$defs/cluster_group" },
"core": { "$ref": "#/$defs/cluster_group" },
"max_partitions": {
"description": "Maximum number of partitions during extract, limits num_partitions for each table",
"type": "integer",
Expand All @@ -394,15 +394,15 @@
"DataPipeline": {
"type": "object",
"properties": {
"role": { "$ref": "#/definitions/identifier" }
"role": { "$ref": "#/$defs/identifier" }
},
"required": [ "role" ],
"additionalProperties": false
},
"RedshiftCluster": {
"type": "object",
"properties": {
"name": { "$ref": "#/definitions/identifier" },
"name": { "$ref": "#/$defs/identifier" },
"max_concurrency": {
"type": "integer",
"minimum": 1
Expand All @@ -423,11 +423,11 @@
"items": {
"type": "object",
"properties": {
"unique-id": { "$ref": "#/definitions/identifier" },
"unique-id": { "$ref": "#/$defs/identifier" },
"description": { "type": "string" },
"start-time": { "type": "string" },
"occurrences": { "type": "integer" },
"patterns": { "$ref": "#/definitions/glob_pattern_list" }
"patterns": { "$ref": "#/$defs/glob_pattern_list" }
},
"required": [ "unique-id", "description" ],
"additionalProperties": false
Expand Down
Loading

0 comments on commit ff28a0b

Please sign in to comment.