diff --git a/package.json b/package.json index cae327103..0ef2497fe 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "url": "https://github.com/aws/amazon-states-language-service" }, "license": "MIT", - "version": "1.15.0", + "version": "1.15.1", "publisher": "aws", "categories": [ "Programming Languages" diff --git a/src/asl-utils/asl/definitions.ts b/src/asl-utils/asl/definitions.ts index 34095e6f7..fa5706114 100644 --- a/src/asl-utils/asl/definitions.ts +++ b/src/asl-utils/asl/definitions.ts @@ -124,8 +124,18 @@ export interface ItemBatcher { MaxInputBytesPerBatchPath?: string } +export enum DistributedMapResultWriterOutputType { + JSON = 'JSON', + JSONL = 'JSONL', +} +export enum DistributedMapResultWriterTransformation { + NONE = 'NONE', + FLATTEN = 'FLATTEN', + COMPACT = 'COMPACT', +} + export interface ResultWriter { - Resource: string + Resource?: string Parameters?: { Bucket?: string Key?: string @@ -142,12 +152,17 @@ export interface ResultWriter { ExpectedBucketOwner?: string } | JSONataExpression + WriterConfig?: { + OutputType?: DistributedMapResultWriterOutputType + Transformation?: DistributedMapResultWriterTransformation + } } export interface ItemReader { ReaderConfig?: { InputType?: ParsingInputType CSVHeaderLocation?: CSVHeaderLocationType + CSVDelimiter?: CSVDelimiterType MaxItems?: number | JSONataExpression MaxItemsPath?: string CSVHeaders?: string[] @@ -185,6 +200,7 @@ export enum MapExecutionType { export enum ParsingInputType { CSV = 'CSV', JSON = 'JSON', + JSONL = 'JSONL', MANIFEST = 'MANIFEST', } @@ -193,6 +209,14 @@ export enum CSVHeaderLocationType { GIVEN = 'GIVEN', } +export enum CSVDelimiterType { + COMMA = 'COMMA', + PIPE = 'PIPE', + SEMICOLON = 'SEMICOLON', + TAB = 'TAB', + SPACE = 'SPACE', +} + export interface ProcessorConfig { Mode: DistributedMapProcessingMode ExecutionType?: MapExecutionType diff --git a/src/json-schema/bundled.json b/src/json-schema/bundled.json index c44379aed..b19bb4b15 100644 --- a/src/json-schema/bundled.json +++ b/src/json-schema/bundled.json @@ -2061,8 +2061,8 @@ "InputType": { "type": "string", "description": "A string that specifies the type of Amazon S3 data source.", - "minLength": 1, - "enum": ["CSV", "JSON", "MANIFEST"] + "enum": ["CSV", "JSON", "JSONL", "MANIFEST"], + "minLength": 1 }, "CSVHeaders": { "type": "array", @@ -2093,6 +2093,12 @@ "MaxItemsPath": { "$ref": "#/properties/States/additionalProperties/oneOf/0/allOf/1/properties/ResultPath/oneOf/0", "description": "A reference path to an integer that limits the number of data items passed to the Map state." + }, + "CSVDelimiter": { + "type": "string", + "description": "A string that specifies the delimiter used to separate values in the CSV file.", + "minLength": 1, + "enum": ["COMMA", "PIPE", "SEMICOLON", "TAB", "SPACE"] } } }, @@ -2234,6 +2240,25 @@ "minLength": 1 } } + }, + "WriterConfig": { + "type": "object", + "description": "A JSON object that specifies the configuration for the results of the map.", + "properties": { + "OutputType": { + "type": "string", + "description": "A string that specifies the type in which Map state results are formatted.", + "minLength": 1, + "enum": ["JSON", "JSONL"] + }, + "Transformation": { + "type": "string", + "description": "A string that specifies how the Map state results transformed before returning.", + "minLength": 1, + "enum": ["NONE", "FLATTENED", "COMPACT"] + } + }, + "required": ["OutputType"] } } }, diff --git a/src/json-schema/partial/map_state.json b/src/json-schema/partial/map_state.json index b9abc12f8..bc45a546e 100755 --- a/src/json-schema/partial/map_state.json +++ b/src/json-schema/partial/map_state.json @@ -56,8 +56,8 @@ "InputType": { "type": "string", "description": "A string that specifies the type of Amazon S3 data source.", - "minLength": 1, - "enum": ["CSV", "JSON", "MANIFEST"] + "enum": ["CSV", "JSON", "JSONL", "MANIFEST"], + "minLength": 1 }, "CSVHeaders": { "type": "array", @@ -88,6 +88,12 @@ "MaxItemsPath": { "$ref": "common.json#/$defs/referencePath", "description": "A reference path to an integer that limits the number of data items passed to the Map state." + }, + "CSVDelimiter": { + "type": "string", + "description": "A string that specifies the delimiter used to separate values in the CSV file.", + "minLength": 1, + "enum": ["COMMA", "PIPE", "SEMICOLON", "TAB", "SPACE"] } } }, @@ -229,6 +235,25 @@ "minLength": 1 } } + }, + "WriterConfig": { + "type": "object", + "description": "A JSON object that specifies the configuration for the results of the map.", + "properties": { + "OutputType": { + "type": "string", + "description": "A string that specifies the type in which Map state results are formatted.", + "minLength": 1, + "enum": ["JSON", "JSONL"] + }, + "Transformation": { + "type": "string", + "description": "A string that specifies how the Map state results transformed before returning.", + "minLength": 1, + "enum": ["NONE", "FLATTENED", "COMPACT"] + } + }, + "required": ["OutputType"] } } }, diff --git a/src/tests/json-strings/validationStrings.ts b/src/tests/json-strings/validationStrings.ts index aa007d593..66a1bbdac 100644 --- a/src/tests/json-strings/validationStrings.ts +++ b/src/tests/json-strings/validationStrings.ts @@ -2413,3 +2413,160 @@ export const documentInvalidFailWithAssign = `{ } } }` + +export const documentMapResultWriter = ` +{ + "StartAt": "Map", + "States": { + "Map": { + "Type": "Map", + "ItemsPath": "$.array", + "ResultPath": "$.array", + "MaxConcurrency": 2, + "Next": "Final State", + "ItemProcessor": { + "StartAt": "Pass", + "States": { + "Pass": { + "Type": "Pass", + "End": true + } + }, + "ProcessorConfig": { + "ExecutionType": "EXPRESS", + "Mode": "DISTRIBUTED" + } + }, + "ResultWriter": { + "WriterConfig": { + "OutputType": "JSON" + } + } + }, + "Final State": { + "Type": "Pass", + "End": true + } + } +} +` + +export const documentMapResultWriterMissingOutputType = ` +{ + "StartAt": "Map", + "States": { + "Map": { + "Type": "Map", + "ItemsPath": "$.array", + "ResultPath": "$.array", + "MaxConcurrency": 2, + "Next": "Final State", + "ItemProcessor": { + "StartAt": "Pass", + "States": { + "Pass": { + "Type": "Pass", + "End": true + } + }, + "ProcessorConfig": { + "ExecutionType": "EXPRESS", + "Mode": "DISTRIBUTED" + } + }, + "ResultWriter": { + "WriterConfig": {} + } + }, + "Final State": { + "Type": "Pass", + "End": true + } + } +} +` + +export const documentMapItemReader = ` +{ + "StartAt": "Map", + "States": { + "Map": { + "Type": "Map", + "ItemsPath": "$.array", + "ResultPath": "$.array", + "MaxConcurrency": 2, + "Next": "Final State", + "ItemProcessor": { + "StartAt": "Pass", + "States": { + "Pass": { + "Type": "Pass", + "End": true + } + }, + "ProcessorConfig": { + "ExecutionType": "EXPRESS", + "Mode": "DISTRIBUTED" + } + }, + "ItemReader": { + "Resource": "arn:aws:states:::s3:getObject", + "ReaderConfig": { + "InputType": "CSV", + "CSVHeaderLocation": "FIRST_ROW" + }, + "Parameters": { + "Bucket": "abc", + "Key": "def" + } + } + }, + "Final State": { + "Type": "Pass", + "End": true + } + } +} +` + +export const documentMapItemReaderWithInputTypeJSONL = ` +{ + "StartAt": "Map", + "States": { + "Map": { + "Type": "Map", + "ItemsPath": "$.array", + "ResultPath": "$.array", + "MaxConcurrency": 2, + "Next": "Final State", + "ItemProcessor": { + "StartAt": "Pass", + "States": { + "Pass": { + "Type": "Pass", + "End": true + } + }, + "ProcessorConfig": { + "ExecutionType": "EXPRESS", + "Mode": "DISTRIBUTED" + } + }, + "ItemReader": { + "Resource": "arn:aws:states:::s3:getObject", + "ReaderConfig": { + "InputType": "JSONL" + }, + "Parameters": { + "Bucket": "abc", + "Key": "def" + } + } + }, + "Final State": { + "Type": "Pass", + "End": true + } + } +} +` diff --git a/src/tests/validation.test.ts b/src/tests/validation.test.ts index c29a09d18..9c88a2669 100644 --- a/src/tests/validation.test.ts +++ b/src/tests/validation.test.ts @@ -83,6 +83,10 @@ import { documentValidParametersJsonPath, documentValidResultSelectorIntrinsicFunction, documentValidResultSelectorJsonPath, + documentMapResultWriter, + documentMapResultWriterMissingOutputType, + documentMapItemReader, + documentMapItemReaderWithInputTypeJSONL, } from './json-strings/validationStrings' import { toDocument } from './utils/testUtilities' @@ -255,6 +259,40 @@ describe('ASL context-aware validation', () => { ], }) }) + + test("Doesn't show diagnostics for valid ResultWriter", async () => { + await testValidations({ + json: documentMapResultWriter, + diagnostics: [], + }) + }) + + test('Shows diagnostics for valid ResultWriter', async () => { + await testValidations({ + json: documentMapResultWriterMissingOutputType, + diagnostics: [ + { + message: 'Missing property "OutputType".', + start: [24, 12], + end: [24, 26], + }, + ], + }) + }) + + test("Doesn't show diagnostics for ItemReader", async () => { + await testValidations({ + json: documentMapItemReader, + diagnostics: [], + }) + }) + + test("Doesn't show diagnostics for valid InputType JSONL", async () => { + await testValidations({ + json: documentMapItemReaderWithInputTypeJSONL, + diagnostics: [], + }) + }) }) describe('Next', () => {