Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support Enhanced IO fields in Distributed map #165

Merged
merged 1 commit into from
Feb 25, 2025
+275 −6
Merged

Support Enhanced IO fields in Distributed map #165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"url": "https://github.com/aws/amazon-states-language-service"
},
"license": "MIT",
"version": "1.15.0",
"version": "1.16.0",
"publisher": "aws",
"categories": [
"Programming Languages"
Expand Down
26 changes: 25 additions & 1 deletion src/asl-utils/asl/definitions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,18 @@ export interface ItemBatcher {
MaxInputBytesPerBatchPath?: string
}

export enum DistributedMapResultWriterOutputType {
JSON = 'JSON',
JSONL = 'JSONL',
}
export enum DistributedMapResultWriterTransformation {
NONE = 'NONE',
FLATTEN = 'FLATTEN',
COMPACT = 'COMPACT',
}

export interface ResultWriter {
Resource: string
Resource?: string
Parameters?: {
Bucket?: string
Key?: string
Expand All @@ -142,12 +152,17 @@ export interface ResultWriter {
ExpectedBucketOwner?: string
}
| JSONataExpression
WriterConfig?: {
OutputType?: DistributedMapResultWriterOutputType
Transformation?: DistributedMapResultWriterTransformation
}
}

export interface ItemReader {
ReaderConfig?: {
InputType?: ParsingInputType
CSVHeaderLocation?: CSVHeaderLocationType
CSVDelimiter?: CSVDelimiterType
MaxItems?: number | JSONataExpression
MaxItemsPath?: string
CSVHeaders?: string[]
Expand Down Expand Up @@ -185,6 +200,7 @@ export enum MapExecutionType {
export enum ParsingInputType {
CSV = 'CSV',
JSON = 'JSON',
JSONL = 'JSONL',
MANIFEST = 'MANIFEST',
}

Expand All @@ -193,6 +209,14 @@ export enum CSVHeaderLocationType {
GIVEN = 'GIVEN',
}

export enum CSVDelimiterType {
COMMA = 'COMMA',
PIPE = 'PIPE',
SEMICOLON = 'SEMICOLON',
TAB = 'TAB',
SPACE = 'SPACE',
}

export interface ProcessorConfig {
Mode: DistributedMapProcessingMode
ExecutionType?: MapExecutionType
Expand Down
29 changes: 27 additions & 2 deletions src/json-schema/bundled.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2061,8 +2061,8 @@
"InputType": {
"type": "string",
"description": "A string that specifies the type of Amazon S3 data source.",
"minLength": 1,
"enum": ["CSV", "JSON", "MANIFEST"]
"enum": ["CSV", "JSON", "JSONL", "MANIFEST"],
"minLength": 1
},
"CSVHeaders": {
"type": "array",
Expand Down Expand Up @@ -2093,6 +2093,12 @@
"MaxItemsPath": {
"$ref": "#/properties/States/additionalProperties/oneOf/0/allOf/1/properties/ResultPath/oneOf/0",
"description": "A reference path to an integer that limits the number of data items passed to the Map state."
},
"CSVDelimiter": {
"type": "string",
"description": "A string that specifies the delimiter used to separate values in the CSV file.",
"minLength": 1,
"enum": ["COMMA", "PIPE", "SEMICOLON", "TAB", "SPACE"]
}
}
},
Expand Down Expand Up @@ -2234,6 +2240,25 @@
"minLength": 1
}
}
},
"WriterConfig": {
"type": "object",
"description": "A JSON object that specifies the configuration for the results of the map.",
"properties": {
"OutputType": {
"type": "string",
"description": "A string that specifies the type in which Map state results are formatted.",
"minLength": 1,
"enum": ["JSON", "JSONL"]
},
"Transformation": {
"type": "string",
"description": "A string that specifies how the Map state results transformed before returning.",
"minLength": 1,
"enum": ["NONE", "FLATTENED", "COMPACT"]
}
},
"required": ["OutputType"]
}
}
},
Expand Down
29 changes: 27 additions & 2 deletions src/json-schema/partial/map_state.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@
"InputType": {
"type": "string",
"description": "A string that specifies the type of Amazon S3 data source.",
"minLength": 1,
"enum": ["CSV", "JSON", "MANIFEST"]
"enum": ["CSV", "JSON", "JSONL", "MANIFEST"],
"minLength": 1
},
"CSVHeaders": {
"type": "array",
Expand Down Expand Up @@ -88,6 +88,12 @@
"MaxItemsPath": {
"$ref": "common.json#/$defs/referencePath",
"description": "A reference path to an integer that limits the number of data items passed to the Map state."
},
"CSVDelimiter": {
"type": "string",
"description": "A string that specifies the delimiter used to separate values in the CSV file.",
"minLength": 1,
"enum": ["COMMA", "PIPE", "SEMICOLON", "TAB", "SPACE"]
}
}
},
Expand Down Expand Up @@ -229,6 +235,25 @@
"minLength": 1
}
}
},
"WriterConfig": {
"type": "object",
"description": "A JSON object that specifies the configuration for the results of the map.",
"properties": {
"OutputType": {
"type": "string",
"description": "A string that specifies the type in which Map state results are formatted.",
"minLength": 1,
"enum": ["JSON", "JSONL"]
},
"Transformation": {
"type": "string",
"description": "A string that specifies how the Map state results transformed before returning.",
"minLength": 1,
"enum": ["NONE", "FLATTENED", "COMPACT"]
}
},
"required": ["OutputType"]
}
}
},
Expand Down
157 changes: 157 additions & 0 deletions src/tests/json-strings/validationStrings.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2413,3 +2413,160 @@ export const documentInvalidFailWithAssign = `{
}
}
}`

export const documentMapResultWriter = `
{
"StartAt": "Map",
"States": {
"Map": {
"Type": "Map",
"ItemsPath": "$.array",
"ResultPath": "$.array",
"MaxConcurrency": 2,
"Next": "Final State",
"ItemProcessor": {
"StartAt": "Pass",
"States": {
"Pass": {
"Type": "Pass",
"End": true
}
},
"ProcessorConfig": {
"ExecutionType": "EXPRESS",
"Mode": "DISTRIBUTED"
}
},
"ResultWriter": {
"WriterConfig": {
"OutputType": "JSON"
}
}
},
"Final State": {
"Type": "Pass",
"End": true
}
}
}
`

export const documentMapResultWriterMissingOutputType = `
{
"StartAt": "Map",
"States": {
"Map": {
"Type": "Map",
"ItemsPath": "$.array",
"ResultPath": "$.array",
"MaxConcurrency": 2,
"Next": "Final State",
"ItemProcessor": {
"StartAt": "Pass",
"States": {
"Pass": {
"Type": "Pass",
"End": true
}
},
"ProcessorConfig": {
"ExecutionType": "EXPRESS",
"Mode": "DISTRIBUTED"
}
},
"ResultWriter": {
"WriterConfig": {}
}
},
"Final State": {
"Type": "Pass",
"End": true
}
}
}
`

export const documentMapItemReader = `
{
"StartAt": "Map",
"States": {
"Map": {
"Type": "Map",
"ItemsPath": "$.array",
"ResultPath": "$.array",
"MaxConcurrency": 2,
"Next": "Final State",
"ItemProcessor": {
"StartAt": "Pass",
"States": {
"Pass": {
"Type": "Pass",
"End": true
}
},
"ProcessorConfig": {
"ExecutionType": "EXPRESS",
"Mode": "DISTRIBUTED"
}
},
"ItemReader": {
"Resource": "arn:aws:states:::s3:getObject",
"ReaderConfig": {
"InputType": "CSV",
"CSVHeaderLocation": "FIRST_ROW"
},
"Parameters": {
"Bucket": "abc",
"Key": "def"
}
}
},
"Final State": {
"Type": "Pass",
"End": true
}
}
}
`

export const documentMapItemReaderWithInputTypeJSONL = `
{
"StartAt": "Map",
"States": {
"Map": {
"Type": "Map",
"ItemsPath": "$.array",
"ResultPath": "$.array",
"MaxConcurrency": 2,
"Next": "Final State",
"ItemProcessor": {
"StartAt": "Pass",
"States": {
"Pass": {
"Type": "Pass",
"End": true
}
},
"ProcessorConfig": {
"ExecutionType": "EXPRESS",
"Mode": "DISTRIBUTED"
}
},
"ItemReader": {
"Resource": "arn:aws:states:::s3:getObject",
"ReaderConfig": {
"InputType": "JSONL"
},
"Parameters": {
"Bucket": "abc",
"Key": "def"
}
}
},
"Final State": {
"Type": "Pass",
"End": true
}
}
}
`
38 changes: 38 additions & 0 deletions src/tests/validation.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ import {
documentValidParametersJsonPath,
documentValidResultSelectorIntrinsicFunction,
documentValidResultSelectorJsonPath,
documentMapResultWriter,
documentMapResultWriterMissingOutputType,
documentMapItemReader,
documentMapItemReaderWithInputTypeJSONL,
} from './json-strings/validationStrings'
import { toDocument } from './utils/testUtilities'

Expand Down Expand Up @@ -255,6 +259,40 @@ describe('ASL context-aware validation', () => {
],
})
})

test("Doesn't show diagnostics for valid ResultWriter", async () => {
await testValidations({
json: documentMapResultWriter,
diagnostics: [],
})
})

test('Shows diagnostics for valid ResultWriter', async () => {
await testValidations({
json: documentMapResultWriterMissingOutputType,
diagnostics: [
{
message: 'Missing property "OutputType".',
start: [24, 12],
end: [24, 26],
},
],
})
})

test("Doesn't show diagnostics for ItemReader", async () => {
await testValidations({
json: documentMapItemReader,
diagnostics: [],
})
})

test("Doesn't show diagnostics for valid InputType JSONL", async () => {
await testValidations({
json: documentMapItemReaderWithInputTypeJSONL,
diagnostics: [],
})
})
})

describe('Next', () => {
Expand Down