Skip to content

Commit

Permalink
Add EnhancedIO schema support for Dmap
Browse files Browse the repository at this point in the history 
Bump version to 1.15.1
  • Loading branch information
Silvia Chen committed Feb 24, 2025
1 parent 903c284 commit 90820d5
Show file tree
Hide file tree
Showing 6 changed files with 275 additions and 6 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"url": "https://github.com/aws/amazon-states-language-service"
},
"license": "MIT",
"version": "1.15.0",
"version": "1.15.1",
"publisher": "aws",
"categories": [
"Programming Languages"
Expand Down
26 changes: 25 additions & 1 deletion src/asl-utils/asl/definitions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,18 @@ export interface ItemBatcher {
MaxInputBytesPerBatchPath?: string
}

export enum DistributedMapResultWriterOutputType {
JSON = 'JSON',
JSONL = 'JSONL',
}
export enum DistributedMapResultWriterTransformation {
NONE = 'NONE',
FLATTEN = 'FLATTEN',
COMPACT = 'COMPACT',
}

export interface ResultWriter {
Resource: string
Resource?: string
Parameters?: {
Bucket?: string
Key?: string
Expand All @@ -142,12 +152,17 @@ export interface ResultWriter {
ExpectedBucketOwner?: string
}
| JSONataExpression
WriterConfig?: {
OutputType?: DistributedMapResultWriterOutputType
Transformation?: DistributedMapResultWriterTransformation
}
}

export interface ItemReader {
ReaderConfig?: {
InputType?: ParsingInputType
CSVHeaderLocation?: CSVHeaderLocationType
CSVDelimiter?: CSVDelimiterType
MaxItems?: number | JSONataExpression
MaxItemsPath?: string
CSVHeaders?: string[]
Expand Down Expand Up @@ -185,6 +200,7 @@ export enum MapExecutionType {
export enum ParsingInputType {
CSV = 'CSV',
JSON = 'JSON',
JSONL = 'JSONL',
MANIFEST = 'MANIFEST',
}

Expand All @@ -193,6 +209,14 @@ export enum CSVHeaderLocationType {
GIVEN = 'GIVEN',
}

export enum CSVDelimiterType {
COMMA = 'COMMA',
PIPE = 'PIPE',
SEMICOLON = 'SEMICOLON',
TAB = 'TAB',
SPACE = 'SPACE',
}

export interface ProcessorConfig {
Mode: DistributedMapProcessingMode
ExecutionType?: MapExecutionType
Expand Down
29 changes: 27 additions & 2 deletions src/json-schema/bundled.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2061,8 +2061,8 @@
"InputType": {
"type": "string",
"description": "A string that specifies the type of Amazon S3 data source.",
"minLength": 1,
"enum": ["CSV", "JSON", "MANIFEST"]
"enum": ["CSV", "JSON", "JSONL", "MANIFEST"],
"minLength": 1
},
"CSVHeaders": {
"type": "array",
Expand Down Expand Up @@ -2093,6 +2093,12 @@
"MaxItemsPath": {
"$ref": "#/properties/States/additionalProperties/oneOf/0/allOf/1/properties/ResultPath/oneOf/0",
"description": "A reference path to an integer that limits the number of data items passed to the Map state."
},
"CSVDelimiter": {
"type": "string",
"description": "A string that specifies the delimiter used to separate values in the CSV file.",
"minLength": 1,
"enum": ["COMMA", "PIPE", "SEMICOLON", "TAB", "SPACE"]
}
}
},
Expand Down Expand Up @@ -2234,6 +2240,25 @@
"minLength": 1
}
}
},
"WriterConfig": {
"type": "object",
"description": "A JSON object that specifies the configuration for the results of the map.",
"properties": {
"OutputType": {
"type": "string",
"description": "A string that specifies the type in which Map state results are formatted.",
"minLength": 1,
"enum": ["JSON", "JSONL"]
},
"Transformation": {
"type": "string",
"description": "A string that specifies how the Map state results transformed before returning.",
"minLength": 1,
"enum": ["NONE", "FLATTENED", "COMPACT"]
}
},
"required": ["OutputType"]
}
}
},
Expand Down
29 changes: 27 additions & 2 deletions src/json-schema/partial/map_state.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@
"InputType": {
"type": "string",
"description": "A string that specifies the type of Amazon S3 data source.",
"minLength": 1,
"enum": ["CSV", "JSON", "MANIFEST"]
"enum": ["CSV", "JSON", "JSONL", "MANIFEST"],
"minLength": 1
},
"CSVHeaders": {
"type": "array",
Expand Down Expand Up @@ -88,6 +88,12 @@
"MaxItemsPath": {
"$ref": "common.json#/$defs/referencePath",
"description": "A reference path to an integer that limits the number of data items passed to the Map state."
},
"CSVDelimiter": {
"type": "string",
"description": "A string that specifies the delimiter used to separate values in the CSV file.",
"minLength": 1,
"enum": ["COMMA", "PIPE", "SEMICOLON", "TAB", "SPACE"]
}
}
},
Expand Down Expand Up @@ -229,6 +235,25 @@
"minLength": 1
}
}
},
"WriterConfig": {
"type": "object",
"description": "A JSON object that specifies the configuration for the results of the map.",
"properties": {
"OutputType": {
"type": "string",
"description": "A string that specifies the type in which Map state results are formatted.",
"minLength": 1,
"enum": ["JSON", "JSONL"]
},
"Transformation": {
"type": "string",
"description": "A string that specifies how the Map state results transformed before returning.",
"minLength": 1,
"enum": ["NONE", "FLATTENED", "COMPACT"]
}
},
"required": ["OutputType"]
}
}
},
Expand Down
157 changes: 157 additions & 0 deletions src/tests/json-strings/validationStrings.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2413,3 +2413,160 @@ export const documentInvalidFailWithAssign = `{
}
}
}`

export const documentMapResultWriter = `
{
"StartAt": "Map",
"States": {
"Map": {
"Type": "Map",
"ItemsPath": "$.array",
"ResultPath": "$.array",
"MaxConcurrency": 2,
"Next": "Final State",
"ItemProcessor": {
"StartAt": "Pass",
"States": {
"Pass": {
"Type": "Pass",
"End": true
}
},
"ProcessorConfig": {
"ExecutionType": "EXPRESS",
"Mode": "DISTRIBUTED"
}
},
"ResultWriter": {
"WriterConfig": {
"OutputType": "JSON"
}
}
},
"Final State": {
"Type": "Pass",
"End": true
}
}
}
`

export const documentMapResultWriterMissingOutputType = `
{
"StartAt": "Map",
"States": {
"Map": {
"Type": "Map",
"ItemsPath": "$.array",
"ResultPath": "$.array",
"MaxConcurrency": 2,
"Next": "Final State",
"ItemProcessor": {
"StartAt": "Pass",
"States": {
"Pass": {
"Type": "Pass",
"End": true
}
},
"ProcessorConfig": {
"ExecutionType": "EXPRESS",
"Mode": "DISTRIBUTED"
}
},
"ResultWriter": {
"WriterConfig": {}
}
},
"Final State": {
"Type": "Pass",
"End": true
}
}
}
`

export const documentMapItemReader = `
{
"StartAt": "Map",
"States": {
"Map": {
"Type": "Map",
"ItemsPath": "$.array",
"ResultPath": "$.array",
"MaxConcurrency": 2,
"Next": "Final State",
"ItemProcessor": {
"StartAt": "Pass",
"States": {
"Pass": {
"Type": "Pass",
"End": true
}
},
"ProcessorConfig": {
"ExecutionType": "EXPRESS",
"Mode": "DISTRIBUTED"
}
},
"ItemReader": {
"Resource": "arn:aws:states:::s3:getObject",
"ReaderConfig": {
"InputType": "CSV",
"CSVHeaderLocation": "FIRST_ROW"
},
"Parameters": {
"Bucket": "abc",
"Key": "def"
}
}
},
"Final State": {
"Type": "Pass",
"End": true
}
}
}
`

export const documentMapItemReaderWithInputTypeJSONL = `
{
"StartAt": "Map",
"States": {
"Map": {
"Type": "Map",
"ItemsPath": "$.array",
"ResultPath": "$.array",
"MaxConcurrency": 2,
"Next": "Final State",
"ItemProcessor": {
"StartAt": "Pass",
"States": {
"Pass": {
"Type": "Pass",
"End": true
}
},
"ProcessorConfig": {
"ExecutionType": "EXPRESS",
"Mode": "DISTRIBUTED"
}
},
"ItemReader": {
"Resource": "arn:aws:states:::s3:getObject",
"ReaderConfig": {
"InputType": "JSONL"
},
"Parameters": {
"Bucket": "abc",
"Key": "def"
}
}
},
"Final State": {
"Type": "Pass",
"End": true
}
}
}
`
38 changes: 38 additions & 0 deletions src/tests/validation.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ import {
documentValidParametersJsonPath,
documentValidResultSelectorIntrinsicFunction,
documentValidResultSelectorJsonPath,
documentMapResultWriter,
documentMapResultWriterMissingOutputType,
documentMapItemReader,
documentMapItemReaderWithInputTypeJSONL,
} from './json-strings/validationStrings'
import { toDocument } from './utils/testUtilities'

Expand Down Expand Up @@ -255,6 +259,40 @@ describe('ASL context-aware validation', () => {
],
})
})

test("Doesn't show diagnostics for valid ResultWriter", async () => {
await testValidations({
json: documentMapResultWriter,
diagnostics: [],
})
})

test('Shows diagnostics for valid ResultWriter', async () => {
await testValidations({
json: documentMapResultWriterMissingOutputType,
diagnostics: [
{
message: 'Missing property "OutputType".',
start: [24, 12],
end: [24, 26],
},
],
})
})

test("Doesn't show diagnostics for ItemReader", async () => {
await testValidations({
json: documentMapItemReader,
diagnostics: [],
})
})

test("Doesn't show diagnostics for valid InputType JSONL", async () => {
await testValidations({
json: documentMapItemReaderWithInputTypeJSONL,
diagnostics: [],
})
})
})

describe('Next', () => {
Expand Down

0 comments on commit 90820d5

Please sign in to comment.