diff --git a/1.architectures/3.aws-batch/0.aws-batch-distributed-training-p5.yaml b/1.architectures/3.aws-batch/0.aws-batch-distributed-training-p5.yaml
new file mode 100644
index 00000000..12eb57f5
--- /dev/null
+++ b/1.architectures/3.aws-batch/0.aws-batch-distributed-training-p5.yaml
@@ -0,0 +1,699 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT-0
+
+AWSTemplateFormatVersion: '2010-09-09'
+Description: >
+ Setup for P5 instances on AWS Batch with CEs, JQ, ECR and Job Definition.
+ Author: Sean Smith - seaam@
+
+Parameters:
+ VPCStackParameter:
+ Type: String
+ Description: Private subnets will be retrieved for the compute environment
+ Default: 'aws-batch-vpc'
+ CapacityBlockId:
+ Description: ID of the Capacity Block
+ Type: String
+ Default: ''
+
+Metadata:
+ AWS::CloudFormation::Interface:
+ ParameterGroups:
+ - Label:
+ default: General configuration
+ Parameters:
+ - VPCStackParameter
+ - Label:
+ default: AWS Batch Configuration
+ Parameters:
+ - CapacityBlockId
+ ParameterLabels:
+ VPCStackParameter:
+ default: Name of the VPC Stack
+ CapacityBlockId:
+ default: Capacity Block ID
+
+
+Resources:
+ ###################
+ ## EC2 Resources ##
+ ###################
+ DistributedDeepLearningLT:
+ Type: AWS::EC2::LaunchTemplate
+ Properties:
+ LaunchTemplateData:
+ InstanceMarketOptions:
+ MarketType: "capacity-block"
+ CapacityReservationSpecification:
+ CapacityReservationTarget:
+ CapacityReservationId: !Ref CapacityBlockId
+ NetworkInterfaces:
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 0
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 1
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 2
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 3
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 4
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 5
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 6
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 7
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 8
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 9
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 10
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 11
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 12
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 13
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 14
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 15
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 16
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 17
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 18
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 19
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 20
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 21
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 22
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 23
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 24
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 25
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 26
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 27
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 28
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 29
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 30
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+ - Description: EFA Interface
+ Groups: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-SecurityGroup
+ NetworkCardIndex: 31
+ DeviceIndex: 0
+ DeleteOnTermination: true
+ InterfaceType: efa
+
+ ########################
+ ## Batch Architecture ##
+ ########################
+
+ ##
+ ## IAM Roles for AWS Batch
+ ##
+ # Configure IAM roles for Batch
+ BatchInstanceRole:
+ Type: AWS::IAM::Role
+ Properties:
+ AssumeRolePolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ -
+ Effect: Allow
+ Principal:
+ Service:
+ - batch.amazonaws.com
+ Action:
+ - sts:AssumeRole
+ ManagedPolicyArns:
+ - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole
+
+ ECSTaskServiceRole:
+ Type: AWS::IAM::Role
+ Properties:
+ AssumeRolePolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ -
+ Effect: Allow
+ Principal:
+ Service:
+ - ec2.amazonaws.com
+ Action:
+ - sts:AssumeRole
+ ManagedPolicyArns:
+ - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role
+
+ ECSTaskInstanceProfile:
+ Type: AWS::IAM::InstanceProfile
+ Properties:
+ Path: /
+ Roles:
+ - !Ref ECSTaskServiceRole
+ InstanceProfileName: !Join [ "", [ "ECSTaskInstanceProfileIAM-", !Ref AWS::StackName ] ]
+
+ ##
+ ## Compute Environment and Job Definition
+ ##
+ # Build the AWS Batch CEs
+ DistributedDeepLearningCE:
+ Type: AWS::Batch::ComputeEnvironment
+ Properties:
+ Type: MANAGED
+ ServiceRole: !Ref BatchInstanceRole
+ ComputeResources:
+ AllocationStrategy: BEST_FIT
+ MaxvCpus: 100000
+ DesiredvCpus: 0
+ MinvCpus: 0
+ Subnets: !Split
+ - ','
+ - Fn::ImportValue: !Sub ${VPCStackParameter}-PrivateSubnet
+ Type: EC2
+ InstanceRole: !Ref ECSTaskInstanceProfile
+ LaunchTemplate:
+ LaunchTemplateId: !Ref DistributedDeepLearningLT
+ Version: $Latest
+ InstanceTypes:
+ - p5.48xlarge
+ State: ENABLED
+ Tags:
+ Name: Batch Deep Learning
+
+ # Job Queue
+ DistributedDeepLearningJQ:
+ Type: AWS::Batch::JobQueue
+ Properties:
+ ComputeEnvironmentOrder:
+ - ComputeEnvironment: !Ref DistributedDeepLearningCE
+ Order: 1
+ Priority: 1
+ State: "ENABLED"
+
+ ##
+ ## ECR and AWS Batch Job definition
+ ##
+ NCCLTestRepository:
+ Type: AWS::ECR::Repository
+
+ NCCLTest:
+ Type: AWS::Batch::JobDefinition
+ Properties:
+ Type: multinode
+ NodeProperties:
+ MainNode: 0
+ NumNodes: 2
+ NodeRangeProperties:
+ - TargetNodes: '0:'
+ Container:
+ InstanceType: p5.48xlarge
+ Image: public.ecr.aws/hpc-cloud/nccl-tests:latest
+ Command:
+ - /opt/amazon/openmpi/bin/mpirun
+ - --allow-run-as-root
+ - --tag-output
+ - -np
+ - "8"
+ - -N
+ - "8"
+ - --bind-to
+ - none
+ - -x
+ - PATH
+ - -x
+ - LD_LIBRARY_PATH
+ - -x
+ - FI_PROVIDER=efa
+ - -x
+ - FI_EFA_USE_DEVICE_RDMA=1
+ - -x
+ - FI_EFA_FORK_SAFE=1
+ - -x
+ - NCCL_DEBUG=INFO
+ - -x
+ - NCCL_BUFFSIZE=8388608
+ - -x
+ - NCCL_P2P_NET_CHUNKSIZE=524288
+ - -x
+ - NCCL_TUNER_PLUGIN=/opt/aws-ofi-nccl/install/lib/libnccl-ofi-tuner.so
+ - --mca
+ - pml
+ - ^cm,ucx
+ - --mca
+ - btl
+ - tcp,self
+ - --mca
+ - btl_tcp_if_exclude
+ - lo,docker0,veth_def_agent
+ - /opt/nccl-tests/build/all_reduce_perf
+ - -b
+ - "8"
+ - -e
+ - "16G"
+ - -f
+ - "2"
+ - -g
+ - "1"
+ - -c
+ - "1"
+ - -n
+ - "100"
+ Environment:
+ - Name: LD_LIBRARY_PATH
+ Value: /opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/nvidia/lib:$LD_LIBRARY_PATH
+ - Name: PATH
+ Value: $PATH:/opt/amazon/efa/bin:/usr/bin
+ ResourceRequirements:
+ - Type: VCPU
+ Value: 192
+ - Type: GPU
+ Value: 8
+ - Type: MEMORY
+ Value: 1049000
+ Ulimits:
+ - Name: memlock
+ HardLimit: -1
+ SoftLimit: -1
+ - Name: stack
+ HardLimit: 67108864
+ SoftLimit: 67108864
+ - Name: nofile
+ HardLimit: 1024000
+ SoftLimit: 1024000
+ LinuxParameters:
+ SharedMemorySize: 49152
+ Devices:
+ - HostPath: /dev/infiniband/uverbs0
+ ContainerPath: /dev/infiniband/uverbs0
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs1
+ ContainerPath: /dev/infiniband/uverbs1
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs2
+ ContainerPath: /dev/infiniband/uverbs2
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs3
+ ContainerPath: /dev/infiniband/uverbs3
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs4
+ ContainerPath: /dev/infiniband/uverbs4
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs5
+ ContainerPath: /dev/infiniband/uverbs5
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs6
+ ContainerPath: /dev/infiniband/uverbs6
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs7
+ ContainerPath: /dev/infiniband/uverbs7
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs8
+ ContainerPath: /dev/infiniband/uverbs8
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs9
+ ContainerPath: /dev/infiniband/uverbs9
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs10
+ ContainerPath: /dev/infiniband/uverbs10
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs11
+ ContainerPath: /dev/infiniband/uverbs11
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs12
+ ContainerPath: /dev/infiniband/uverbs12
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs13
+ ContainerPath: /dev/infiniband/uverbs13
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs14
+ ContainerPath: /dev/infiniband/uverbs14
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs15
+ ContainerPath: /dev/infiniband/uverbs15
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs16
+ ContainerPath: /dev/infiniband/uverbs16
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs17
+ ContainerPath: /dev/infiniband/uverbs17
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs18
+ ContainerPath: /dev/infiniband/uverbs18
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs19
+ ContainerPath: /dev/infiniband/uverbs19
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs20
+ ContainerPath: /dev/infiniband/uverbs20
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs21
+ ContainerPath: /dev/infiniband/uverbs21
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs22
+ ContainerPath: /dev/infiniband/uverbs22
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs23
+ ContainerPath: /dev/infiniband/uverbs23
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs24
+ ContainerPath: /dev/infiniband/uverbs24
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs25
+ ContainerPath: /dev/infiniband/uverbs25
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs26
+ ContainerPath: /dev/infiniband/uverbs26
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs27
+ ContainerPath: /dev/infiniband/uverbs27
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs28
+ ContainerPath: /dev/infiniband/uverbs28
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs29
+ ContainerPath: /dev/infiniband/uverbs29
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs30
+ ContainerPath: /dev/infiniband/uverbs30
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ - HostPath: /dev/infiniband/uverbs31
+ ContainerPath: /dev/infiniband/uverbs31
+ Permissions:
+ - READ
+ - WRITE
+ - MKNOD
+ PropagateTags: true
+ RetryStrategy:
+ Attempts: 1
+
+Outputs:
+
+ ECRRepository:
+ Description: ECR Repository for the containers
+ Value: !Ref NCCLTestRepository
+
+ ECRRepositoryUrl:
+ Description: ECR Repository for the containers
+ Value: !Join ['', [!Ref 'AWS::AccountId','.dkr.ecr.', !Ref 'AWS::Region', '.amazonaws.com/', !Ref NCCLTestRepository ] ]
+
+ JobDefinitionMultiInstance:
+ Description: Job definition for Multi-node Parallel Jobs
+ Value: !Ref NCCLTest
+
+ DistributedDeepLearningJQ:
+ Description: Job Queue
+ Value: !Ref DistributedDeepLearningJQ
diff --git a/1.architectures/3.aws-batch/README.md b/1.architectures/3.aws-batch/README.md
index 25cf773e..af494aee 100644
--- a/1.architectures/3.aws-batch/README.md
+++ b/1.architectures/3.aws-batch/README.md
@@ -42,11 +42,11 @@ The templates takes parameters that are mandatory and optional, see below for mo
If you'd like to deploy through the AWS CLI instead of the quick create link above, the command to deploy the template is shown below. Please edit the parameters values with your own configuration.
```bash
-aws cloudformation create-stack --stack-name batch-distributed-training \
- --template-body file://0.aws-batch-distributed-training.yaml \
- --parameters ParameterKey=VPCStackParameter,ParameterValue="vpc-stack-ml" \
- ParameterKey=CapacityReservationId,ParameterValue="cr-123567890abc" \
- --capabilities CAPABILITY_IAM
+aws cloudformation create-stack --stack-name aws-batch-p5 \
+ --template-body file://0.aws-batch-distributed-training-p5.yaml \
+ --parameters ParameterKey=VPCStackParameter,ParameterValue="aws-batch-vpc" \
+ ParameterKey=CapacityReservationId,ParameterValue="cr-1234567890" \
+ --capabilities CAPABILITY_NAMED_IAM
```
## Gotchas
diff --git a/micro-benchmarks/nccl-tests/aws-batch/README.md b/micro-benchmarks/nccl-tests/aws-batch/README.md
new file mode 100644
index 00000000..58433c63
--- /dev/null
+++ b/micro-benchmarks/nccl-tests/aws-batch/README.md
@@ -0,0 +1,65 @@
+# NCCL Tests on AWS Batch
+
+1. First create a VPC and Subnet in the same AZ as your capacity block. You can use the following template:
+
+[
1-Click Deploy 🚀
](https://console.aws.amazon.com/cloudformation/home?#/stacks/quickcreate?templateURL=https%3A%2F%2Fawsome-distributed-training.s3.amazonaws.com%2Ftemplates%2F2.vpc-one-az.yaml&stackName=aws-batch-vpc)
+
+2. Next you can deploy the AWS Batch template included in this PR, where `cr-1234567890` is the id of your capacity block and `aws-batch-vpc` is the name of the vpc stack you created above.
+
+```bash
+git clone https://github.com/aws-samples/awsome-distributed-training.git
+cd awsome-distributed-training/1.architectures/3.aws-batch
+aws cloudformation create-stack --stack-name aws-batch-p5 \
+ --template-body file://0.aws-batch-distributed-training-p5.yaml \
+ --parameters ParameterKey=VPCStackParameter,ParameterValue="aws-batch-vpc" \
+ ParameterKey=CapacityBlockId,ParameterValue="cr-1234567890" \
+ --capabilities CAPABILITY_NAMED_IAM
+```
+
+3. Next you can submit a job in the aws batch console, by default the NCCLTest Job Definition uses the pre-built container image:
+
+```
+public.ecr.aws/hpc-cloud/nccl-tests:latest
+```
+
+4. Check the output by going to `Job` > `Nodes` > `Log Stream`
+
+```
+[1,0]:# out-of-place in-place
+[1,0]:# size count type redop root time algbw busbw #wrong time algbw busbw #wrong
+[1,0]:# (B) (elements) (us) (GB/s) (GB/s) (us) (GB/s) (GB/s)
+[1,0]: 8 2 float sum -1 12.90 0.00 0.00 0 12.87 0.00 0.00 0
+[1,0]: 16 4 float sum -1 12.88 0.00 0.00 0 12.83 0.00 0.00 0
+[1,0]: 32 8 float sum -1 22.25 0.00 0.00 0 13.34 0.00 0.00 0
+[1,0]: 64 16 float sum -1 14.48 0.00 0.01 0 14.37 0.00 0.01 0
+[1,0]: 128 32 float sum -1 15.47 0.01 0.01 0 15.27 0.01 0.01 0
+[1,0]: 256 64 float sum -1 15.72 0.02 0.03 0 15.33 0.02 0.03 0
+[1,0]: 512 128 float sum -1 16.19 0.03 0.06 0 15.52 0.03 0.06 0
+[1,0]: 1024 256 float sum -1 16.43 0.06 0.11 0 15.61 0.07 0.11 0
+[1,0]: 2048 512 float sum -1 16.62 0.12 0.22 0 15.90 0.13 0.23 0
+[1,0]: 4096 1024 float sum -1 16.76 0.24 0.43 0 16.06 0.26 0.45 0
+[1,0]: 8192 2048 float sum -1 16.85 0.49 0.85 0 16.20 0.51 0.88 0
+[1,0]: 16384 4096 float sum -1 17.46 0.94 1.64 0 16.78 0.98 1.71 0
+[1,0]: 32768 8192 float sum -1 18.78 1.74 3.05 0 18.09 1.81 3.17 0
+[1,0]: 65536 16384 float sum -1 19.09 3.43 6.01 0 18.26 3.59 6.28 0
+[1,0]: 131072 32768 float sum -1 19.37 6.77 11.84 0 18.57 7.06 12.35 0
+[1,0]: 262144 65536 float sum -1 20.27 12.94 22.64 0 20.13 13.02 22.78 0
+[1,0]: 524288 131072 float sum -1 21.17 24.77 43.35 0 20.96 25.01 43.77 0
+[1,0]: 1048576 262144 float sum -1 28.56 36.71 64.24 0 28.37 36.96 64.68 0
+[1,0]: 2097152 524288 float sum -1 43.45 48.27 84.47 0 43.39 48.34 84.59 0
+[1,0]: 4194304 1048576 float sum -1 55.44 75.66 132.41 0 55.00 76.27 133.46 0
+[1,0]: 8388608 2097152 float sum -1 84.29 99.52 174.16 0 83.22 100.80 176.40 0
+[1,0]: 16777216 4194304 float sum -1 124.0 135.33 236.82 0 123.4 135.92 237.86 0
+[1,0]: 33554432 8388608 float sum -1 198.6 168.93 295.63 0 198.3 169.17 296.06 0
+[1,0]: 67108864 16777216 float sum -1 326.6 205.45 359.54 0 326.5 205.55 359.71 0
+[1,0]: 134217728 33554432 float sum -1 592.8 226.39 396.19 0 592.1 226.69 396.71 0
+[1,0]: 268435456 67108864 float sum -1 1121.4 239.38 418.91 0 1121.2 239.43 419.00 0
+[1,0]: 536870912 134217728 float sum -1 2165.3 247.94 433.90 0 2166.4 247.81 433.67 0
+[1,0]: 1073741824 268435456 float sum -1 4253.0 252.47 441.81 0 4255.4 252.33 441.57 0
+[1,0]: 2147483648 536870912 float sum -1 8463.7 253.73 444.02 0 8466.5 253.65 443.88 0
+[1,0]: 4294967296 1073741824 float sum -1 15791 271.98 475.97 0 15773 272.29 476.51 0
+[1,0]: 8589934592 2147483648 float sum -1 31289 274.53 480.43 0 31294 274.49 480.36 0
+[1,0]: 17179869184 4294967296 float sum -1 62239 276.03 483.05 0 62200 276.20 483.35 0
+[1,0]:# Out of bounds values : 0 OK
+[1,0]:# Avg bus bandwidth : 156.742
+```
\ No newline at end of file