Add support for ParallelCluster 3.7.0

DEVELOPMENT.md

@@ -6,6 +6,7 @@ To run AWS ParallelCluster UI locally, start by setting the following environmen
 export AWS_ACCESS_KEY_ID=[...]
 export AWS_SECRET_ACCESS_KEY=[...]
 export AWS_DEFAULT_REGION=us-east-2
+export API_VERSION="3.7.0"
 export API_BASE_URL=https://[API_ID].execute-api.us-east-2.amazonaws.com/prod  # get this from ParallelClusterApi stack outputs
 export ENV=dev
 ```
@@ -51,7 +52,9 @@ export AUDIENCE=<the value of the Client ID noted in the previous step>
 export AUTH_PATH=<the UserPoolAuthDomain output of the ParallelClusterCognito nested stack>
 ```
 
-Set `DISABLE_AUTH=False` in `api/utils.py` to facilitate live reloading.
+Set `DISABLE_AUTH=True` in `api/utils.py` to facilitate live reloading.
+Notice that cluster creation (both concrete and dryrun) is expected to fail
+when authentication is disabled.
 
 Start the API backend by running:
 

api/PclusterApiHandler.py

@@ -498,6 +498,13 @@ def get_aws_config():
     except:
         pass
 
+    file_caches = []
+    try:
+        file_caches = list(filter(lambda file_cache: (file_cache["Lifecycle"] == "AVAILABLE"),
+                                  fsx.describe_file_caches()["FileCaches"]))
+    except:
+        pass
+
     efs_filesystems = []
     try:
         efs_filesystems = efs.describe_file_systems()["FileSystems"]
@@ -518,6 +525,7 @@ def get_aws_config():
         "region": region,
         "fsx_filesystems": fsx_filesystems,
         "fsx_volumes": fsx_volumes,
+        "file_caches": file_caches,
         "efs_filesystems": efs_filesystems,
         "efa_instance_types": efa_instance_types,
     }

frontend/locales/en/strings.json

@@ -780,6 +780,7 @@
         "storageTypes": "Storage types",
         "storageTypesPlaceholder": "Select file system types",
         "volumePlaceholder": "Select a volume",
+        "cachePlaceholder": "Select a cache",
         "addStorage": "Add storage",
         "removeStorage": "Remove storage",
         "sourceTitle": "Source {{index}} - {{name}}"
@@ -820,7 +821,8 @@
         "existing": {
           "fsxLustre": "FSx Lustre Filesystem",
           "fsxOpenZfs": "FSx OpenZFS volume",
-          "fsxOnTap": "FSx NetApp ONTAP volume"
+          "fsxOnTap": "FSx NetApp ONTAP volume",
+          "fileCache": "Amazon File Cache"
         },
         "capacity": {
           "label": "Storage capacity (GB)",

frontend/src/__tests__/storageCreationValidation.test.tsx

@@ -10,9 +10,12 @@ describe('Given a function to determine whether we can create a storage of a giv
     })
   })
   describe('when the storage type does not support creation', () => {
-    it('should not allow the creation of a new storage', () => {
+    it('should not allow the creation of a new storage with type FsxOntap', () => {
       expect(canCreateStorage('FsxOntap', [], [])).toBeFalsy()
     })
+    it('should not allow the creation of a new storage with type FileCache', () => {
+      expect(canCreateStorage('FileCache', [], [])).toBeFalsy()
+    })
   })
   describe('when the attached storages are not available', () => {
     it('should allow the creation of a new storage', () => {
@@ -60,9 +63,14 @@ describe('Given a function to determine whether we can attach an existing storag
     })
   })
   describe('when the attached storages are not available', () => {
-    it('should allow the attachment of an existing storage', () => {
+    it('should allow the attachment of an existing storage with type Efs', () => {
       expect(canAttachExistingStorage('Efs', null as any, [])).toBeTruthy()
     })
+    it('should allow the attachment of an existing storage with type FileCache', () => {
+      expect(
+        canAttachExistingStorage('FileCache', null as any, []),
+      ).toBeTruthy()
+    })
   })
   describe('when the ui storages details are not available', () => {
     it('should allow the attachment of an existing storage', () => {

frontend/src/feature-flags/__tests__/FeatureFlagsProvider.test.ts

@@ -91,9 +91,35 @@ describe('given a feature flags provider and a list of rules', () => {
     })
   })
 
-  describe('when the version is above and 3.6.0', () => {
+  describe('when the version is between 3.6.0 and 3.7.0', () => {
     it('should return the list of available features', async () => {
-      const features = await subject('3.6.0', region)
+      const features = await subject('3.6.1', region)
+      expect(features).toEqual<AvailableFeature[]>([
+        'multiuser_cluster',
+        'fsx_ontap',
+        'fsx_openzsf',
+        'lustre_persistent2',
+        'memory_based_scheduling',
+        'slurm_queue_update_strategy',
+        'ebs_deletion_policy',
+        'cost_monitoring',
+        'slurm_accounting',
+        'queues_multiple_instance_types',
+        'dynamic_fs_mount',
+        'efs_deletion_policy',
+        'lustre_deletion_policy',
+        'imds_support',
+        'multi_az',
+        'on_node_updated',
+        'rhel8',
+        'new_resources_limits',
+      ])
+    })
+  })
+
+  describe('when the version is above and 3.7.0', () => {
+    it('should return the list of available features', async () => {
+      const features = await subject('3.7.0', region)
       expect(features).toEqual<AvailableFeature[]>([
         'multiuser_cluster',
         'fsx_ontap',
@@ -113,6 +139,10 @@ describe('given a feature flags provider and a list of rules', () => {
         'on_node_updated',
         'rhel8',
         'new_resources_limits',
+        'ubuntu22',
+        'login_nodes',
+        'amazon_file_cache',
+        'job_exclusive_allocation',
       ])
     })
   })

frontend/src/feature-flags/featureFlagsProvider.ts

@@ -32,6 +32,12 @@ const versionToFeaturesMap: Record<string, AvailableFeature[]> = {
   ],
   '3.4.0': ['multi_az', 'on_node_updated'],
   '3.6.0': ['rhel8', 'new_resources_limits'],
+  '3.7.0': [
+    'ubuntu22',
+    'login_nodes',
+    'amazon_file_cache',
+    'job_exclusive_allocation',
+  ],
 }
 
 const featureToUnsupportedRegionsMap: Partial<

frontend/src/feature-flags/types.ts

@@ -8,6 +8,7 @@
 // OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
 // limitations under the License.
 
+// We must track here the features that may be under feature flagging in the wizard.
 export type AvailableFeature =
   | 'fsx_ontap'
   | 'fsx_openzsf'
@@ -27,3 +28,7 @@ export type AvailableFeature =
   | 'rhel8'
   | 'cost_monitoring'
   | 'new_resources_limits'
+  | 'ubuntu22'
+  | 'login_nodes'
+  | 'amazon_file_cache'
+  | 'job_exclusive_allocation'

frontend/src/model.tsx

@@ -623,10 +623,12 @@ function LoadAwsConfig(region?: string, callback?: Callback) {
     .then((response: any) => {
       if (response.status === 200) {
         console.log('aws', response.data)
-        const {fsx_filesystems, fsx_volumes, ...data} = response.data
+        const {fsx_filesystems, fsx_volumes, file_caches, ...data} =
+          response.data
         setState(['aws'], {
           fsxFilesystems: extractFsxFilesystems(fsx_filesystems),
           fsxVolumes: extractFsxVolumes(fsx_volumes),
+          fileCaches: extractFileCaches(file_caches),
           ...data,
         })
         GetInstanceTypes(region)
@@ -661,6 +663,23 @@ const extractFsxFilesystems = (filesystems: any) => {
   }
 }
 
+const extractFileCaches = (file_caches: any) => {
+  const mappedFileCaches = file_caches
+    .map((fc: any) => ({
+      id: fc.FileCacheId,
+      name: nameFromFilesystem(fc),
+      type: fc.FileCacheType,
+    }))
+    .map((fc: any) => ({
+      ...fc,
+      displayName: `${fc.id} ${fc.name}`,
+    }))
+
+  return {
+    lustre: mappedFileCaches.filter((fc: any) => fc.type === 'LUSTRE'),
+  }
+}
+
 const nameFromFilesystem = (filesystem: any) => {
   const {Tags: tags} = filesystem
   if (!tags) {

frontend/src/old-pages/Configure/Storage.tsx

@@ -96,8 +96,11 @@ function storageValidate() {
   if (storages) {
     storages.forEach((storage: Storage, index: number) => {
       const settings = `${storage.StorageType}Settings`
+      //TODO Add idType as an attribute within STORAGE_TYPE_PROPS to avoid nested ternary operations.
       const idType = STORAGE_TYPE_PROPS[storage.StorageType].mountFilesystem
         ? 'FileSystemId'
+        : storage.StorageType == 'FileCache'
+        ? 'FileCacheId'
         : 'VolumeId'
       const useExisting =
         getState(['app', 'wizard', 'storage', 'ui', index, 'useExisting']) ||
@@ -872,12 +875,18 @@ function StorageInstance({index}: any) {
   const useExisting =
     useState([...uiSettingsForStorage, 'useExisting']) ||
     !(STORAGE_TYPE_PROPS[storageType].maxToCreate > 0)
+  //TODO Add idType as an attribute within STORAGE_TYPE_PROPS to avoid nested ternary operations.
   const existingPath = STORAGE_TYPE_PROPS[storageType].mountFilesystem
     ? [...settingsPath, 'FileSystemId']
+    : storageType == 'FileCache'
+    ? [...settingsPath, 'FileCacheId']
     : [...settingsPath, 'VolumeId']
+  //TODO Add idType as an attribute within STORAGE_TYPE_PROPS to avoid nested ternary operations.
   const existingPathError = useState(
     STORAGE_TYPE_PROPS[storageType].mountFilesystem
       ? [...errorsInstancePath, 'FileSystemId']
+      : storageType == 'FileCache'
+      ? [...errorsInstancePath, 'FileCacheId']
       : [...errorsInstancePath, 'VolumeId'],
   )
   const existingId = useState(existingPath) || ''
@@ -887,6 +896,7 @@ function StorageInstance({index}: any) {
   const {t} = useTranslation()
 
   const fsxFilesystems = useState(['aws', 'fsxFilesystems'])
+  const fileCaches = useState(['aws', 'fileCaches'])
   const fsxVolumes = useState(['aws', 'fsxVolumes'])
   const efsFilesystems = useState(['aws', 'efs_filesystems']) || []
 
@@ -1115,6 +1125,29 @@ function StorageInstance({index}: any) {
                     />
                   </FormField>
                 ),
+                FileCache: (
+                  <FormField
+                    label={t('wizard.storage.Fsx.existing.fileCache')}
+                    errorText={existingPathError}
+                  >
+                    <Select
+                      placeholder={t(
+                        'wizard.storage.container.cachePlaceholder',
+                      )}
+                      selectedOption={existingId && idToOption(existingId)}
+                      onChange={({detail}) => {
+                        setState(existingPath, detail.selectedOption.value)
+                      }}
+                      options={fileCaches.lustre.map((x: any) => {
+                        return {
+                          value: x.id,
+                          label: x.id + (x.Name ? ` (${x.Name})` : ''),
+                        }
+                      })}
+                      empty={t('wizard.storage.instance.useExisting.empty')}
+                    />
+                  </FormField>
+                ),
                 Efs: (
                   <FormField
                     label="EFS Filesystem"
@@ -1149,6 +1182,7 @@ function StorageInstance({index}: any) {
             Ebs: <EbsSettings index={index} />,
             FsxOntap: null,
             FsxOpenZfs: null,
+            FileCache: null,
           }[storageType]}
       </SpaceBetween>
     </Container>
@@ -1159,6 +1193,7 @@ const ALL_STORAGES: StorageTypeOption[] = [
   ['FsxLustre', 'Amazon FSx for Lustre (FSX)'],
   ['FsxOntap', 'Amazon FSx for NetApp ONTAP (FSX)'],
   ['FsxOpenZfs', 'Amazon FSx for OpenZFS (FSX)'],
+  ['FileCache', 'Amazon File Cache'],
   ['Efs', 'Amazon Elastic File System (EFS)'],
   ['Ebs', 'Amazon Elastic Block Store (EBS)'],
 ]
@@ -1169,6 +1204,7 @@ function Storage() {
   const uiStorageSettings = useState(['app', 'wizard', 'storage', 'ui'])
   const isFsxOnTapActive = useFeatureFlag('fsx_ontap')
   const isFsxOpenZsfActive = useFeatureFlag('fsx_openzsf')
+  const isFileCacheActive = useFeatureFlag('amazon_file_cache')
   const canEditFilesystems = useDynamicStorage()
 
   const hasAddedStorage = storages?.length > 0
@@ -1179,6 +1215,7 @@ function Storage() {
     FsxLustre: 21,
     FsxOntap: 20,
     FsxOpenZfs: 20,
+    FileCache: 20,
     Efs: 21,
     Ebs: 5,
   }
@@ -1190,6 +1227,9 @@ function Storage() {
     if (storageType === 'FsxOpenZfs' && !isFsxOpenZsfActive) {
       return false
     }
+    if (storageType === 'FileCache' && !isFileCacheActive) {
+      return false
+    }
     return true
   })
 

frontend/src/old-pages/Configure/Storage.types.ts

@@ -19,6 +19,11 @@ export const STORAGE_TYPE_PROPS = {
     maxToCreate: 0,
     maxExistingToAttach: 20,
   },
+  FileCache: {
+    mountFilesystem: false,
+    maxToCreate: 0,
+    maxExistingToAttach: 20,
+  },
   Efs: {
     mountFilesystem: true,
     maxToCreate: 1,
@@ -105,6 +110,10 @@ export interface FsxOpenZfsSettings {
   VolumeId: string
 }
 
+export interface FileCacheSettings {
+  FileCacheId: string
+}
+
 interface CommonSharedStorageDetails {
   Name: string
   MountDir: string
@@ -135,12 +144,18 @@ export interface FsxOpenZfsStorage extends CommonSharedStorageDetails {
   FsxOpenZfsSettings?: FsxOpenZfsSettings
 }
 
+export interface FileCacheStorage extends CommonSharedStorageDetails {
+  StorageType: 'FileCache'
+  FileCacheSettings?: FileCacheSettings
+}
+
 export type Storage =
   | EbsStorage
   | EfsStorage
   | FsxLustreStorage
   | FsxOnTapStorage
   | FsxOpenZfsStorage
+  | FileCacheStorage
 
 export type Storages = Storage[]
 

frontend/src/old-pages/Configure/Storage/storage.mapper.ts

@@ -22,6 +22,8 @@ function mapStorageToUiSetting(storage: Storage): UIStorageSettings[0] {
       return {useExisting: !!storage.FsxOntapSettings?.VolumeId}
     case 'FsxOpenZfs':
       return {useExisting: !!storage.FsxOpenZfsSettings?.VolumeId}
+    case 'FileCache':
+      return {useExisting: !!storage.FileCacheSettings?.FileCacheId}
   }
 }
 
@@ -39,6 +41,8 @@ export function mapStorageToExternalFileSystem(
       return storage.FsxOntapSettings?.VolumeId
     case 'FsxOpenZfs':
       return storage.FsxOpenZfsSettings?.VolumeId
+    case 'FileCache':
+      return storage.FileCacheSettings?.FileCacheId
     default:
       return undefined
   }

infrastructure/environments/demo-cfn-update-args.yaml

@@ -1,7 +1,7 @@
 TemplateURL: BUCKET_URL_PLACEHOLDER/parallelcluster-ui.yaml
 Parameters:
   - ParameterKey: Version
-    ParameterValue: 3.6.0
+    ParameterValue: 3.7.0
   - ParameterKey: InfrastructureBucket
     ParameterValue: BUCKET_URL_PLACEHOLDER
   - ParameterKey: UserPoolId

infrastructure/parallelcluster-ui.yaml

@@ -22,7 +22,7 @@ Parameters:
   Version:
     Description: Version of AWS ParallelCluster to deploy
     Type: String
-    Default: 3.6.0
+    Default: 3.7.0
   ImageBuilderVpcId:
     Description: (Optional) Select the VPC to use for building the container images. If not selected, default VPC will be used.
     Type: String