Skip to content

Commit

Permalink
Added automatic cluster creation safeguards & feedback section for fa…
Browse files Browse the repository at this point in the history
…iled kaito installations (#1209)
  • Loading branch information
tejhan authored Feb 6, 2025
1 parent 7b71f31 commit 34ddf5d
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 17 deletions.
72 changes: 56 additions & 16 deletions src/panels/KaitoPanel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,24 @@ export class KaitoPanelDataProvider implements PanelDataProvider<"kaito"> {
}

private async handleKaitoInstallation(webview: MessageSink<ToWebViewMsgDef>) {
// Get current json
const currentJson = await longRunning(`Get current cluster information.`, () => {
return this.resourceManagementClient.resources.getById(this.armId, "2023-08-01");
});

// Prevent KAITO installation on automatic clusters
const skuName = currentJson.sku?.name;
if (skuName === "Automatic") {
webview.postKaitoInstallProgressUpdate({
operationDescription: "Automatic Cluster Detected",
event: 3,
errorMessage:
"KAITO cannot be installed on automatic clusters. Please try installing KAITO on a standard cluster.",
models: [],
});
return;
}

// Get the feature registration state
const getFeatureClientRegisterState = await longRunning(
`Getting the AIToolchainOperator registration state.`,
Expand All @@ -161,11 +179,6 @@ export class KaitoPanelDataProvider implements PanelDataProvider<"kaito"> {
}
}

// Get current json
const currentJson = await longRunning(`Get current cluster information.`, () => {
return this.resourceManagementClient.resources.getById(this.armId, "2023-08-01");
});

// Install kaito enablement
const kaitoInstallationResult = await longRunning(
`Enabling the KAITO for cluster '${this.clusterName}'.`,
Expand All @@ -191,9 +204,18 @@ export class KaitoPanelDataProvider implements PanelDataProvider<"kaito"> {
);

if (failed(installKaitoFederatedCredentialsAndRoleAssignments)) {
//installing federated credentionals failed
const errorMessage = installKaitoFederatedCredentialsAndRoleAssignments.error;
vscode.window.showErrorMessage(
`Error installing KAITO Federated Credentials and role Assignments: ${installKaitoFederatedCredentialsAndRoleAssignments.error}`,
`Error installing KAITO Federated Credentials and role Assignments: ${errorMessage}`,
);
webview.postKaitoInstallProgressUpdate({
operationDescription: "Installing Federated Credentials Failed",
event: 3,
errorMessage: errorMessage,
models: [],
});
return;
}

//kaito installation succeeded
Expand Down Expand Up @@ -304,13 +326,18 @@ export class KaitoPanelDataProvider implements PanelDataProvider<"kaito"> {
return { succeeded: false, error: clusterInfo.error };
}

await this.installKaitoRoleAssignments(
const roleAssignmentsResult = await this.installKaitoRoleAssignments(
clusterInfo.result.nodeResourceGroup!,
this.subscriptionId,
this.resourceGroupName,
this.clusterName,
);

//halt installation if role assignments creation failed
if (failed(roleAssignmentsResult)) {
return { succeeded: false, error: roleAssignmentsResult.error };
}

const aksOidcIssuerUrl = clusterInfo.result.oidcIssuerProfile?.issuerURL;
if (!aksOidcIssuerUrl) {
vscode.window.showErrorMessage(
Expand All @@ -321,12 +348,18 @@ export class KaitoPanelDataProvider implements PanelDataProvider<"kaito"> {
error: "Error getting aks oidc issuer url, oidc issuer url is undefined/null/empty",
};
}
await this.installKaitoFederatedCredentials(

const kaitoFederatedCredentialsResult = await this.installKaitoFederatedCredentials(
clusterInfo.result.nodeResourceGroup!,
this.clusterName,
aksOidcIssuerUrl,
);

//halt installation if federated credentials installation failed
if (failed(kaitoFederatedCredentialsResult)) {
return { succeeded: false, error: kaitoFederatedCredentialsResult.error };
}

//kubectl rollout restart deployment kaito-gpu-provisioner -n kube-system
const command = `rollout restart deployment kaito-gpu-provisioner -n kube-system`;
const kubectlresult = await invokeKubectlCommand(this.kubectl, this.kubeConfigFilePath, command);
Expand Down Expand Up @@ -404,14 +437,14 @@ export class KaitoPanelDataProvider implements PanelDataProvider<"kaito"> {
subscriptionId: string,
resourceGroupName: string,
clusterName: string,
) {
): Promise<Errorable<string>> {
// get principal id of managed service identity
const identityName = `ai-toolchain-operator-${clusterName}`;
const identityResult = await getIdentity(this.managedServiceIdentityClient, mcResourceGroup, identityName);

if (failed(identityResult)) {
vscode.window.showErrorMessage(`Error getting identity: ${identityResult.error}`);
return;
return { succeeded: false, error: identityResult.error };
}

const roleAssignment = await createRoleAssignment(
Expand All @@ -423,17 +456,22 @@ export class KaitoPanelDataProvider implements PanelDataProvider<"kaito"> {
);

if (failed(roleAssignment)) {
vscode.window.showWarningMessage(
`Error installing KAITO Federated Credentials and role Assignments: ${roleAssignment.error}`,
);
// Don't cancel installation if role assignments already exist, user could be attempting to reinstall
if (roleAssignment.error?.includes("already exists")) {
return { succeeded: true, result: "Role assignments already exist" };
} else {
// cancel installation only if there is an alternate error that conflicts with further steps
return { succeeded: false, error: roleAssignment.error };
}
}
return { succeeded: true, result: "Role assignments created successfully" };
}

private async installKaitoFederatedCredentials(
nodeResourceGroup: string,
clusterName: string,
aksOidcIssuerUrl: string,
) {
): Promise<Errorable<string>> {
const result = await createFederatedCredential(
this.managedServiceIdentityClient,
nodeResourceGroup,
Expand All @@ -445,8 +483,10 @@ export class KaitoPanelDataProvider implements PanelDataProvider<"kaito"> {
);

if (failed(result)) {
vscode.window.showErrorMessage(`Error creating federated credentials: ${result.error}`);
return;
// vscode.window.showErrorMessage(`Error creating federated credentials: ${result.error}`);
return { succeeded: false, error: result.error };
} else {
return { succeeded: true, result: "Federated credentials created successfully" };
}
}
}
Expand Down
4 changes: 4 additions & 0 deletions webview-ui/src/Kaito/Kaito.module.css
Original file line number Diff line number Diff line change
Expand Up @@ -123,3 +123,7 @@
.installBlurb {
margin: 0.5rem 0rem 0.06rem 0rem;
}
.errorMessage {
font-weight: 400;
width: 90%;
}
8 changes: 7 additions & 1 deletion webview-ui/src/Kaito/Kaito.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ export function Kaito(initialState: InitialState) {
</p>
</ul>
</div>
<div>
<div className={styles.installationDiv}>
{state.kaitoInstallStatus === ProgressEventType.NotStarted && (
<button className={styles.button} onClick={handleClick} disabled={isDisabled}>
Install KAITO
Expand Down Expand Up @@ -111,6 +111,12 @@ export function Kaito(initialState: InitialState) {
</div>
</div>
)}
{state.kaitoInstallStatus === ProgressEventType.Failed && (
<div className={styles.postInstall}>
<p>Error installing KAITO.</p>
<p className={styles.errorMessage}>{state.errors}</p>
</div>
)}
</div>
</div>
</>
Expand Down

0 comments on commit 34ddf5d

Please sign in to comment.