From e9057a22fc960311b58a0f03eb87fdc5c78db664 Mon Sep 17 00:00:00 2001 From: Shamil Mubarakshin <127750046+shamil-mubarakshin@users.noreply.github.com> Date: Wed, 27 Dec 2023 18:39:58 +0100 Subject: [PATCH] [ubuntu] Refactor PowerShell build scripts (#9064) * [ubuntu] Refactor PowerShell build scripts * Add Module import * Add Invoke-DownloadWithRetry function * Fix temp download dir * Update function to Add-GlobalEnvironmentVariable --- .../scripts/build/Configure-Toolset.ps1 | 112 ++++++++---------- .../build/Install-PowerShellAzModules.ps1 | 26 ++-- .../build/Install-PowerShellModules.ps1 | 18 ++- .../ubuntu/scripts/build/Install-Toolset.ps1 | 35 +++--- .../scripts/helpers/Common.Helpers.psm1 | 76 +++++++++++- 5 files changed, 162 insertions(+), 105 deletions(-) diff --git a/images/ubuntu/scripts/build/Configure-Toolset.ps1 b/images/ubuntu/scripts/build/Configure-Toolset.ps1 index 657915f3b33b..df7e8efe3ef8 100644 --- a/images/ubuntu/scripts/build/Configure-Toolset.ps1 +++ b/images/ubuntu/scripts/build/Configure-Toolset.ps1 @@ -4,91 +4,79 @@ ## Desc: Configure toolset ################################################################################ -Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1" -DisableNameChecking - -function Get-ToolsetToolFullPath -{ - param - ( - [Parameter(Mandatory)] [string] $ToolName, - [Parameter(Mandatory)] [string] $ToolVersion, - [Parameter(Mandatory)] [string] $ToolArchitecture +Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1" + +function Get-TCToolVersionPath { + param( + [Parameter(Mandatory)] + [string] $ToolName, + [Parameter(Mandatory)] + [string] $ToolVersion, + [Parameter(Mandatory)] + [string] $ToolArchitecture ) - $toolPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath $toolName - $toolPathVersion = Join-Path -Path $toolPath -ChildPath $toolVersion - $foundVersion = Get-Item $toolPathVersion | Sort-Object -Property {[version]$_.name} -Descending | Select-Object -First 1 - $installationDir = Join-Path -Path $foundVersion -ChildPath $toolArchitecture + $toolPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath $ToolName + $toolPathVersion = Join-Path -Path $toolPath -ChildPath $ToolVersion + $foundVersion = Get-Item $toolPathVersion | Sort-Object -Property { [version] $_.name } -Descending | Select-Object -First 1 + $installationDir = Join-Path -Path $foundVersion -ChildPath $ToolArchitecture + return $installationDir } -function Add-EnvironmentVariable -{ - param - ( - [Parameter(Mandatory)] [string] $Name, - [Parameter(Mandatory)] [string] $Value, +function Add-GlobalEnvironmentVariable { + param( + [Parameter(Mandatory)] + [string] $Name, + [Parameter(Mandatory)] + [string] $Value, [string] $FilePath = "/etc/environment" ) - $envVar = "{0}={1}" -f $name, $value - Tee-Object -InputObject $envVar -FilePath $filePath -Append + $envVar = "{0}={1}" -f $Name, $Value + Tee-Object -InputObject $envVar -FilePath $FilePath -Append } $ErrorActionPreference = "Stop" -Write-Host "Configure toolset tools environment..." -$toolsEnvironment = @{ +Write-Host "Configure toolcache tools environment..." +$toolEnvConfigs = @{ go = @{ - command = "ln -s {0}/bin/* /usr/bin/" + command = "ln -s {0}/bin/* /usr/bin/" variableTemplate = "GOROOT_{0}_{1}_X64" } } -$toolset = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw | ConvertFrom-Json - -foreach ($tool in $toolset.toolcache) -{ - $toolName = $tool.name - $toolArch = $tool.arch - $toolEnvironment = $toolsEnvironment[$toolName] +# Get toolcache content from toolset +$tools = (Get-ToolsetContent).toolcache | Where-Object { $toolEnvConfigs.Keys -contains $_.name } - if (-not $toolEnvironment) - { - continue - } +foreach ($tool in $tools) { + $toolEnvConfig = $toolEnvConfigs[$tool.name] - foreach ($toolVersion in $tool.versions) - { - Write-Host "Set $toolName $toolVersion environment variable..." - $toolPath = Get-ToolsetToolFullPath -ToolName $toolName -ToolVersion $toolVersion -ToolArchitecture $toolArch - $envName = $toolEnvironment.variableTemplate -f $toolVersion.split(".") + if (-not ([string]::IsNullOrEmpty($toolEnvConfig.variableTemplate))) { + foreach ($toolVersion in $tool.versions) { + Write-Host "Set $($tool.name) $toolVersion environment variable..." + $toolPath = Get-TCToolVersionPath -ToolName $tool.name -ToolVersion $toolVersion -ToolArchitecture $tool.arch + $envVariableName = $toolEnvConfig.variableTemplate -f $toolVersion.split(".") - # Add environment variable name=value - Add-EnvironmentVariable -Name $envName -Value $toolPath + Add-GlobalEnvironmentVariable -Name $envVariableName -Value $toolPath + } } # Invoke command and add env variable for the default tool version - $toolDefVersion = $tool.default - if (-not $toolDefVersion) - { - continue - } - - $envDefName = $toolEnvironment.defaultVariable - $toolPath = Get-ToolsetToolFullPath -ToolName $toolName -ToolVersion $toolDefVersion -ToolArchitecture $toolArch - - if ($envDefName) - { - Write-Host "Set default $envDefName for $toolName $toolDefVersion environment variable..." - Add-EnvironmentVariable -Name $envDefName -Value $toolPath - } - - if ($toolEnvironment.command) - { - $command = $toolEnvironment.command -f $toolPath - Write-Host "Invoke $command command for default $toolName $toolDefVersion..." - Invoke-Expression -Command $command + if (-not ([string]::IsNullOrEmpty($tool.default))) { + $toolDefaultPath = Get-TCToolVersionPath -ToolName $tool.name -ToolVersion $tool.default -ToolArchitecture $tool.arch + + if (-not ([string]::IsNullOrEmpty($toolEnvConfig.defaultVariable))) { + Write-Host "Set default $($toolEnvConfig.defaultVariable) for $($tool.name) $($tool.default) environment variable..." + Add-GlobalEnvironmentVariable -Name $toolEnvConfig.defaultVariable -Value $toolDefaultPath + } + + if (-not ([string]::IsNullOrEmpty($toolEnvConfig.command))) { + $command = $toolEnvConfig.command -f $toolDefaultPath + Write-Host "Invoke $command command for default $($tool.name) $($tool.default) ..." + Invoke-Expression -Command $command + } } } diff --git a/images/ubuntu/scripts/build/Install-PowerShellAzModules.ps1 b/images/ubuntu/scripts/build/Install-PowerShellAzModules.ps1 index 081c8c45b95a..6dd5fef883e1 100644 --- a/images/ubuntu/scripts/build/Install-PowerShellAzModules.ps1 +++ b/images/ubuntu/scripts/build/Install-PowerShellAzModules.ps1 @@ -6,18 +6,17 @@ $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" -Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1" -DisableNameChecking +Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1" # Get modules content from toolset $modules = (Get-ToolsetContent).azureModules $installPSModulePath = "/usr/share" -foreach ($module in $modules) -{ +foreach ($module in $modules) { $moduleName = $module.name + Write-Host "Installing ${moduleName} to the ${installPSModulePath} path..." - foreach ($version in $module.versions) - { + foreach ($version in $module.versions) { $modulePath = Join-Path -Path $installPSModulePath -ChildPath "${moduleName}_${version}" Write-Host " - $version [$modulePath]" Save-Module -Path $modulePath -Name $moduleName -RequiredVersion $version -Force @@ -28,17 +27,16 @@ foreach ($module in $modules) # Get github release asset for each version foreach ($toolVersion in $module.zip_versions) { $asset = $assets | Where-Object version -eq $toolVersion ` - | Select-Object -ExpandProperty files ` - | Select-Object -First 1 - - Write-Host "Installing $($module.name) $toolVersion ..." - if ($null -ne $asset) { - Write-Host "Download $($asset.filename)" - wget $asset.download_url -nv --retry-connrefused --tries=10 -P $installPSModulePath - } else { - Write-Host "Asset was not found in versions manifest" + | Select-Object -ExpandProperty files ` + | Select-Object -First 1 + + if (-not $asset) { + Write-Host "Asset for ${moduleName} ${toolVersion} was not found in versions manifest" exit 1 } + + Write-Host "Downloading asset for ${moduleName} ${toolVersion}: $($asset.filename)" + Invoke-DownloadWithRetry $asset.download_url -Destination "$installPSModulePath/$($asset.filename)" } } diff --git a/images/ubuntu/scripts/build/Install-PowerShellModules.ps1 b/images/ubuntu/scripts/build/Install-PowerShellModules.ps1 index 82e09f3840c9..14bb757e483c 100644 --- a/images/ubuntu/scripts/build/Install-PowerShellModules.ps1 +++ b/images/ubuntu/scripts/build/Install-PowerShellModules.ps1 @@ -6,7 +6,7 @@ $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" -Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1" -DisableNameChecking +Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1" # Specifies the installation policy Set-PSRepository -InstallationPolicy Trusted -Name PSGallery @@ -18,22 +18,18 @@ Update-Module -Name PowerShellGet -Force # Install PowerShell modules $modules = (Get-ToolsetContent).powershellModules -foreach($module in $modules) -{ +foreach($module in $modules) { $moduleName = $module.name - Write-Host "Installing ${moduleName} module" - if ($module.versions) - { - foreach ($version in $module.versions) - { + Write-Host "Installing ${moduleName} module" + if ($module.versions) { + foreach ($version in $module.versions) { Write-Host " - $version" Install-Module -Name $moduleName -RequiredVersion $version -Scope AllUsers -SkipPublisherCheck -Force } - continue + } else { + Install-Module -Name $moduleName -Scope AllUsers -SkipPublisherCheck -Force } - - Install-Module -Name $moduleName -Scope AllUsers -SkipPublisherCheck -Force } Invoke-PesterTests -TestFile "PowerShellModules" -TestName "PowerShellModules" diff --git a/images/ubuntu/scripts/build/Install-Toolset.ps1 b/images/ubuntu/scripts/build/Install-Toolset.ps1 index 87c14f512ea8..6c53b6089aaf 100644 --- a/images/ubuntu/scripts/build/Install-Toolset.ps1 +++ b/images/ubuntu/scripts/build/Install-Toolset.ps1 @@ -4,19 +4,21 @@ ## Desc: Install toolset ################################################################################ -Function Install-Asset { +Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1" + +function Install-Asset { param( [Parameter(Mandatory = $true)] [object] $ReleaseAsset ) Write-Host "Download $($ReleaseAsset.filename)" - wget $ReleaseAsset.download_url -nv --retry-connrefused --tries=10 + $assetArchivePath = Invoke-DownloadWithRetry $ReleaseAsset.download_url Write-Host "Extract $($ReleaseAsset.filename) content..." - $assetFolderPath = Join-Path "/tmp" $($ReleaseAsset.filename) - New-Item -ItemType Directory -Path $assetFolderPath - tar -xzf $ReleaseAsset.filename -C $assetFolderPath + $assetFolderPath = Join-Path "/tmp" "$($ReleaseAsset.filename)-temp-dir" + New-Item -ItemType Directory -Path $assetFolderPath | Out-Null + tar -xzf $assetArchivePath -C $assetFolderPath Write-Host "Invoke installation script..." Push-Location -Path $assetFolderPath @@ -26,10 +28,8 @@ Function Install-Asset { $ErrorActionPreference = "Stop" -# Get toolset content -$toolset = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw - -$tools = ConvertFrom-Json -InputObject $toolset | Select-Object -ExpandProperty toolcache | Where-Object {$_.url -ne $null } +# Get toolcache content from toolset +$tools = (Get-ToolsetContent).toolcache | Where-Object { $_.url -ne $null } foreach ($tool in $tools) { # Get versions manifest for current tool @@ -38,17 +38,18 @@ foreach ($tool in $tools) { # Get github release asset for each version foreach ($toolVersion in $tool.versions) { $asset = $assets | Where-Object version -like $toolVersion ` - | Select-Object -ExpandProperty files ` - | Where-Object { ($_.platform -eq $tool.platform) -and ($_.platform_version -eq $tool.platform_version)} ` - | Select-Object -First 1 + | Select-Object -ExpandProperty files ` + | Where-Object { ($_.platform -eq $tool.platform) -and ($_.platform_version -eq $tool.platform_version)} ` + | Select-Object -First 1 - Write-Host "Installing $($tool.name) $toolVersion $($tool.arch)..." - if ($null -ne $asset) { - Install-Asset -ReleaseAsset $asset - } else { - Write-Host "Asset was not found in versions manifest" + if (-not $asset) { + Write-Host "Asset for $($tool.name) $toolVersion $($tool.arch) not found in versions manifest" exit 1 } + + Write-Host "Installing $($tool.name) $toolVersion $($tool.arch)..." + Install-Asset -ReleaseAsset $asset } + chown -R "$($env:SUDO_USER):$($env:SUDO_USER)" "/opt/hostedtoolcache/$($tool.name)" } diff --git a/images/ubuntu/scripts/helpers/Common.Helpers.psm1 b/images/ubuntu/scripts/helpers/Common.Helpers.psm1 index 22376cd4fa3a..1f740345228b 100644 --- a/images/ubuntu/scripts/helpers/Common.Helpers.psm1 +++ b/images/ubuntu/scripts/helpers/Common.Helpers.psm1 @@ -24,7 +24,7 @@ function Get-CommandResult { This command runs "ls -la" in bash and returns the output and exit code as hashtable. #> - param ( + param( [Parameter(Mandatory=$true)] [string] $Command, [int[]] $ExpectedExitCode = 0, @@ -76,3 +76,77 @@ function Get-ToolsetContent { $toolsetJson = Get-Content -Path $toolsetPath -Raw ConvertFrom-Json -InputObject $toolsetJson } + +function Invoke-DownloadWithRetry { + <# + .SYNOPSIS + Downloads a file from a given URL with retry functionality. + + .DESCRIPTION + The Invoke-DownloadWithRetry function downloads a file from the specified URL + to the specified path. It includes retry functionality in case the download fails. + + .PARAMETER Url + The URL of the file to download. + + .PARAMETER Path + The path where the downloaded file will be saved. If not provided, a temporary path + will be used. + + .EXAMPLE + Invoke-DownloadWithRetry -Url "https://example.com/file.zip" -Path "/usr/local/bin" + Downloads the file from the specified URL and saves it to the specified path. + + .EXAMPLE + Invoke-DownloadWithRetry -Url "https://example.com/file.zip" + Downloads the file from the specified URL and saves it to a temporary path. + + .OUTPUTS + The path where the downloaded file is saved. + #> + param( + [Parameter(Mandatory)] + [string] $Url, + [Alias("Destination")] + [string] $DestinationPath + ) + + if (-not $DestinationPath) { + $invalidChars = [IO.Path]::GetInvalidFileNameChars() -join '' + $re = "[{0}]" -f [RegEx]::Escape($invalidChars) + $fileName = [IO.Path]::GetFileName($Url) -replace $re + + if ([String]::IsNullOrEmpty($fileName)) { + $fileName = [System.IO.Path]::GetRandomFileName() + } + $DestinationPath = Join-Path -Path "/tmp" -ChildPath $fileName + } + + Write-Host "Downloading package from $Url to $DestinationPath..." + + $interval = 30 + $downloadStartTime = Get-Date + for ($retries = 20; $retries -gt 0; $retries--) { + try { + $attemptStartTime = Get-Date + (New-Object System.Net.WebClient).DownloadFile($Url, $DestinationPath) + $attemptSeconds = [math]::Round(($(Get-Date) - $attemptStartTime).TotalSeconds, 2) + Write-Host "Package downloaded in $attemptSeconds seconds" + break + } catch { + $attemptSeconds = [math]::Round(($(Get-Date) - $attemptStartTime).TotalSeconds, 2) + Write-Warning "Package download failed in $attemptSeconds seconds" + Write-Warning $_.Exception.Message + } + + if ($retries -eq 0) { + $totalSeconds = [math]::Round(($(Get-Date) - $downloadStartTime).TotalSeconds, 2) + throw "Package download failed after $totalSeconds seconds" + } + + Write-Warning "Waiting $interval seconds before retrying (retries left: $retries)..." + Start-Sleep -Seconds $interval + } + + return $DestinationPath +}