<# .SYNOPSIS Upload files to Azure blob storage. .DESCRIPTION Wrapper around AzCopy.exe to upload files to Azure blob storage with retry logic. .PARAMETER Path The folder containing data to upload. (Example: C:\) .PARAMETER KeyFile Azure storage account access key, encoded as a SecureString object and exported to XML with Export-CliXml. (Example: C:\keyfile.xml) .PARAMETER Destination The URL of the storage account with container name. (Example: https://mystorageaccount.blob.core.windows.net/mycontainer) .PARAMETER AzCopyBinPath The full file path to AzCopy.exe. -- https://aka.ms/azcopy .PARAMETER BlobType The type of blob to upload. Must be either block, page, or append. (Default: block) .PARAMETER LoopCount The number of times to attempt the upload. (Default: 100) .PARAMETER Pattern The file pattern of files to upload. (Example: fileprefix*.txt) .LINK https://aka.ms/azcopy #> Param( [Parameter(Mandatory=$False, HelpMessage='The path to a directory to upload')] [Alias('Source')] [String] $Path = 'C:\Users\Public\Documents\Hyper-V\Virtual hard disks', [Parameter(Mandatory=$False, HelpMessage='File path to a storage account key SecureString object exported as a Powershell XML')] [Alias('Key')] [String] $KeyFile = 'C:\keyfile.xml', [Parameter(Mandatory=$False, HelpMessage='The destination URL')] [Alias('Dest')] [String] $Destination = 'https://mystorageaccount.blob.core.windows.net/mycontainer', [Parameter(Mandatory=$False, HelpMessage='The full file path to AzCopy.exe')] [String] $AzCopyBinPath = [System.IO.Path]::Combine(${Env:ProgramFiles(x86)}, 'Microsoft SDKs', 'Azure', 'AzCopy', 'AzCopy.exe'), [Parameter(Mandatory=$False, HelpMessage='The blob type: either block, page, or append (default: block)')] [ValidateSet('append', 'block', 'page')] [String] $BlobType = 'block', [Parameter(Mandatory=$False, HelpMessage='The number of times to attempt to upload')] [UInt16] $LoopCount = 100, [Parameter(Mandatory=$False, HelpMessage='The pattern of file or files to upload')] [Alias('Filter')] [String] $Pattern = '*.vhdx' ) $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2 Set-Variable -Option ReadOnly -Name time_str -Value 'yyyy-MM-dd HH:mm:ss.ffffff' Function LogError { Param([String] $Message) Write-Error ("{0} {1}" -f ((Get-Date).ToString($time_str), $Message)) Return } Function Log { Param([String] $Message) Write-Host ("{0} {1}" -f ((Get-Date).ToString($time_str), $Message)) Return } Function LogTimeSpan { Param([String] $Name, [TimeSpan] $TimeSpan) $hours = [System.Math]::Floor($TimeSpan.TotalMinutes/60) Log ("{0} time: {1:0} hr {2:0} min ({3:0} sec)" -f ($Name.PadRight(5,' '), $hours, $TimeSpan.Minutes, $TimeSpan.TotalSeconds)) Return } Function Summary { Param() $out = 0 $sum = 0 $files = @(Get-Childitem -Path $Path -Filter $Pattern -ErrorAction SilentlyContinue | Where-Object { -not $_.PSIsContainer }) foreach ($f in $files) { $sum += $f.Length } if (1 -gt $out) { $out = $sum / [Math]::Pow(2,40); $unit = 'TiB' } if (1 -gt $out) { $out = $sum / [Math]::Pow(2,30); $unit = 'GiB' } if (1 -gt $out) { $out = $sum / [Math]::Pow(2,20); $unit = 'MiB' } if (1 -gt $out) { $out = $sum / [Math]::Pow(2,10); $unit = 'KiB' } Log ("uploading {0:0.0} {1} in {2} file(s)" -f ($out, $unit, $files.Count)) Return } Function Main { Param() if (-not (Test-Path -Path $Path)) { LogError ("source dir {0} not found" -f ($Path)) Return } if (-not (Test-Path -Path $AzCopyBinPath)) { LogError 'AzCopy.exe not found -- https://aka.ms/azcopy' Return } $ts_copy = New-Object -TypeName System.TimeSpan $sleep_sec = 30 # key file created like so: # $sec_str = ConvertTo-SecureString -AsPlainText -Force -String 'REMOVED' # Export-CliXml -InputObject $sec_str -Force -Encoding 'UTF8' -Path 'keyfile.xml' $sa_key_sec_str = Import-CliXml -Path (Get-Item -Path $KeyFile).FullName $sa_key_sec_ptr = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($sa_key_sec_str) $sa_key = [System.Runtime.InteropServices.Marshal]::PtrToSTringBSTR($sa_key_sec_ptr) $azcopy_args = @('/V', '/Y', '/XO', "/BlobType:$BlobType", "/Source:`"$Path`"", "/Dest:$Destination", "/DestKey:$sa_key", "/Pattern:$Pattern") Clear-Variable -Name sa_key [System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($sa_key_sec_ptr) Summary Log 'start' $copy_start = Get-Date :AZCOPYLOOP for ($i = 0; $i -lt $LoopCount; $i++) { Log ("run {0}" -f ((1+$i))) if (0 -lt $i) { Log ("sleep {0} sec" -f ($sleep_sec)) # AzCopy 5.2.0 has an automatic sleep built-in Start-Sleep -Seconds $sleep_sec } # wait for any other running instances of AzCopy to stop :WAITLOOP while ($True) { $procs = @(Get-Process -Name ([System.IO.Path]::GetFileNameWithoutExtension($AzCopyBinPath)) -ErrorAction SilentlyContinue) if (0 -eq $procs.Count) { break :WAITLOOP } $procs[0].WaitForExit() } $loop_start = Get-Date # "This cmdlet generates a System.Diagnostics.Process object, if you # specify the PassThru parameter. Otherwise, this cmdlet does not # return any output." # $proc = Start-Process -FilePath $AzCopyBinPath -ArgumentList $azcopy_args -PassThru Log ("pid {0} waiting" -f ($proc.Id)) $proc.WaitForExit() Log ("pid {0} exitcode {1}" -f ($proc.Id, $proc.ExitCode)) $ts = (Get-Date) - $loop_start if (0 -gt $ts.Ticks) { LogError '$ts < 0: time travel has been invented' Return } LogTimeSpan -Name 'run' -TimeSpan $ts $ts_copy += $ts if (0 -eq $proc.ExitCode) { break :AZCOPYLOOP } # AzCopy v5.2.0 exits 0 on success } $ts_total = (Get-Date) - $copy_start if (0 -gt $ts_total.Ticks) { LogError '$ts_total < 0: time travel has been invented' Return } Log 'end' LogTimeSpan -Name 'copy' -TimeSpan $ts_copy LogTimeSpan -Name 'total' -TimeSpan $ts_total Return } . Main Exit