From b2a7561a7f7863f597c1a7653b9d1ff02d218676 Mon Sep 17 00:00:00 2001 From: Viktor Utkin Date: Tue, 5 Mar 2019 13:37:00 +0300 Subject: [PATCH] [win_get_url] feature: Add support `checksum` to module win_get_url (#51986) * set valid_until equal to current time + spot_wait_timeout * Add checksum check for downloaded file. * refactoring * fix typo * add fixes * mart try,catch handling * revert lib/ansible/modules/cloud/amazon/ec2.py from upstream * refactoring * remove empty lines * add checksum verification for existing file * fix current file check * refactoring destination file check * add handling exceptions * refactoring * Added download file hash data from url * fix string aligning * fix bug with uri * Added get hash from multy-string file * Added URI support for checksum file location * refactoing * Remove any non-alphanumeric characters for hash from url * fix discussions; add support for PS3 * refactoring * add size return value * checkout from upstream for lib/ansible/modules/cloud/amazon/ec2.py * add Ansible.ModuleUtils.Legacy support; refactoring * Copyright added * Checking files size before and after downloading added. * remove unused code * Corrected regexp for dotted slashed file name prefix in hash-file * hotfix typo error; add int tests * remove legacy module support; split checksum to checksum, checksum_algorithm, checksum_url * changed default hash algorithm * Fixed case for ContentLength = -1 * Old comment removed * fix typo * Remove file size check before downloading * add alias to ; fix tests * adjust tests; fix lint warnings from PSScritpAnalyzer * workaround for bug in win_chocolatey module on win2008 * remove win_get_url.ps1 from /test/sanity/pslint/ignore.txt * add checksum_algorithm as retuen value * first normalise before return Result * resolve discussions Signed-off-by: Viktor Utkin * fix discussions fix http tests as discussed * fix last discussions * Reduce code duplication and add idempotency check * fix sanity issue and remove testing code * move back to using tmp file for checksum comparison --- .../fragments/win_get_url-checksum.yaml | 3 + lib/ansible/modules/windows/win_get_url.ps1 | 383 +++++++++++++----- lib/ansible/modules/windows/win_get_url.py | 66 ++- .../targets/win_get_url/defaults/main.yml | 3 + .../targets/win_get_url/tasks/main.yml | 16 +- .../win_get_url/tasks/tests_checksum.yml | 202 +++++++++ .../targets/win_get_url/tasks/tests_url.yml | 17 +- test/sanity/pslint/ignore.txt | 2 +- 8 files changed, 576 insertions(+), 116 deletions(-) create mode 100644 changelogs/fragments/win_get_url-checksum.yaml create mode 100644 test/integration/targets/win_get_url/tasks/tests_checksum.yml diff --git a/changelogs/fragments/win_get_url-checksum.yaml b/changelogs/fragments/win_get_url-checksum.yaml new file mode 100644 index 00000000000..1816f9936a8 --- /dev/null +++ b/changelogs/fragments/win_get_url-checksum.yaml @@ -0,0 +1,3 @@ +minor_changes: +- win_get_url - Add the ``checksum`` option to verify the integrity of a downloaded file. +- win_get_url - Add idempotency check if the remote file has the same contents as the dest file. diff --git a/lib/ansible/modules/windows/win_get_url.ps1 b/lib/ansible/modules/windows/win_get_url.ps1 index ff94c6dd15e..b94ef64881b 100644 --- a/lib/ansible/modules/windows/win_get_url.ps1 +++ b/lib/ansible/modules/windows/win_get_url.ps1 @@ -3,10 +3,13 @@ # Copyright: (c) 2015, Paul Durivage # Copyright: (c) 2015, Tal Auslander # Copyright: (c) 2017, Dag Wieers +# Copyright: (c) 2019, Viktor Utkin +# Copyright: (c) 2019, Uladzimir Klybik # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) #AnsibleRequires -CSharpUtil Ansible.Basic #Requires -Module Ansible.ModuleUtils.AddType +#Requires -Module Ansible.ModuleUtils.FileUtil $spec = @{ options = @{ @@ -23,7 +26,13 @@ $spec = @{ proxy_username = @{ type='str' } proxy_password = @{ type='str'; no_log=$true } force = @{ type='bool'; default=$true } + checksum = @{ type='str' } + checksum_algorithm = @{ type='str'; default='sha1'; choices = @("md5", "sha1", "sha256", "sha384", "sha512") } + checksum_url = @{ type='str' } } + mutually_exclusive = @( + ,@('checksum', 'checksum_url') + ) supports_check_mode = $true } @@ -42,145 +51,271 @@ $proxy_url = $module.Params.proxy_url $proxy_username = $module.Params.proxy_username $proxy_password = $module.Params.proxy_password $force = $module.Params.force +$checksum = $module.Params.checksum +$checksum_algorithm = $module.Params.checksum_algorithm +$checksum_url = $module.Params.checksum_url -Add-CSharpType -AnsibleModule $module -References @' - using System.Net; - public class ExtendedWebClient : WebClient { - public int Timeout; - - public ExtendedWebClient() { - Timeout = 600000; // Default timeout value - } - - protected override WebRequest GetWebRequest(System.Uri address) { - WebRequest request = base.GetWebRequest(address); - request.Timeout = Timeout; - return request; - } - } -'@ - - -Function CheckModified-File($module, $url, $dest, $headers, $credentials, $timeout, $use_proxy, $proxy) { - - $fileLastMod = ([System.IO.FileInfo]$dest).LastWriteTimeUtc - $webLastMod = $null +$module.Result.elapsed = 0 +$module.Result.url = $url - $webRequest = [System.Net.WebRequest]::Create($url) +Function Invoke-AnsibleWebRequest { + <# + .SYNOPSIS + Creates a WebRequest and invokes a ScriptBlock with the response passed in. + It handles the common module options like credential, timeout, proxy options + in a single location to reduce code duplication. + #> + param( + [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module, + [Parameter(Mandatory=$true)][Uri]$Uri, + [Parameter(Mandatory=$true)][Hashtable]$Method, + [Parameter(Mandatory=$true)][ScriptBlock]$Script, # Invoked in this cmdlet + [System.Collections.IDictionary]$Headers, + [Int32]$Timeout, + [Switch]$UseProxy, + [System.Net.WebProxy]$Proxy, + $Credential # Either a String (force_basic_auth) or NetCredentials + ) + + $web_request = [System.Net.WebRequest]::Create($Uri) + $web_request.Method = $Method.($web_request.GetType().Name) foreach ($header in $headers.GetEnumerator()) { - $webRequest.Headers.Add($header.Name, $header.Value) + $web_request.Headers.Add($header.Key, $header.Value) } if ($timeout) { - $webRequest.Timeout = $timeout * 1000 + $web_request.Timeout = $timeout * 1000 } - if (-not $use_proxy) { - # Ignore the system proxy settings - $webRequest.Proxy = $null - } elseif ($proxy) { - $webRequest.Proxy = $proxy + if (-not $UseProxy) { + $web_request.Proxy = $null + } elseif ($ProxyUri) { + $web_request.Proxy = $Proxy } - if ($credentials) { - if ($force_basic_auth) { - $webRequest.Headers.Add("Authorization", "Basic $credentials") + if ($Credential) { + if ($Credential -is [String]) { + # force_basic_auth=yes is set + $web_request.Headers.Add("Authorization", "Basic $Credential") } else { - $webRequest.Credentials = $credentials + $web_request.Credentials = $Credential } } - if ($webRequest -is [System.Net.FtpWebRequest]) { - $webRequest.Method = [System.Net.WebRequestMethods+Ftp]::GetDateTimestamp - } else { - $webRequest.Method = [System.Net.WebRequestMethods+Http]::Head + try { + $web_response = $web_request.GetResponse() + $response_stream = $web_response.GetResponseStream() + try { + # Invoke the ScriptBlock and pass in the WebResponse and ResponseStream + &$Script -Response $web_response -Stream $response_stream + } finally { + $response_stream.Dispose() + } + + if ($Uri.IsFile) { + # A FileWebResponse won't have these properties set + $module.Result.msg = "OK" + $module.Result.status_code = 200 + } else { + $module.Result.msg = [string]$web_response.StatusDescription + $module.Result.status_code = [int]$web_response.StatusCode + } + } catch [System.Net.WebException] { + $module.Result.status_code = [int]$_.Exception.Response.StatusCode + $module.FailJson("Error requesting '$Uri'. $($_.Exception.Message)", $_) + } finally { + if ($web_response) { + $web_response.Close() + } } +} + +Function Get-ChecksumFromUri { + param( + [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module, + [Parameter(Mandatory=$true)][Uri]$Uri, + [Uri]$SourceUri, + [Hashtable]$RequestParams + ) + + $script = { + param($Response, $Stream) + + $read_stream = New-Object -TypeName System.IO.StreamReader -ArgumentList $Stream + $web_checksum = $read_stream.ReadToEnd() + $basename = (Split-Path -Path $SourceUri.LocalPath -Leaf) + $basename = [regex]::Escape($basename) + $web_checksum_str = $web_checksum -split '\r?\n' | Select-String -Pattern $("\s+\.?\/?\\?" + $basename + "\s*$") + if (-not $web_checksum_str) { + $Module.FailJson("Checksum record not found for file name '$basename' in file from url: '$Uri'") + } + + $web_checksum_str_splitted = $web_checksum_str[0].ToString().split(" ", 2) + $hash_from_file = $web_checksum_str_splitted[0].Trim() + # Remove any non-alphanumeric characters + $hash_from_file = $hash_from_file -replace '\W+', '' - # FIXME: Split both try-statements and single-out catched exceptions with more specific error messages - Try { - $webResponse = $webRequest.GetResponse() - $webLastMod = $webResponse.LastModified - } Catch [System.Net.WebException] { - $module.Result.status_code = [int] $_.Exception.Response.StatusCode - $module.FailJson("Error requesting '$url'. $($_.Exception.Message)", $_) - } Catch { - $module.FailJson("Error when requesting 'Last-Modified' date from '$url'. $($_.Exception.Message)", $_) + Write-Output -InputObject $hash_from_file } - $module.Result.status_code = [int] $webResponse.StatusCode - $module.Result.msg = [string] $webResponse.StatusDescription - $webResponse.Close() + $invoke_args = @{ + Module = $Module + Uri = $Uri + Method = @{ + FileWebRequest = [System.Net.WebRequestMethods+File]::DownloadFile + FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::DownloadFile + HttpWebRequest = [System.Net.WebRequestMethods+Http]::Get + } + Script = $script + } + try { + Invoke-AnsibleWebRequest @invoke_args @RequestParams + } catch { + $Module.FailJson("Error when getting the remote checksum from '$Uri'. $($_.Exception.Message)", $_) + } +} - if ($webLastMod -and ((Get-Date -Date $webLastMod).ToUniversalTime() -lt $fileLastMod)) { - return $false +Function Compare-ModifiedFile { + <# + .SYNOPSIS + Compares the remote URI resource against the local Dest resource. Will + return true if the LastWriteTime/LastModificationDate of the remote is + newer than the local resource date. + #> + param( + [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module, + [Parameter(Mandatory=$true)][Uri]$Uri, + [Parameter(Mandatory=$true)][String]$Dest, + [Hashtable]$RequestParams + ) + + $dest_last_mod = (Get-AnsibleItem -Path $Dest).LastWriteTimeUtc + + # If the URI is a file we don't need to go through the whole WebRequest + if ($Uri.IsFile) { + $src_last_mod = (Get-AnsibleItem -Path $Uri.AbsolutePath).LastWriteTimeUtc } else { - return $true + $invoke_args = @{ + Module = $Module + Uri = $Uri + Method = @{ + FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::GetDateTimestamp + HttpWebRequest = [System.Net.WebRequestMethods+Http]::Head + } + Script = { param($Response, $Stream); $Response.LastModified } + } + try { + $src_last_mod = Invoke-AnsibleWebRequest @invoke_args @RequestParams + } catch { + $Module.FailJson("Error when requesting 'Last-Modified' date from '$Uri'. $($_.Exception.Message)", $_) + } } + + # Return $true if the Uri LastModification date is newer than the Dest LastModification date + ((Get-Date -Date $src_last_mod).ToUniversalTime() -gt $dest_last_mod) } +Function Get-Checksum { + param( + [Parameter(Mandatory=$true)][String]$Path, + [String]$Algorithm = "sha1" + ) + + switch ($Algorithm) { + 'md5' { $sp = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider } + 'sha1' { $sp = New-Object -TypeName System.Security.Cryptography.SHA1CryptoServiceProvider } + 'sha256' { $sp = New-Object -TypeName System.Security.Cryptography.SHA256CryptoServiceProvider } + 'sha384' { $sp = New-Object -TypeName System.Security.Cryptography.SHA384CryptoServiceProvider } + 'sha512' { $sp = New-Object -TypeName System.Security.Cryptography.SHA512CryptoServiceProvider } + } -Function Download-File($module, $url, $dest, $headers, $credentials, $timeout, $use_proxy, $proxy) { + $fs = [System.IO.File]::Open($Path, [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read, + [System.IO.FileShare]::ReadWrite) + try { + $hash = [System.BitConverter]::ToString($sp.ComputeHash($fs)).Replace("-", "").ToLower() + } finally { + $fs.Dispose() + } + return $hash +} - $module_start = Get-Date +Function Invoke-DownloadFile { + param( + [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module, + [Parameter(Mandatory=$true)][Uri]$Uri, + [Parameter(Mandatory=$true)][String]$Dest, + [String]$Checksum, + [String]$ChecksumAlgorithm, + [Hashtable]$RequestParams + ) # Check $dest parent folder exists before attempting download, which avoids unhelpful generic error message. - $dest_parent = Split-Path -LiteralPath $dest + $dest_parent = Split-Path -LiteralPath $Dest if (-not (Test-Path -LiteralPath $dest_parent -PathType Container)) { - $module.FailJson("The path '$dest_parent' does not exist for destination '$dest', or is not visible to the current user. Ensure download destination folder exists (perhaps using win_file state=directory) before win_get_url runs.") + $module.FailJson("The path '$dest_parent' does not exist for destination '$Dest', or is not visible to the current user. Ensure download destination folder exists (perhaps using win_file state=directory) before win_get_url runs.") } - # TODO: Replace this with WebRequest - $extWebClient = New-Object ExtendedWebClient - - foreach ($header in $headers.GetEnumerator()) { - $extWebClient.Headers.Add($header.Name, $header.Value) - } + $download_script = { + param($Response, $Stream) + + # Download the file to a temporary directory so we can compare it + $tmp_dest = Join-Path -Path $Module.Tmpdir -ChildPath ([System.IO.Path]::GetRandomFileName()) + $fs = [System.IO.File]::Create($tmp_dest) + try { + $Stream.CopyTo($fs) + $fs.Flush() + } finally { + $fs.Dispose() + } + $tmp_checksum = Get-Checksum -Path $tmp_dest -Algorithm $ChecksumAlgorithm + $Module.Result.checksum_src = $tmp_checksum - if ($timeout) { - $extWebClient.Timeout = $timeout * 1000 - } + # If the checksum has been set, verify the checksum of the remote against the input checksum. + if ($Checksum -and $Checksum -ne $tmp_checksum) { + $Module.FailJson(("The checksum for {0} did not match '{1}', it was '{2}'" -f $Uri, $Checksum, $tmp_checksum)) + } - if (-not $use_proxy) { - # Ignore the system proxy settings - $extWebClient.Proxy = $null - } elseif ($proxy) { - $extWebClient.Proxy = $proxy - } + $download = $true + if (Test-Path -LiteralPath $Dest) { + # Validate the remote checksum against the existing downloaded file + $dest_checksum = Get-Checksum -Path $Dest -Algorithm $ChecksumAlgorithm + + # If we don't need to download anything, save the dest checksum so we don't waste time calculating it + # again at the end of the script + if ($dest_checksum -eq $tmp_checksum) { + $download = $false + $Module.Result.checksum_dest = $dest_checksum + $Module.Result.size = (Get-AnsibleItem -Path $Dest).Length + } + } - if ($credentials) { - if ($force_basic_auth) { - $extWebClient.Headers.Add("Authorization","Basic $credentials") - } else { - $extWebClient.Credentials = $credentials + if ($download) { + Copy-Item -Path $tmp_dest -Destination $Dest -Force -WhatIf:$Module.CheckMode > $null + $Module.Result.changed = $true } } - if (-not $module.CheckMode) { - # FIXME: Single-out catched exceptions with more specific error messages - Try { - $extWebClient.DownloadFile($url, $dest) - } Catch [System.Net.WebException] { - $module.Result.status_code = [int] $_.Exception.Response.StatusCode - $module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds - $module.FailJson("Error downloading '$url' to '$dest': $($_.Exception.Message)", $_) - } Catch { - $module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds - $module.FailJson("Unknown error downloading '$url' to '$dest': $($_.Exception.Message)", $_) + $invoke_args = @{ + Module = $Module + Uri = $Uri + Method = @{ + FileWebRequest = [System.Net.WebRequestMethods+File]::DownloadFile + FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::DownloadFile + HttpWebRequest = [System.Net.WebRequestMethods+Http]::Get } + Script = $download_script } - $module.Result.status_code = 200 - $module.Result.changed = $true - $module.Result.msg = 'OK' - $module.Result.dest = $dest - $module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds - + $module_start = Get-Date + try { + Invoke-AnsibleWebRequest @invoke_args @RequestParams + } catch { + $Module.FailJson("Unknown error downloading '$Uri' to '$Dest': $($_.Exception.Message)", $_) + } finally { + $Module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds + } } -$module.Result.dest = $dest -$module.Result.elapsed = 0 -$module.Result.url = $url - if (-not $use_proxy -and ($proxy_url -or $proxy_username -or $proxy_password)) { $module.Warn("Not using a proxy on request, however a 'proxy_url', 'proxy_username' or 'proxy_password' was defined.") } @@ -224,6 +359,7 @@ if (Test-Path -LiteralPath $dest -PathType Container) { # We have a trailing path separator $module.FailJson("The destination path '$dest' does not exist, or is not visible to the current user. Ensure download destination folder exists (perhaps using win_file state=directory) before win_get_url runs.") } + $module.Result.dest = $dest # Enable TLS1.1/TLS1.2 if they're available but disabled (eg. .NET 4.5) @@ -236,22 +372,53 @@ if ([Net.SecurityProtocolType].GetMember("Tls12").Count -gt 0) { } [Net.ServicePointManager]::SecurityProtocol = $security_protocols -if ($force -or -not (Test-Path -LiteralPath $dest)) { +$request_params = @{ + Credential = $credentials + Headers = $headers + Timeout = $timeout + UseProxy = $use_proxy + Proxy = $proxy +} - Download-File -module $module -url $url -dest $dest -credentials $credentials ` - -headers $headers -timeout $timeout -use_proxy $use_proxy -proxy $proxy +if ($checksum) { + $checksum = $checksum.Trim().toLower() +} +if ($checksum_algorithm) { + $checksum_algorithm = $checksum_algorithm.Trim().toLower() +} +if ($checksum_url) { + $checksum_url = $checksum_url.Trim().toLower() +} -} else { +# Check for case $checksum variable contain url. If yes, get file data from url and replace original value in $checksum +if ($checksum_url) { + $checksum_uri = [System.Uri]$checksum_url + if ($checksum_uri.Scheme -notin @("file", "ftp", "http", "https")) { + $module.FailJson("Unsupported 'checksum_url' value for '$dest': '$checksum_url'") + } - $is_modified = CheckModified-File -module $module -url $url -dest $dest -credentials $credentials ` - -headers $headers -timeout $timeout -use_proxy $use_proxy -proxy $proxy + $checksum = Get-ChecksumFromUri -Module $Module -Uri $checksum_uri -SourceUri $url -RequestParams $request_params +} +if ($force -or -not (Test-Path -LiteralPath $dest)) { + # force=yes or dest does not exist, download the file + # Note: Invoke-DownloadFile will compare the checksums internally if dest exists + Invoke-DownloadFile -Module $module -Uri $url -Dest $dest -Checksum $checksum ` + -ChecksumAlgorithm $checksum_algorithm -RequestParams $request_params +} else { + # force=no, we want to check the last modified dates and only download if they don't match + $is_modified = Compare-ModifiedFile -Module $module -Uri $url -Dest $dest -RequestParams $request_params if ($is_modified) { + Invoke-DownloadFile -Module $module -Uri $url -Dest $dest -Checksum $checksum ` + -ChecksumAlgorithm $checksum_algorithm -RequestParams $request_params + } +} - Download-File -module $module -url $url -dest $dest -credentials $credentials ` - -headers $headers -timeout $timeout -use_proxy $use_proxy -proxy $proxy - - } +if ((-not $module.Result.ContainsKey("checksum_dest")) -and (Test-Path -LiteralPath $dest)) { + # Calculate the dest file checksum if it hasn't already been done + $module.Result.checksum_dest = Get-Checksum -Path $dest -Algorithm $checksum_algorithm + $module.Result.size = (Get-AnsibleItem -Path $dest).Length } $module.ExitJson() + diff --git a/lib/ansible/modules/windows/win_get_url.py b/lib/ansible/modules/windows/win_get_url.py index f15fff8c6ff..16ff6fd785c 100644 --- a/lib/ansible/modules/windows/win_get_url.py +++ b/lib/ansible/modules/windows/win_get_url.py @@ -34,7 +34,7 @@ options: required: yes force: description: - - If C(yes), will always download the file. If C(no), will only + - If C(yes), will download the file every time and replace the file if the contents change. If C(no), will only download the file if it does not exist or the remote file has been modified more recently than the local file. - This works by sending an http HEAD request to retrieve last modified @@ -73,6 +73,36 @@ options: type: bool default: yes version_added: '2.4' + checksum: + description: + - If a I(checksum) is passed to this parameter, the digest of the + destination file will be calculated after it is downloaded to ensure + its integrity and verify that the transfer completed successfully. + - This option cannot be set with I(checksum_url). + type: str + version_added: "2.8" + checksum_algorithm: + description: + - Specifies the hashing algorithm used when calculating the checksum of + the remote and destination file. + type: str + choices: + - md5 + - sha1 + - sha256 + - sha385 + - sha512 + default: sha1 + version_added: "2.8" + checksum_url: + description: + - Specifies a URL that contains the checksum values for the resource at + I(url). + - Like C(checksum), this is used to verify the integrity of the remote + transfer. + - This option cannot be set with I(checksum). + type: str + version_added: "2.8" proxy_url: description: - The full URL of the proxy server to download through. @@ -105,6 +135,9 @@ notes: - If your URL includes an escaped slash character (%2F) this module will convert it to a real slash. This is a result of the behaviour of the System.Uri class as described in L(the documentation,https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/network/schemesettings-element-uri-settings#remarks). +- Since Ansible 2.8, the module will skip reporting a change if the remote + checksum is the same as the local local even when C(force=yes). This is to + better align with M(get_url). seealso: - module: get_url - module: uri @@ -140,6 +173,22 @@ EXAMPLES = r''' dest: '%TEMP%\ftp-file.txt' url_username: ftp-user url_password: ftp-password + +- name: Download src with sha256 checksum url + win_get_url: + url: http://www.example.com/earthrise.jpg + dest: C:\temp\earthrise.jpg + checksum_url: http://www.example.com/sha256sum.txt + checksum_algorithm: sha256 + force: True + +- name: Download src with sha256 checksum url + win_get_url: + url: http://www.example.com/earthrise.jpg + dest: C:\temp\earthrise.jpg + checksum: a97e6837f60cec6da4491bab387296bbcd72bdba + checksum_algorithm: sha1 + force: True ''' RETURN = r''' @@ -148,11 +197,26 @@ dest: returned: always type: str sample: C:\Users\RandomUser\earthrise.jpg +checksum_dest: + description: checksum of the file after the download + returned: success and dest has been downloaded + type: str + sample: 6e642bb8dd5c2e027bf21dd923337cbb4214f827 +checksum_src: + description: checksum of the remote resource + returned: force=yes or dest did not exist + type: str + sample: 6e642bb8dd5c2e027bf21dd923337cbb4214f827 elapsed: description: The elapsed seconds between the start of poll and the end of the module. returned: always type: float sample: 2.1406487 +size: + description: size of the dest file + returned: success + type: int + sample: 1220 url: description: requested url returned: always diff --git a/test/integration/targets/win_get_url/defaults/main.yml b/test/integration/targets/win_get_url/defaults/main.yml index 4ee191003c4..54976cfc49b 100644 --- a/test/integration/targets/win_get_url/defaults/main.yml +++ b/test/integration/targets/win_get_url/defaults/main.yml @@ -2,3 +2,6 @@ test_win_get_url_path: '{{win_output_dir}}\win_get_url' test_win_get_url_host: www.redhat.com test_win_get_url_env_var: WIN_GET_URL + +remote_tmp_path: '{{win_output_dir}}\win_get_url_dest' +remote_http_path: '{{test_win_get_url_path}}\http' diff --git a/test/integration/targets/win_get_url/tasks/main.yml b/test/integration/targets/win_get_url/tasks/main.yml index 8ba5cc45339..9fa8f22870e 100644 --- a/test/integration/targets/win_get_url/tasks/main.yml +++ b/test/integration/targets/win_get_url/tasks/main.yml @@ -1,8 +1,12 @@ --- - name: ensure testing folder is present win_file: - path: '{{test_win_get_url_path}}' + path: '{{ item }}' state: directory + loop: + - '{{ test_win_get_url_path }}' + - '{{ remote_http_path }}' + - '{{ remote_tmp_path }}' - name: copy across testing files win_copy: @@ -41,6 +45,9 @@ - name: run FTP tests include_tasks: tests_ftp.yml + - name: run checksum tests + include_tasks: tests_checksum.yml + always: - name: remove SlimFTPd service win_service: @@ -53,7 +60,10 @@ level: machine state: absent - - name: remove testing folder + - name: remove testing folders win_file: - path: '{{test_win_get_url_path}}' + path: '{{ item }}' state: absent + loop: + - '{{ test_win_get_url_path }}' + - '{{ remote_tmp_path }}' diff --git a/test/integration/targets/win_get_url/tasks/tests_checksum.yml b/test/integration/targets/win_get_url/tasks/tests_checksum.yml new file mode 100644 index 00000000000..e53ca3057ab --- /dev/null +++ b/test/integration/targets/win_get_url/tasks/tests_checksum.yml @@ -0,0 +1,202 @@ +--- +- name: create src file + win_copy: + dest: '{{ remote_http_path }}\27617.txt' + content: 'ptux' + +- name: create sha1 checksum file of src + win_copy: + dest: '{{ remote_http_path }}\sha1sum.txt' + content: 'a97e6837f60cec6da4491bab387296bbcd72bdba 27617.txt' + +- name: add sha1 checksum not going to be downloaded + win_lineinfile: + dest: '{{ remote_http_path }}\sha1sum.txt' + line: '{{ item }}' + loop: + - '3911340502960ca33aece01129234460bfeb2791 not_target1.txt' + - '1b4b6adf30992cedb0f6edefd6478ff0a593b2e4 not_target2.txt' + +- name: create sha256 checksum file of src + win_copy: + dest: '{{ remote_http_path }}\sha256sum.txt' + content: 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. 27617.txt' + +- name: add sha256 checksum not going to be downloaded + win_lineinfile: + dest: '{{ remote_http_path }}\sha256sum.txt' + line: '{{ item }}' + loop: + - '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892 not_target1.txt' + - 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b not_target2.txt' + +- name: create sha256 checksum file of src with a dot leading path + win_copy: + dest: '{{ remote_http_path }}\sha256sum_with_dot.txt' + content: 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. ./27617.txt' + +- name: add sha256 checksums with dot leading path not going to be downloaded + win_lineinfile: + dest: '{{ remote_http_path }}\sha256sum_with_dot.txt' + line: '{{ item }}' + loop: + - '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892 ./not_target1.txt' + - 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b ./not_target2.txt' + +- name: copy files to ftp + win_copy: + src: '{{ remote_http_path }}\' + dest: '{{ test_win_get_url_path }}\ftp\{{ item }}\' + remote_src: yes + loop: '{{ dest_folders }}' + vars: + dest_folders: + - anon + - user + - user-pass + +- name: download src with sha1 checksum url + win_get_url: + url: 'ftp://localhost/anon/27617.txt' + dest: '{{ remote_tmp_path }}' + checksum_url: 'ftp://localhost/anon/sha1sum.txt' + force: True + register: result_sha1 + +- name: download src with sha1 checksum value + win_get_url: + url: 'ftp://localhost/anon/27617.txt' + dest: '{{ remote_tmp_path }}' + checksum: 'a97e6837f60cec6da4491bab387296bbcd72bdba' + force: True + register: result_sha1_alt + +- win_stat: + path: '{{ remote_tmp_path }}\27617.txt' + register: stat_result_sha1 + +- name: download src with sha256 checksum url + win_get_url: + url: 'ftp://localhost/anon/27617.txt' + dest: '{{ remote_tmp_path }}\27617sha256.txt' + checksum_url: 'ftp://localhost/anon/sha256sum.txt' + checksum_algorithm: sha256 + force: True + register: result_sha256 + +- win_stat: + path: '{{ remote_tmp_path }}\27617.txt' + register: stat_result_sha256 + +- name: download src with sha256 checksum url with dot leading paths + win_get_url: + url: 'ftp://localhost/anon/27617.txt' + dest: '{{ remote_tmp_path }}\27617sha256_with_dot.txt' + checksum_url: 'ftp://localhost/anon/sha256sum_with_dot.txt' + checksum_algorithm: sha256 + force: True + register: result_sha256_with_dot + +- win_stat: + path: '{{ remote_tmp_path }}\27617sha256_with_dot.txt' + register: stat_result_sha256_with_dot + +- name: Assert that the file was downloaded + assert: + that: + - result_sha1 is changed + - result_sha1.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba' + - result_sha1_alt is succeeded + - not result_sha1_alt is changed + - result_sha1_alt.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba' + - result_sha256 is changed + - result_sha256_with_dot is changed + - stat_result_sha1.stat.exists | bool + - stat_result_sha256.stat.exists | bool + - stat_result_sha256_with_dot.stat.exists | bool + +# Check download with force: False + +- name: download src with sha1 checksum url + win_get_url: + url: 'ftp://localhost/anon/27617.txt' + dest: '{{ remote_tmp_path }}' + checksum_url: 'ftp://localhost/anon/sha1sum.txt' + checksum_algorithm: sha1 + force: False + register: result_sha1_no_force + +- name: download src with sha256 checksum url + win_get_url: + url: 'ftp://localhost/anon/27617.txt' + dest: '{{ remote_tmp_path }}/27617sha256.txt' + checksum_url: 'ftp://localhost/anon/sha256sum.txt' + checksum_algorithm: sha256 + force: False + register: result_sha256_no_force + +- name: assert download single file with force no + assert: + that: + - result_sha1_no_force is not changed + - result_sha256_no_force is not changed + +# Check download with check_mode: True + +- name: download src with sha1 checksum url | checkmode + win_get_url: + url: 'ftp://localhost/anon/27617.txt' + dest: '{{ remote_tmp_path }}' + checksum_url: 'ftp://localhost/anon/sha1sum.txt' + checksum_algorithm: sha1 + check_mode: True + register: result_sha1_check_mode + +- name: download src with sha256 checksum url | checkmode + win_get_url: + url: 'ftp://localhost/anon/27617.txt' + dest: '{{ remote_tmp_path }}\27617sha256.txt' + checksum_url: 'ftp://localhost/anon/sha256sum.txt' + checksum_algorithm: sha256 + check_mode: True + register: result_sha256_check_mode + +- name: assert download single file with force no + assert: + that: + - result_sha1_check_mode.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba' + - result_sha256_check_mode.checksum_dest == 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006' + - result_sha1_check_mode.checksum_src | length + - result_sha256_check_mode.checksum_src | length + +# Check download with wrong checksum + +- name: download src with sha1 checksum value | fail + win_get_url: + url: 'ftp://localhost/anon/27617.txt' + dest: '{{ remote_tmp_path }}' + checksum: 'redacted' + checksum_algorithm: sha1 + failed_when: + - '"did not match" not in result_sha1_failed.msg' + - '"The checksum" not in result_sha1_failed.msg' + - '"Unknown error downloading" not in result_sha1_failed.msg' + register: result_sha1_failed + +- name: download src with sha256 checksum value | fail + win_get_url: + url: 'ftp://localhost/anon/27617.txt' + dest: '{{ remote_tmp_path }}\27617sha256.txt' + checksum: 'redacted' + checksum_algorithm: sha256 + failed_when: + - '"did not match" not in result_sha256_failed.msg' + - '"The checksum" not in result_sha256_failed.msg' + - '"Unknown error downloading" not in result_sha256_failed.msg' + register: result_sha256_failed + +- name: assert failed downloads + assert: + that: + - result_sha1_failed is succeeded + - result_sha256_failed is succeeded diff --git a/test/integration/targets/win_get_url/tasks/tests_url.yml b/test/integration/targets/win_get_url/tasks/tests_url.yml index d8706f633f6..539aba6a63c 100644 --- a/test/integration/targets/win_get_url/tasks/tests_url.yml +++ b/test/integration/targets/win_get_url/tasks/tests_url.yml @@ -39,7 +39,16 @@ - http_download.dest - http_download_result.stat.exists -# TODO: add check for idempotent run once it is added with force: yes +- name: download single file (idempotent) + win_get_url: + url: https://{{test_win_get_url_host}} + dest: '{{test_win_get_url_path}}\web.html' + register: http_download_again + +- name: assert download single file (idempotent) + assert: + that: + - not http_download_again is changed - name: download single file with force no win_get_url: @@ -53,8 +62,10 @@ that: - http_download_no_force is not changed -- name: manually change last modified time on FTP source to older datetime - win_shell: (Get-Item -Path '{{test_win_get_url_path}}\web.html').LastWriteTime = (Get-Date -Date "01/01/1970") +- name: manually change the content and last modified time on FTP source to older datetime + win_shell: | + Set-Content -Path '{{test_win_get_url_path}}\web.html' -Value 'abc' + (Get-Item -Path '{{test_win_get_url_path}}\web.html').LastWriteTime = (Get-Date -Date "01/01/1970") - name: download newer file with force no win_get_url: diff --git a/test/sanity/pslint/ignore.txt b/test/sanity/pslint/ignore.txt index 2f9327bcb01..51ef60bb5c7 100644 --- a/test/sanity/pslint/ignore.txt +++ b/test/sanity/pslint/ignore.txt @@ -38,7 +38,7 @@ lib/ansible/modules/windows/win_find.ps1 PSAvoidUsingEmptyCatchBlock lib/ansible/modules/windows/win_find.ps1 PSAvoidUsingWMICmdlet lib/ansible/modules/windows/win_firewall_rule.ps1 PSAvoidUsingCmdletAliases lib/ansible/modules/windows/win_firewall_rule.ps1 PSUseApprovedVerbs -lib/ansible/modules/windows/win_get_url.ps1 PSUseApprovedVerbs +lib/ansible/modules/windows/win_get_url.ps1 PSUsePSCredentialType # Credential param can take a base64 encoded string as well as a PSCredential lib/ansible/modules/windows/win_hotfix.ps1 PSUseApprovedVerbs lib/ansible/modules/windows/win_iis_webbinding.ps1 PSUseApprovedVerbs lib/ansible/modules/windows/win_iis_website.ps1 PSAvoidUsingCmdletAliases