[win_get_url] feature: Add support `checksum` to module win_get_url (#51986)

* set valid_until equal to current time + spot_wait_timeout * Add checksum check for downloaded file. * refactoring * fix typo * add fixes * mart try,catch handling * revert lib/ansible/modules/cloud/amazon/ec2.py from upstream * refactoring * remove empty lines * add checksum verification for existing file * fix current file check * refactoring destination file check * add handling exceptions * refactoring * Added download file hash data from url * fix string aligning * fix bug with uri * Added get hash from multy-string file * Added URI support for checksum file location * refactoing * Remove any non-alphanumeric characters for hash from url * fix discussions; add support for PS3 * refactoring * add size return value * checkout from upstream for lib/ansible/modules/cloud/amazon/ec2.py * add Ansible.ModuleUtils.Legacy support; refactoring * Copyright added * Checking files size before and after downloading added. * remove unused code * Corrected regexp for dotted slashed file name prefix in hash-file * hotfix typo error; add int tests * remove legacy module support; split checksum to checksum, checksum_algorithm, checksum_url * changed default hash algorithm * Fixed case for ContentLength = -1 * Old comment removed * fix typo * Remove file size check before downloading * add alias to ; fix tests * adjust tests; fix lint warnings from PSScritpAnalyzer * workaround for bug in win_chocolatey module on win2008 * remove win_get_url.ps1 from /test/sanity/pslint/ignore.txt * add checksum_algorithm as retuen value * first normalise before return Result * resolve discussions Signed-off-by: Viktor Utkin <viktor.utkin7@yandex.ru> * fix discussions fix http tests as discussed * fix last discussions * Reduce code duplication and add idempotency check * fix sanity issue and remove testing code * move back to using tmp file for checksum comparison
6 years ago · b2a7561a7f
parent aafc5538bc
commit b2a7561a7f
8 changed files with 576 additions and 116 deletions
--- a/changelogs/fragments/win_get_url-checksum.yaml
+++ b/changelogs/fragments/win_get_url-checksum.yaml
@ -0,0 +1,3 @@
+minor_changes:
+- win_get_url - Add the ``checksum`` option to verify the integrity of a downloaded file.
+- win_get_url - Add idempotency check if the remote file has the same contents as the dest file.
--- a/lib/ansible/modules/windows/win_get_url.ps1
+++ b/lib/ansible/modules/windows/win_get_url.ps1
@ -3,10 +3,13 @@
 # Copyright: (c) 2015, Paul Durivage <paul.durivage@rackspace.com>
 # Copyright: (c) 2015, Tal Auslander <tal@cloudshare.com>
 # Copyright: (c) 2017, Dag Wieers <dag@wieers.com>
+# Copyright: (c) 2019, Viktor Utkin <viktor_utkin@epam.com>
+# Copyright: (c) 2019, Uladzimir Klybik <uladzimir_klybik@epam.com>
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

 #AnsibleRequires -CSharpUtil Ansible.Basic
 #Requires -Module Ansible.ModuleUtils.AddType
+#Requires -Module Ansible.ModuleUtils.FileUtil

 $spec = @{
    options = @{
@ -23,7 +26,13 @@ $spec = @{
        proxy_username = @{ type='str' }
        proxy_password = @{ type='str'; no_log=$true }
        force = @{ type='bool'; default=$true }
+        checksum = @{ type='str' }
+        checksum_algorithm = @{ type='str'; default='sha1'; choices = @("md5", "sha1", "sha256", "sha384", "sha512") }
+        checksum_url = @{ type='str' }
    }
+    mutually_exclusive = @(
+        ,@('checksum', 'checksum_url')
+    )
    supports_check_mode = $true
 }

@ -42,145 +51,271 @@ $proxy_url = $module.Params.proxy_url
 $proxy_username = $module.Params.proxy_username
 $proxy_password = $module.Params.proxy_password
 $force = $module.Params.force
+$checksum = $module.Params.checksum
+$checksum_algorithm = $module.Params.checksum_algorithm
+$checksum_url = $module.Params.checksum_url

-Add-CSharpType -AnsibleModule $module -References @'
-    using System.Net;
-    public class ExtendedWebClient : WebClient {
-        public int Timeout;
-
-        public ExtendedWebClient() {
-            Timeout = 600000; // Default timeout value
-        }
-
-        protected override WebRequest GetWebRequest(System.Uri address) {
-            WebRequest request = base.GetWebRequest(address);
-            request.Timeout = Timeout;
-            return request;
-        }
-    }
-'@
-
-
-Function CheckModified-File($module, $url, $dest, $headers, $credentials, $timeout, $use_proxy, $proxy) {
-
-    $fileLastMod = ([System.IO.FileInfo]$dest).LastWriteTimeUtc
-    $webLastMod = $null
+$module.Result.elapsed = 0
+$module.Result.url = $url

-    $webRequest = [System.Net.WebRequest]::Create($url)
+Function Invoke-AnsibleWebRequest {
+    <#
+    .SYNOPSIS
+    Creates a WebRequest and invokes a ScriptBlock with the response passed in.
+    It handles the common module options like credential, timeout, proxy options
+    in a single location to reduce code duplication.
+    #>
+    param(
+        [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
+        [Parameter(Mandatory=$true)][Uri]$Uri,
+        [Parameter(Mandatory=$true)][Hashtable]$Method,
+        [Parameter(Mandatory=$true)][ScriptBlock]$Script, # Invoked in this cmdlet
+        [System.Collections.IDictionary]$Headers,
+        [Int32]$Timeout,
+        [Switch]$UseProxy,
+        [System.Net.WebProxy]$Proxy,
+        $Credential  # Either a String (force_basic_auth) or NetCredentials
+    )
+
+    $web_request = [System.Net.WebRequest]::Create($Uri)
+    $web_request.Method = $Method.($web_request.GetType().Name)

    foreach ($header in $headers.GetEnumerator()) {
-        $webRequest.Headers.Add($header.Name, $header.Value)
+        $web_request.Headers.Add($header.Key, $header.Value)
    }

    if ($timeout) {
-        $webRequest.Timeout = $timeout * 1000
+        $web_request.Timeout = $timeout * 1000
    }

-    if (-not $use_proxy) {
-        # Ignore the system proxy settings
-        $webRequest.Proxy = $null
-    } elseif ($proxy) {
-        $webRequest.Proxy = $proxy
+    if (-not $UseProxy) {
+        $web_request.Proxy = $null
+    } elseif ($ProxyUri) {
+        $web_request.Proxy = $Proxy
    }

-    if ($credentials) {
-        if ($force_basic_auth) {
-            $webRequest.Headers.Add("Authorization", "Basic $credentials")
+    if ($Credential) {
+        if ($Credential -is [String]) {
+            # force_basic_auth=yes is set
+            $web_request.Headers.Add("Authorization", "Basic $Credential")
        } else {
-            $webRequest.Credentials = $credentials
+            $web_request.Credentials = $Credential
+        }
    }
+
+    try {
+        $web_response = $web_request.GetResponse()
+        $response_stream = $web_response.GetResponseStream()
+        try {
+            # Invoke the ScriptBlock and pass in the WebResponse and ResponseStream
+            &$Script -Response $web_response -Stream $response_stream
+        } finally {
+            $response_stream.Dispose()
        }

-    if ($webRequest -is [System.Net.FtpWebRequest]) {
-        $webRequest.Method = [System.Net.WebRequestMethods+Ftp]::GetDateTimestamp
+        if ($Uri.IsFile) {
+            # A FileWebResponse won't have these properties set
+            $module.Result.msg = "OK"
+            $module.Result.status_code = 200
        } else {
-        $webRequest.Method = [System.Net.WebRequestMethods+Http]::Head
+            $module.Result.msg = [string]$web_response.StatusDescription
+            $module.Result.status_code = [int]$web_response.StatusCode
+        }
+    } catch [System.Net.WebException] {
+        $module.Result.status_code = [int]$_.Exception.Response.StatusCode
+        $module.FailJson("Error requesting '$Uri'. $($_.Exception.Message)", $_)
+    } finally {
+        if ($web_response) {
+            $web_response.Close()
+        }
    }
+}
+
+Function Get-ChecksumFromUri {
+    param(
+        [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
+        [Parameter(Mandatory=$true)][Uri]$Uri,
+        [Uri]$SourceUri,
+        [Hashtable]$RequestParams
+    )
+
+    $script = {
+        param($Response, $Stream)
+
+        $read_stream = New-Object -TypeName System.IO.StreamReader -ArgumentList $Stream
+        $web_checksum = $read_stream.ReadToEnd()
+        $basename = (Split-Path -Path $SourceUri.LocalPath -Leaf)
+        $basename = [regex]::Escape($basename)
+        $web_checksum_str = $web_checksum -split '\r?\n' | Select-String -Pattern $("\s+\.?\/?\\?" + $basename + "\s*$")
+        if (-not $web_checksum_str) {
+            $Module.FailJson("Checksum record not found for file name '$basename' in file from url: '$Uri'")
+        }
+
+        $web_checksum_str_splitted = $web_checksum_str[0].ToString().split(" ", 2)
+        $hash_from_file = $web_checksum_str_splitted[0].Trim()
+        # Remove any non-alphanumeric characters
+        $hash_from_file = $hash_from_file -replace '\W+', ''

-    # FIXME: Split both try-statements and single-out catched exceptions with more specific error messages
-    Try {
-        $webResponse = $webRequest.GetResponse()
-        $webLastMod = $webResponse.LastModified
-    } Catch [System.Net.WebException] {
-        $module.Result.status_code = [int] $_.Exception.Response.StatusCode
-        $module.FailJson("Error requesting '$url'. $($_.Exception.Message)", $_)
-    } Catch {
-        $module.FailJson("Error when requesting 'Last-Modified' date from '$url'. $($_.Exception.Message)", $_)
+        Write-Output -InputObject $hash_from_file
    }
-    $module.Result.status_code = [int] $webResponse.StatusCode
-    $module.Result.msg = [string] $webResponse.StatusDescription
-    $webResponse.Close()
+    $invoke_args = @{
+        Module = $Module
+        Uri = $Uri
+        Method = @{
+            FileWebRequest = [System.Net.WebRequestMethods+File]::DownloadFile
+            FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::DownloadFile
+            HttpWebRequest = [System.Net.WebRequestMethods+Http]::Get
+        }
+        Script = $script
+    }
+    try {
+        Invoke-AnsibleWebRequest @invoke_args @RequestParams
+    } catch {
+        $Module.FailJson("Error when getting the remote checksum from '$Uri'. $($_.Exception.Message)", $_)
+    }
+}

-    if ($webLastMod -and ((Get-Date -Date $webLastMod).ToUniversalTime() -lt $fileLastMod)) {
-        return $false
+Function Compare-ModifiedFile {
+    <#
+    .SYNOPSIS
+    Compares the remote URI resource against the local Dest resource. Will
+    return true if the LastWriteTime/LastModificationDate of the remote is
+    newer than the local resource date.
+    #>
+    param(
+        [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
+        [Parameter(Mandatory=$true)][Uri]$Uri,
+        [Parameter(Mandatory=$true)][String]$Dest,
+        [Hashtable]$RequestParams
+    )
+
+    $dest_last_mod = (Get-AnsibleItem -Path $Dest).LastWriteTimeUtc
+
+    # If the URI is a file we don't need to go through the whole WebRequest
+    if ($Uri.IsFile) {
+        $src_last_mod = (Get-AnsibleItem -Path $Uri.AbsolutePath).LastWriteTimeUtc
    } else {
-        return $true
+        $invoke_args = @{
+            Module = $Module
+            Uri = $Uri
+            Method = @{
+                FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::GetDateTimestamp
+                HttpWebRequest = [System.Net.WebRequestMethods+Http]::Head
            }
+            Script = { param($Response, $Stream); $Response.LastModified }
+        }
+        try {
+            $src_last_mod = Invoke-AnsibleWebRequest @invoke_args @RequestParams
+        } catch {
+            $Module.FailJson("Error when requesting 'Last-Modified' date from '$Uri'. $($_.Exception.Message)", $_)
+        }
+    }
+
+    # Return $true if the Uri LastModification date is newer than the Dest LastModification date
+    ((Get-Date -Date $src_last_mod).ToUniversalTime() -gt $dest_last_mod)
 }

+Function Get-Checksum {
+    param(
+        [Parameter(Mandatory=$true)][String]$Path,
+        [String]$Algorithm = "sha1"
+    )
+
+    switch ($Algorithm) {
+        'md5' { $sp = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider }
+        'sha1' { $sp = New-Object -TypeName System.Security.Cryptography.SHA1CryptoServiceProvider }
+        'sha256' { $sp = New-Object -TypeName System.Security.Cryptography.SHA256CryptoServiceProvider }
+        'sha384' { $sp = New-Object -TypeName System.Security.Cryptography.SHA384CryptoServiceProvider }
+        'sha512' { $sp = New-Object -TypeName System.Security.Cryptography.SHA512CryptoServiceProvider }
+    }

-Function Download-File($module, $url, $dest, $headers, $credentials, $timeout, $use_proxy, $proxy) {
+    $fs = [System.IO.File]::Open($Path, [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read,
+        [System.IO.FileShare]::ReadWrite)
+    try {
+        $hash = [System.BitConverter]::ToString($sp.ComputeHash($fs)).Replace("-", "").ToLower()
+    } finally {
+        $fs.Dispose()
+    }
+    return $hash
+}

-    $module_start = Get-Date
+Function Invoke-DownloadFile {
+    param(
+        [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
+        [Parameter(Mandatory=$true)][Uri]$Uri,
+        [Parameter(Mandatory=$true)][String]$Dest,
+        [String]$Checksum,
+        [String]$ChecksumAlgorithm,
+        [Hashtable]$RequestParams
+    )

    # Check $dest parent folder exists before attempting download, which avoids unhelpful generic error message.
-    $dest_parent = Split-Path -LiteralPath $dest
+    $dest_parent = Split-Path -LiteralPath $Dest
    if (-not (Test-Path -LiteralPath $dest_parent -PathType Container)) {
-        $module.FailJson("The path '$dest_parent' does not exist for destination '$dest', or is not visible to the current user.  Ensure download destination folder exists (perhaps using win_file state=directory) before win_get_url runs.")
+        $module.FailJson("The path '$dest_parent' does not exist for destination '$Dest', or is not visible to the current user.  Ensure download destination folder exists (perhaps using win_file state=directory) before win_get_url runs.")
    }

-    # TODO: Replace this with WebRequest
-    $extWebClient = New-Object ExtendedWebClient
-
-    foreach ($header in $headers.GetEnumerator()) {
-        $extWebClient.Headers.Add($header.Name, $header.Value)
+    $download_script = {
+        param($Response, $Stream)
+
+        # Download the file to a temporary directory so we can compare it
+        $tmp_dest = Join-Path -Path $Module.Tmpdir -ChildPath ([System.IO.Path]::GetRandomFileName())
+        $fs = [System.IO.File]::Create($tmp_dest)
+        try {
+            $Stream.CopyTo($fs)
+            $fs.Flush()
+        } finally {
+            $fs.Dispose()
        }
+        $tmp_checksum = Get-Checksum -Path $tmp_dest -Algorithm $ChecksumAlgorithm
+        $Module.Result.checksum_src = $tmp_checksum

-    if ($timeout) {
-        $extWebClient.Timeout = $timeout * 1000
+        # If the checksum has been set, verify the checksum of the remote against the input checksum.
+        if ($Checksum -and $Checksum -ne $tmp_checksum) {
+            $Module.FailJson(("The checksum for {0} did not match '{1}', it was '{2}'" -f $Uri, $Checksum, $tmp_checksum))
        }

-    if (-not $use_proxy) {
-        # Ignore the system proxy settings
-        $extWebClient.Proxy = $null
-    } elseif ($proxy) {
-        $extWebClient.Proxy = $proxy
+        $download = $true
+        if (Test-Path -LiteralPath $Dest) {
+            # Validate the remote checksum against the existing downloaded file
+            $dest_checksum = Get-Checksum -Path $Dest -Algorithm $ChecksumAlgorithm
+
+            # If we don't need to download anything, save the dest checksum so we don't waste time calculating it
+            # again at the end of the script
+            if ($dest_checksum -eq $tmp_checksum) {
+                $download = $false
+                $Module.Result.checksum_dest = $dest_checksum
+                $Module.Result.size = (Get-AnsibleItem -Path $Dest).Length
+            }
        }

-    if ($credentials) {
-        if ($force_basic_auth) {
-            $extWebClient.Headers.Add("Authorization","Basic $credentials")
-        } else {
-            $extWebClient.Credentials = $credentials
+        if ($download) {
+            Copy-Item -Path $tmp_dest -Destination $Dest -Force -WhatIf:$Module.CheckMode > $null
+            $Module.Result.changed = $true
        }
    }

-    if (-not $module.CheckMode) {
-        # FIXME: Single-out catched exceptions with more specific error messages
-        Try {
-            $extWebClient.DownloadFile($url, $dest)
-        } Catch [System.Net.WebException] {
-            $module.Result.status_code = [int] $_.Exception.Response.StatusCode
-            $module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
-            $module.FailJson("Error downloading '$url' to '$dest': $($_.Exception.Message)", $_)
-        } Catch {
-            $module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
-            $module.FailJson("Unknown error downloading '$url' to '$dest': $($_.Exception.Message)", $_)
+    $invoke_args = @{
+        Module = $Module
+        Uri = $Uri
+        Method = @{
+            FileWebRequest = [System.Net.WebRequestMethods+File]::DownloadFile
+            FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::DownloadFile
+            HttpWebRequest = [System.Net.WebRequestMethods+Http]::Get
        }
+        Script = $download_script
    }

-    $module.Result.status_code = 200
-    $module.Result.changed = $true
-    $module.Result.msg = 'OK'
-    $module.Result.dest = $dest
-    $module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
-
+    $module_start = Get-Date
+    try {
+        Invoke-AnsibleWebRequest @invoke_args @RequestParams
+    } catch {
+        $Module.FailJson("Unknown error downloading '$Uri' to '$Dest': $($_.Exception.Message)", $_)
+    } finally {
+        $Module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
+    }
 }

-$module.Result.dest = $dest
-$module.Result.elapsed = 0
-$module.Result.url = $url
-
 if (-not $use_proxy -and ($proxy_url -or $proxy_username -or $proxy_password)) {
    $module.Warn("Not using a proxy on request, however a 'proxy_url', 'proxy_username' or 'proxy_password' was defined.")
 }
@ -224,6 +359,7 @@ if (Test-Path -LiteralPath $dest -PathType Container) {
    # We have a trailing path separator
    $module.FailJson("The destination path '$dest' does not exist, or is not visible to the current user.  Ensure download destination folder exists (perhaps using win_file state=directory) before win_get_url runs.")
 }
+
 $module.Result.dest = $dest

 # Enable TLS1.1/TLS1.2 if they're available but disabled (eg. .NET 4.5)
@ -236,22 +372,53 @@ if ([Net.SecurityProtocolType].GetMember("Tls12").Count -gt 0) {
 }
 [Net.ServicePointManager]::SecurityProtocol = $security_protocols

-if ($force -or -not (Test-Path -LiteralPath $dest)) {
+$request_params = @{
+    Credential = $credentials
+    Headers = $headers
+    Timeout = $timeout
+    UseProxy = $use_proxy
+    Proxy = $proxy
+}

-    Download-File -module $module -url $url -dest $dest -credentials $credentials `
-                  -headers $headers -timeout $timeout -use_proxy $use_proxy -proxy $proxy
+if ($checksum) {
+    $checksum = $checksum.Trim().toLower()
+}
+if ($checksum_algorithm) {
+    $checksum_algorithm = $checksum_algorithm.Trim().toLower()
+}
+if ($checksum_url) {
+    $checksum_url = $checksum_url.Trim().toLower()
+}

-} else {
+# Check for case $checksum variable contain url. If yes, get file data from url and replace original value in $checksum
+if ($checksum_url) {
+    $checksum_uri = [System.Uri]$checksum_url
+    if ($checksum_uri.Scheme -notin @("file", "ftp", "http", "https")) {
+        $module.FailJson("Unsupported 'checksum_url' value for '$dest': '$checksum_url'")
+    }

-    $is_modified = CheckModified-File -module $module -url $url -dest $dest -credentials $credentials `
-                                      -headers $headers -timeout $timeout -use_proxy $use_proxy -proxy $proxy
+    $checksum = Get-ChecksumFromUri -Module $Module -Uri $checksum_uri -SourceUri $url -RequestParams $request_params
+}

+if ($force -or -not (Test-Path -LiteralPath $dest)) {
+    # force=yes or dest does not exist, download the file
+    # Note: Invoke-DownloadFile will compare the checksums internally if dest exists
+    Invoke-DownloadFile -Module $module -Uri $url -Dest $dest -Checksum $checksum `
+        -ChecksumAlgorithm $checksum_algorithm -RequestParams $request_params
+} else {
+    # force=no, we want to check the last modified dates and only download if they don't match
+    $is_modified = Compare-ModifiedFile -Module $module -Uri $url -Dest $dest -RequestParams $request_params
    if ($is_modified) {
-
-        Download-File -module $module -url $url -dest $dest -credentials $credentials `
-                      -headers $headers -timeout $timeout -use_proxy $use_proxy -proxy $proxy
-
+        Invoke-DownloadFile -Module $module -Uri $url -Dest $dest -Checksum $checksum `
+            -ChecksumAlgorithm $checksum_algorithm -RequestParams $request_params
   }
 }

+if ((-not $module.Result.ContainsKey("checksum_dest")) -and (Test-Path -LiteralPath $dest)) {
+    # Calculate the dest file checksum if it hasn't already been done
+    $module.Result.checksum_dest = Get-Checksum -Path $dest -Algorithm $checksum_algorithm
+    $module.Result.size = (Get-AnsibleItem -Path $dest).Length
+}
+
 $module.ExitJson()
+
--- a/lib/ansible/modules/windows/win_get_url.py
+++ b/lib/ansible/modules/windows/win_get_url.py
@ -34,7 +34,7 @@ options:
    required: yes
  force:
    description:
-    - If C(yes), will always download the file. If C(no), will only
+    - If C(yes), will download the file every time and replace the file if the contents change. If C(no), will only
      download the file if it does not exist or the remote file has been
      modified more recently than the local file.
    - This works by sending an http HEAD request to retrieve last modified
@ -73,6 +73,36 @@ options:
    type: bool
    default: yes
    version_added: '2.4'
+  checksum:
+    description:
+      - If a I(checksum) is passed to this parameter, the digest of the
+        destination file will be calculated after it is downloaded to ensure
+        its integrity and verify that the transfer completed successfully.
+      - This option cannot be set with I(checksum_url).
+    type: str
+    version_added: "2.8"
+  checksum_algorithm:
+    description:
+      - Specifies the hashing algorithm used when calculating the checksum of
+        the remote and destination file.
+    type: str
+    choices:
+      - md5
+      - sha1
+      - sha256
+      - sha385
+      - sha512
+    default: sha1
+    version_added: "2.8"
+  checksum_url:
+    description:
+      - Specifies a URL that contains the checksum values for the resource at
+        I(url).
+      - Like C(checksum), this is used to verify the integrity of the remote
+        transfer.
+      - This option cannot be set with I(checksum).
+    type: str
+    version_added: "2.8"
  proxy_url:
    description:
    - The full URL of the proxy server to download through.
@ -105,6 +135,9 @@ notes:
 - If your URL includes an escaped slash character (%2F) this module will convert it to a real slash.
  This is a result of the behaviour of the System.Uri class as described in
  L(the documentation,https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/network/schemesettings-element-uri-settings#remarks).
+- Since Ansible 2.8, the module will skip reporting a change if the remote
+  checksum is the same as the local local even when C(force=yes). This is to
+  better align with M(get_url).
 seealso:
 - module: get_url
 - module: uri
@ -140,6 +173,22 @@ EXAMPLES = r'''
    dest: '%TEMP%\ftp-file.txt'
    url_username: ftp-user
    url_password: ftp-password
+
+- name: Download src with sha256 checksum url
+  win_get_url:
+    url: http://www.example.com/earthrise.jpg
+    dest: C:\temp\earthrise.jpg
+    checksum_url: http://www.example.com/sha256sum.txt
+    checksum_algorithm: sha256
+    force: True
+
+- name: Download src with sha256 checksum url
+  win_get_url:
+    url: http://www.example.com/earthrise.jpg
+    dest: C:\temp\earthrise.jpg
+    checksum: a97e6837f60cec6da4491bab387296bbcd72bdba
+    checksum_algorithm: sha1
+    force: True
 '''

 RETURN = r'''
@ -148,11 +197,26 @@ dest:
    returned: always
    type: str
    sample: C:\Users\RandomUser\earthrise.jpg
+checksum_dest:
+    description: <algorithm> checksum of the file after the download
+    returned: success and dest has been downloaded
+    type: str
+    sample: 6e642bb8dd5c2e027bf21dd923337cbb4214f827
+checksum_src:
+    description: <algorithm> checksum of the remote resource
+    returned: force=yes or dest did not exist
+    type: str
+    sample: 6e642bb8dd5c2e027bf21dd923337cbb4214f827
 elapsed:
    description: The elapsed seconds between the start of poll and the end of the module.
    returned: always
    type: float
    sample: 2.1406487
+size:
+    description: size of the dest file
+    returned: success
+    type: int
+    sample: 1220
 url:
    description: requested url
    returned: always
--- a/test/integration/targets/win_get_url/defaults/main.yml
+++ b/test/integration/targets/win_get_url/defaults/main.yml
@ -2,3 +2,6 @@
 test_win_get_url_path: '{{win_output_dir}}\win_get_url'
 test_win_get_url_host: www.redhat.com
 test_win_get_url_env_var: WIN_GET_URL
+
+remote_tmp_path: '{{win_output_dir}}\win_get_url_dest'
+remote_http_path: '{{test_win_get_url_path}}\http'
--- a/test/integration/targets/win_get_url/tasks/main.yml
+++ b/test/integration/targets/win_get_url/tasks/main.yml
@ -1,8 +1,12 @@
 ---
 - name: ensure testing folder is present
  win_file:
-    path: '{{test_win_get_url_path}}'
+    path: '{{ item }}'
    state: directory
+  loop:
+    - '{{ test_win_get_url_path }}'
+    - '{{ remote_http_path }}'
+    - '{{ remote_tmp_path }}'

 - name: copy across testing files
  win_copy:
@ -41,6 +45,9 @@
  - name: run FTP tests
    include_tasks: tests_ftp.yml

+  - name: run checksum tests
+    include_tasks: tests_checksum.yml
+
  always:
  - name: remove SlimFTPd service
    win_service:
@ -53,7 +60,10 @@
      level: machine
      state: absent

-  - name: remove testing folder
+  - name: remove testing folders
    win_file:
-      path: '{{test_win_get_url_path}}'
+      path: '{{ item }}'
      state: absent
+    loop:
+      - '{{ test_win_get_url_path }}'
+      - '{{ remote_tmp_path }}'
--- a/test/integration/targets/win_get_url/tasks/tests_checksum.yml
+++ b/test/integration/targets/win_get_url/tasks/tests_checksum.yml
@ -0,0 +1,202 @@
+---
+- name: create src file
+  win_copy:
+    dest: '{{ remote_http_path }}\27617.txt'
+    content: 'ptux'
+
+- name: create sha1 checksum file of src
+  win_copy:
+    dest: '{{ remote_http_path }}\sha1sum.txt'
+    content: 'a97e6837f60cec6da4491bab387296bbcd72bdba  27617.txt'
+
+- name: add sha1 checksum not going to be downloaded
+  win_lineinfile:
+    dest: '{{ remote_http_path }}\sha1sum.txt'
+    line: '{{ item }}'
+  loop:
+    - '3911340502960ca33aece01129234460bfeb2791  not_target1.txt'
+    - '1b4b6adf30992cedb0f6edefd6478ff0a593b2e4  not_target2.txt'
+
+- name: create sha256 checksum file of src
+  win_copy:
+    dest: '{{ remote_http_path }}\sha256sum.txt'
+    content: 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  27617.txt'
+
+- name: add sha256 checksum not going to be downloaded
+  win_lineinfile:
+    dest: '{{ remote_http_path }}\sha256sum.txt'
+    line: '{{ item }}'
+  loop:
+    - '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892  not_target1.txt'
+    - 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b  not_target2.txt'
+
+- name: create sha256 checksum file of src with a dot leading path
+  win_copy:
+    dest: '{{ remote_http_path }}\sha256sum_with_dot.txt'
+    content: 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  ./27617.txt'
+
+- name: add sha256 checksums with dot leading path not going to be downloaded
+  win_lineinfile:
+    dest: '{{ remote_http_path }}\sha256sum_with_dot.txt'
+    line: '{{ item }}'
+  loop:
+    - '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892  ./not_target1.txt'
+    - 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b  ./not_target2.txt'
+
+- name: copy files to ftp
+  win_copy:
+    src: '{{ remote_http_path }}\'
+    dest: '{{ test_win_get_url_path }}\ftp\{{ item }}\'
+    remote_src: yes
+  loop: '{{ dest_folders }}'
+  vars:
+    dest_folders:
+      - anon
+      - user
+      - user-pass
+
+- name: download src with sha1 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum_url: 'ftp://localhost/anon/sha1sum.txt'
+    force: True
+  register: result_sha1
+
+- name: download src with sha1 checksum value
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum: 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+    force: True
+  register: result_sha1_alt
+
+- win_stat:
+    path: '{{ remote_tmp_path }}\27617.txt'
+  register: stat_result_sha1
+
+- name: download src with sha256 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum.txt'
+    checksum_algorithm: sha256
+    force: True
+  register: result_sha256
+
+- win_stat:
+    path: '{{ remote_tmp_path }}\27617.txt'
+  register: stat_result_sha256
+
+- name: download src with sha256 checksum url with dot leading paths
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256_with_dot.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum_with_dot.txt'
+    checksum_algorithm: sha256
+    force: True
+  register: result_sha256_with_dot
+
+- win_stat:
+    path: '{{ remote_tmp_path }}\27617sha256_with_dot.txt'
+  register: stat_result_sha256_with_dot
+
+- name: Assert that the file was downloaded
+  assert:
+    that:
+      - result_sha1 is changed
+      - result_sha1.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+      - result_sha1_alt is succeeded
+      - not result_sha1_alt is changed
+      - result_sha1_alt.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+      - result_sha256 is changed
+      - result_sha256_with_dot is changed
+      - stat_result_sha1.stat.exists | bool
+      - stat_result_sha256.stat.exists | bool
+      - stat_result_sha256_with_dot.stat.exists | bool
+
+# Check download with force: False
+
+- name: download src with sha1 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum_url: 'ftp://localhost/anon/sha1sum.txt'
+    checksum_algorithm: sha1
+    force: False
+  register: result_sha1_no_force
+
+- name: download src with sha256 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}/27617sha256.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum.txt'
+    checksum_algorithm: sha256
+    force: False
+  register: result_sha256_no_force
+
+- name: assert download single file with force no
+  assert:
+    that:
+      - result_sha1_no_force is not changed
+      - result_sha256_no_force is not changed
+
+# Check download with check_mode: True
+
+- name: download src with sha1 checksum url | checkmode
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum_url: 'ftp://localhost/anon/sha1sum.txt'
+    checksum_algorithm: sha1
+  check_mode: True
+  register: result_sha1_check_mode
+
+- name: download src with sha256 checksum url | checkmode
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum.txt'
+    checksum_algorithm: sha256
+  check_mode: True
+  register: result_sha256_check_mode
+
+- name: assert download single file with force no
+  assert:
+    that:
+      - result_sha1_check_mode.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+      - result_sha256_check_mode.checksum_dest == 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006'
+      - result_sha1_check_mode.checksum_src | length
+      - result_sha256_check_mode.checksum_src | length
+
+# Check download with wrong checksum
+
+- name: download src with sha1 checksum value | fail
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum: 'redacted'
+    checksum_algorithm: sha1
+  failed_when:
+    - '"did not match" not in result_sha1_failed.msg'
+    - '"The checksum" not in result_sha1_failed.msg'
+    - '"Unknown error downloading" not in result_sha1_failed.msg'
+  register: result_sha1_failed
+
+- name: download src with sha256 checksum value | fail
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256.txt'
+    checksum: 'redacted'
+    checksum_algorithm: sha256
+  failed_when:
+    - '"did not match" not in result_sha256_failed.msg'
+    - '"The checksum" not in result_sha256_failed.msg'
+    - '"Unknown error downloading" not in result_sha256_failed.msg'
+  register: result_sha256_failed
+
+- name: assert failed downloads
+  assert:
+    that:
+      - result_sha1_failed is succeeded
+      - result_sha256_failed is succeeded
--- a/test/integration/targets/win_get_url/tasks/tests_url.yml
+++ b/test/integration/targets/win_get_url/tasks/tests_url.yml
@ -39,7 +39,16 @@
    - http_download.dest
    - http_download_result.stat.exists

-# TODO: add check for idempotent run once it is added with force: yes
+- name: download single file (idempotent)
+  win_get_url:
+    url: https://{{test_win_get_url_host}}
+    dest: '{{test_win_get_url_path}}\web.html'
+  register: http_download_again
+
+- name: assert download single file (idempotent)
+  assert:
+    that:
+    - not http_download_again is changed

 - name: download single file with force no
  win_get_url:
@ -53,8 +62,10 @@
    that:
    - http_download_no_force is not changed

- name: manually change last modified time on FTP source to older datetime
-  win_shell: (Get-Item -Path '{{test_win_get_url_path}}\web.html').LastWriteTime = (Get-Date -Date "01/01/1970")
+- name: manually change the content and last modified time on FTP source to older datetime
+  win_shell: |
+    Set-Content -Path '{{test_win_get_url_path}}\web.html' -Value 'abc'
+    (Get-Item -Path '{{test_win_get_url_path}}\web.html').LastWriteTime = (Get-Date -Date "01/01/1970")

 - name: download newer file with force no
  win_get_url:
--- a/test/sanity/pslint/ignore.txt
+++ b/test/sanity/pslint/ignore.txt
@ -38,7 +38,7 @@ lib/ansible/modules/windows/win_find.ps1 PSAvoidUsingEmptyCatchBlock
 lib/ansible/modules/windows/win_find.ps1 PSAvoidUsingWMICmdlet
 lib/ansible/modules/windows/win_firewall_rule.ps1 PSAvoidUsingCmdletAliases
 lib/ansible/modules/windows/win_firewall_rule.ps1 PSUseApprovedVerbs
-lib/ansible/modules/windows/win_get_url.ps1 PSUseApprovedVerbs
+lib/ansible/modules/windows/win_get_url.ps1 PSUsePSCredentialType  # Credential param can take a base64 encoded string as well as a PSCredential
 lib/ansible/modules/windows/win_hotfix.ps1 PSUseApprovedVerbs
 lib/ansible/modules/windows/win_iis_webbinding.ps1 PSUseApprovedVerbs
 lib/ansible/modules/windows/win_iis_website.ps1 PSAvoidUsingCmdletAliases