WIP: net_command: Initial tests for VyOS (#20147)

* Conditional include on ansible_network_os

* copy & paste error

* More tests

* More tests

* junos tests (based on vyos)

* remove excessive whitespace

* Pass in ansible_network_os

* net_command for ios

* consistent debug

* wrapp line

* ansible-test changes made in another PR

* ansible-test changes made in another PR

test/integration/targets/net_command/tasks/ios.yml

@@ -0,0 +1,144 @@
+---
+# Test code for the net_command module.
+# (c) 2017, Red Hat
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+
+
+#########################################
+# Positive: Basic Golden path
+#
+- name: net_command - Valid command
+  net_command: show version
+  connection: network_cli
+  register: result
+
+- name: Display result
+  debug:
+    msg: "{{ result }}"
+
+- name: Ensure output is valid
+  assert:
+    that:
+      # net_command should always return "changed", it isn't idempotent
+      - result|changed
+      - result|success
+      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+      - result.rc == 0
+      - result.stderr == ""
+      # Check for platform specific string in output
+      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - "'Cisco Systems' in result.stdout"
+      - "'Cisco' in result.stdout_lines[0]"
+
+
+##########################################
+## Positive: stdout and stderr with rc:0
+##
+#- name: net_command - stdout and stderr, rc:0
+#  net_command: python -c 'import os; os.write(2, "Oh no, this is an error\n"); os.write(1, "This is stdout\n") '
+#  connection: network_cli
+#  register: result
+#
+#- name: Display result
+#  debug:
+#    msg: "{{ result }}"
+#
+#- name: Ensure output is valid
+#  assert:
+#    that:
+#      # net_command should always return "changed", it isn't idempotent
+#      - result|changed
+#      - result|success
+#      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+#      - result.rc == 0
+#      - result.stderr == ""
+#      # Check for platform specific string in output
+#      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+#      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+#      - "'This is stdout' in result.stdout"
+#      - "'Oh no, this is an error' in result.stderr"
+#      - "'Version' in result.stdout_lines[0]"
+#  # FIXME BUG https://github.com/ansible/ansible/issues/20233
+#  # FIXME Once working change test to check for exact return, rather than "in"
+#  ignore_errors: true
+
+#########################################
+# Negative: None zero exit value and stderr
+
+- name: net_command - None zero exit value and stderr
+  net_command: show foobar
+  connection: network_cli
+  register: result
+  ignore_errors: true
+
+- name: Display result
+  debug:
+    msg: "{{ result }}"
+
+- name: Check we failed
+  assert:
+    that:
+      # net_command should always return "changed", it isn't idempotent
+      - result|changed
+      - result|failed
+      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+      - result.rc == 1
+      - "'Invalid input detected at' in result.stderr"
+      # Check for platform specific string in output
+      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.stdout == ""
+      # FIXME BUG: https://github.com/ansible/ansible/issues/20296
+      #- "'Version' in result.stdout_lines[0]"
+
+#########################################
+
+- name: net_command - Missing command
+  net_command:
+  connection: network_cli
+  register: result
+  ignore_errors: true
+
+- name: Display result
+  debug:
+    msg: "{{ result }}"
+
+- name: Check we failed
+  assert:
+    that:
+      # net_command should always return "changed", it isn't idempotent
+      - result|changed
+      - result|failed
+      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+      - result.rc == 1
+      # Check for platform specific string in output
+      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.stdout == ""
+      # FIXME BUG: https://github.com/ansible/ansible/issues/20294
+      - "'Invalid command:' in result.stderr"
+  ignore_errors: true
+
+# Quoting and escape tests - look at command/shell test cases
+#
+#
+# Blocked Test Cases
+#   * Run a command that returns stdout and stderr
+#
+# Run through ansible-lint

test/integration/targets/net_command/tasks/junos.yml

@@ -0,0 +1,144 @@
+---
+# Test code for the net_command module.
+# (c) 2017, Red Hat
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+
+
+#########################################
+# Positive: Basic Golden path
+#
+- name: net_command - Valid command
+  net_command: show version
+  connection: network_cli
+  register: result
+
+- name: Display result
+  debug:
+    msg: "{{ result }}"
+
+- name: Ensure output is valid
+  assert:
+    that:
+      # net_command should always return "changed", it isn't idempotent
+      - result|changed
+      - result|success
+      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+      - result.rc == 0
+      - result.stderr == ""
+      # Check for platform specific string in output
+      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - "'JUNOS Software Release' in result.stdout"
+      - "'Hostname' in result.stdout_lines[0]"
+
+
+##########################################
+## Positive: stdout and stderr with rc:0
+##
+#- name: net_command - stdout and stderr, rc:0
+#  net_command: python -c 'import os; os.write(2, "Oh no, this is an error\n"); os.write(1, "This is stdout\n") '
+#  connection: network_cli
+#  register: result
+#
+#- name: Display result
+#  debug:
+#    msg: "{{ result }}"
+#
+#- name: Ensure output is valid
+#  assert:
+#    that:
+#      # net_command should always return "changed", it isn't idempotent
+#      - result|changed
+#      - result|success
+#      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+#      - result.rc == 0
+#      - result.stderr == ""
+#      # Check for platform specific string in output
+#      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+#      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+#      - "'This is stdout' in result.stdout"
+#      - "'Oh no, this is an error' in result.stderr"
+#      - "'Version' in result.stdout_lines[0]"
+#  # FIXME BUG https://github.com/ansible/ansible/issues/20233
+#  # FIXME Once working change test to check for exact return, rather than "in"
+#  ignore_errors: true
+
+#########################################
+# Negative: None zero exit value and stderr
+
+- name: net_command - None zero exit value and stderr
+  net_command: show foobar
+  connection: network_cli
+  register: result
+  ignore_errors: true
+
+- name: Display result
+  debug:
+    msg: "{{ result }}"
+
+- name: Check we failed
+  assert:
+    that:
+      # net_command should always return "changed", it isn't idempotent
+      - result|changed
+      - result|failed
+      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+      - result.rc == 1
+      - "'syntax error, expecting <command>.' in result.stderr"
+      # Check for platform specific string in output
+      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.stdout == ""
+      # FIXME BUG: https://github.com/ansible/ansible/issues/20296
+      #- "'Version' in result.stdout_lines[0]"
+
+#########################################
+
+- name: net_command - Missing command
+  net_command:
+  connection: network_cli
+  register: result
+  ignore_errors: true
+
+- name: Display result
+  debug:
+    msg: "{{ result }}"
+
+- name: Check we failed
+  assert:
+    that:
+      # net_command should always return "changed", it isn't idempotent
+      - result|changed
+      - result|failed
+      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+      - result.rc == 1
+      # Check for platform specific string in output
+      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.stdout == ""
+      # FIXME BUG: https://github.com/ansible/ansible/issues/20294
+      - "'Invalid command:' in result.stderr"
+  ignore_errors: true
+
+# Quoting and escape tests - look at command/shell test cases
+#
+#
+# Blocked Test Cases
+#   * Run a command that returns stdout and stderr
+#
+# Run through ansible-lint

test/integration/targets/net_command/tasks/main.yml

@@ -20,16 +20,27 @@
 
 # This is a placeholder test, will be fleshed out by gundalow at a later point
 
-- name: Check we can communicate using net_command
-  net_command: show version
-  connection: network_cli
-  register: result
-
-- debug:
-    msg: "{{ result.stdout }}"
-
-- name: Ensure output is valid
-  assert:
-    that:
-      - result.changed == true
-      - "'VyOS' in result.stdout"
+
+# Check if  ansible_network_os being undef or incorrect
+# Without this we wouldn't run any tests, and not actually test Ansible
+
+# FIXME One we can get ansible_network_os from env remove it from 
+# test/runner/johnb@x260:~/git/ansible-inc/ansible/test/runner/lib/executor.py
+#
+
+- name: Check for acceptable ansible_network_os
+  fail:
+      msg: "ansible_network_os is undefined or not an acceptable platform: '{{ ansible_network_os }}'"
+  when: ansible_network_os not in [ 'vyos', 'junos', 'ios' ]
+
+- name: vyos Specific includes
+  include: vyos.yml
+  when: ansible_network_os == 'vyos'
+
+- name: junos Specific includes
+  include: junos.yml
+  when: ansible_network_os == 'junos'
+
+- name: ios Specific includes
+  include: ios.yml
+  when: ansible_network_os == 'ios'

test/integration/targets/net_command/tasks/vyos.yml

@@ -0,0 +1,144 @@
+---
+# Test code for the net_command module.
+# (c) 2017, Red Hat
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+
+
+#########################################
+# Positive: Basic Golden path
+#
+- name: net_command - Valid command
+  net_command: show version
+  connection: network_cli
+  register: result
+
+- name: Display result
+  debug:
+    msg: "{{ result }}"
+
+- name: Ensure output is valid
+  assert:
+    that:
+      # net_command should always return "changed", it isn't idempotent
+      - result|changed
+      - result|success
+      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+      - result.rc == 0
+      - result.stderr == ""
+      # Check for platform specific string in output
+      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - "'VyOS' in result.stdout"
+      - "'Version' in result.stdout_lines[0]"
+
+
+#########################################
+# Positive: stdout and stderr with rc:0
+#
+- name: net_command - stdout and stderr, rc:0
+  net_command: python -c 'import os; os.write(2, "Oh no, this is an error\n"); os.write(1, "This is stdout\n") '
+  connection: network_cli
+  register: result
+
+- name: Display result
+  debug:
+    msg: "{{ result }}"
+
+- name: Ensure output is valid
+  assert:
+    that:
+      # net_command should always return "changed", it isn't idempotent
+      - result|changed
+      - result|success
+      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+      - result.rc == 0
+      - result.stderr == ""
+      # Check for platform specific string in output
+      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - "'This is stdout' in result.stdout"
+      - "'Oh no, this is an error' in result.stderr"
+      - "'Version' in result.stdout_lines[0]"
+  # FIXME BUG https://github.com/ansible/ansible/issues/20233
+  # FIXME Once working change test to check for exact return, rather than "in"
+  ignore_errors: true
+
+#########################################
+# Negative: None zero exit value and stderr
+
+- name: net_command - None zero exit value and stderr
+  net_command: show foobar
+  connection: network_cli
+  register: result
+  ignore_errors: true
+
+- name: Display result
+  debug:
+    msg: "{{ result }}"
+
+- name: Check we failed
+  assert:
+    that:
+      # net_command should always return "changed", it isn't idempotent
+      - result|changed
+      - result|failed
+      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+      - result.rc == 1
+      - "'Invalid command:' in result.stderr"
+      # Check for platform specific string in output
+      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.stdout == ""
+      # FIXME BUG: https://github.com/ansible/ansible/issues/20296
+      #- "'Version' in result.stdout_lines[0]"
+
+#########################################
+
+- name: net_command - Missing command
+  net_command:
+  connection: network_cli
+  register: result
+  ignore_errors: true
+
+- name: Display result
+  debug:
+    msg: "{{ result }}"
+
+- name: Check we failed
+  assert:
+    that:
+      # net_command should always return "changed", it isn't idempotent
+      - result|changed
+      - result|failed
+      - result.delta is match('^\d:(\d){2}:(\d){2}.(\d){6}$')
+      - result.rc == 1
+      # Check for platform specific string in output
+      - result.end is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.start is match('^(\d){4}\-(\d){2}\-(\d){2} (\d){2}:(\d){2}:(\d){2}.(\d){6}$')
+      - result.stdout == ""
+      # FIXME BUG: https://github.com/ansible/ansible/issues/20294
+      - "'Invalid command:' in result.stderr"
+  ignore_errors: true
+
+# Quoting and escape tests - look at command/shell test cases
+#
+#
+# Blocked Test Cases
+#   * Run a command that returns stdout and stderr
+#
+# Run through ansible-lint