WIP: Resolve CI issues for stable-2.3. (#32605)

* Revert "If pip install requirements.txt fails, upgrade pip (#32399)"

This reverts commit 2fb4f547a9.

* Avoid pep8 user warning in 1.7.1.
* Limit cryptography version for Windows tests.
* Upgrade pip in virtualenv for pip test.
* Upgrade pip in virtualenv for groupby_filter test.
* Upgrade pip in virtualenv for template_jinja2_latest test.

CODING_GUIDELINES.md

@@ -14,15 +14,15 @@ Language
   * While not all components of Ansible must be in Python, core contributions to the Ansible repo must be written in Python.  This is to maximize the ability of everyone to contribute.
   * If you want to write non-Python ansible modules or inventory scripts, that's fine, but they are not going to get merged in most likely.  Sorry!!  
 
-PEP 8 and basic style checks
+PEP8 and basic style checks
 ===========================
 
-  * [PEP 8](https://www.python.org/dev/peps/pep-0008/) is a great Python style guide, which you should read.
+  * PEP8 is a great Python style guide, which you should read.
-  * PEP 8 must not be strictly followed in all aspects, but most of it is good advice
+  * PEP8 must not be strictly followed in all aspects, but most of it is good advice
   * In particular, we don't really care about line lengths.  Buy a bigger monitor!
-  * To run checks for things we care about, use [ansible-test](http://docs.ansible.com/ansible/dev_guide/testing_pep8.html#running-locally).
+  * To run checks for things we care about, run "make pep8"
   * Similarly, additional checks can be made with "make pyflakes"
-  * There is no need to submit code changes for PEP 8 and pyflakes fixes, as these break attribution history.  Project leadership will make these periodically.
+  * There is no need to submit code changes for pep8 and pyflake fixes, as these break attribution history.  Project leadership will make these periodically.
   * Do not submit pull requests that simply adjust whitespace in the code
 
 Testing

lib/ansible/modules/packaging/os/apt_repository.py

@@ -210,7 +210,6 @@ class SourcesList(object):
             if filename is not None:
                 return filename
             return '_'.join(re.sub('[^a-zA-Z0-9]', ' ', s).split())
-
         def _strip_username_password(s):
             if '@' in s:
                 s = s.split('@', 1)

test/integration/targets/groupby_filter/runme.sh

@@ -16,6 +16,9 @@ virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/jinja2"
 
 source "${MYTMPDIR}/jinja2/bin/activate"
 
+curl https://bootstrap.pypa.io/get-pip.py > "${MYTMPDIR}/get-pip.py"
+python "${MYTMPDIR}/get-pip.py"
+
 pip install -U jinja2==2.9.4
 
 ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"

test/integration/targets/pip/tasks/pip.yml

@@ -151,6 +151,15 @@
       - "not setuptools_check_mode.changed"
 
 
+# Upgrade pip in virtualenv
+- name: download get-pip.py
+  get_url:
+    url: https://bootstrap.pypa.io/get-pip.py
+    dest: "{{ output_dir }}"
+
+- name: install pip
+  command: "{{ output_dir }}/pipenv/bin/python {{ output_dir }}/get-pip.py"
+
 # Normal case
 - name: check for q package
   pip: name=q virtualenv={{ output_dir }}/pipenv state=present

test/integration/targets/template_jinja2_latest/runme.sh

@@ -16,6 +16,9 @@ virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/jinja2"
 
 source "${MYTMPDIR}/jinja2/bin/activate"
 
+curl https://bootstrap.pypa.io/get-pip.py > "${MYTMPDIR}/get-pip.py"
+python "${MYTMPDIR}/get-pip.py"
+
 pip install -U jinja2
 
 ansible-playbook -i ../../inventory main.yml -e @../../integration_config.yml -v "$@"

test/runner/lib/executor.py

@@ -156,11 +156,16 @@ def install_command_requirements(args):
 
     try:
         run_command(args, cmd)
-    except SubprocessError:
+    except SubprocessError as ex:
-        # Gundalow/Mattclay: Revisit logic here https://github.com/ansible/ansible/issues/32398
+        if ex.status != 2:
-        # If the install fails for any reason, upgrade pip and retry.
+            raise
-        # We don't wish to unconditionally upgrade pip, as that would reduce test coverage
+
+        # If pip is too old it won't understand the arguments we passed in, so we'll need to upgrade it.
 
+        # Installing "coverage" on ubuntu 16.04 fails with the error:
+        # AttributeError: 'Requirement' object has no attribute 'project_name'
+        # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258
+        # Upgrading pip works around the issue.
         run_command(args, ['pip', 'install', '--upgrade', 'pip'])
         run_command(args, cmd)
 

test/runner/lib/sanity.py

@@ -316,7 +316,7 @@ def command_sanity_pep8(args, targets):
         return SanitySkipped(test)
 
     cmd = [
-        'pycodestyle',
+        'pep8',
         '--max-line-length', '160',
         '--config', '/dev/null',
         '--ignore', ','.join(sorted(current_ignore)),

test/runner/requirements/constraints.txt

@@ -5,3 +5,4 @@ sphinx < 1.6 ; python_version < '2.7' # sphinx 1.6 and later require python 2.7
 yamllint < 1.8 ; python_version < '2.7' # yamllint 1.8 and later require python 2.7 or later
 wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later
 isort < 4.2.8 # 4.2.8 changes import sort order requirements which breaks previously passing pylint tests
+pep8 <= 1.7.0 # pep8 1.7.0 is the last "real" release, avoid the user warning added in 1.7.1

test/runner/requirements/sanity.txt

@@ -1,12 +1,9 @@
-cryptography
 jinja2
 mock
-paramiko
+pep8
-pycodestyle
 pylint
 pytest
 rstcheck
 sphinx
-virtualenv
 voluptuous
 yamllint

test/runner/requirements/windows-integration.txt

@@ -1,3 +1,4 @@
+cryptography >= 1.3.4, < 2.1 # cryptography 2.1 requires pip 8.1.2+ (sanity_ok)
 jinja2
 junit-xml
 pywinrm

test/sanity/pep8/current-ignore.txt

@@ -22,13 +22,9 @@ E266
 E301
 E302
 E303
-E305
-E306
 E402
 E502
 E713
-E722
 E731
-E741
 W391
 W503