WIP: Resolve CI issues for stable-2.3. (#32605)

* Revert "If pip install requirements.txt fails, upgrade pip (#32399)"

This reverts commit 2fb4f547a9.

* Avoid pep8 user warning in 1.7.1.
* Limit cryptography version for Windows tests.
* Upgrade pip in virtualenv for pip test.
* Upgrade pip in virtualenv for groupby_filter test.
* Upgrade pip in virtualenv for template_jinja2_latest test.

CODING_GUIDELINES.md

@@ -17,12 +17,12 @@ Language
 PEP8 and basic style checks
 ===========================
 
-  * [PEP 8](https://www.python.org/dev/peps/pep-0008/) is a great Python style guide, which you should read.
+  * PEP8 is a great Python style guide, which you should read.
   * PEP8 must not be strictly followed in all aspects, but most of it is good advice
   * In particular, we don't really care about line lengths.  Buy a bigger monitor!
-  * To run checks for things we care about, use [ansible-test](http://docs.ansible.com/ansible/dev_guide/testing_pep8.html#running-locally).
+  * To run checks for things we care about, run "make pep8"
   * Similarly, additional checks can be made with "make pyflakes"
-  * There is no need to submit code changes for PEP 8 and pyflakes fixes, as these break attribution history.  Project leadership will make these periodically.
+  * There is no need to submit code changes for pep8 and pyflake fixes, as these break attribution history.  Project leadership will make these periodically.
   * Do not submit pull requests that simply adjust whitespace in the code
 
 Testing

lib/ansible/modules/packaging/os/apt_repository.py

@@ -210,7 +210,6 @@ class SourcesList(object):
             if filename is not None:
                 return filename
             return '_'.join(re.sub('[^a-zA-Z0-9]', ' ', s).split())
-
         def _strip_username_password(s):
             if '@' in s:
                 s = s.split('@', 1)

test/integration/targets/groupby_filter/runme.sh

@@ -16,6 +16,9 @@ virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/jinja2"
 
 source "${MYTMPDIR}/jinja2/bin/activate"
 
+curl https://bootstrap.pypa.io/get-pip.py > "${MYTMPDIR}/get-pip.py"
+python "${MYTMPDIR}/get-pip.py"
+
 pip install -U jinja2==2.9.4
 
 ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"

test/integration/targets/pip/tasks/pip.yml

@@ -151,6 +151,15 @@
       - "not setuptools_check_mode.changed"
 
 
+# Upgrade pip in virtualenv
+- name: download get-pip.py
+  get_url:
+    url: https://bootstrap.pypa.io/get-pip.py
+    dest: "{{ output_dir }}"
+
+- name: install pip
+  command: "{{ output_dir }}/pipenv/bin/python {{ output_dir }}/get-pip.py"
+
 # Normal case
 - name: check for q package
   pip: name=q virtualenv={{ output_dir }}/pipenv state=present

test/integration/targets/template_jinja2_latest/runme.sh

@@ -16,6 +16,9 @@ virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/jinja2"
 
 source "${MYTMPDIR}/jinja2/bin/activate"
 
+curl https://bootstrap.pypa.io/get-pip.py > "${MYTMPDIR}/get-pip.py"
+python "${MYTMPDIR}/get-pip.py"
+
 pip install -U jinja2
 
 ansible-playbook -i ../../inventory main.yml -e @../../integration_config.yml -v "$@"

test/runner/lib/executor.py

@@ -156,11 +156,16 @@ def install_command_requirements(args):
 
     try:
         run_command(args, cmd)
-    except SubprocessError:
-        # Gundalow/Mattclay: Revisit logic here https://github.com/ansible/ansible/issues/32398
-        # If the install fails for any reason, upgrade pip and retry.
-        # We don't wish to unconditionally upgrade pip, as that would reduce test coverage
+    except SubprocessError as ex:
+        if ex.status != 2:
+            raise
+
+        # If pip is too old it won't understand the arguments we passed in, so we'll need to upgrade it.
 
+        # Installing "coverage" on ubuntu 16.04 fails with the error:
+        # AttributeError: 'Requirement' object has no attribute 'project_name'
+        # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258
+        # Upgrading pip works around the issue.
         run_command(args, ['pip', 'install', '--upgrade', 'pip'])
         run_command(args, cmd)
 

test/runner/lib/sanity.py

@@ -316,7 +316,7 @@ def command_sanity_pep8(args, targets):
         return SanitySkipped(test)
 
     cmd = [
-        'pycodestyle',
+        'pep8',
         '--max-line-length', '160',
         '--config', '/dev/null',
         '--ignore', ','.join(sorted(current_ignore)),

test/runner/requirements/constraints.txt

@@ -5,3 +5,4 @@ sphinx < 1.6 ; python_version < '2.7' # sphinx 1.6 and later require python 2.7
 yamllint < 1.8 ; python_version < '2.7' # yamllint 1.8 and later require python 2.7 or later
 wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later
 isort < 4.2.8 # 4.2.8 changes import sort order requirements which breaks previously passing pylint tests
+pep8 <= 1.7.0 # pep8 1.7.0 is the last "real" release, avoid the user warning added in 1.7.1

test/runner/requirements/sanity.txt

@@ -1,12 +1,9 @@
-cryptography
 jinja2
 mock
-paramiko
-pycodestyle
+pep8
 pylint
 pytest
 rstcheck
 sphinx
-virtualenv
 voluptuous
 yamllint

test/runner/requirements/windows-integration.txt

@@ -1,3 +1,4 @@
+cryptography >= 1.3.4, < 2.1 # cryptography 2.1 requires pip 8.1.2+ (sanity_ok)
 jinja2
 junit-xml
 pywinrm

test/sanity/pep8/current-ignore.txt

@@ -22,13 +22,9 @@ E266
 E301
 E302
 E303
-E305
-E306
 E402
 E502
 E713
-E722
 E731
-E741
 W391
 W503