diff --git a/test/integration/targets/pyyaml/aliases b/test/integration/targets/pyyaml/aliases
new file mode 100644
index 00000000000..1d28bdb2aa3
--- /dev/null
+++ b/test/integration/targets/pyyaml/aliases
@@ -0,0 +1,2 @@
+shippable/posix/group5
+context/controller
diff --git a/test/integration/targets/pyyaml/runme.sh b/test/integration/targets/pyyaml/runme.sh
new file mode 100755
index 00000000000..0361835a0d6
--- /dev/null
+++ b/test/integration/targets/pyyaml/runme.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eu
+source virtualenv.sh
+set +x
+
+# deps are already installed, using --no-deps to avoid re-installing them
+# Install PyYAML without libyaml to validate ansible can run
+PYYAML_FORCE_LIBYAML=0 pip install --no-binary PyYAML --ignore-installed --no-cache-dir --no-deps PyYAML
+
+ansible --version | tee /dev/stderr | grep 'libyaml = False'