Fix Ansible plugin loading

This corrects a security vulnerability related to loading Ansible
plugins under the `ansible.builtin.*` aliases.

Change-Id: I3a394904765e22080aa038c44bfe26e07a1e86c7
Story: 2009941

releasenotes/notes/ansible-plugin-security-90ecd81635e12179.yaml

@@ -0,0 +1,31 @@
+---
+security:
+  - |
+    A vulnerability which allowed the execution of untrusted code on
+    the executor was fixed.
+
+    Zuul restricts the Ansible modules and plugins which can be used
+    in the `untrusted` security context (i.e., untrusted projects).
+    It also prohibits running programs on the Zuul executor in the
+    untrusted security context.
+
+    Ansible 2.8 and later versions support referencing builtin modules
+    using the `ansible.builtin.<name>` alias.  Playbooks which use
+    this mechanism can bypass Zuul's security restrictions and run
+    arbitrary local code or otherwise restricted modules.
+
+    Zuul's use of bubblewrap means that any commands executed via this
+    vulnerability would still be contained within the restricted
+    environment, meaning that they can not access files outside of the
+    build directory or continue running longer than the playbook.  But
+    they may have been able to access files within the build directory
+    but outside of the `work/` directory, as well as potentially
+    exploit any kernel or hypervisor privilege escalation
+    vulnerabilities.
+
+    The Zuul team now considers the restricted Ansible environment to
+    be ineffective as a security mechanism and is developing plans to
+    remove the restrictions and rely entirely on bubblewrap in the
+    future.  These changes will occur in a future release of Zuul
+    (likely 6.0.0) and will be preceded by more details about the
+    change.

tests/fixtures/config/ansible/git/org_plugin-project/playbooks/collections_bad.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  tasks:
+    - ansible.builtin.command:
+        cmd: 'echo foobar'

tests/unit/test_v3.py

@@ -3774,6 +3774,12 @@ class FunctionalAnsibleMixIn(object):
         # TODOv3(jeblair): parse the ansible output and verify we're
         # getting the exception we expect.
 
+    def test_plugins_collections(self):
+        plugin_tests = [
+            ('collections_bad', 'FAILURE'),
+        ]
+        self._test_plugins(plugin_tests)
+
     def test_plugins_1(self):
         '''
         Split plugin tests to avoid timeouts and exceeding subunit

zuul/ansible/base/callback/zuul_json.py

@@ -27,6 +27,7 @@ import datetime
 import json
 import os
 
+from ansible.plugins.loader import PluginLoader
 from ansible.plugins.callback import CallbackBase
 try:
     # It's here in 2.3
@@ -202,3 +203,33 @@ class CallbackModule(CallbackBase):
         outfile.write('\n]\n')
 
     v2_runner_on_unreachable = v2_runner_on_ok
+
+
+# Using 'ansible.builtin.command' instead of 'command' bypasses our
+# custom plugins, so rewrite any uses of ansible.builtin.X to just X.
+# This workaround is temporary until we remove our custom plugins.
+
+# This happens here because Ansible will load the zuul_json plugin for
+# any invocation where we care about restricting access, and this is
+# the earliest in the Ansible startup procedure we can access.
+
+# Monkepatch some PluginLoader methods to rewrite the modules it is
+# loading.
+orig_get = PluginLoader.get
+orig_find_plugin = PluginLoader.find_plugin
+
+
+def mp_get(self, name, *args, **kwargs):
+    name = name.rsplit('.', 1)[-1]
+    ret = orig_get(self, name, *args, **kwargs)
+    return ret
+
+
+def mp_find_plugin(self, name, *args, **kwargs):
+    name = name.rsplit('.', 1)[-1]
+    ret = orig_find_plugin(self, name, *args, **kwargs)
+    return ret
+
+
+PluginLoader.get = mp_get
+PluginLoader.find_plugin = mp_find_plugin

zuul/lib/ansible-config.conf

@@ -1,7 +1,8 @@
 # This file describes the currently supported ansible versions
 [common]
 default_version = 2.9
-requirements = ara>=0.16.5,<1.0.0 openstacksdk openshift jmespath google-cloud-storage pywinrm boto3
+# Jinja2 pinned due to 3.1.0 breaking ara
+requirements = ara>=0.16.5,<1.0.0 Jinja2<3.1.0 openstacksdk openshift jmespath google-cloud-storage pywinrm boto3
 
 [2.8]
 # Ansible 2.8.16 breaks the k8s connection plugin