Require Jinja2 3.0.0 (#75881)

* Require Jinja2 3.0.0

ci_complete

* Fix sanity

* Remove Jinja min/max tests

* ansible-test changes

* ci_complete

* More cleanup

ci_complete

* Revert _count_newlines_from_end :( and other stuff

* Fix sanity

* It's using host_vars ...

* Unused import

* Remove overridden groupby filter

* environmentfilter -> pass_environment

* Explain preserve_trailing_newlines

* Add changelog

* ci_complete

* Deprecated ANSIBLE_JINJA2_NATIVE_WARNING

* native_helpers.py cleanup

* More cleanup in the find intgration test

changelogs/fragments/75881-jinja2-3-min.yml

@@ -0,0 +1,3 @@
+major_changes:
+- Jinja2 Controller Requirement - Jinja2 3.0.0 or newer is required for the control node (the machine that runs Ansible)
+  (https://github.com/ansible/ansible/pull/75881)

lib/ansible/__init__.py

@@ -22,17 +22,6 @@
 # make vendored top-level modules accessible EARLY
 import ansible._vendor
 
-# patch Jinja2 >= 3.0 for backwards compatibility
-try:
-    import sys as _sys
-    from jinja2.filters import pass_context as _passctx, pass_environment as _passenv, pass_eval_context as _passevalctx
-    _mod = _sys.modules['jinja2.filters']
-    _mod.contextfilter = _passctx
-    _mod.environmentfilter = _passenv
-    _mod.evalcontextfilter = _passevalctx
-except ImportError:
-    _sys = None
-
 # Note: Do not add any code to this file.  The ansible module may be
 # a namespace package when using Ansible-2.1+ Anything in this file may not be
 # available if one of the other packages in the namespace is loaded first.

lib/ansible/config/base.yml

@@ -791,7 +791,7 @@ DEFAULT_JINJA2_EXTENSIONS:
 DEFAULT_JINJA2_NATIVE:
   name: Use Jinja2's NativeEnvironment for templating
   default: False
-  description: This option preserves variable types during template operations. This requires Jinja2 >= 2.10.
+  description: This option preserves variable types during template operations.
   env: [{name: ANSIBLE_JINJA2_NATIVE}]
   ini:
   - {key: jinja2_native, section: defaults}
@@ -1656,6 +1656,19 @@ INVENTORY_UNPARSED_IS_FAILED:
   ini:
   - {key: unparsed_is_failed, section: inventory}
   type: bool
+JINJA2_NATIVE_WARNING:
+   name: Running older than required Jinja version for jinja2_native warning
+   default: True
+   description: Toggle to control showing warnings related to running a Jinja version
+                older than required for jinja2_native
+   env:
+    - name: ANSIBLE_JINJA2_NATIVE_WARNING
+      deprecated:
+        why: This option is no longer used in the Ansible Core code base.
+        version: "2.17"
+   ini:
+   - {key: jinja2_native_warning, section: defaults}
+   type: boolean
 MAX_FILE_SIZE_FOR_DIFF:
   name: Diff maximum file size
   default: 104448
@@ -1664,15 +1677,6 @@ MAX_FILE_SIZE_FOR_DIFF:
   ini:
   - {key: max_diff_size, section: defaults}
   type: int
-JINJA2_NATIVE_WARNING:
-  name: Running older than required Jinja version for jinja2_native warning
-  default: True
-  description: Toggle to control showing warnings related to running a Jinja version
-               older than required for jinja2_native
-  env: [{name: ANSIBLE_JINJA2_NATIVE_WARNING}]
-  ini:
-  - {key: jinja2_native_warning, section: defaults}
-  type: boolean
 NETWORK_GROUP_MODULES:
   name: Network module families
   default: [eos, nxos, ios, iosxr, junos, enos, ce, vyos, sros, dellos9, dellos10, dellos6, asa, aruba, aireos, bigip, ironware, onyx, netconf, exos, voss, slxos]

lib/ansible/plugins/action/template.py

@@ -65,18 +65,6 @@ def run(self, tmp=None, task_vars=None):
         comment_end_string = self._task.args.get('comment_end_string', None)
         output_encoding = self._task.args.get('output_encoding', 'utf-8') or 'utf-8'
 
-        # Option `lstrip_blocks' was added in Jinja2 version 2.7.
-        if lstrip_blocks:
-            try:
-                import jinja2.defaults
-            except ImportError:
-                raise AnsibleError('Unable to import Jinja2 defaults for determining Jinja2 features.')
-
-            try:
-                jinja2.defaults.LSTRIP_BLOCKS
-            except AttributeError:
-                raise AnsibleError("Option `lstrip_blocks' is only available in Jinja2 versions >=2.7")
-
         wrong_sequences = ["\\n", "\\r", "\\r\\n"]
         allowed_sequences = ["\n", "\r", "\r\n"]
 

lib/ansible/plugins/doc_fragments/template_common.py

@@ -90,7 +90,6 @@ class ModuleDocFragment(object):
     description:
     - Determine when leading spaces and tabs should be stripped.
     - When set to C(yes) leading spaces and tabs are stripped from the start of a line to a block.
-    - This functionality requires Jinja 2.7 or newer.
     type: bool
     default: no
     version_added: '2.6'

lib/ansible/plugins/filter/core.py

@@ -21,7 +21,7 @@
 from functools import partial
 from random import Random, SystemRandom, shuffle
 
-from jinja2.filters import environmentfilter, do_groupby as _do_groupby
+from jinja2.filters import pass_environment
 
 from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleFilterTypeError
 from ansible.module_utils.six import string_types, integer_types, reraise, text_type
@@ -32,7 +32,7 @@
 from ansible.module_utils.common.yaml import yaml_load, yaml_load_all
 from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.parsing.yaml.dumper import AnsibleDumper
-from ansible.template import AnsibleUndefined, recursive_check_defined
+from ansible.template import recursive_check_defined
 from ansible.utils.display import Display
 from ansible.utils.encrypt import passlib_or_crypt
 from ansible.utils.hashing import md5s, checksum_s
@@ -217,7 +217,7 @@ def from_yaml_all(data):
     return data
 
 
-@environmentfilter
+@pass_environment
 def rand(environment, end, start=None, step=None, seed=None):
     if seed is None:
         r = SystemRandom()
@@ -421,7 +421,7 @@ def comment(text, style='plain', **kw):
         str_end)
 
 
-@environmentfilter
+@pass_environment
 def extract(environment, item, container, morekeys=None):
     if morekeys is None:
         keys = [item]
@@ -437,26 +437,6 @@ def extract(environment, item, container, morekeys=None):
     return value
 
 
-@environmentfilter
-def do_groupby(environment, value, attribute):
-    """Overridden groupby filter for jinja2, to address an issue with
-    jinja2>=2.9.0,<2.9.5 where a namedtuple was returned which
-    has repr that prevents ansible.template.safe_eval.safe_eval from being
-    able to parse and eval the data.
-
-    jinja2<2.9.0,>=2.9.5 is not affected, as <2.9.0 uses a tuple, and
-    >=2.9.5 uses a standard tuple repr on the namedtuple.
-
-    The adaptation here, is to run the jinja2 `do_groupby` function, and
-    cast all of the namedtuples to a regular tuple.
-
-    See https://github.com/ansible/ansible/issues/20098
-
-    We may be able to remove this in the future.
-    """
-    return [tuple(t) for t in _do_groupby(environment, value, attribute)]
-
-
 def b64encode(string, encoding='utf-8'):
     return to_text(base64.b64encode(to_bytes(string, encoding=encoding, errors='surrogate_or_strict')))
 
@@ -571,9 +551,6 @@ class FilterModule(object):
 
     def filters(self):
         return {
-            # jinja2 overrides
-            'groupby': do_groupby,
-
             # base 64
             'b64decode': b64decode,
             'b64encode': b64encode,

lib/ansible/plugins/filter/mathstuff.py

@@ -26,7 +26,7 @@
 import itertools
 import math
 
-from jinja2.filters import environmentfilter
+from jinja2.filters import pass_environment
 
 from ansible.errors import AnsibleFilterError, AnsibleFilterTypeError
 from ansible.module_utils.common.text import formatters
@@ -42,16 +42,11 @@
 except ImportError:
     HAS_UNIQUE = False
 
-try:
-    from jinja2.filters import do_max, do_min
-    HAS_MIN_MAX = True
-except ImportError:
-    HAS_MIN_MAX = False
 
 display = Display()
 
 
-@environmentfilter
+@pass_environment
 # Use case_sensitive=None as a sentinel value, so we raise an error only when
 # explicitly set and cannot be handle (by Jinja2 w/o 'unique' or fallback version)
 def unique(environment, a, case_sensitive=None, attribute=None):
@@ -88,7 +83,7 @@ def _do_fail(e):
     return c
 
 
-@environmentfilter
+@pass_environment
 def intersect(environment, a, b):
     if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) & set(b)
@@ -97,7 +92,7 @@ def intersect(environment, a, b):
     return c
 
 
-@environmentfilter
+@pass_environment
 def difference(environment, a, b):
     if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) - set(b)
@@ -106,7 +101,7 @@ def difference(environment, a, b):
     return c
 
 
-@environmentfilter
+@pass_environment
 def symmetric_difference(environment, a, b):
     if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) ^ set(b)
@@ -116,7 +111,7 @@ def symmetric_difference(environment, a, b):
     return c
 
 
-@environmentfilter
+@pass_environment
 def union(environment, a, b):
     if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) | set(b)
@@ -125,30 +120,6 @@ def union(environment, a, b):
     return c
 
 
-@environmentfilter
-def min(environment, a, **kwargs):
-    if HAS_MIN_MAX:
-        return do_min(environment, a, **kwargs)
-    else:
-        if kwargs:
-            raise AnsibleFilterError("Ansible's min filter does not support any keyword arguments. "
-                                     "You need Jinja2 2.10 or later that provides their version of the filter.")
-        _min = __builtins__.get('min')
-        return _min(a)
-
-
-@environmentfilter
-def max(environment, a, **kwargs):
-    if HAS_MIN_MAX:
-        return do_max(environment, a, **kwargs)
-    else:
-        if kwargs:
-            raise AnsibleFilterError("Ansible's max filter does not support any keyword arguments. "
-                                     "You need Jinja2 2.10 or later that provides their version of the filter.")
-        _max = __builtins__.get('max')
-        return _max(a)
-
-
 def logarithm(x, base=math.e):
     try:
         if base == 10:
@@ -251,10 +222,6 @@ class FilterModule(object):
 
     def filters(self):
         filters = {
-            # general math
-            'min': min,
-            'max': max,
-
             # exponents and logarithms
             'log': logarithm,
             'pow': power,

lib/ansible/plugins/filter/urls.py

@@ -6,64 +6,15 @@
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
-from ansible.module_utils.six import PY3, iteritems, string_types
-from ansible.module_utils.six.moves.urllib.parse import quote, quote_plus, unquote_plus
-from ansible.module_utils._text import to_bytes, to_text
+from functools import partial
 
-try:
-    from jinja2.filters import do_urlencode
-    HAS_URLENCODE = True
-except ImportError:
-    HAS_URLENCODE = False
-
-
-def unicode_urldecode(string):
-    if PY3:
-        return unquote_plus(string)
-    return to_text(unquote_plus(to_bytes(string)))
-
-
-def do_urldecode(string):
-    return unicode_urldecode(string)
-
-
-# NOTE: We implement urlencode when Jinja2 is older than v2.7
-def unicode_urlencode(string, for_qs=False):
-    safe = b'' if for_qs else b'/'
-    if for_qs:
-        quote_func = quote_plus
-    else:
-        quote_func = quote
-    if PY3:
-        return quote_func(string, safe)
-    return to_text(quote_func(to_bytes(string), safe))
-
-
-def do_urlencode(value):
-    itemiter = None
-    if isinstance(value, dict):
-        itemiter = iteritems(value)
-    elif not isinstance(value, string_types):
-        try:
-            itemiter = iter(value)
-        except TypeError:
-            pass
-    if itemiter is None:
-        return unicode_urlencode(value)
-    return u'&'.join(unicode_urlencode(k) + '=' +
-                     unicode_urlencode(v, for_qs=True)
-                     for k, v in itemiter)
+from urllib.parse import unquote_plus
 
 
 class FilterModule(object):
     ''' Ansible core jinja2 filters '''
 
     def filters(self):
-        filters = {
-            'urldecode': do_urldecode,
+        return {
+            'urldecode': partial(unquote_plus),
         }
-
-        if not HAS_URLENCODE:
-            filters['urlencode'] = do_urlencode
-
-        return filters

lib/ansible/plugins/lookup/template.py

@@ -79,13 +79,15 @@
 from copy import deepcopy
 import os
 
+import ansible.constants as C
+
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.module_utils._text import to_bytes, to_text
-from ansible.template import generate_ansible_template_vars, AnsibleEnvironment, USE_JINJA2_NATIVE
+from ansible.template import generate_ansible_template_vars, AnsibleEnvironment
 from ansible.utils.display import Display
 
-if USE_JINJA2_NATIVE:
+if C.DEFAULT_JINJA2_NATIVE:
     from ansible.utils.native_jinja import NativeJinjaText
 
 
@@ -109,7 +111,7 @@ def run(self, terms, variables, **kwargs):
         comment_start_string = self.get_option('comment_start_string')
         comment_end_string = self.get_option('comment_end_string')
 
-        if USE_JINJA2_NATIVE and not jinja2_native:
+        if C.DEFAULT_JINJA2_NATIVE and not jinja2_native:
             templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment)
         else:
             templar = self._templar
@@ -152,7 +154,7 @@ def run(self, terms, variables, **kwargs):
                     res = templar.template(template_data, preserve_trailing_newlines=True,
                                            convert_data=convert_data_p, escape_backslashes=False)
 
-                if USE_JINJA2_NATIVE and not jinja2_native:
+                if C.DEFAULT_JINJA2_NATIVE and not jinja2_native:
                     # jinja2_native is true globally but off for the lookup, we need this text
                     # not to be processed by literal_eval anywhere in Ansible
                     res = NativeJinjaText(res)