Skip to content
Permalink
Browse files
Require Jinja2 3.0.0 (#75881)
* Require Jinja2 3.0.0

ci_complete

* Fix sanity

* Remove Jinja min/max tests

* ansible-test changes

* ci_complete

* More cleanup

ci_complete

* Revert _count_newlines_from_end :( and other stuff

* Fix sanity

* It's using host_vars ...

* Unused import

* Remove overridden groupby filter

* environmentfilter -> pass_environment

* Explain preserve_trailing_newlines

* Add changelog

* ci_complete

* Deprecated ANSIBLE_JINJA2_NATIVE_WARNING

* native_helpers.py cleanup

* More cleanup in the find intgration test
  • Loading branch information
mkrizek committed Oct 20, 2021
1 parent 835fe71 commit 7621784b947adb497a24725259318506a102d1a4
Showing 46 changed files with 98 additions and 523 deletions.
@@ -0,0 +1,3 @@
major_changes:
- Jinja2 Controller Requirement - Jinja2 3.0.0 or newer is required for the control node (the machine that runs Ansible)
(https://github.com/ansible/ansible/pull/75881)
@@ -22,17 +22,6 @@
# make vendored top-level modules accessible EARLY
import ansible._vendor

# patch Jinja2 >= 3.0 for backwards compatibility
try:
import sys as _sys
from jinja2.filters import pass_context as _passctx, pass_environment as _passenv, pass_eval_context as _passevalctx
_mod = _sys.modules['jinja2.filters']
_mod.contextfilter = _passctx
_mod.environmentfilter = _passenv
_mod.evalcontextfilter = _passevalctx
except ImportError:
_sys = None

# Note: Do not add any code to this file. The ansible module may be
# a namespace package when using Ansible-2.1+ Anything in this file may not be
# available if one of the other packages in the namespace is loaded first.
@@ -791,7 +791,7 @@ DEFAULT_JINJA2_EXTENSIONS:
DEFAULT_JINJA2_NATIVE:
name: Use Jinja2's NativeEnvironment for templating
default: False
description: This option preserves variable types during template operations. This requires Jinja2 >= 2.10.
description: This option preserves variable types during template operations.
env: [{name: ANSIBLE_JINJA2_NATIVE}]
ini:
- {key: jinja2_native, section: defaults}
@@ -1656,6 +1656,19 @@ INVENTORY_UNPARSED_IS_FAILED:
ini:
- {key: unparsed_is_failed, section: inventory}
type: bool
JINJA2_NATIVE_WARNING:
name: Running older than required Jinja version for jinja2_native warning
default: True
description: Toggle to control showing warnings related to running a Jinja version
older than required for jinja2_native
env:
- name: ANSIBLE_JINJA2_NATIVE_WARNING
deprecated:
why: This option is no longer used in the Ansible Core code base.
version: "2.17"
ini:
- {key: jinja2_native_warning, section: defaults}
type: boolean
MAX_FILE_SIZE_FOR_DIFF:
name: Diff maximum file size
default: 104448
@@ -1664,15 +1677,6 @@ MAX_FILE_SIZE_FOR_DIFF:
ini:
- {key: max_diff_size, section: defaults}
type: int
JINJA2_NATIVE_WARNING:
name: Running older than required Jinja version for jinja2_native warning
default: True
description: Toggle to control showing warnings related to running a Jinja version
older than required for jinja2_native
env: [{name: ANSIBLE_JINJA2_NATIVE_WARNING}]
ini:
- {key: jinja2_native_warning, section: defaults}
type: boolean
NETWORK_GROUP_MODULES:
name: Network module families
default: [eos, nxos, ios, iosxr, junos, enos, ce, vyos, sros, dellos9, dellos10, dellos6, asa, aruba, aireos, bigip, ironware, onyx, netconf, exos, voss, slxos]
@@ -65,18 +65,6 @@ def run(self, tmp=None, task_vars=None):
comment_end_string = self._task.args.get('comment_end_string', None)
output_encoding = self._task.args.get('output_encoding', 'utf-8') or 'utf-8'

# Option `lstrip_blocks' was added in Jinja2 version 2.7.
if lstrip_blocks:
try:
import jinja2.defaults
except ImportError:
raise AnsibleError('Unable to import Jinja2 defaults for determining Jinja2 features.')

try:
jinja2.defaults.LSTRIP_BLOCKS
except AttributeError:
raise AnsibleError("Option `lstrip_blocks' is only available in Jinja2 versions >=2.7")

wrong_sequences = ["\\n", "\\r", "\\r\\n"]
allowed_sequences = ["\n", "\r", "\r\n"]

@@ -90,7 +90,6 @@ class ModuleDocFragment(object):
description:
- Determine when leading spaces and tabs should be stripped.
- When set to C(yes) leading spaces and tabs are stripped from the start of a line to a block.
- This functionality requires Jinja 2.7 or newer.
type: bool
default: no
version_added: '2.6'
@@ -21,7 +21,7 @@
from functools import partial
from random import Random, SystemRandom, shuffle

from jinja2.filters import environmentfilter, do_groupby as _do_groupby
from jinja2.filters import pass_environment

from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleFilterTypeError
from ansible.module_utils.six import string_types, integer_types, reraise, text_type
@@ -32,7 +32,7 @@
from ansible.module_utils.common.yaml import yaml_load, yaml_load_all
from ansible.parsing.ajson import AnsibleJSONEncoder
from ansible.parsing.yaml.dumper import AnsibleDumper
from ansible.template import AnsibleUndefined, recursive_check_defined
from ansible.template import recursive_check_defined
from ansible.utils.display import Display
from ansible.utils.encrypt import passlib_or_crypt
from ansible.utils.hashing import md5s, checksum_s
@@ -217,7 +217,7 @@ def from_yaml_all(data):
return data


@environmentfilter
@pass_environment
def rand(environment, end, start=None, step=None, seed=None):
if seed is None:
r = SystemRandom()
@@ -421,7 +421,7 @@ def comment(text, style='plain', **kw):
str_end)


@environmentfilter
@pass_environment
def extract(environment, item, container, morekeys=None):
if morekeys is None:
keys = [item]
@@ -437,26 +437,6 @@ def extract(environment, item, container, morekeys=None):
return value


@environmentfilter
def do_groupby(environment, value, attribute):
"""Overridden groupby filter for jinja2, to address an issue with
jinja2>=2.9.0,<2.9.5 where a namedtuple was returned which
has repr that prevents ansible.template.safe_eval.safe_eval from being
able to parse and eval the data.
jinja2<2.9.0,>=2.9.5 is not affected, as <2.9.0 uses a tuple, and
>=2.9.5 uses a standard tuple repr on the namedtuple.
The adaptation here, is to run the jinja2 `do_groupby` function, and
cast all of the namedtuples to a regular tuple.
See https://github.com/ansible/ansible/issues/20098
We may be able to remove this in the future.
"""
return [tuple(t) for t in _do_groupby(environment, value, attribute)]


def b64encode(string, encoding='utf-8'):
return to_text(base64.b64encode(to_bytes(string, encoding=encoding, errors='surrogate_or_strict')))

@@ -571,9 +551,6 @@ class FilterModule(object):

def filters(self):
return {
# jinja2 overrides
'groupby': do_groupby,

# base 64
'b64decode': b64decode,
'b64encode': b64encode,
@@ -26,7 +26,7 @@
import itertools
import math

from jinja2.filters import environmentfilter
from jinja2.filters import pass_environment

from ansible.errors import AnsibleFilterError, AnsibleFilterTypeError
from ansible.module_utils.common.text import formatters
@@ -42,16 +42,11 @@
except ImportError:
HAS_UNIQUE = False

try:
from jinja2.filters import do_max, do_min
HAS_MIN_MAX = True
except ImportError:
HAS_MIN_MAX = False

display = Display()


@environmentfilter
@pass_environment
# Use case_sensitive=None as a sentinel value, so we raise an error only when
# explicitly set and cannot be handle (by Jinja2 w/o 'unique' or fallback version)
def unique(environment, a, case_sensitive=None, attribute=None):
@@ -88,7 +83,7 @@ def _do_fail(e):
return c


@environmentfilter
@pass_environment
def intersect(environment, a, b):
if isinstance(a, Hashable) and isinstance(b, Hashable):
c = set(a) & set(b)
@@ -97,7 +92,7 @@ def intersect(environment, a, b):
return c


@environmentfilter
@pass_environment
def difference(environment, a, b):
if isinstance(a, Hashable) and isinstance(b, Hashable):
c = set(a) - set(b)
@@ -106,7 +101,7 @@ def difference(environment, a, b):
return c


@environmentfilter
@pass_environment
def symmetric_difference(environment, a, b):
if isinstance(a, Hashable) and isinstance(b, Hashable):
c = set(a) ^ set(b)
@@ -116,7 +111,7 @@ def symmetric_difference(environment, a, b):
return c


@environmentfilter
@pass_environment
def union(environment, a, b):
if isinstance(a, Hashable) and isinstance(b, Hashable):
c = set(a) | set(b)
@@ -125,30 +120,6 @@ def union(environment, a, b):
return c


@environmentfilter
def min(environment, a, **kwargs):
if HAS_MIN_MAX:
return do_min(environment, a, **kwargs)
else:
if kwargs:
raise AnsibleFilterError("Ansible's min filter does not support any keyword arguments. "
"You need Jinja2 2.10 or later that provides their version of the filter.")
_min = __builtins__.get('min')
return _min(a)


@environmentfilter
def max(environment, a, **kwargs):
if HAS_MIN_MAX:
return do_max(environment, a, **kwargs)
else:
if kwargs:
raise AnsibleFilterError("Ansible's max filter does not support any keyword arguments. "
"You need Jinja2 2.10 or later that provides their version of the filter.")
_max = __builtins__.get('max')
return _max(a)


def logarithm(x, base=math.e):
try:
if base == 10:
@@ -251,10 +222,6 @@ class FilterModule(object):

def filters(self):
filters = {
# general math
'min': min,
'max': max,

# exponents and logarithms
'log': logarithm,
'pow': power,
@@ -6,64 +6,15 @@
from __future__ import absolute_import, division, print_function
__metaclass__ = type

from ansible.module_utils.six import PY3, iteritems, string_types
from ansible.module_utils.six.moves.urllib.parse import quote, quote_plus, unquote_plus
from ansible.module_utils._text import to_bytes, to_text
from functools import partial

try:
from jinja2.filters import do_urlencode
HAS_URLENCODE = True
except ImportError:
HAS_URLENCODE = False


def unicode_urldecode(string):
if PY3:
return unquote_plus(string)
return to_text(unquote_plus(to_bytes(string)))


def do_urldecode(string):
return unicode_urldecode(string)


# NOTE: We implement urlencode when Jinja2 is older than v2.7
def unicode_urlencode(string, for_qs=False):
safe = b'' if for_qs else b'/'
if for_qs:
quote_func = quote_plus
else:
quote_func = quote
if PY3:
return quote_func(string, safe)
return to_text(quote_func(to_bytes(string), safe))


def do_urlencode(value):
itemiter = None
if isinstance(value, dict):
itemiter = iteritems(value)
elif not isinstance(value, string_types):
try:
itemiter = iter(value)
except TypeError:
pass
if itemiter is None:
return unicode_urlencode(value)
return u'&'.join(unicode_urlencode(k) + '=' +
unicode_urlencode(v, for_qs=True)
for k, v in itemiter)
from urllib.parse import unquote_plus


class FilterModule(object):
''' Ansible core jinja2 filters '''

def filters(self):
filters = {
'urldecode': do_urldecode,
return {
'urldecode': partial(unquote_plus),
}

if not HAS_URLENCODE:
filters['urlencode'] = do_urlencode

return filters
@@ -79,13 +79,15 @@
from copy import deepcopy
import os

import ansible.constants as C

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.module_utils._text import to_bytes, to_text
from ansible.template import generate_ansible_template_vars, AnsibleEnvironment, USE_JINJA2_NATIVE
from ansible.template import generate_ansible_template_vars, AnsibleEnvironment
from ansible.utils.display import Display

if USE_JINJA2_NATIVE:
if C.DEFAULT_JINJA2_NATIVE:
from ansible.utils.native_jinja import NativeJinjaText


@@ -109,7 +111,7 @@ def run(self, terms, variables, **kwargs):
comment_start_string = self.get_option('comment_start_string')
comment_end_string = self.get_option('comment_end_string')

if USE_JINJA2_NATIVE and not jinja2_native:
if C.DEFAULT_JINJA2_NATIVE and not jinja2_native:
templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment)
else:
templar = self._templar
@@ -152,7 +154,7 @@ def run(self, terms, variables, **kwargs):
res = templar.template(template_data, preserve_trailing_newlines=True,
convert_data=convert_data_p, escape_backslashes=False)

if USE_JINJA2_NATIVE and not jinja2_native:
if C.DEFAULT_JINJA2_NATIVE and not jinja2_native:
# jinja2_native is true globally but off for the lookup, we need this text
# not to be processed by literal_eval anywhere in Ansible
res = NativeJinjaText(res)

0 comments on commit 7621784

Please sign in to comment.