habsi/ansible
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modules moved to use best_parsable_locale (#75250)

Browse Source 
* modules moved to use best_parsable_locale

* fixed invocations

* better better

* also module_utils

* converted to function as per fb

* patch testt

* whitespace
This commit is contained in:
Brian Coca
2021-07-15 15:18:01 -04:00
committed by GitHub
parent d527be8a52
commit 61900c7672
18 changed files with 126 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
changelogs/fragments/best_locale.yml Normal file
View File

@ -0,0 +1,2 @@
minor_changes:
- move all builtin modules to use the best possible locale function instead of hardcoding 'C'.

5
lib/ansible/module_utils/basic.py
View File

@ -1245,10 +1245,7 @@ class AnsibleModule(object):
# fallback to the 'best' locale, per the function
# final fallback is 'C', which may cause unicode issues
# but is preferable to simply failing on unknown locale
try:
best_locale = get_best_parsable_locale(self)
except RuntimeError:
best_locale = 'C'
best_locale = get_best_parsable_locale(self)
# need to set several since many tools choose to ignore documented precedence and scope
locale.setlocale(locale.LC_ALL, best_locale)

54
lib/ansible/module_utils/common/locale.py
View File

@ -7,7 +7,7 @@ __metaclass__ = type
from ansible.module_utils._text import to_native
def get_best_parsable_locale(module, preferences=None):
def get_best_parsable_locale(module, preferences=None, raise_on_locale=False):
'''
Attempts to return the best possible locale for parsing output in English
useful for scraping output with i18n tools. When this raises an exception
@ -15,35 +15,43 @@ def get_best_parsable_locale(module, preferences=None):
:param module: an AnsibleModule instance
:param preferences: A list of preferred locales, in order of preference
:param raise_on_locale: boolean that determines if we raise exception or not
due to locale CLI issues
:returns: The first matched preferred locale or 'C' which is the default
'''
locale = module.get_bin_path("locale")
if not locale:
# not using required=true as that forces fail_json
raise RuntimeWarning("Could not find 'locale' tool")
available = []
found = 'C' # default posix, its ascii but always there
try:
locale = module.get_bin_path("locale")
if not locale:
# not using required=true as that forces fail_json
raise RuntimeWarning("Could not find 'locale' tool")
if preferences is None:
# new POSIX standard or English cause those are messages core team expects
# yes, the last 2 are the same but some systems are weird
preferences = ['C.utf8', 'en_US.utf8', 'C', 'POSIX']
available = []
rc, out, err = module.run_command([locale, '-a'])
if preferences is None:
# new POSIX standard or English cause those are messages core team expects
# yes, the last 2 are the same but some systems are weird
preferences = ['C.utf8', 'en_US.utf8', 'C', 'POSIX']
if rc == 0:
if out:
available = out.strip().splitlines()
rc, out, err = module.run_command([locale, '-a'])
if rc == 0:
if out:
available = out.strip().splitlines()
else:
raise RuntimeWarning("No output from locale, rc=%s: %s" % (rc, to_native(err)))
else:
raise RuntimeWarning("No output from locale, rc=%s: %s" % (rc, to_native(err)))
else:
raise RuntimeWarning("Unable to get locale information, rc=%s: %s" % (rc, to_native(err)))
raise RuntimeWarning("Unable to get locale information, rc=%s: %s" % (rc, to_native(err)))
if available:
for pref in preferences:
if pref in available:
found = pref
break
except RuntimeWarning:
if raise_on_locale:
raise
if available:
for pref in preferences:
if pref in available:
found = pref
break
return found

6
lib/ansible/module_utils/facts/hardware/linux.py
View File

@ -29,11 +29,12 @@ from multiprocessing import cpu_count
from multiprocessing.pool import ThreadPool
from ansible.module_utils._text import to_text
from ansible.module_utils.six import iteritems
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.common.process import get_bin_path
from ansible.module_utils.common.text.formatters import bytes_to_human
from ansible.module_utils.facts.hardware.base import Hardware, HardwareCollector
from ansible.module_utils.facts.utils import get_file_content, get_file_lines, get_mount_size
from ansible.module_utils.six import iteritems
# import this as a module to ensure we get the same module instance
from ansible.module_utils.facts import timeout
@ -85,7 +86,8 @@ class LinuxHardware(Hardware):
def populate(self, collected_facts=None):
hardware_facts = {}
self.module.run_command_environ_update = {'LANG': 'C', 'LC_ALL': 'C', 'LC_NUMERIC': 'C'}
locale = get_best_parsable_locale(self.module)
self.module.run_command_environ_update = {'LANG': locale, 'LC_ALL': locale, 'LC_NUMERIC': locale}
cpu_facts = self.get_cpu_facts(collected_facts=collected_facts)
memory_facts = self.get_memory_facts()

9
lib/ansible/module_utils/facts/hardware/sunos.py
View File

@ -19,14 +19,12 @@ __metaclass__ = type
import re
import time
from ansible.module_utils.six.moves import reduce
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.common.text.formatters import bytes_to_human
from ansible.module_utils.facts.utils import get_file_content, get_mount_size
from ansible.module_utils.facts.hardware.base import Hardware, HardwareCollector
from ansible.module_utils.facts import timeout
from ansible.module_utils.six.moves import reduce
class SunOSHardware(Hardware):
@ -42,7 +40,8 @@ class SunOSHardware(Hardware):
# FIXME: could pass to run_command(environ_update), but it also tweaks the env
# of the parent process instead of altering an env provided to Popen()
# Use C locale for hardware collection helpers to avoid locale specific number formatting (#24542)
self.module.run_command_environ_update = {'LANG': 'C', 'LC_ALL': 'C', 'LC_NUMERIC': 'C'}
locale = get_best_parsable_locale(self.module)
self.module.run_command_environ_update = {'LANG': locale, 'LC_ALL': locale, 'LC_NUMERIC': locale}
cpu_facts = self.get_cpu_facts()
memory_facts = self.get_memory_facts()

26
lib/ansible/modules/apt.py
View File

@ -327,23 +327,12 @@ import tempfile
import time
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.common.respawn import has_respawned, probe_interpreters_for_module, respawn_module
from ansible.module_utils._text import to_bytes, to_native
from ansible.module_utils.six import PY3
from ansible.module_utils.urls import fetch_file
# APT related constants
APT_ENV_VARS = dict(
DEBIAN_FRONTEND='noninteractive',
DEBIAN_PRIORITY='critical',
# We screenscrape apt-get and aptitude output for information so we need
# to make sure we use the C locale when running commands
LANG='C',
LC_ALL='C',
LC_MESSAGES='C',
LC_CTYPE='C',
)
DPKG_OPTIONS = 'force-confdef,force-confold'
APT_GET_ZERO = "\n0 upgraded, 0 newly installed"
APTITUDE_ZERO = "\n0 packages upgraded, 0 newly installed"
@ -1092,6 +1081,19 @@ def main():
supports_check_mode=True,
)
# We screenscrape apt-get and aptitude output for information so we need
# to make sure we use the best parsable locale when running commands
# also set apt specific vars for desired behaviour
locale = get_best_parsable_locale(module)
# APT related constants
APT_ENV_VARS = dict(
DEBIAN_FRONTEND='noninteractive',
DEBIAN_PRIORITY='critical',
LANG=locale,
LC_ALL=locale,
LC_MESSAGES=locale,
LC_CTYPE=locale,
)
module.run_command_environ_update = APT_ENV_VARS
if not HAS_PYTHON_APT:

24
lib/ansible/modules/apt_key.py
View File

@ -153,14 +153,24 @@ import os
# FIXME: standardize into module_common
from traceback import format_exc
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils._text import to_native
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.urls import fetch_url
apt_key_bin = None
gpg_bin = None
lang_env = dict(LANG='C', LC_ALL='C', LC_MESSAGES='C')
locale = None
def lang_env(module):
if not hasattr(lang_env, 'result'):
locale = get_best_parsable_locale(module)
lang_env.result = dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale)
return lang_env.result
def find_needed_binaries(module):
@ -288,12 +298,11 @@ def get_key_id_from_file(module, filename, data=None):
native_data = to_native(data)
is_armored = native_data.find("-----BEGIN PGP PUBLIC KEY BLOCK-----") >= 0
global lang_env
key = None
cmd = [gpg_bin, '--with-colons', filename]
(rc, out, err) = module.run_command(cmd, environ_update=lang_env, data=(native_data if is_armored else data), binary_data=not is_armored)
(rc, out, err) = module.run_command(cmd, environ_update=lang_env(module), data=(native_data if is_armored else data), binary_data=not is_armored)
if rc != 0:
module.fail_json(msg="Unable to extract key from '%s'" % ('inline data' if data is not None else filename), stdout=out, stderr=err)
@ -311,7 +320,6 @@ def get_key_id_from_data(module, data):
def import_key(module, keyring, keyserver, key_id):
global lang_env
if keyring:
cmd = "%s --keyring %s adv --no-tty --keyserver %s" % (apt_key_bin, keyring, keyserver)
else:
@ -324,17 +332,17 @@ def import_key(module, keyring, keyserver, key_id):
cmd = "%s --recv %s" % (cmd, key_id)
for retry in range(5):
(rc, out, err) = module.run_command(cmd, environ_update=lang_env)
(rc, out, err) = module.run_command(cmd, environ_update=lang_env(module))
if rc == 0:
break
else:
# Out of retries
if rc == 2 and 'not found on keyserver' in out:
msg = 'Key %s not found on keyserver %s' % (key_id, keyserver)
module.fail_json(cmd=cmd, msg=msg, forced_environment=lang_env)
module.fail_json(cmd=cmd, msg=msg, forced_environment=lang_env(module))
else:
msg = "Error fetching key %s from keyserver: %s" % (key_id, keyserver)
module.fail_json(cmd=cmd, msg=msg, forced_environment=lang_env, rc=rc, stdout=out, stderr=err)
module.fail_json(cmd=cmd, msg=msg, forced_environment=lang_env(module), rc=rc, stdout=out, stderr=err)
return True

6
lib/ansible/modules/copy.py
View File

@ -285,9 +285,10 @@ import stat
import tempfile
import traceback
from ansible.module_utils._text import to_bytes, to_native
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.common.process import get_bin_path
from ansible.module_utils._text import to_bytes, to_native
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.six import PY3
@ -309,7 +310,8 @@ def clear_facls(path):
# FIXME "setfacl -b" is available on Linux and FreeBSD. There is "setfacl -D e" on z/OS. Others?
acl_command = [setfacl, '-b', path]
b_acl_command = [to_bytes(x) for x in acl_command]
rc, out, err = module.run_command(b_acl_command, environ_update=dict(LANG='C', LC_ALL='C', LC_MESSAGES='C'))
locale = get_best_parsable_locale(module)
rc, out, err = module.run_command(b_acl_command, environ_update=dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale))
if rc != 0:
raise RuntimeError('Error running "{0}": stdout: "{1}"; stderr: "{2}"'.format(' '.join(b_acl_command), out, err))

10
lib/ansible/modules/git.py
View File

@ -325,10 +325,11 @@ import shutil
import tempfile
from ansible.module_utils.compat.version import LooseVersion
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.six import b, string_types
from ansible.module_utils._text import to_native, to_text
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.common.process import get_bin_path
from ansible.module_utils.six import b, string_types
def relocate_repo(module, result, repo_dir, old_repo_dir, worktree_dir):
@ -1201,7 +1202,7 @@ def main():
umask = int(umask, 8)
except Exception:
module.fail_json(msg="umask must be an octal integer",
details=str(sys.exc_info()[1]))
details=to_text(sys.exc_info()[1]))
os.umask(umask)
# Certain features such as depth require a file:/// protocol for path based urls
@ -1211,7 +1212,8 @@ def main():
# We screenscrape a huge amount of git commands so use C locale anytime we
# call run_command()
module.run_command_environ_update = dict(LANG='C', LC_ALL='C', LC_MESSAGES='C', LC_CTYPE='C')
locale = get_best_parsable_locale(module)
module.run_command_environ_update = dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale, LC_CTYPE=locale)
if separate_git_dir:
separate_git_dir = os.path.realpath(separate_git_dir)

4
lib/ansible/modules/package_facts.py
View File

@ -211,6 +211,7 @@ import re
from ansible.module_utils._text import to_native, to_text
from ansible.module_utils.basic import AnsibleModule, missing_required_lib
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.common.process import get_bin_path
from ansible.module_utils.common.respawn import has_respawned, probe_interpreters_for_module, respawn_module
from ansible.module_utils.facts.packages import LibMgr, CLIMgr, get_all_pkg_managers
@ -310,7 +311,8 @@ class PACMAN(CLIMgr):
CLI = 'pacman'
def list_installed(self):
rc, out, err = module.run_command([self._cli, '-Qi'], environ_update=dict(LC_ALL='C'))
locale = get_best_parsable_locale(module)
rc, out, err = module.run_command([self._cli, '-Qi'], environ_update=dict(LC_ALL=locale))
if rc != 0 or err:
raise Exception("Unable to list packages rc=%s : %s" % (rc, err))
return out.split("\n\n")[:-1]

6
lib/ansible/modules/pip.py
View File

@ -275,8 +275,9 @@ except ImportError:
HAS_SETUPTOOLS = False
SETUPTOOLS_IMP_ERR = traceback.format_exc()
from ansible.module_utils.basic import AnsibleModule, is_executable, missing_required_lib
from ansible.module_utils._text import to_native
from ansible.module_utils.basic import AnsibleModule, is_executable, missing_required_lib
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.six import PY3
@ -354,7 +355,8 @@ def _get_packages(module, pip, chdir):
'''Return results of pip command to get packages.'''
# Try 'pip list' command first.
command = '%s list --format=freeze' % pip
lang_env = {'LANG': 'C', 'LC_ALL': 'C', 'LC_MESSAGES': 'C'}
locale = get_best_parsable_locale(module)
lang_env = {'LANG': locale, 'LC_ALL': locale, 'LC_MESSAGES': locale}
rc, out, err = module.run_command(command, cwd=chdir, environ_update=lang_env)
# If there was an error (pip version too old) then use 'pip freeze'.

6
lib/ansible/modules/service.py
View File

@ -156,6 +156,7 @@ if platform.system() != 'SunOS':
from ansible.module_utils._text import to_bytes, to_text
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.common.sys_info import get_platform_subclass
from ansible.module_utils.service import fail_if_missing
from ansible.module_utils.six import PY2, b
@ -224,11 +225,13 @@ class Service(object):
def execute_command(self, cmd, daemonize=False):
locale = get_best_parsable_locale(self.module)
lang_env = dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale)
# Most things don't need to be daemonized
if not daemonize:
# chkconfig localizes messages and we're screen scraping so make
# sure we use the C locale
lang_env = dict(LANG='C', LC_ALL='C', LC_MESSAGES='C')
return self.module.run_command(cmd, environ_update=lang_env)
# This is complex because daemonization is hard for people.
@ -273,7 +276,6 @@ class Service(object):
# chkconfig localizes messages and we're screen scraping so make
# sure we use the C locale
lang_env = dict(LANG='C', LC_ALL='C', LC_MESSAGES='C')
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=lang_env, preexec_fn=lambda: os.close(pipe[1]))
stdout = b("")
stderr = b("")

4
lib/ansible/modules/service_facts.py
View File

@ -85,6 +85,7 @@ ansible_facts:
import platform
import re
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.common.locale import get_best_parsable_locale
class BaseService(object):
@ -308,7 +309,8 @@ class OpenBSDScanService(BaseService):
def main():
module = AnsibleModule(argument_spec=dict(), supports_check_mode=True)
module.run_command_environ_update = dict(LANG="C", LC_ALL="C")
locale = get_best_parsable_locale(module)
module.run_command_environ_update = dict(LANG=locale, LC_ALL=locale)
service_modules = (ServiceScanService, SystemctlScanService, AIXScanService, OpenBSDScanService)
all_services = {}
incomplete_warning = False

7
lib/ansible/modules/subversion.py
View File

@ -128,9 +128,9 @@ RETURN = r'''#'''
import os
import re
from ansible.module_utils.compat.version import LooseVersion
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.compat.version import LooseVersion
class Subversion(object):
@ -320,7 +320,8 @@ def main():
# We screenscrape a huge amount of svn commands so use C locale anytime we
# call run_command()
module.run_command_environ_update = dict(LANG='C', LC_MESSAGES='C')
locale = get_best_parsable_locale(module)
module.run_command_environ_update = dict(LANG=locale, LC_MESSAGES=locale)
if not dest and (checkout or update or export):
module.fail_json(msg="the destination directory must be specified unless checkout=no, update=no, and export=no")

14
lib/ansible/modules/unarchive.py
View File

@ -235,10 +235,11 @@ import traceback
from functools import partial
from zipfile import ZipFile, BadZipfile
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.urls import fetch_file
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.common.process import get_bin_path
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.urls import fetch_file
try: # python 3.3+
from shlex import quote
@ -766,7 +767,8 @@ class TgzArchive(object):
if self.include_files:
cmd.extend(self.include_files)
rc, out, err = self.module.run_command(cmd, cwd=self.b_dest, environ_update=dict(LANG='C', LC_ALL='C', LC_MESSAGES='C'))
locale = get_best_parsable_locale(self.module)
rc, out, err = self.module.run_command(cmd, cwd=self.b_dest, environ_update=dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale))
if rc != 0:
raise UnarchiveError('Unable to list files in the archive')
@ -810,7 +812,8 @@ class TgzArchive(object):
cmd.extend(['-f', self.src])
if self.include_files:
cmd.extend(self.include_files)
rc, out, err = self.module.run_command(cmd, cwd=self.b_dest, environ_update=dict(LANG='C', LC_ALL='C', LC_MESSAGES='C'))
locale = get_best_parsable_locale(self.module)
rc, out, err = self.module.run_command(cmd, cwd=self.b_dest, environ_update=dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale))
# Check whether the differences are in something that we're
# setting anyway
@ -863,7 +866,8 @@ class TgzArchive(object):
cmd.extend(['-f', self.src])
if self.include_files:
cmd.extend(self.include_files)
rc, out, err = self.module.run_command(cmd, cwd=self.b_dest, environ_update=dict(LANG='C', LC_ALL='C', LC_MESSAGES='C'))
locale = get_best_parsable_locale(self.module)
rc, out, err = self.module.run_command(cmd, cwd=self.b_dest, environ_update=dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale))
return dict(cmd=cmd, rc=rc, out=out, err=err)
def can_handle_archive(self):

3
lib/ansible/modules/user.py
View File

@ -461,6 +461,7 @@ import math
from ansible.module_utils import distro
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.common.sys_info import get_platform_subclass
try:
@ -1134,7 +1135,7 @@ class User(object):
master_out_fd, slave_out_fd = pty.openpty()
master_err_fd, slave_err_fd = pty.openpty()
env = os.environ.copy()
env['LC_ALL'] = 'C'
env['LC_ALL'] = get_best_parsable_locale(self.module)
try:
p = subprocess.Popen([to_bytes(c) for c in cmd],
stdin=slave_in_fd,

15
lib/ansible/modules/yum.py
View File

@ -376,6 +376,7 @@ EXAMPLES = '''
'''
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.common.locale import get_best_parsable_locale
from ansible.module_utils.common.respawn import has_respawned, respawn_module
from ansible.module_utils._text import to_native, to_text
from ansible.module_utils.urls import fetch_url
@ -601,8 +602,9 @@ class YumModule(YumDnf):
if self.installroot != '/':
cmd.extend(['--root', self.installroot])
# rpm localizes messages and we're screen scraping so make sure we use
# the C locale
lang_env = dict(LANG='C', LC_ALL='C', LC_MESSAGES='C')
# an appropriate locale
locale = get_best_parsable_locale(self.module)
lang_env = dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale)
rc, out, err = self.module.run_command(cmd, environ_update=lang_env)
if rc != 0 and 'is not installed' not in out:
self.module.fail_json(msg='Error from rpm: %s: %s' % (cmd, err))
@ -939,7 +941,8 @@ class YumModule(YumDnf):
else:
res['changes'] = dict(installed=pkgs)
lang_env = dict(LANG='C', LC_ALL='C', LC_MESSAGES='C')
locale = get_best_parsable_locale(self.module)
lang_env = dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale)
rc, out, err = self.module.run_command(cmd, environ_update=lang_env)
if rc == 1:
@ -1500,7 +1503,8 @@ class YumModule(YumDnf):
elif self.update_only:
if pkgs['update']:
cmd = self.yum_basecmd + ['update'] + pkgs['update']
lang_env = dict(LANG='C', LC_ALL='C', LC_MESSAGES='C')
locale = get_best_parsable_locale(self.module)
lang_env = dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale)
rc, out, err = self.module.run_command(cmd, environ_update=lang_env)
out_lower = out.strip().lower()
if not out_lower.endswith("no packages marked for update") and \
@ -1510,7 +1514,8 @@ class YumModule(YumDnf):
rc, out, err = [0, '', '']
elif pkgs['install'] or will_update and not self.update_only:
cmd = self.yum_basecmd + ['install'] + pkgs['install'] + pkgs['update']
lang_env = dict(LANG='C', LC_ALL='C', LC_MESSAGES='C')
locale = get_best_parsable_locale(self.module)
lang_env = dict(LANG=locale, LC_ALL=locale, LC_MESSAGES=locale)
rc, out, err = self.module.run_command(cmd, environ_update=lang_env)
out_lower = out.strip().lower()
if not out_lower.endswith("no packages marked for update") and \

5
test/units/modules/test_apt_key.py
View File

@ -9,9 +9,14 @@ from units.compat import unittest
from ansible.modules import apt_key
def returnc(x):
return 'C'
class AptKeyTestCase(unittest.TestCase):
@mock.patch.object(apt_key, 'apt_key_bin', '/usr/bin/apt-key')
@mock.patch.object(apt_key, 'lang_env', returnc)
@mock.patch.dict(os.environ, {'HTTP_PROXY': 'proxy.example.com'})
def test_import_key_with_http_proxy(self):
m_mock = mock.Mock()