Modernize install (#76021)

Co-authored-by: Matt Clay <matt@mystile.com>
Co-authored-by: Matt Davis <mrd@redhat.com>
Co-authored-by: Sviatoslav Sydorenko <wk.cvs.github@sydorenko.org.ua>

MANIFEST.in

@@ -7,6 +7,10 @@ include docs/docsite/rst/collections/all_plugins.rst
 exclude docs/docsite/rst_warnings
 exclude docs/docsite/rst/conf.py
 exclude docs/docsite/rst/index.rst
+exclude docs/docsite/rst/dev_guide/testing/sanity/bin-symlinks.rst
+exclude docs/docsite/rst/dev_guide/testing/sanity/botmeta.rst
+exclude docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst
+exclude docs/docsite/rst/dev_guide/testing/sanity/release-names.rst
 recursive-exclude docs/docsite/_build *
 recursive-exclude docs/docsite/_extensions *.pyc *.pyo
 include examples/hosts
@@ -31,10 +35,15 @@ recursive-include test/lib/ansible_test/_util/controller/sanity/validate-modules
 recursive-include test/sanity *.json *.py *.txt
 recursive-include test/support *.py *.ps1 *.psm1 *.cs
 exclude test/sanity/code-smell/botmeta.*
+exclude test/sanity/code-smell/release-names.*
+exclude test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py
+exclude test/lib/ansible_test/_internal/commands/sanity/integration_aliases.py
 recursive-include test/units *
 include Makefile
 include MANIFEST.in
 include changelogs/CHANGELOG*.rst
 include changelogs/changelog.yaml
 recursive-include hacking/build_library *.py
 include hacking/build-ansible.py
+include hacking/test-module.py
+include bin/*

bin/ansible

@@ -1 +1 @@
-../lib/ansible/cli/scripts/ansible_cli_stub.py
+../lib/ansible/cli/adhoc.py

bin/ansible-config

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/config.py

bin/ansible-console

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/console.py

bin/ansible-doc

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/doc.py

bin/ansible-galaxy

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/galaxy.py

bin/ansible-inventory

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/inventory.py

bin/ansible-playbook

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/playbook.py

bin/ansible-pull

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/pull.py

bin/ansible-vault

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/vault.py

changelogs/fragments/modernize-install.yaml

@@ -0,0 +1,3 @@
+minor_changes:
+- Installation - modernize our python installation, to reduce dynamic code in setup.py, and migrate
+  what is feasible to setup.cfg. This will enable shipping wheels in the future.

hacking/env-setup

@@ -48,10 +48,12 @@ FULL_PATH=$($PYTHON_BIN -c "import os; print(os.path.realpath('$HACKING_DIR'))")
 export ANSIBLE_HOME="$(dirname "$FULL_PATH")"
 
 PREFIX_PYTHONPATH="$ANSIBLE_HOME/lib"
+ANSIBLE_TEST_PREFIX_PYTHONPATH="$ANSIBLE_HOME/test/lib"
 PREFIX_PATH="$ANSIBLE_HOME/bin"
 PREFIX_MANPATH="$ANSIBLE_HOME/docs/man"
 
 expr "$PYTHONPATH" : "${PREFIX_PYTHONPATH}.*" > /dev/null || prepend_path PYTHONPATH "$PREFIX_PYTHONPATH"
+expr "$PYTHONPATH" : "${ANSIBLE_TEST_PREFIX_PYTHONPATH}.*" > /dev/null || prepend_path PYTHONPATH "$ANSIBLE_TEST_PREFIX_PYTHONPATH"
 expr "$PATH" : "${PREFIX_PATH}.*" > /dev/null || prepend_path PATH "$PREFIX_PATH"
 expr "$MANPATH" : "${PREFIX_MANPATH}.*" > /dev/null || prepend_path MANPATH "$PREFIX_MANPATH"
 

hacking/env-setup.fish

@@ -5,6 +5,7 @@ set HACKING_DIR (dirname (status -f))
 set FULL_PATH (python -c "import os; print(os.path.realpath('$HACKING_DIR'))")
 set ANSIBLE_HOME (dirname $FULL_PATH)
 set PREFIX_PYTHONPATH $ANSIBLE_HOME/lib
+set ANSIBLE_TEST_PREFIX_PYTHONPATH $ANSIBLE_HOME/test/lib
 set PREFIX_PATH $ANSIBLE_HOME/bin
 set PREFIX_MANPATH $ANSIBLE_HOME/docs/man
 
@@ -31,6 +32,16 @@ else
     end
 end
 
+# Set ansible_test PYTHONPATH
+switch PYTHONPATH
+    case "$ANSIBLE_TEST_PREFIX_PYTHONPATH*"
+    case "*"
+        if not [ $QUIET ]
+            echo "Appending PYTHONPATH"
+        end
+        set -gx PYTHONPATH "$ANSIBLE_TEST_PREFIX_PYTHONPATH:$PYTHONPATH"
+end
+
 # Set PATH
 if not contains $PREFIX_PATH $PATH
     set -gx PATH $PREFIX_PATH $PATH

lib/ansible/__main__.py

@@ -0,0 +1,41 @@
+# Copyright: (c) 2021, Matt Martz <matt@sivel.net>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+import argparse
+import importlib
+import os
+import sys
+
+from importlib.metadata import distribution
+
+
+def _short_name(name):
+    return name.replace('ansible-', '').replace('ansible', 'adhoc')
+
+
+def main():
+    dist = distribution('ansible-core')
+    ep_map = {_short_name(ep.name): ep for ep in dist.entry_points if ep.group == 'console_scripts'}
+
+    parser = argparse.ArgumentParser(prog='python -m ansible', add_help=False)
+    parser.add_argument('entry_point', choices=list(ep_map) + ['test'])
+    args, extra = parser.parse_known_args()
+
+    if args.entry_point == 'test':
+        ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        source_root = os.path.join(ansible_root, 'test', 'lib')
+
+        if os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', '__init__.py')):
+            # running from source, use that version of ansible-test instead of any version that may already be installed
+            sys.path.insert(0, source_root)
+
+        module = importlib.import_module('ansible_test._util.target.cli.ansible_test_cli_stub')
+        main = module.main
+    else:
+        main = ep_map[args.entry_point].load()
+
+    main([args.entry_point] + extra)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/__init__.py

@@ -7,17 +7,36 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import sys
+
+# Used for determining if the system is running a new enough python version
+# and should only restrict on our documented minimum versions
+if sys.version_info < (3, 8):
+    raise SystemExit(
+        'ERROR: Ansible requires Python 3.8 or newer on the controller. '
+        'Current version: %s' % ''.join(sys.version.splitlines())
+    )
+
+import errno
 import getpass
 import os
 import subprocess
-import sys
-
+import traceback
 from abc import ABCMeta, abstractmethod
+from pathlib import Path
+
+try:
+    from ansible import constants as C
+    from ansible.utils.display import Display, initialize_locale
+    initialize_locale()
+    display = Display()
+except Exception as e:
+    print('ERROR: %s' % e, file=sys.stderr)
+    sys.exit(5)
 
-from ansible.cli.arguments import option_helpers as opt_help
-from ansible import constants as C
 from ansible import context
-from ansible.errors import AnsibleError
+from ansible.cli.arguments import option_helpers as opt_help
+from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
 from ansible.inventory.manager import InventoryManager
 from ansible.module_utils.six import with_metaclass, string_types, PY3
 from ansible.module_utils._text import to_bytes, to_text
@@ -27,7 +46,6 @@
 from ansible.release import __version__
 from ansible.utils.collection_loader import AnsibleCollectionConfig
 from ansible.utils.collection_loader._collection_finder import _get_collection_name_from_path
-from ansible.utils.display import Display
 from ansible.utils.path import unfrackpath
 from ansible.utils.unsafe_proxy import to_unsafe_text
 from ansible.vars.manager import VariableManager
@@ -39,9 +57,6 @@
     HAS_ARGCOMPLETE = False
 
 
-display = Display()
-
-
 class CLI(with_metaclass(ABCMeta, object)):
     ''' code behind bin/ansible* programs '''
 
@@ -292,7 +307,7 @@ def init_parser(self):
                 ansible.arguments.option_helpers.add_runas_options(self.parser)
                 self.parser.add_option('--my-option', dest='my_option', action='store')
         """
-        self.parser = opt_help.create_base_parser(os.path.basename(self.args[0]), usage=usage, desc=desc, epilog=epilog, )
+        self.parser = opt_help.create_base_parser(self.name, usage=usage, desc=desc, epilog=epilog)
 
     @abstractmethod
     def post_process_args(self, options):
@@ -532,3 +547,74 @@ def get_password_from_file(pwd_file):
             raise AnsibleError('Empty password was provided from file (%s)' % pwd_file)
 
         return to_unsafe_text(secret)
+
+    @classmethod
+    def cli_executor(cls, args=None):
+        if args is None:
+            args = sys.argv
+
+        try:
+            display.debug("starting run")
+
+            ansible_dir = Path("~/.ansible").expanduser()
+            try:
+                ansible_dir.mkdir(mode=0o700)
+            except OSError as exc:
+                if exc.errno != errno.EEXIST:
+                    display.warning(
+                        "Failed to create the directory '%s': %s" % (ansible_dir, to_text(exc, errors='surrogate_or_replace'))
+                    )
+            else:
+                display.debug("Created the '%s' directory" % ansible_dir)
+
+            try:
+                args = [to_text(a, errors='surrogate_or_strict') for a in args]
+            except UnicodeError:
+                display.error('Command line args are not in utf-8, unable to continue.  Ansible currently only understands utf-8')
+                display.display(u"The full traceback was:\n\n%s" % to_text(traceback.format_exc()))
+                exit_code = 6
+            else:
+                cli = cls(args)
+                exit_code = cli.run()
+
+        except AnsibleOptionsError as e:
+            cli.parser.print_help()
+            display.error(to_text(e), wrap_text=False)
+            exit_code = 5
+        except AnsibleParserError as e:
+            display.error(to_text(e), wrap_text=False)
+            exit_code = 4
+    # TQM takes care of these, but leaving comment to reserve the exit codes
+    #    except AnsibleHostUnreachable as e:
+    #        display.error(str(e))
+    #        exit_code = 3
+    #    except AnsibleHostFailed as e:
+    #        display.error(str(e))
+    #        exit_code = 2
+        except AnsibleError as e:
+            display.error(to_text(e), wrap_text=False)
+            exit_code = 1
+        except KeyboardInterrupt:
+            display.error("User interrupted execution")
+            exit_code = 99
+        except Exception as e:
+            if C.DEFAULT_DEBUG:
+                # Show raw stacktraces in debug mode, It also allow pdb to
+                # enter post mortem mode.
+                raise
+            have_cli_options = bool(context.CLIARGS)
+            display.error("Unexpected Exception, this is probably a bug: %s" % to_text(e), wrap_text=False)
+            if not have_cli_options or have_cli_options and context.CLIARGS['verbosity'] > 2:
+                log_only = False
+                if hasattr(e, 'orig_exc'):
+                    display.vvv('\nexception type: %s' % to_text(type(e.orig_exc)))
+                    why = to_text(e.orig_exc)
+                    if to_text(e) != why:
+                        display.vvv('\noriginal msg: %s' % why)
+            else:
+                display.display("to see the full traceback, use -vvv")
+                log_only = True
+            display.display(u"the full traceback was:\n\n%s" % to_text(traceback.format_exc()), log_only=log_only)
+            exit_code = 250
+
+        sys.exit(exit_code)

lib/ansible/cli/adhoc.py

@@ -1,13 +1,16 @@
+#!/usr/bin/env python
 # Copyright: (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
 # Copyright: (c) 2018, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
 from ansible import constants as C
 from ansible import context
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.executor.task_queue_manager import TaskQueueManager
@@ -25,6 +28,8 @@ class AdHocCLI(CLI):
         this command allows you to define and run a single task 'playbook' against a set of hosts
     '''
 
+    name = 'ansible'
+
     def init_parser(self):
         ''' create an options parser for bin/ansible '''
         super(AdHocCLI, self).init_parser(usage='%prog <host-pattern> [options]',
@@ -179,3 +184,11 @@ def run(self):
                 loader.cleanup_all_tmp_files()
 
         return result
+
+
+def main(args=None):
+    AdHocCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/config.py

@@ -1,9 +1,14 @@
+#!/usr/bin/env python
 # Copyright: (c) 2017, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
+
 import os
 import shlex
 import subprocess
@@ -13,7 +18,6 @@
 import ansible.plugins.loader as plugin_loader
 
 from ansible import constants as C
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.config.manager import ConfigManager, Setting
 from ansible.errors import AnsibleError, AnsibleOptionsError
@@ -33,6 +37,8 @@
 class ConfigCLI(CLI):
     """ Config command line class """
 
+    name = 'ansible-config'
+
     def __init__(self, args, callback=None):
 
         self.config_file = None
@@ -469,3 +475,11 @@ def execute_dump(self):
             text = self._get_plugin_configs(context.CLIARGS['type'], context.CLIARGS['args'])
 
         self.pager(to_text('\n'.join(text), errors='surrogate_or_strict'))
+
+
+def main(args=None):
+    ConfigCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()