New release v2.6.0

* Skip if insertbefore BOF until later (#41767)

If a line match is found in the file and no regexp is specified, insertbefore would improperly try to add a line if set to BOF.

Add tests for this scenario.

(cherry picked from commit eaae1318f8)

* Add changelog fragment

.gitignore

@@ -103,3 +103,4 @@ htmlcov/
 # ansible-test coverage results
 test/units/.coverage.*
 /test/integration/cloud-config-azure.yml
+changelogs/.plugin-cache.yaml

Makefile

@@ -239,9 +239,9 @@ sdist: clean docs
 sdist_upload: clean docs
 	$(PYTHON) setup.py sdist upload 2>&1 |tee upload.log
 
-.PHONY: changelog_reno
-changelog_reno:
-	reno -d changelogs/ report --title 'Ansible $(MAJOR_VERSION) "$(CODENAME)" Release Notes' --collapse-pre-release --no-show-source --earliest-version v$(MAJOR_VERSION).0a1 --output changelogs/CHANGELOG-v$(MAJOR_VERSION).rst
+.PHONY: changelog
+changelog:
+	packaging/release/changelogs/changelog.py release -vv && packaging/release/changelogs/changelog.py generate -vv
 
 .PHONY: rpmcommon
 rpmcommon: sdist

changelogs/.changes.yaml

@@ -0,0 +1,333 @@
+releases:
+  2.6.0:
+    codename: Heartbreaker
+    fragments:
+    - v2.6.0_summary.yaml
+    release_date: '2018-06-28'
+  2.6.0a1:
+    codename: Heartbreaker
+    fragments:
+    - ansible-managed-encoding.yaml
+    - azure-lb-sku.yaml
+    - base64_filter_encoding.yaml
+    - command-support-argv.yaml
+    - copy-files-default-follow.yaml
+    - copy-no-chmod.yaml
+    - copy-recursive-copy.yml
+    - encoding-docs-plugin-parsing.yaml
+    - file-disallow-src.yaml
+    - file-nonexistent-link.yaml
+    - file-nonexitent-link-recurse.yaml
+    - file-succeed-if-already-absent.yaml
+    - gem-custom-home.yaml
+    - grafana-deprecation-warnings.yaml
+    - includes.yaml
+    - mode-preserve.yaml
+    - pause-ctrl-c.yaml
+    - postgresql_user-encrypted-fixes.yml
+    - py36-spwd.yaml
+    - template-filename-encoding.yaml
+    - user-expires.yaml
+    - v2.6.0a1_catchup.yaml
+    - v2.6.0a1_summary.yaml
+    - win_environment-do-not-delete-on-null.yaml
+    - win_file-special-char-fix.yaml
+    - win_get_url-ftp-support.yaml
+    - win_module_utils_sid-conversion.yaml
+    - win_template-dest-dir-fix.yaml
+    - win_updates_typo_whitelist_fix.yaml
+    - window_become-better-errors.yaml
+    - windows-deprecation-removal.yaml
+    - winrm-encoding-fix
+    - winrm-get_correct_user.yaml
+    - yaml-callback-unicode.yaml
+    modules:
+    - aci_l3out
+    - avi_autoscalelaunchconfig
+    - avi_l4policyset
+    - avi_useraccount
+    - aws_caller_facts
+    - aws_inspector_target
+    - aws_ses_identity_policy
+    - aws_sgw_facts
+    - azure_rm_aks
+    - azure_rm_aks_facts
+    - bigip_data_group
+    - bigip_device_license
+    - bigip_gtm_global
+    - bigip_gtm_monitor_bigip
+    - bigip_gtm_monitor_firepass
+    - bigip_gtm_monitor_http
+    - bigip_gtm_monitor_https
+    - bigip_gtm_monitor_tcp
+    - bigip_gtm_monitor_tcp_half_open
+    - bigip_gtm_pool_member
+    - bigip_gtm_virtual_server
+    - bigip_log_destination
+    - bigip_log_publisher
+    - bigip_management_route
+    - bigip_profile_dns
+    - bigip_profile_tcp
+    - bigip_profile_udp
+    - bigip_service_policy
+    - bigip_smtp
+    - bigip_snmp_community
+    - bigip_timer_policy
+    - bigip_trunk
+    - bigiq_application_fasthttp
+    - bigiq_application_fastl4_tcp
+    - bigiq_application_fastl4_udp
+    - bigiq_application_http
+    - bigiq_application_https_offload
+    - bigiq_application_https_waf
+    - bigiq_regkey_license_assignment
+    - cnos_command
+    - cnos_config
+    - cs_role_permission
+    - digital_ocean_account_facts
+    - digital_ocean_certificate_facts
+    - digital_ocean_domain_facts
+    - digital_ocean_image_facts
+    - digital_ocean_load_balancer_facts
+    - digital_ocean_region_facts
+    - digital_ocean_size_facts
+    - digital_ocean_snapshot_facts
+    - digital_ocean_tag_facts
+    - digital_ocean_volume_facts
+    - ec2_eip_facts
+    - ec2_vpc_vpn_facts
+    - exos_command
+    - fortios_webfilter
+    - gcp_compute_address
+    - gcp_compute_backend_bucket
+    - gcp_compute_backend_service
+    - gcp_compute_health_check
+    - gcp_compute_http_health_check
+    - gcp_compute_https_health_check
+    - gcp_compute_instance
+    - gcp_compute_instance_template
+    - gcp_compute_network
+    - gcp_compute_route
+    - gcp_compute_subnetwork
+    - gcp_compute_target_http_proxy
+    - gcp_compute_target_https_proxy
+    - gcp_compute_target_tcp_proxy
+    - gcp_container_cluster
+    - gcp_container_node_pool
+    - gcp_dns_resource_record_set
+    - gcp_pubsub_subscription
+    - gcp_pubsub_topic
+    - gcp_storage_bucket
+    - gcp_storage_bucket_access_control
+    - gitlab_deploy_key
+    - gitlab_hooks
+    - k8s
+    - ldap_passwd
+    - memset_zone
+    - meraki_network
+    - meraki_organization
+    - na_ontap_broadcast_domain
+    - net_get
+    - net_put
+    - netconf_get
+    - one_host
+    - one_image
+    - one_image_facts
+    - one_vm
+    - os_server_metadata
+    - os_volume_snapshot
+    - psexec
+    - purefa_ds
+    - purefa_facts
+    - purefa_pgsnap
+    - purefb_fs
+    - purefb_snap
+    - rds_instance_facts
+    - rds_snapshot_facts
+    - scaleway_compute
+    - scaleway_sshkey
+    - slxos_command
+    - slxos_config
+    - slxos_facts
+    - slxos_interface
+    - slxos_l2_interface
+    - slxos_linkagg
+    - spectrum_device
+    - sysvinit
+    - vmware_cluster_facts
+    - vmware_datastore_cluster
+    - vmware_datastore_maintenancemode
+    - vmware_guest_disk_facts
+    - vmware_guest_snapshot_facts
+    - vmware_host_capability_facts
+    - vmware_host_powerstate
+    - vmware_local_user_facts
+    - vmware_portgroup_facts
+    - vmware_resource_pool_facts
+    - vmware_tag
+    - vmware_tag_facts
+    - vmware_vswitch_facts
+    - win_domain_computer
+    - win_hostname
+    - win_pester
+    - yarn
+    - zabbix_group_facts
+    plugins:
+      callback:
+      - cgroup_memory_recap
+      - grafana_annotations
+      - sumologic
+      connection:
+      - httpapi
+      inventory:
+      - foreman
+      - generator
+      - nmap
+      lookup:
+      - onepassword
+      - onepassword_raw
+    release_date: '2018-05-21'
+  2.6.0a2:
+    codename: Heartbreaker
+    fragments:
+    - always_run_removal.yaml
+    - aws_ses_identity_check_mode.yaml
+    - powershell_use_set_tmpdir.yaml
+    - selinux-check-mode.yaml
+    - synchronize_remote_is_local.yaml
+    - v2.6.0a2_catchup.yaml
+    - v2.6.0a2_summary.yaml
+    - win_updates-add-scheduled-tasks-back.yaml
+    - winrm_kinit_error-fix.yaml
+    - xenserver-facts-fix.yaml
+    modules:
+    - acme_account
+    - aws_config_aggregation_authorization
+    - aws_config_aggregator
+    - aws_config_delivery_channel
+    - aws_config_recorder
+    - aws_config_rule
+    - aws_glue_connection
+    - aws_glue_job
+    - azure_rm_resource
+    - azure_rm_resource_facts
+    - bigip_gtm_monitor_external
+    - bigip_monitor_external
+    - bigiq_utility_license
+    - cloud_init_data_facts
+    - elb_network_lb
+    - flatpak
+    - flatpak_remote
+    - gcp_compute_disk
+    - gcp_compute_firewall
+    - gcp_compute_forwarding_rule
+    - gcp_compute_global_address
+    - gcp_compute_global_forwarding_rule
+    - gcp_compute_image
+    - gcp_compute_instance_group
+    - gcp_compute_instance_group_manager
+    - gcp_compute_ssl_certificate
+    - gcp_compute_target_pool
+    - gcp_compute_target_ssl_proxy
+    - gcp_compute_url_map
+    - gluster_peer
+    - heroku_collaborator
+    - memset_dns_reload
+    - memset_zone_domain
+    - memset_zone_record
+    - meraki_admin
+    - meraki_snmp
+    - na_ontap_aggregate
+    - na_ontap_broadcast_domain_ports
+    - na_ontap_cifs
+    - na_ontap_cifs_acl
+    - na_ontap_cifs_server
+    - na_ontap_cluster
+    - na_ontap_cluster_ha
+    - na_ontap_export_policy
+    - na_ontap_export_policy_rule
+    - na_ontap_igroup
+    - na_ontap_interface
+    - na_ontap_iscsi
+    - na_ontap_job_schedule
+    - na_ontap_license
+    - na_ontap_lun
+    - na_ontap_lun_map
+    - na_ontap_net_ifgrp
+    - na_ontap_net_port
+    - na_ontap_net_routes
+    - na_ontap_net_vlan
+    - na_ontap_nfs
+    - na_ontap_ntp
+    - na_ontap_qtree
+    - na_ontap_service_processor_network
+    - na_ontap_snapshot
+    - na_ontap_snmp
+    - na_ontap_svm
+    - na_ontap_ucadapter
+    - na_ontap_user
+    - na_ontap_user_role
+    - na_ontap_volume
+    - na_ontap_volume_clone
+    - netconf_rpc
+    - one_service
+    - slxos_l3_interface
+    - slxos_vlan
+    plugins:
+      inventory:
+      - gcp_compute
+    release_date: '2018-05-30'
+  2.6.0rc1:
+    codename: Heartbreaker
+    fragments:
+    - v2.6.0rc1_catchup.yaml
+    - v2.6.0rc1_summary.yaml
+    release_date: '2018-06-05'
+  2.6.0rc2:
+    codename: Heartbreaker
+    fragments:
+    - 41167_ansible_doc_regression.yaml
+    - admin-users-default-change.yaml
+    - aws_s3_async_fix.yaml
+    - aws_s3_decryption_fix.yaml
+    - aws_s3_fix_custom_endpoints.yaml
+    - aws_s3_fix_exception_handling.yaml
+    - ec2_ami_fix_block_device_mapping_volume_size_type.yaml
+    - v2.6.0rc2_summary.yaml
+    release_date: '2018-06-08'
+  2.6.0rc3:
+    codename: Heartbreaker
+    fragments:
+    - digital_ocean_block_storage_fix.yaml
+    - gce-sort-fix.yaml
+    - iosxr_config_route_policy_objects_fix.yaml
+    - no_log_fix_for_connection_exceptions.yaml
+    - reset_con_fix.yml
+    - v2.6.0rc3_summary.yaml
+    release_date: '2018-06-18'
+  2.6.0rc4:
+    codename: Heartbreaker
+    fragments:
+    - 41530-apt-mark-deb6.yaml
+    - debug_fixes.yml
+    - eos_vrf_failure-fix.yml
+    - file_touch_diff.yaml
+    - fix_config_required.yml
+    - fix_yaml_inv_nulls.yml
+    - missing_raise.yml
+    - v2.6.0rc4_summary.yaml
+    - vars_lk.yml
+    - virtualbox_inventory.yaml
+    - win_updates-async-fix.yml
+    - winrm_kinit-remove-pass-log.yml
+    release_date: '2018-06-25'
+  2.6.0rc5:
+    codename: Heartbreaker
+    fragments:
+    - fix_options.yml
+    - fix_password_lookup_loading.yaml
+    - group_by_localhost.yaml
+    - lineinfile-insertbefore-bof-bugfix.yaml
+    - v2.6.0rc5_summary.yaml
+    - win_iis_webapppool-output-fix.yml
+    release_date: '2018-06-26'

changelogs/CHANGELOG-v2.6.rst

@@ -2,130 +2,518 @@
 Ansible 2.6 "Heartbreaker" Release Notes
 ========================================
 
-.. _Ansible 2.6 "Heartbreaker" Release Notes_v2.6.0a1:
-
-v2.6.0a1
-========
-
-.. _Ansible 2.6 "Heartbreaker" Release Notes_v2.6.0a1_Release Summary:
+v2.6.0
+======
 
 Release Summary
 ---------------
 
-| Release Date: 2018-05-21
+| Release Date: 2018-06-28
 | `Porting Guide <https://docs.ansible.com/ansible/devel/porting_guides.html>`_
 
 
-.. _Ansible 2.6 "Heartbreaker" Release Notes_v2.6.0a1_Minor Changes:
-
 Minor Changes
 -------------
 
-- azure_rm_loadbalancer - add support for sku
-
-- azure_rm_publicipaddress - add support for sku
-
 - Added an ``encoding`` option to the ``b64encode`` and ``b64decode`` filters to specify the encoding of the string that is base64 encoded.
-
-- import/include - Cache task_vars to speed up IncludedFile.process_include_results (https://github.com/ansible/ansible/pull/39026)
-
 - PowerShell modules that use Convert-ToSID in Ansible.ModuleUtils.SID.psm1 like win_user_right now accept an actual SID as an input string. This means any local or domain accounts that are named like a SID need to be prefixed with the domain, hostname, or . to ensure it converts to that accounts SID https://github.com/ansible/ansible/issues/38502
+- Raise AnsibleParserError which was missing previously
+- The aws_ses_identity module supports check mode
+- `postgresql_user` module changed `encrypted=yes` to be the default. This shouldn't break any current playbooks, the module will just store passwords hashed by default. This change was done because Postgres 10 dropped support for `UNENCRYPTED` passwords and because all versions since Postgres 7.2 support storing encrypted passwords.
+- azure_rm_loadbalancer - add support for sku
+- azure_rm_publicipaddress - add support for sku
+- cloudflare_dns module - Removed restriction from protocol to allow other protocols than tcp and udp to be specified.
+- command module - Added argv option to allow command to be specified as a list vs. a string (https://github.com/ansible/ansible/issues/19392)
+- gem - add ability to specify a custom directory for installing gems (https://github.com/ansible/ansible/pull/38195)
+- import/include - Cache task_vars to speed up IncludedFile.process_include_results (https://github.com/ansible/ansible/pull/39026)
+- postgresql_user module - Changed encrypted=yes to be the default. This shouldn't break any current playbooks, the module will just store passwords hashed by default. This change was done because Postgres 10 dropped support for UNENCRYPTED passwords and because all versions since Postgres 7.2 support storing encrypted passwords.
+- vmware_target_canonical_facts module - The target_id parameter is an optional parameter.
 
+Deprecated Features
+-------------------
 
-.. _Ansible 2.6 "Heartbreaker" Release Notes_v2.6.0a1_Removed Features (previously deprecated):
+- nxos_igmp_interface module - The oif_prefix and oif_source properties are deprecated. Use the oif_ps parameter with a dictionary of prefix and source to values instead.
 
 Removed Features (previously deprecated)
 ----------------------------------------
 
+- removed the deprecated always_run task option, please use `check_mode: no` instead
 - win_chocolatey - removed deprecated upgrade option and choco_* output return values
-
 - win_feature - removed deprecated reboot option
-
 - win_iis_webapppool - removed the ability to supply attributes as a string in favour of a dictionary
-
 - win_package - removed deprecated name option
-
 - win_regedit - removed deprecated support for specifying HKCC as HCCC
 
-
-.. _Ansible 2.6 "Heartbreaker" Release Notes_v2.6.0a1_Bugfixes:
-
 Bugfixes
 --------
 
-- template - Fix for encoding issues when a template path contains non-ascii characters and using the template path in ansible_managed (https://github.com/ansible/ansible/issues/27262)
-
-- copy - fixed copy to only follow symlinks for files in the non-recursive case
-
-- file - fixed the default follow behaviour of file to be true
-
-- copy module - The copy module was attempting to change the mode of files for remote_src=True even if mode was not set as a parameter.  This failed on filesystems which do not have permission bits (https://github.com/ansible/ansible/pull/40099)
-
-- Fix an encoding issue when parsing the examples from a plugins' documentation
-
-- file module - The file module allowed the user to specify src as a parameter when state was not link or hard.  This is documented as only applying to state=link or state=hard but in previous Ansible, this could have an effect in rare cornercases.  For instance, "ansible -m file -a 'state=directory path=/tmp src=/var/lib'" would create /tmp/lib.  This has been disabled and a warning emitted (will change to an error in Ansible-2.10).
-
-- file module - Fix error when running a task which assures a symlink to a nonexistent file exists for the second and subsequent times (https://github.com/ansible/ansible/issues/39558)
-
-- file module - Fix error when recursively assigning permissions and a symlink to a nonexistent file is present in the directory tree (https://github.com/ansible/ansible/issues/39456)
-
-- file module - Eliminate an error if we're asked to remove a file but something removes it while we are processing the request (https://github.com/ansible/ansible/pull/39466)
-
-- Various grafana_* modules - Port away from the deprecated b64encodestring function to the b64encode function instead. https://github.com/ansible/ansible/pull/38388
-
-- import/include - Update TaskInclude _raw_params with the expanded/templated path to file allowing nested includes using host vars in file (https://github.com/ansible/ansible/pull/39365)
-
-- dynamic includes - Don't treat undefined vars for conditional includes as truthy (https://github.com/ansible/ansible/pull/39377)
-
-- import/include - Ensure role handlers have the proper parent, allowing for correct attribute inheritance (https://github.com/ansible/ansible/pull/39426)
-
-- include_role/import_role - Use the computed role name for include_role/import_role so to diffentiate between names computed from host vars (https://github.com/ansible/ansible/pull/39516)
-
-- import_playbook - Pass vars applied to import_playbook into parsing of the playbook as they may be needed to parse the imported plays (https://github.com/ansible/ansible/pull/39521)
-
-- include_role/import_role - Don't overwrite included role handlers with play handlers on parse (https://github.com/ansible/ansible/pull/39563)
-
-- dynamic includes - Improved performance by fixing re-parenting on copy (https://github.com/ansible/ansible/pull/38747)
-
-- dynamic includes - Fix IncludedFile comparison for free strategy (https://github.com/ansible/ansible/pull/37083)
-
-- dynamic includes - Allow inheriting attributes from static parents (https://github.com/ansible/ansible/pull/38827)
-
-- include_role/import_role - improved performance and recursion depth (https://github.com/ansible/ansible/pull/36470)
-
-- include_role/import_role - Fix parameter templating (https://github.com/ansible/ansible/pull/36372)
-
-- dynamic includes - Use the copied and merged task for calculating task vars (https://github.com/ansible/ansible/pull/39762)
-
-- Implement mode=preserve for the template module
-
-- Fix mode=preserve with remote_src=True for the copy module
-
+- **Security Fix** - Some connection exceptions would cause no_log specified on a task to be ignored.  If this happened, the task information, including any private information could have been displayed to stdout and (if enabled, not the default) logged to a log file specified in ansible.cfg's log_path. Additionally, sites which redirected stdout from ansible runs to a log file may have stored that private information onto disk that way as well. (https://github.com/ansible/ansible/pull/41414)
+- Changed the admin_users config option to not include "admin" by default as admin is frequently used for a non-privileged account  (https://github.com/ansible/ansible/pull/41164)
+- Changed the output to "text" for "show vrf" command as default "json" output format with respect to "eapi" transport was failing (https://github.com/ansible/ansible/pull/41470)
 - Document mode=preserve for both the copy and template module
-
-- pause - ensure ctrl+c interrupt works in all cases (https://github.com/ansible/ansible/issues/35372)
-
-- spwd - With python 3.6 spwd.getspnam returns PermissionError instead of KeyError if user does not have privileges (https://github.com/ansible/ansible/issues/39472)
-
-- template action plugin - fix the encoding of filenames to avoid tracebacks on Python2 when characters that are not present in the user's locale are present. (https://github.com/ansible/ansible/pull/39424)
-
-- user - only change the expiration time when necessary (https://github.com/ansible/ansible/issues/13235)
-
-- win_environment - Fix for issue where the environment value was deleted when a null value or empty string was set - https://github.com/ansible/ansible/issues/40450
-
-- win_file - fix issue where special chars like [ and ] were not being handled correctly https://github.com/ansible/ansible/pull/37901
-
-- win_get_url - fixed a few bugs around authentication and force no when using an FTP URL
-
-- win_template - fix when specifying the dest option as a directory with and without the trailing slash https://github.com/ansible/ansible/issues/39886
-
-- win_updates - Fix typo that hid the download error when a download failed
-
-- win_updates - Fix logic when using a whitelist for multiple updates
-
-- windows become - Show better error messages when the become process fails
-
-- winrm - allow `ansible_user` or `ansible_winrm_user` to override `ansible_ssh_user` when both are defined in an inventory - https://github.com/ansible/ansible/issues/39844
-
+- Fix added for Digital Ocean Volumes API change causing Ansible to recieve an unexpected value in the response. (https://github.com/ansible/ansible/pull/41431)
+- Fix an encoding issue when parsing the examples from a plugins' documentation
+- Fix iosxr_config module to handle route-policy, community-set, prefix-set, as-path-set and rd-set blocks. All these blocks are part of route-policy language of iosxr.
+- Fix mode=preserve with remote_src=True for the copy module
+- Implement mode=preserve for the template module
 - The yaml callback plugin now allows non-ascii characters to be displayed.
+- Various grafana_* modules - Port away from the deprecated b64encodestring function to the b64encode function instead. https://github.com/ansible/ansible/pull/38388
+- added missing 'raise' to exception definition https://github.com/ansible/ansible/pull/41690
+- allow custom endpoints to be used in the aws_s3 module (https://github.com/ansible/ansible/pull/36832)
+- allow set_options to be called multiple times https://github.com/ansible/ansible/pull/41913
+- ansible-doc - fixed traceback on missing plugins (https://github.com/ansible/ansible/pull/41167)
+- cast the device_mapping volume size to an int in the ec2_ami module (https://github.com/ansible/ansible/pull/40938)
+- copy - fixed copy to only follow symlinks for files in the non-recursive case
+- copy module - The copy module was attempting to change the mode of files for remote_src=True even if mode was not set as a parameter.  This failed on filesystems which do not have permission bits (https://github.com/ansible/ansible/pull/40099)
+- copy module - fixed recursive copy with relative paths (https://github.com/ansible/ansible/pull/40166)
+- correct debug display for all cases https://github.com/ansible/ansible/pull/41331
+- correctly check hostvars for vars term https://github.com/ansible/ansible/pull/41819
+- correctly handle yaml inventory files when entries are null dicts https://github.com/ansible/ansible/issues/41692
+- dynamic includes - Allow inheriting attributes from static parents (https://github.com/ansible/ansible/pull/38827)
+- dynamic includes - Don't treat undefined vars for conditional includes as truthy (https://github.com/ansible/ansible/pull/39377)
+- dynamic includes - Fix IncludedFile comparison for free strategy (https://github.com/ansible/ansible/pull/37083)
+- dynamic includes - Improved performance by fixing re-parenting on copy (https://github.com/ansible/ansible/pull/38747)
+- dynamic includes - Use the copied and merged task for calculating task vars (https://github.com/ansible/ansible/pull/39762)
+- file - fixed the default follow behaviour of file to be true
+- file module - Eliminate an error if we're asked to remove a file but something removes it while we are processing the request (https://github.com/ansible/ansible/pull/39466)
+- file module - Fix error when recursively assigning permissions and a symlink to a nonexistent file is present in the directory tree (https://github.com/ansible/ansible/issues/39456)
+- file module - Fix error when running a task which assures a symlink to a nonexistent file exists for the second and subsequent times (https://github.com/ansible/ansible/issues/39558)
+- file module - The file module allowed the user to specify src as a parameter when state was not link or hard.  This is documented as only applying to state=link or state=hard but in previous Ansible, this could have an effect in rare cornercases.  For instance, "ansible -m file -a 'state=directory path=/tmp src=/var/lib'" would create /tmp/lib.  This has been disabled and a warning emitted (will change to an error in Ansible-2.10).
+- file module - The touch subcommand had its diff output broken during the 2.6.x development cycle.  This is now fixed (https://github.com/ansible/ansible/issues/41755)
+- fix BotoCoreError exception handling
+- fix apt-mark on debian6 (https://github.com/ansible/ansible/pull/41530)
+- fix async for the aws_s3 module by adding async support to the action plugin (https://github.com/ansible/ansible/pull/40826)
+- fix decrypting vault files for the aws_s3 module (https://github.com/ansible/ansible/pull/39634)
+- fix errors with S3-compatible APIs if they cannot use ACLs for buckets or objects
+- fix permission handling to try to download a file even if the user does not have permission to list all objects in the bucket
+- fixed config required handling, specifically for _terms in lookups https://github.com/ansible/ansible/pull/41740
+- gce_net - Fix sorting of allowed ports (https://github.com/ansible/ansible/pull/41567)
+- group_by - support implicit localhost (https://github.com/ansible/ansible/pull/41860)
+- import/include - Ensure role handlers have the proper parent, allowing for correct attribute inheritance (https://github.com/ansible/ansible/pull/39426)
+- import_playbook - Pass vars applied to import_playbook into parsing of the playbook as they may be needed to parse the imported plays (https://github.com/ansible/ansible/pull/39521)
+- include_role/import_role - Don't overwrite included role handlers with play handlers on parse (https://github.com/ansible/ansible/pull/39563)
+- include_role/import_role - Fix parameter templating (https://github.com/ansible/ansible/pull/36372)
+- include_role/import_role - Use the computed role name for include_role/import_role so to diffentiate between names computed from host vars (https://github.com/ansible/ansible/pull/39516)
+- include_role/import_role - improved performance and recursion depth (https://github.com/ansible/ansible/pull/36470)
+- lineinfile - fix insertbefore when used with BOF to not insert duplicate lines (https://github.com/ansible/ansible/issues/38219)
+- password lookup - Do not load password lookup in network filters, allowing the password lookup to be overriden (https://github.com/ansible/ansible/pull/41907)
+- pause - ensure ctrl+c interrupt works in all cases (https://github.com/ansible/ansible/issues/35372)
+- powershell - use the tmpdir set by `remote_tmp` for become/async tasks instead of the generic $env:TEMP - https://github.com/ansible/ansible/pull/40210
+- selinux - correct check mode behavior to report same changes as normal mode (https://github.com/ansible/ansible/pull/40721)
+- spwd - With python 3.6 spwd.getspnam returns PermissionError instead of KeyError if user does not have privileges (https://github.com/ansible/ansible/issues/39472)
+- synchronize - Ensure the local connection created by synchronize uses _remote_is_local=True, which causes ActionBase to build a local tmpdir (https://github.com/ansible/ansible/pull/40833)
+- template - Fix for encoding issues when a template path contains non-ascii characters and using the template path in ansible_managed (https://github.com/ansible/ansible/issues/27262)
+- template action plugin - fix the encoding of filenames to avoid tracebacks on Python2 when characters that are not present in the user's locale are present. (https://github.com/ansible/ansible/pull/39424)
+- user - only change the expiration time when necessary (https://github.com/ansible/ansible/issues/13235)
+- uses correct conn info for reset_connection  https://github.com/ansible/ansible/issues/27520
+- win_environment - Fix for issue where the environment value was deleted when a null value or empty string was set - https://github.com/ansible/ansible/issues/40450
+- win_file - fix issue where special chars like [ and ] were not being handled correctly https://github.com/ansible/ansible/pull/37901
+- win_get_url - fixed a few bugs around authentication and force no when using an FTP URL
+- win_iis_webapppool - redirect some module output to null so Ansible can read the output JSON https://github.com/ansible/ansible/issues/40874
+- win_template - fix when specifying the dest option as a directory with and without the trailing slash https://github.com/ansible/ansible/issues/39886
+- win_updates - Added the ability to run on a scheduled task for older hosts so async starts working again - https://github.com/ansible/ansible/issues/38364
+- win_updates - Fix logic when using a whitelist for multiple updates
+- win_updates - Fix typo that hid the download error when a download failed
+- win_updates - Fixed issue where running win_updates on async fails without any error
+- windows become - Show better error messages when the become process fails
+- winrm - Add better error handling when the kinit process fails
+- winrm - allow `ansible_user` or `ansible_winrm_user` to override `ansible_ssh_user` when both are defined in an inventory - https://github.com/ansible/ansible/issues/39844
+- winrm - ensure pexpect is set to not echo the input on a failure and have a manual sanity check afterwards https://github.com/ansible/ansible/issues/41865
+- winrm connection plugin - Fix exception messages sometimes raising a traceback when the winrm connection plugin encounters an unrecoverable error.  https://github.com/ansible/ansible/pull/39333
+- xenserver_facts - ensure module works with newer versions of XenServer (https://github.com/ansible/ansible/pull/35821)
 
+New Plugins
+-----------
+
+Callback
+~~~~~~~~
+
+- cgroup_memory_recap - Profiles maximum memory usage of tasks and full execution using cgroups
+- grafana_annotations - send ansible events as annotations on charts to grafana over http api.
+- sumologic - Sends task result events to Sumologic
+
+Connection
+~~~~~~~~~~
+
+- httpapi - Use httpapi to run command on network appliances
+
+Inventory
+~~~~~~~~~
+
+- foreman - foreman inventory source
+- gcp_compute - Google Cloud Compute Engine inventory source
+- generator - Uses Jinja2 to construct hosts and groups from patterns
+- nmap - Uses nmap to find hosts to target
+
+Lookup
+~~~~~~
+
+- onepassword - fetch field values from 1Password
+- onepassword_raw - fetch raw json data from 1Password
+
+New Modules
+-----------
+
+Cloud
+~~~~~
+
+amazon
+^^^^^^
+
+- aws_caller_facts - Get facts about the user and account being used to make AWS calls.
+- aws_config_aggregation_authorization - Manage cross-account AWS Config authorizations
+- aws_config_aggregator - Manage AWS Config aggregations across multiple accounts
+- aws_config_delivery_channel - Manage AWS Config delivery channels
+- aws_config_recorder - Manage AWS Config Recorders
+- aws_config_rule - Manage AWS Config resources
+- aws_glue_connection - Manage an AWS Glue connection
+- aws_glue_job - Manage an AWS Glue job
+- aws_inspector_target - Create, Update and Delete Amazon Inspector Assessment Targets
+- aws_ses_identity_policy - Manages SES sending authorization policies
+- aws_sgw_facts - Fetch AWS Storage Gateway facts
+- ec2_eip_facts - List EC2 EIP details
+- ec2_vpc_vpn_facts - Gather facts about VPN Connections in AWS.
+- elb_network_lb - Manage a Network Load Balancer
+- rds_instance_facts - obtain facts about one or more RDS instances
+- rds_snapshot_facts - obtain facts about one or more RDS snapshots
+
+azure
+^^^^^
+
+- azure_rm_aks - Manage a managed Azure Container Service (AKS) Instance.
+- azure_rm_aks_facts - Get Azure Kubernetes Service facts.
+- azure_rm_resource - Create any Azure resource.
+- azure_rm_resource_facts - Generic facts of Azure resources.
+
+cloudstack
+^^^^^^^^^^
+
+- cs_role_permission - Manages role permissions on Apache CloudStack based clouds.
+
+digital_ocean
+^^^^^^^^^^^^^
+
+- digital_ocean_account_facts - Gather facts about DigitalOcean User account
+- digital_ocean_certificate_facts - Gather facts about DigitalOcean certificates
+- digital_ocean_domain_facts - Gather facts about DigitalOcean Domains
+- digital_ocean_image_facts - Gather facts about DigitalOcean images
+- digital_ocean_load_balancer_facts - Gather facts about DigitalOcean load balancers
+- digital_ocean_region_facts - Gather facts about DigitalOcean regions
+- digital_ocean_size_facts - Gather facts about DigitalOcean Droplet sizes
+- digital_ocean_snapshot_facts - Gather facts about DigitalOcean Snapshot
+- digital_ocean_tag_facts - Gather facts about DigitalOcean tags
+- digital_ocean_volume_facts - Gather facts about DigitalOcean volumes
+
+google
+^^^^^^
+
+- gcp_compute_address - Creates a GCP Address
+- gcp_compute_backend_bucket - Creates a GCP BackendBucket
+- gcp_compute_backend_service - Creates a GCP BackendService
+- gcp_compute_disk - Creates a GCP Disk
+- gcp_compute_firewall - Creates a GCP Firewall
+- gcp_compute_forwarding_rule - Creates a GCP ForwardingRule
+- gcp_compute_global_address - Creates a GCP GlobalAddress
+- gcp_compute_global_forwarding_rule - Creates a GCP GlobalForwardingRule
+- gcp_compute_health_check - Creates a GCP HealthCheck
+- gcp_compute_http_health_check - Creates a GCP HttpHealthCheck
+- gcp_compute_https_health_check - Creates a GCP HttpsHealthCheck
+- gcp_compute_image - Creates a GCP Image
+- gcp_compute_instance - Creates a GCP Instance
+- gcp_compute_instance_group - Creates a GCP InstanceGroup
+- gcp_compute_instance_group_manager - Creates a GCP InstanceGroupManager
+- gcp_compute_instance_template - Creates a GCP InstanceTemplate
+- gcp_compute_network - Creates a GCP Network
+- gcp_compute_route - Creates a GCP Route
+- gcp_compute_ssl_certificate - Creates a GCP SslCertificate
+- gcp_compute_subnetwork - Creates a GCP Subnetwork
+- gcp_compute_target_http_proxy - Creates a GCP TargetHttpProxy
+- gcp_compute_target_https_proxy - Creates a GCP TargetHttpsProxy
+- gcp_compute_target_pool - Creates a GCP TargetPool
+- gcp_compute_target_ssl_proxy - Creates a GCP TargetSslProxy
+- gcp_compute_target_tcp_proxy - Creates a GCP TargetTcpProxy
+- gcp_compute_url_map - Creates a GCP UrlMap
+- gcp_container_cluster - Creates a GCP Cluster
+- gcp_container_node_pool - Creates a GCP NodePool
+- gcp_dns_resource_record_set - Creates a GCP ResourceRecordSet
+- gcp_pubsub_subscription - Creates a GCP Subscription
+- gcp_pubsub_topic - Creates a GCP Topic
+- gcp_storage_bucket - Creates a GCP Bucket
+- gcp_storage_bucket_access_control - Creates a GCP BucketAccessControl
+
+heroku
+^^^^^^
+
+- heroku_collaborator - Add or delete app collaborators on Heroku
+
+memset
+^^^^^^
+
+- memset_dns_reload - Request reload of Memset's DNS infrastructure,
+- memset_zone - Creates and deletes Memset DNS zones.
+- memset_zone_domain - Create and delete domains in Memset DNS zones.
+- memset_zone_record - Create and delete records in Memset DNS zones.
+
+misc
+^^^^
+
+- cloud_init_data_facts - Retrieve facts of cloud-init.
+
+opennebula
+^^^^^^^^^^
+
+- one_host - Manages OpenNebula Hosts
+- one_image - Manages OpenNebula images
+- one_image_facts - Gather facts about OpenNebula images
+- one_service - Deploy and manage OpenNebula services
+- one_vm - Creates or terminates OpenNebula instances
+
+openstack
+^^^^^^^^^
+
+- os_server_metadata - Add/Update/Delete Metadata in Compute Instances from OpenStack
+- os_volume_snapshot - Create/Delete Cinder Volume Snapshots
+
+scaleway
+^^^^^^^^
+
+- scaleway_compute - Scaleway compute management module
+- scaleway_sshkey - Scaleway SSH keys management module
+
+vmware
+^^^^^^
+
+- vmware_cluster_facts - Gather facts about clusters available in given vCenter
+- vmware_datastore_cluster - Manage VMware vSphere datastore clusters
+- vmware_datastore_maintenancemode - Place a datastore into maintenance mode
+- vmware_guest_disk_facts - Gather facts about disks of given virtual machine
+- vmware_guest_snapshot_facts - Gather facts about virtual machine's snapshots in vCenter
+- vmware_host_capability_facts - Gathers facts about an ESXi host's capability information
+- vmware_host_powerstate - Manages power states of host systems in vCenter
+- vmware_local_user_facts - Gather facts about users on the given ESXi host
+- vmware_portgroup_facts - Gathers facts about an ESXi host's portgroup configuration
+- vmware_resource_pool_facts - Gathers facts about resource pool information
+- vmware_tag - Manage VMware tags
+- vmware_tag_facts - Manage VMware tag facts
+- vmware_vswitch_facts - Gathers facts about an ESXi host's vswitch configurations
+
+Clustering
+~~~~~~~~~~
+
+k8s
+^^^
+
+- k8s - Manage Kubernetes (K8s) objects
+
+Commands
+~~~~~~~~
+
+- psexec - Runs commands on a remote Windows host based on the PsExec model
+
+Monitoring
+~~~~~~~~~~
+
+- spectrum_device - Creates/deletes devices in CA Spectrum.
+
+zabbix
+^^^^^^
+
+- zabbix_group_facts - Gather facts about Zabbix hostgroup
+
+Net Tools
+~~~~~~~~~
+
+ldap
+^^^^
+
+- ldap_passwd - Set passwords in LDAP.
+
+Network
+~~~~~~~
+
+aci
+^^^
+
+- aci_l3out - Manage Layer 3 Outside (L3Out) objects (l3ext:Out)
+
+avi
+^^^
+
+- avi_autoscalelaunchconfig - Module for setup of AutoScaleLaunchConfig Avi RESTful Object
+- avi_l4policyset - Module for setup of L4PolicySet Avi RESTful Object
+- avi_useraccount - Avi UserAccount Module
+
+cnos
+^^^^
+
+- cnos_command - Run arbitrary commands on Lenovo CNOS devices
+- cnos_config - Manage Lenovo CNOS configuration sections
+
+exos
+^^^^
+
+- exos_command - Run commands on remote devices running Extreme EXOS
+
+f5
+^^
+
+- bigip_data_group - Manage data groups on a BIG-IP
+- bigip_device_license - Manage license installation and activation on BIG-IP devices
+- bigip_gtm_global - Manages global GTM settings
+- bigip_gtm_monitor_bigip - Manages F5 BIG-IP GTM BIG-IP monitors
+- bigip_gtm_monitor_external - Manages external GTM monitors on a BIG-IP
+- bigip_gtm_monitor_firepass - Manages F5 BIG-IP GTM FirePass monitors
+- bigip_gtm_monitor_http - Manages F5 BIG-IP GTM http monitors
+- bigip_gtm_monitor_https - Manages F5 BIG-IP GTM https monitors
+- bigip_gtm_monitor_tcp - Manages F5 BIG-IP GTM tcp monitors
+- bigip_gtm_monitor_tcp_half_open - Manages F5 BIG-IP GTM tcp half-open monitors
+- bigip_gtm_pool_member - Manage GTM pool member settings
+- bigip_gtm_virtual_server - Manages F5 BIG-IP GTM virtual servers
+- bigip_log_destination - Manages log destinations on a BIG-IP.
+- bigip_log_publisher - Manages log publishers on a BIG-IP
+- bigip_management_route - Manage system management routes on a BIG-IP
+- bigip_monitor_external - Manages external LTM monitors on a BIG-IP
+- bigip_profile_dns - Manage DNS profiles on a BIG-IP
+- bigip_profile_tcp - Manage TCP profiles on a BIG-IP
+- bigip_profile_udp - Manage UDP profiles on a BIG-IP
+- bigip_service_policy - Manages service policies on a BIG-IP.
+- bigip_smtp - Manages SMTP settings on the BIG-IP
+- bigip_snmp_community - Manages SNMP communities on a BIG-IP.
+- bigip_timer_policy - Manage timer policies on a BIG-IP
+- bigip_trunk - Manage trunks on a BIG-IP
+- bigiq_application_fasthttp - Manages BIG-IQ FastHTTP applications
+- bigiq_application_fastl4_tcp - Manages BIG-IQ FastL4 TCP applications
+- bigiq_application_fastl4_udp - Manages BIG-IQ FastL4 UDP applications
+- bigiq_application_http - Manages BIG-IQ HTTP applications
+- bigiq_application_https_offload - Manages BIG-IQ HTTPS offload applications
+- bigiq_application_https_waf - Manages BIG-IQ HTTPS WAF applications
+- bigiq_regkey_license_assignment - Manage regkey license assignment on BIG-IPs from a BIG-IQ.
+- bigiq_utility_license - Manage utility licenses on a BIG-IQ
+
+files
+^^^^^
+
+- net_get - Copy files from a network device to Ansible Controller
+- net_put - Copy files from Ansibe controller to a network device
+
+fortios
+^^^^^^^
+
+- fortios_webfilter - Configure webfilter capabilities of FortiGate and FortiOS.
+
+meraki
+^^^^^^
+
+- meraki_admin - Manage administrators in the Meraki cloud
+- meraki_network - Manage networks in the Meraki cloud
+- meraki_organization - Manage organizations in the Meraki cloud
+- meraki_snmp - Manage organizations in the Meraki cloud
+
+netconf
+^^^^^^^
+
+- netconf_get - Fetch configuration/state data from NETCONF enabled network devices.
+- netconf_rpc - Execute operations on NETCONF enabled network devices.
+
+slxos
+^^^^^
+
+- slxos_command - Run commands on remote devices running Extreme Networks SLX-OS
+- slxos_config - Manage Extreme Networks SLX-OS configuration sections
+- slxos_facts - Collect facts from devices running Extreme SLX-OS
+- slxos_interface - Manage Interfaces on Extreme SLX-OS network devices
+- slxos_l2_interface - Manage Layer-2 interface on Extreme Networks SLXOS devices.
+- slxos_l3_interface - Manage L3 interfaces on Extreme Networks SLXOS network devices.
+- slxos_linkagg - Manage link aggregation groups on Extreme Networks SLXOS network devices
+- slxos_vlan - Manage VLANs on Extreme Networks SLX-OS network devices
+
+Packaging
+~~~~~~~~~
+
+language
+^^^^^^^^
+
+- yarn - Manage node.js packages with Yarn
+
+os
+^^
+
+- flatpak - Manage flatpaks
+- flatpak_remote - Manage flatpak repository remotes
+
+Source Control
+~~~~~~~~~~~~~~
+
+- gitlab_deploy_key - Manages GitLab project deploy keys.
+- gitlab_hooks - Manages GitLab project hooks.
+
+Storage
+~~~~~~~
+
+glusterfs
+^^^^^^^^^
+
+- gluster_peer - Attach/Detach peers to/from the cluster
+
+netapp
+^^^^^^
+
+- na_ontap_aggregate - Manage NetApp ONTAP aggregates.
+- na_ontap_broadcast_domain - Manage NetApp ONTAP broadcast domains.
+- na_ontap_broadcast_domain_ports - Manage NetApp Ontap broadcast domain ports
+- na_ontap_cifs - Manage NetApp cifs-share
+- na_ontap_cifs_acl - Manage NetApp cifs-share-access-control
+- na_ontap_cifs_server - cifs server configuration
+- na_ontap_cluster - Create/Join ONTAP cluster. Apply license to cluster
+- na_ontap_cluster_ha - Manage HA status for cluster
+- na_ontap_export_policy - Manage NetApp ONTAP export-policy
+- na_ontap_export_policy_rule - Manage ONTAP Export rules
+- na_ontap_igroup - ONTAP iSCSI igroup configuration
+- na_ontap_interface - ONTAP LIF configuration
+- na_ontap_iscsi - Manage NetApp Ontap iscsi service
+- na_ontap_job_schedule - Manage NetApp Ontap Job Schedule
+- na_ontap_license - Manage NetApp ONTAP protocol and feature licenses
+- na_ontap_lun - Manage  NetApp Ontap luns
+- na_ontap_lun_map - Manage NetApp Ontap lun maps
+- na_ontap_net_ifgrp - Create, modify, destroy the network interface group
+- na_ontap_net_port - Manage NetApp Ontap network ports.
+- na_ontap_net_routes - Manage NetApp Ontap network routes
+- na_ontap_net_vlan - Manage NetApp Ontap network vlan
+- na_ontap_nfs - Manage Ontap NFS status
+- na_ontap_ntp - Create/Delete/modify_version ONTAP NTP server
+- na_ontap_qtree - Manage qtrees
+- na_ontap_service_processor_network - Manage NetApp Ontap service processor network
+- na_ontap_snapshot - Manage NetApp Sanpshots
+- na_ontap_snmp - Manage NetApp SNMP community
+- na_ontap_svm - Manage NetApp Ontap svm
+- na_ontap_ucadapter - ONTAP UC adapter configuration
+- na_ontap_user - useradmin configuration and management
+- na_ontap_user_role - useradmin configuration and management
+- na_ontap_volume - Manage NetApp ONTAP volumes.
+- na_ontap_volume_clone - Manage NetApp Ontap volume clones.
+
+purestorage
+^^^^^^^^^^^
+
+- purefa_ds - Configure FlashArray Directory Service
+- purefa_facts - Collect facts from Pure Storage FlashArray
+- purefa_pgsnap - Manage protection group snapshots on Pure Storage FlashArrays
+- purefb_fs - Manage filesystemon Pure Storage FlashBlade`
+- purefb_snap - Manage filesystem snapshots on Pure Storage FlashBlades
+
+System
+~~~~~~
+
+- sysvinit - Manage SysV services.
+
+Web Infrastructure
+~~~~~~~~~~~~~~~~~~
+
+- acme_account - Create, modify or delete accounts with Let's Encrypt
+
+Windows
+~~~~~~~
+
+- win_domain_computer - Manage computers in Active Directory
+- win_hostname - Manages local Windows computer name.
+- win_pester - Run Pester tests on Windows hosts

changelogs/config.yaml

@@ -3,16 +3,11 @@ release_tag_re: '(v(?:[\d.ab\-]|rc)+)'
 pre_release_tag_re: '(?P<pre_release>(?:[ab]|rc)+\d*)$'
 notesdir: fragments
 prelude_section_name: release_summary
+new_plugins_after_name: removed_features
 sections:
 - ['major_changes', 'Major Changes']
 - ['minor_changes', 'Minor Changes']
 - ['deprecated_features', 'Deprecated Features']
 - ['removed_features', 'Removed Features (previously deprecated)']
-- ['new_lookup_plugins', 'New Lookup Plugins']
-- ['new_callback_plugins', 'New Callback Plugins']
-- ['new_connection_plugins', 'New Connection Plugins']
-- ['new_test_plugins', 'New Test Plugins']
-- ['new_filter_plugins', 'New Filter Plugins']
-- ['new_modules', 'New Modules']
 - ['bugfixes', 'Bugfixes']
 - ['known_issues', 'Known Issues']

changelogs/fragments/41167_ansible_doc_regression.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+- ansible-doc - fixed traceback on missing plugins (https://github.com/ansible/ansible/pull/41167)

changelogs/fragments/41530-apt-mark-deb6.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+  - fix apt-mark on debian6 (https://github.com/ansible/ansible/pull/41530)

changelogs/fragments/admin-users-default-change.yaml

@@ -0,0 +1,4 @@
+---
+bugfixes:
+  - Changed the admin_users config option to not include "admin" by default as
+    admin is frequently used for a non-privileged account  (https://github.com/ansible/ansible/pull/41164)

changelogs/fragments/always_run_removal.yaml

@@ -1,2 +1,2 @@
 removed_features:
-- removed the deprecated always_run task option, please use `check_mode: no` instead
+- "removed the deprecated always_run task option, please use `check_mode: no` instead"

changelogs/fragments/aws_s3_async_fix.yaml

@@ -0,0 +1,3 @@
+---
+bugfixes:
+- fix async for the aws_s3 module by adding async support to the action plugin (https://github.com/ansible/ansible/pull/40826)

changelogs/fragments/aws_s3_decryption_fix.yaml

@@ -0,0 +1,3 @@
+---
+bugfixes:
+- fix decrypting vault files for the aws_s3 module (https://github.com/ansible/ansible/pull/39634)

changelogs/fragments/aws_s3_fix_custom_endpoints.yaml

@@ -0,0 +1,3 @@
+---
+bugfixes:
+- allow custom endpoints to be used in the aws_s3 module (https://github.com/ansible/ansible/pull/36832)

changelogs/fragments/aws_s3_fix_exception_handling.yaml

@@ -0,0 +1,5 @@
+---
+bugfixes:
+  - fix permission handling to try to download a file even if the user does not have permission to list all objects in the bucket
+  - fix BotoCoreError exception handling
+  - fix errors with S3-compatible APIs if they cannot use ACLs for buckets or objects

changelogs/fragments/command-support-argv.yaml

@@ -1,4 +1,4 @@
 ---
-features:
+minor_changes:
   - command module - Added argv option to allow command to be specified as a list vs. a string
     (https://github.com/ansible/ansible/issues/19392)

changelogs/fragments/debug_fixes.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - correct debug display for all cases https://github.com/ansible/ansible/pull/41331

changelogs/fragments/digital_ocean_block_storage_fix.yaml

@@ -0,0 +1,3 @@
+---
+bugfixes:
+- Fix added for Digital Ocean Volumes API change causing Ansible to recieve an unexpected value in the response. (https://github.com/ansible/ansible/pull/41431)

changelogs/fragments/ec2_ami_fix_block_device_mapping_volume_size_type.yaml

@@ -0,0 +1,3 @@
+---
+bugfixes:
+- cast the device_mapping volume size to an int in the ec2_ami module (https://github.com/ansible/ansible/pull/40938)

changelogs/fragments/eos_vrf_failure-fix.yml

@@ -0,0 +1,4 @@
+---
+bugfixes:
+  - Changed the output to "text" for "show vrf" command as default "json" output format
+    with respect to "eapi" transport was failing (https://github.com/ansible/ansible/pull/41470)

changelogs/fragments/file_touch_diff.yaml

@@ -0,0 +1,4 @@
+---
+bugfixes:
+- file module - The touch subcommand had its diff output broken during the
+  2.6.x development cycle.  This is now fixed (https://github.com/ansible/ansible/issues/41755)

changelogs/fragments/fix_config_required.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - fixed config required handling, specifically for _terms in lookups https://github.com/ansible/ansible/pull/41740

changelogs/fragments/fix_options.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - allow set_options to be called multiple times https://github.com/ansible/ansible/pull/41913

changelogs/fragments/fix_password_lookup_loading.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+- password lookup - Do not load password lookup in network filters, allowing the password lookup to be overriden (https://github.com/ansible/ansible/pull/41907)

changelogs/fragments/fix_yaml_inv_nulls.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - correctly handle yaml inventory files when entries are null dicts https://github.com/ansible/ansible/issues/41692

changelogs/fragments/gce-sort-fix.yaml

@@ -0,0 +1,3 @@
+---
+bugfixes:
+- gce_net - Fix sorting of allowed ports (https://github.com/ansible/ansible/pull/41567)

changelogs/fragments/gem-custom-home.yaml

@@ -1,2 +1,2 @@
-new_features:
+minor_changes:
   - gem - add ability to specify a custom directory for installing gems (https://github.com/ansible/ansible/pull/38195)

changelogs/fragments/group_by_localhost.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+- group_by - support implicit localhost (https://github.com/ansible/ansible/pull/41860)

changelogs/fragments/includes.yaml

@@ -3,7 +3,6 @@ minor_changes:
 - import/include - Cache task_vars to speed up IncludedFile.process_include_results (https://github.com/ansible/ansible/pull/39026)
 
 bugfixes:
-- import/include - Update TaskInclude _raw_params with the expanded/templated path to file allowing nested includes using host vars in file (https://github.com/ansible/ansible/pull/39365)
 - dynamic includes - Don't treat undefined vars for conditional includes as truthy (https://github.com/ansible/ansible/pull/39377)
 - import/include - Ensure role handlers have the proper parent, allowing for correct attribute inheritance (https://github.com/ansible/ansible/pull/39426)
 - include_role/import_role - Use the computed role name for include_role/import_role so to diffentiate between names computed from host vars (https://github.com/ansible/ansible/pull/39516)

changelogs/fragments/iosxr_config_route_policy_objects_fix.yaml

@@ -0,0 +1,5 @@
+---
+bugfixes:
+- Fix iosxr_config module to handle route-policy, community-set, prefix-set,
+  as-path-set and rd-set blocks. All these blocks are part of route-policy
+  language of iosxr.

changelogs/fragments/lineinfile-insertbefore-bof-bugfix.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+  - lineinfile - fix insertbefore when used with BOF to not insert duplicate lines (https://github.com/ansible/ansible/issues/38219)

changelogs/fragments/missing_raise.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - added missing 'raise' to exception definition https://github.com/ansible/ansible/pull/41690

changelogs/fragments/new-gcp-modules.yml

@@ -1,34 +0,0 @@
-new_features:
-- 'New Google Cloud module: gcp_compute_address'
-- 'New Google Cloud module: gcp_compute_backend_bucket'
-- 'New Google Cloud module: gcp_compute_backend_service'
-- 'New Google Cloud module: gcp_compute_firewall'
-- 'New Google Cloud module: gcp_compute_forwarding_rule'
-- 'New Google Cloud module: gcp_compute_global_address'
-- 'New Google Cloud module: gcp_compute_global_forwarding_rule'
-- 'New Google Cloud module: gcp_compute_health_check'
-- 'New Google Cloud module: gcp_compute_http_health_check'
-- 'New Google Cloud module: gcp_compute_https_health_check'
-- 'New Google Cloud module: gcp_compute_image'
-- 'New Google Cloud module: gcp_compute_instance'
-- 'New Google Cloud module: gcp_compute_instance_group'
-- 'New Google Cloud module: gcp_compute_instance_group_manager'
-- 'New Google Cloud module: gcp_compute_instance_template'
-- 'New Google Cloud module: gcp_compute_network'
-- 'New Google Cloud module: gcp_compute_route'
-- 'New Google Cloud module: gcp_compute_ssl_certificate'
-- 'New Google Cloud module: gcp_compute_subnetwork'
-- 'New Google Cloud module: gcp_compute_target_http_proxy'
-- 'New Google Cloud module: gcp_compute_target_https_proxy'
-- 'New Google Cloud module: gcp_compute_target_pool'
-- 'New Google Cloud module: gcp_compute_target_ssl_proxy'
-- 'New Google Cloud module: gcp_compute_target_tcp_proxy'
-- 'New Google Cloud module: gcp_compute_url_map'
-- 'New Google Cloud module: gcp_container_cluster'
-- 'New Google Cloud module: gcp_container_node_pool'
-- 'New Google Cloud module: gcp_dns_managed_zone'
-- 'New Google Cloud module: gcp_dns_resource_record_set'
-- 'New Google Cloud module: gcp_pubsub_subscription'
-- 'New Google Cloud module: gcp_pubsub_topic'
-- 'New Google Cloud module: gcp_storage_bucket'
-- 'New Google Cloud module: gcp_storage_bucket_access_control'

changelogs/fragments/no_log_fix_for_connection_exceptions.yaml

@@ -0,0 +1,9 @@
+---
+bugfixes:
+- '**Security Fix** - Some connection exceptions would cause no_log specified on
+  a task to be ignored.  If this happened, the task information, including any
+  private information could have been displayed to stdout and (if enabled, not
+  the default) logged to a log file specified in ansible.cfg''s log_path.
+  Additionally, sites which redirected stdout from ansible runs to a log file
+  may have stored that private information onto disk that way as well.
+  (https://github.com/ansible/ansible/pull/41414)'

changelogs/fragments/onepassword-lookup.yaml

@@ -1,2 +0,0 @@
-features:
-  - onepassword lookup - add lookup plugins onepassword and onepassword_raw to retrieve secrets from 1Password vault (https://github.com/ansible/ansible/pull/37207)

changelogs/fragments/reset_con_fix.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - uses correct conn info for reset_connection  https://github.com/ansible/ansible/issues/27520

changelogs/fragments/synchronize_remote_is_local.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+- synchronize - Ensure the local connection created by synchronize uses _remote_is_local=True, which causes ActionBase to build a local tmpdir (https://github.com/ansible/ansible/pull/40833)

changelogs/fragments/v2.6.0_summary.yaml

@@ -0,0 +1,3 @@
+release_summary: |
+   | Release Date: 2018-06-28
+   | `Porting Guide <https://docs.ansible.com/ansible/devel/porting_guides.html>`_

changelogs/fragments/v2.6.0a1_catchup.yaml

@@ -1 +1,7 @@
-{}
+deprecated_features:
+- nxos_igmp_interface module - The oif_prefix and oif_source properties are deprecated. Use the oif_ps parameter with a dictionary of prefix and source to values instead.
+
+minor_changes:
+- cloudflare_dns module - Removed restriction from protocol to allow other protocols than tcp and udp to be specified.
+- vmware_target_canonical_facts module - The target_id parameter is an optional parameter.
+- postgresql_user module - Changed encrypted=yes to be the default. This shouldn't break any current playbooks, the module will just store passwords hashed by default. This change was done because Postgres 10 dropped support for UNENCRYPTED passwords and because all versions since Postgres 7.2 support storing encrypted passwords.

changelogs/fragments/v2.6.0a2_catchup.yaml

@@ -0,0 +1 @@
+{}

changelogs/fragments/v2.6.0a2_summary.yaml

@@ -0,0 +1,3 @@
+release_summary: |
+   | Release Date: 2018-05-30
+   | `Porting Guide <https://docs.ansible.com/ansible/devel/porting_guides.html>`_

changelogs/fragments/v2.6.0rc1_catchup.yaml

@@ -0,0 +1 @@
+{}

changelogs/fragments/v2.6.0rc1_summary.yaml

@@ -0,0 +1,3 @@
+release_summary: |
+   | Release Date: 2018-06-05
+   | `Porting Guide <https://docs.ansible.com/ansible/devel/porting_guides.html>`_

changelogs/fragments/v2.6.0rc2_summary.yaml

@@ -0,0 +1,3 @@
+release_summary: |
+   | Release Date: 2018-06-08
+   | `Porting Guide <https://docs.ansible.com/ansible/devel/porting_guides.html>`_

changelogs/fragments/v2.6.0rc3_summary.yaml

@@ -0,0 +1,3 @@
+release_summary: |
+   | Release Date: 2018-06-18
+   | `Porting Guide <https://docs.ansible.com/ansible/devel/porting_guides.html>`_

changelogs/fragments/v2.6.0rc4_summary.yaml

@@ -0,0 +1,3 @@
+release_summary: |
+   | Release Date: 2018-06-25
+   | `Porting Guide <https://docs.ansible.com/ansible/devel/porting_guides.html>`_

changelogs/fragments/v2.6.0rc5_summary.yaml

@@ -0,0 +1,3 @@
+release_summary: |
+   | Release Date: 2018-06-26
+   | `Porting Guide <https://docs.ansible.com/ansible/devel/porting_guides.html>`_

changelogs/fragments/vars_lk.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - correctly check hostvars for vars term https://github.com/ansible/ansible/pull/41819

changelogs/fragments/virtualbox_inventory.yaml

@@ -0,0 +1,3 @@
+---
+minor_changes:
+- Raise AnsibleParserError which was missing previously

changelogs/fragments/win_iis_webapppool-output-fix.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- win_iis_webapppool - redirect some module output to null so Ansible can read the output JSON https://github.com/ansible/ansible/issues/40874

changelogs/fragments/win_updates-async-fix.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- win_updates - Fixed issue where running win_updates on async fails without any error

changelogs/fragments/winrm_kinit-remove-pass-log.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- winrm - ensure pexpect is set to not echo the input on a failure and have a manual sanity check afterwards https://github.com/ansible/ansible/issues/41865

contrib/inventory/azure_rm.py

@@ -326,10 +326,10 @@ class AzureRM(object):
         # get authentication authority
         # for adfs, user could pass in authority or not.
         # for others, use default authority from cloud environment
-        if self.credentials.get('adfs_authority_url') is None:
-            self._adfs_authority_url = self._cloud_environment.endpoints.active_directory
-        else:
+        if self.credentials.get('adfs_authority_url'):
             self._adfs_authority_url = self.credentials.get('adfs_authority_url')
+        else:
+            self._adfs_authority_url = self._cloud_environment.endpoints.active_directory
 
         # get resource from cloud environment
         self._resource = self._cloud_environment.endpoints.active_directory_resource_id
@@ -627,6 +627,8 @@ class AzureInventory(object):
                             help='Active Directory User')
         parser.add_argument('--password', action='store',
                             help='password')
+        parser.add_argument('--adfs_authority_url', action='store',
+                            help='Azure ADFS authority url')
         parser.add_argument('--cloud_environment', action='store',
                             help='Azure Cloud Environment name or metadata discovery URL')
         parser.add_argument('--resource-groups', action='store',

contrib/inventory/openstack_inventory.py

@@ -191,8 +191,13 @@ def is_cache_stale(cache_file, cache_expiration_time, refresh=False):
 
 
 def get_cache_settings(cloud=None):
-    config = cloud_config.OpenStackConfig(
-        config_files=cloud_config.CONFIG_FILES + CONFIG_FILES).get_one()
+    config_files = cloud_config.CONFIG_FILES + CONFIG_FILES
+    if cloud:
+        config = cloud_config.OpenStackConfig(
+            config_files=config_files).get_one(cloud=cloud)
+    else:
+        config = cloud_config.OpenStackConfig(
+            config_files=config_files).get_all()[0]
     # For inventory-wide caching
     cache_expiration_time = config.get_cache_expiration_time()
     cache_path = config.get_cache_path()

docs/bin/plugin_formatter.py

@@ -85,6 +85,28 @@ pp = PrettyPrinter()
 display = Display()
 
 
+# kludge_ns gives us a kludgey way to set variables inside of loops that need to be visible outside
+# the loop.  We can get rid of this when we no longer need to build docs with less than Jinja-2.10
+# http://jinja.pocoo.org/docs/2.10/templates/#assignments
+# With Jinja-2.10 we can use jinja2's namespace feature, restoring the namespace template portion
+# of: fa5c0282a4816c4dd48e80b983ffc1e14506a1f5
+NS_MAP = {}
+
+
+def to_kludge_ns(key, value):
+    NS_MAP[key] = value
+    return ""
+
+
+def from_kludge_ns(key):
+    return NS_MAP[key]
+
+
+# The max filter was added in Jinja2-2.10.  Until we can require that version, use this
+def do_max(seq):
+    return max(seq)
+
+
 def rst_ify(text):
     ''' convert symbols like I(this is in italics) to valid restructured text '''
 
@@ -298,6 +320,14 @@ def jinja2_environment(template_dir, typ, plugin_type):
                       trim_blocks=True)
     env.globals['xline'] = rst_xline
 
+    # Can be removed (and template switched to use namespace) when we no longer need to build
+    # with <Jinja-2.10
+    env.globals['to_kludge_ns'] = to_kludge_ns
+    env.globals['from_kludge_ns'] = from_kludge_ns
+    if 'max' not in env.filters:
+        # Jinja < 2.10
+        env.filters['max'] = do_max
+
     templates = {}
     if typ == 'rst':
         env.filters['convert_symbols_to_format'] = rst_ify

docs/docsite/_themes/sphinx_rtd_theme/layout.html

@@ -156,7 +156,7 @@
                 <div class="DocSiteProduct-logoText">
                     Ansible  
                     <div class="DocSiteProduct-CurrentVersion" align="right">
-                      devel
+                      2.6
                   </div>
                 </div>
             </div>

docs/docsite/_themes/sphinx_rtd_theme/static/css/theme.css

@@ -421,51 +421,17 @@ table {
        max-width: 100%;
 }
 
-
-.outer-elbow-container {
-    display: flex;
-    height: 100%;
-    flex-direction: row;
-}
-
-.elbow-placeholder {
+.documentation-table td.elbow-placeholder {
     border-left: 1px solid #000;
-    height: 100%;
+    border-top: 0px;
     width: 30px;
+    min-width: 30px;
 }
 
-.elbow-key {
-    height: 100%;
-    padding: 4px;
-    border-top: 1px solid #000;
-    flex-grow: 1;
-    border-left: 1px solid #000;
-}
-
-.elbow-blocker {
-    height: 0;
-    overflow: hidden;
-}
-
-.return-value-column {
-    height: 1px
-}
-
-.return-value-column td {
-    height: inherit
-}
-
-@-moz-document url-prefix() {
-    .return-value-column td {
-        height: 100%
-    }
-}
-
-.cell-border {
+.documentation-table th, .documentation-table td {
     padding: 4px;
     border-left: 1px solid #000;
     border-top: 1px solid #000;
-    height: 100%;
 }
 
 .documentation-table {

docs/docsite/_themes/srtd/static/css/theme.css

@@ -4867,50 +4867,17 @@ table {
     }
 }
 
-.outer-elbow-container {
-    display: flex;
-    height: 100%;
-    flex-direction: row;
-}
-
-.elbow-placeholder {
+.documentation-table td.elbow-placeholder {
     border-left: 1px solid #000;
-    height: 100%;
+    border-top: 0px;
     width: 30px;
+    min-width: 30px;
 }
 
-.elbow-key {
-    height: 100%;
-    padding: 4px;
-    border-top: 1px solid #000;
-    flex-grow: 1;
-    border-left: 1px solid #000;
-}
-
-.elbow-blocker {
-    height: 0;
-    overflow: hidden;
-}
-
-.return-value-column {
-    height: 1px
-}
-
-.return-value-column td {
-    height: inherit
-}
-
-@-moz-document url-prefix() {
-    .return-value-column td {
-        height: 100%
-    }
-}
-
-.cell-border {
+.documentation-table th, .documentation-table td {
     padding: 4px;
     border-left: 1px solid #000;
     border-top: 1px solid #000;
-    height: 100%;
 }
 
 .documentation-table {

docs/docsite/rst/dev_guide/testing/sanity/changelog.rst

@@ -0,0 +1,15 @@
+Sanity Tests » changelog
+========================
+
+Basic linting of changelog fragments with yamllint and rstcheck.
+
+One or more of the following sections are required:
+
+- major_changes
+- minor_changes
+- deprecated_features
+- removed_features
+- bugfixes
+- known_issues
+
+New modules and plugins must not be included in changelog fragments.

docs/docsite/rst/network/user_guide/platform_eos.rst

@@ -4,7 +4,7 @@
 EOS Platform Options
 ***************************************
 
-Arista EOS supports multiple connections. This page offers details on how each connection works in Ansible 2.5 and how to use it. 
+Arista EOS supports multiple connections. This page offers details on how each connection works in Ansible 2.6 and how to use it. 
 
 .. contents:: Topics
 
@@ -41,7 +41,7 @@ Connections Available
 
 For legacy playbooks, EOS still supports ``ansible_connection: local``. We recommend modernizing to use ``ansible_connection: network_cli`` or ``ansible_connection: httpapi`` as soon as possible.
 
-Using CLI in Ansible 2.5
+Using CLI in Ansible 2.6
 ================================================================================
 
 Example CLI ``group_vars/eos.yml``
@@ -76,7 +76,7 @@ Example CLI Task
 
 
 
-Using eAPI in Ansible 2.5
+Using eAPI in Ansible 2.6
 ================================================================================
 
 Enabling eAPI

docs/docsite/rst/network/user_guide/platform_ios.rst

@@ -4,7 +4,7 @@
 IOS Platform Options
 ***************************************
 
-IOS supports Enable Mode (Privilege Escalation). This page offers details on how to use Enable Mode on IOS in Ansible 2.5. 
+IOS supports Enable Mode (Privilege Escalation). This page offers details on how to use Enable Mode on IOS in Ansible 2.6. 
 
 .. contents:: Topics
 
@@ -34,7 +34,7 @@ Connections Available
 
 For legacy playbooks, IOS still supports ``ansible_connection: local``. We recommend modernizing to use ``ansible_connection: network_cli`` as soon as possible.
 
-Using CLI in Ansible 2.5
+Using CLI in Ansible 2.6
 ================================================================================
 
 Example CLI ``group_vars/ios.yml``

docs/docsite/rst/network/user_guide/platform_junos.rst

@@ -4,7 +4,7 @@
 Junos OS Platform Options
 ***************************************
 
-Juniper Junos OS supports multiple connections. This page offers details on how each connection works in Ansible 2.5 and how to use it. 
+Juniper Junos OS supports multiple connections. This page offers details on how each connection works in Ansible 2.6 and how to use it. 
 
 .. contents:: Topics
 
@@ -34,7 +34,7 @@ Connections Available
 
 For legacy playbooks, Ansible still supports ``ansible_connection=local`` on all JUNOS modules. We recommend modernizing to use ``ansible_connection=netconf`` or ``ansible_connection=network_cli`` as soon as possible.
 
-Using CLI in Ansible 2.5
+Using CLI in Ansible 2.6
 ================================================================================
 
 Example CLI inventory ``[junos:vars]``
@@ -65,7 +65,7 @@ Example CLI Task
      when: ansible_network_os == 'junos'
 
 
-Using NETCONF in Ansible 2.5
+Using NETCONF in Ansible 2.6
 ================================================================================
 
 Enabling NETCONF

docs/docsite/rst/network/user_guide/platform_nxos.rst

@@ -4,7 +4,7 @@
 NXOS Platform Options
 ***************************************
 
-Cisco NXOS supports multiple connections. This page offers details on how each connection works in Ansible 2.5 and how to use it.
+Cisco NXOS supports multiple connections. This page offers details on how each connection works in Ansible 2.6 and how to use it.
 
 .. contents:: Topics
 
@@ -22,6 +22,10 @@ Connections Available
 | **Indirect Access**       | via a bastion (jump host)                     | via a web proxy                         |
 +---------------------------+-----------------------------------------------+-----------------------------------------+
 | | **Connection Settings** | | ``ansible_connection: network_cli``         | | ``ansible_connection: httpapi``       |
+| |                         | |                                             | | OR                                    |
+| |                         | |                                             | | ``ansible_connection: local``         |
+| |                         | |                                             | | with ``transport: nxapi``             |
+| |                         | |                                             | | in the ``provider`` dictionary        |
 +---------------------------+-----------------------------------------------+-----------------------------------------+
 | | **Enable Mode**         | | supported - use ``ansible_become: yes``     | | not supported by NX-API               |
 | | (Privilege Escalation)  | | with ``ansible_become_method: enable``      | |                                       |
@@ -32,7 +36,7 @@ Connections Available
 
 For legacy playbooks, NXOS still supports ``ansible_connection: local``. We recommend modernizing to use ``ansible_connection: network_cli`` or ``ansible_connection: httpapi`` as soon as possible.
 
-Using CLI in Ansible 2.5
+Using CLI in Ansible 2.6
 ================================================================================
 
 Example CLI ``group_vars/nxos.yml``
@@ -67,7 +71,7 @@ Example CLI Task
 
 
 
-Using NX-API in Ansible 2.5
+Using NX-API in Ansible 2.6
 ================================================================================
 
 Enabling NX-API

docs/docsite/rst/porting_guides/porting_guide_2.5.rst

@@ -91,6 +91,11 @@ You will run into errors because Ansible reads name in this context as a keyword
 
 For a full list of keywords see ::ref::`Playbook Keywords`.
 
+Migrating from with_X to loop
+-----------------------------
+
+.. include:: ../user_guide/shared_snippets/with2loop.txt
+
 
 Deprecated
 ==========
@@ -197,6 +202,12 @@ desired.
   * The :ref:`blockinfile module <blockinfile_module>` had its ``follow`` parameter removed because
     it inherently modifies the content of an existing file so it makes no sense to operate on the
     link itself.
+  * In Ansible-2.5.3, the :ref:`template module <template_module>` became more strict about its
+    ``src`` file being proper utf-8.  Previously, non-utf8 contents in a template module src file
+    would result in a mangled output file (the non-utf8 characters would be replaced with a unicode
+    replacement character).  Now, on Python2, the module will error out with the message, "Template
+    source files must be utf-8 encoded".  On Python3, the module will first attempt to pass the
+    non-utf8 characters through verbatim and fail if that does not succeed.
 
 Plugins
 =======

docs/docsite/rst/porting_guides/porting_guide_2.6.rst

@@ -42,6 +42,9 @@ Deprecation notices
 
 The following modules will be removed in Ansible 2.10. Please update your playbooks accordingly.
 
+* ``k8s_raw`` use :ref:`k8s <k8s_module>` instead.
+* ``openshift_raw`` use :ref:`k8s <k8s_module>` instead.
+* ``openshift_scale`` use :ref:`k8s_scale <k8s_scale_module>` instead.
 
 Noteworthy module changes
 -------------------------
@@ -62,12 +65,34 @@ Noteworthy module changes
   destination path like this::
 
     $ ansible localhost -m file -a 'path=/tmp/lib state=directory'
+* The ``k8s_raw`` and ``openshift_raw`` modules have been aliased to the new ``k8s`` module.
+* The ``k8s`` module supports all Kubernetes resources including those from Custom Resource Definitions and aggregated API servers. This includes all OpenShift resources.
+* The ``k8s`` module will not accept resources where subkeys have been snake_cased. This was a workaround that was suggested with the ``k8s_raw`` and ``openshift_raw`` modules.
+* The ``k8s`` module may not accept resources where the ``api_version`` has been changed to match the shortened version in the Kubernetes Python client. You should now specify the proper full Kubernetes ``api_version`` for a resource.
+* The ``k8s`` module can now process multi-document YAML files if they are passed with the ``src`` parameter. It will process each document as a separate resource. Resources provided inline with the ``resource_definition`` parameter must still be a single document.
+* The ``k8s`` module will not automatically change ``Project`` creation requests into ``ProjectRequest`` creation requests as the ``openshift_raw`` module did. You must now specify the ``ProjectRequest`` kind explicitly.
+* The ``k8s`` module will not automatically remove secrets from the Ansible return values (and by extension the log). In order to prevent secret values in a task from being logged, specify the ``no_log`` parameter on the task block.
+* The ``k8s_scale`` module now supports scalable OpenShift objects, such as ``DeploymentConfig``.
 
 
 Plugins
 =======
 
-No notable changes.
+Deprecation notices
+-------------------
+
+The following modules will be removed in Ansible 2.10. Please update your playbooks accordingly.
+
+* ``openshift`` use ``k8s`` instead.
+
+
+Noteworthy plugin changes
+-------------------------
+
+* The ``k8s`` lookup plugin now supports all Kubernetes resources including those from Custom Resource Definitions and aggregated API servers. This includes all OpenShift resources.
+* The ``k8s`` lookup plugin may not accept resources where the ``api_version`` has been changed to match the shortened version in the Kubernetes Python client. You should now specify the proper full Kubernetes ``api_version`` for a resource.
+* The ``k8s`` lookup plugin will no longer remove secrets from the Ansible return values (and by extension the log). In order to prevent secret values in a task from being logged, specify the ``no_log`` parameter on the task block.
+
 
 Porting custom scripts
 ======================

docs/docsite/rst/user_guide/playbooks_blocks.rst

@@ -50,7 +50,7 @@ Blocks also introduce the ability to handle errors in a way similar to exception
 
 
   tasks:
-  - name: Attempt and gracefull roll back demo
+  - name: Attempt and graceful roll back demo
     block:
       - debug:
           msg: 'I execute normally'
@@ -65,7 +65,7 @@ Blocks also introduce the ability to handle errors in a way similar to exception
           msg: 'I also never execute :-('
     always:
       - debug:
-          msg: "this always executes"
+          msg: "This always executes"
 
 The tasks in the ``block`` would execute normally, if there is any error the ``rescue`` section would get executed
 with whatever you need to do to recover from the previous error. The ``always`` section runs no matter what previous
@@ -81,7 +81,7 @@ Another example is how to run handlers after an error occurred :
 
 
   tasks:
-    - name: Attempt and gracefull roll back demo
+    - name: Attempt and graceful roll back demo
       block:
         - debug:
             msg: 'I execute normally'
@@ -93,7 +93,7 @@ Another example is how to run handlers after an error occurred :
   handlers:
      - name: run me even after an error
        debug:
-         msg: 'this handler runs even on error'
+         msg: 'This handler runs even on error'
 
 .. seealso::
 

docs/docsite/rst/user_guide/playbooks_loops.rst

@@ -105,7 +105,7 @@ For example, using the 'nested' lookup, you can combine lists::
         priv: "{{ item[1] }}.*:ALL"
         append_privs: yes
         password: "foo"
-      loop: "{{ query('nested', [ 'alice', 'bob' ], [ 'clientdb', 'employeedb', 'providerdb' ]) }}"
+      loop: "{{ ['alice', 'bob'] |product(['clientdb', 'employeedb', 'providerdb'])|list }}"
 
 .. note:: ``with_`` loops are actually a combination of things ``with_`` + ``lookup()``, even ``items`` is a lookup. ``loop`` can be used in the same way as shown above.
 
@@ -121,9 +121,9 @@ In certain situations the ``lookup`` function may not return a list which ``loop
 
 The following invocations are equivalent, using ``wantlist=True`` with ``lookup`` to ensure a return type of a list::
 
-    loop: "{{ query('nested', ['alice', 'bob'], ['clientdb', 'employeedb', 'providerdb']) }}"
+    loop: "{{ query('inventory_hostnames', 'all') }}"
 
-    loop: "{{ lookup('nested', ['alice', 'bob'], ['clientdb', 'employeedb', 'providerdb'], wantlist=True) }}"
+    loop: "{{ lookup('inventory_hostnames', 'all', wantlist=True) }}"
 
 
 .. _do_until_loops:
@@ -330,6 +330,11 @@ If you need to keep track of where you are in a loop, you can use the ``index_va
       loop_control:
         index_var: my_idx
 
+Migrating from with_X to loop
+`````````````````````````````
+
+.. include:: shared_snippets/with2loop.txt
+
 
 .. seealso::
 

docs/docsite/rst/user_guide/playbooks_reuse_includes.rst

@@ -13,7 +13,7 @@ As noted in :doc:`playbooks_reuse`, include and import statements are very simil
 
 Please refer to  :doc:`playbooks_reuse` for documentation concerning the trade-offs one may encounter when using each type.
 
-Also be aware that this behaviour changed in 2.4; prior to that Ansible version only ``include`` was available, and it behaved differently depending on context.
+Also be aware that this behaviour changed in 2.4. Prior to Ansible 2.4, only ``include`` was available and it behaved differently depending on context.
 
 .. versionadded:: 2.4
 
@@ -22,7 +22,6 @@ Importing Playbooks
 
 It is possible to include playbooks inside a master playbook. For example::
 
-    ---
     - import_playbook: webservers.yml
     - import_playbook: databases.yml
 
@@ -36,16 +35,15 @@ Prior to 2.4 only ``include`` was available and worked for both playbooks and ta
 Including and Importing Task Files
 ``````````````````````````````````
 
-Use of included task lists is a great way to define a role that system is going to fulfill. A task include file simply contains a flat list of tasks::
+Breaking tasks up into different files is an excellent way to organize complex sets of tasks or reuse them. A task file simply contains a flat list of tasks::
 
     # common_tasks.yml
-    ---
     - name: placeholder foo
       command: /bin/foo
     - name: placeholder bar
       command: /bin/bar
 
-You can then use ``import_tasks`` or ``include_tasks`` to include this file in your main task list::
+You can then use ``import_tasks`` or ``include_tasks`` to execute the tasks in a file in the main task list::
 
     tasks:
     - import_tasks: common_tasks.yml
@@ -65,27 +63,17 @@ You can also pass variables into imports and includes::
       vars:
         wp_user: bob
 
-Variables can also be passed to include files using an alternative syntax, which also supports structured variables like dictionaries and lists::
+See :ref:`ansible_variable_precedence` for more details on variable inheritance and precedence.
 
-    tasks:
-    - include_tasks: wordpress.yml
-      vars:
-        wp_user: timmy
-        ssh_keys:
-        - "{{ lookup('file', 'keys/one.pub') }}"
-        - "{{ lookup('file', 'keys/two.pub') }}"
-
-Using either syntax, variables passed in can then be used in the included files. These variables will only be available to tasks within the included file. See :ref:`ansible_variable_precedence` for more details on variable inheritance and precedence.
-
-Task include statements can be used at arbitrary depth.
+Task include and import statements can be used at arbitrary depth.
 
 .. note::
-    Static and dynamic can be mixed, however this is not recommended as it may lead to difficult-to-diagnose bugs in your playbooks.
+    - Static and dynamic can be mixed, however this is not recommended as it may lead to difficult-to-diagnose bugs in your playbooks.
+    - The ``key=value`` syntax for passing variables to import and include is deprecated. Use YAML ``vars:`` instead.
 
-Includes and imports can also be used in the ``handlers:`` section; for instance, if you want to define how to restart apache, you only have to do that once for all of your playbooks.  You might make a handlers.yml that looks like::
+Includes and imports can also be used in the ``handlers:`` section. For instance, if you want to define how to restart Apache, you only have to do that once for all of your playbooks. You might make a ``handlers.yml`` that looks like::
 
    # more_handlers.yml
-   ---
    - name: restart apache
      service: name=apache state=restarted
 

docs/docsite/rst/user_guide/playbooks_tests.rst

@@ -119,7 +119,7 @@ You can use `any` and `all` to check if any or all elements in a list are true o
   vars:
     mylist:
         - 1
-        - 3 == 3
+        - "{{ 3 == 3 }}"
         - True
     myotherlist:
         - False

docs/docsite/rst/user_guide/playbooks_variables.rst

@@ -868,7 +868,7 @@ Basically, anything that goes into "role defaults" (the defaults folder inside t
           If you define a variable twice in a play's ``vars:`` section, the second one wins.
 .. note:: The previous describes the default config ``hash_behaviour=replace``, switch to ``merge`` to only partially overwrite.
 .. note:: Group loading follows parent/child relationships. Groups of the same 'patent/child' level are then merged following alphabetical order.
-          This last one can be superceeded by the user via ``ansible_group_priority``, which defaults to 0 for all groups.
+          This last one can be superceeded by the user via ``ansible_group_priority``, which defaults to ``1`` for all groups.
 
 
 Another important thing to consider (for all versions) is that connection variables override config, command line and play/role/task specific options and keywords.  For example::

docs/docsite/rst/user_guide/shared_snippets/with2loop.txt

@@ -0,0 +1,193 @@
+With the release of Ansible 2.5, the recommended way to perform loops is the use the new ``loop`` keyword instead of ``with_X`` style loops.
+
+In many cases, ``loop`` syntax is better expressed using filters instead of more complex use of ``query`` or ``lookup``.
+
+The following examples will show how to convert many common ``with_`` style loops to ``loop`` and filters.
+
+with_list
+---------
+
+``with_list`` is directly replaced by ``loop``.
+
+.. code-block:: yaml+jinja
+
+    - name: with_list
+      debug:
+        msg: "{{ item }}"
+      with_list:
+        - one
+        - two
+
+    - name: with_list -> loop
+      debug:
+        msg: "{{ item }}"
+      loop:
+        - one
+        - two
+
+with_items
+----------
+
+``with_items`` is replaced by ``loop`` and the ``flatten`` filter.
+
+.. code-block:: yaml+jinja
+
+    - name: with_items
+      debug:
+        msg: "{{ item }}"
+      with_items: "{{ items }}"
+
+    - name: with_items -> loop
+      debug:
+        msg: "{{ item }}"
+      loop: "{{ items|flatten(levels=1) }}"
+
+with_indexed_items
+------------------
+
+``with_indexed_items`` is replaced by ``loop``, the ``flatten`` filter and ``loop_control.index_var``.
+
+.. code-block:: yaml+jinja
+
+    - name: with_indexed_items
+      debug:
+        msg: "{{ item.0 }} - {{ item.1 }}"
+      with_indexed_items: "{{ items }}"
+
+    - name: with_indexed_items -> loop
+      debug:
+        msg: "{{ index }} - {{ item }}"
+      loop: "{{ items|flatten(levels=1) }}"
+      loop_control:
+        index_var: index
+
+with_flattened
+--------------
+
+``with_flattened`` is replaced by ``loop`` and the ``flatten`` filter.
+
+.. code-block:: yaml+jinja
+
+    - name: with_flattened
+      debug:
+        msg: "{{ item }}"
+      with_flattened: "{{ items }}"
+
+    - name: with_flattened -> loop
+      debug:
+        msg: "{{ item }}"
+      loop: "{{ items|flatten }}"
+
+with_together
+-------------
+
+``with_together`` is replaced by ``loop`` and the ``zip`` filter.
+
+.. code-block:: yaml+jinja
+
+    - name: with_together
+      debug:
+        msg: "{{ item.0 }} - {{ item.1 }}"
+      with_together:
+        - "{{ list_one }}"
+        - "{{ list_two }}"
+
+    - name: with_together -> loop
+      debug:
+        msg: "{{ item.0 }} - {{ item.1 }}"
+      loop: "{{ list_one|zip(list_two)|list }}"
+
+with_dict
+---------
+
+``with_dict`` can be substituted by ``loop`` and either the ``dictsort`` or ``dict2items`` filters.
+
+.. code-block:: yaml+jinja
+
+    - name: with_dict
+      debug:
+        msg: "{{ item.key }} - {{ item.value }}"
+      with_dict: "{{ dictionary }}"
+
+    - name: with_dict -> loop (option 1)
+      debug:
+        msg: "{{ item.key }} - {{ item.value }}"
+      loop: "{{ dictionary|dict2items }}"
+
+    - name: with_dict -> loop (option 2)
+      debug:
+        msg: "{{ item.0 }} - {{ item.1 }}"
+      loop: "{{ dictionary|dictsort }}"
+
+with_sequence
+-------------
+
+``with_sequence`` is replaced by ``loop`` and the ``range`` function, and potentially the ``format`` filter.
+
+.. code-block:: yaml+jinja
+
+    - name: with_sequence
+      debug:
+        msg: "{{ item }}"
+      with_sequence: start=0 end=4 stride=2 format=testuser%02x
+
+    - name: with_sequence -> loop
+      debug:
+        msg: "{{ 'testuser%02x' | format(item) }}"
+      # range is exclusive of the end point
+      loop: "{{ range(0, 4 + 1, 2)|list }}"
+
+with_subelements
+----------------
+
+``with_subelements`` is replaced by ``loop`` and the ``subelements`` filter.
+
+.. code-block:: yaml+jinja
+
+    - name: with_subelements
+      debug:
+        msg: "{{ item.0.name }} - {{ item.1 }}"
+      with_subelements:
+        - "{{ users }}"
+        - mysql.hosts
+
+    - name: with_subelements -> loop
+      debug:
+        msg: "{{ item.0.name }} - {{ item.1 }}"
+      loop: "{{ users|subelements('mysql.hosts') }}"
+
+with_nested/with_cartesian
+--------------------------
+
+``with_nested`` and ``with_cartesian`` are replaced by loop and the ``product`` filter.
+
+.. code-block:: yaml+jinja
+
+    - name: with_nested
+      debug:
+        msg: "{{ item.0 }} - {{ item.1 }}"
+      with_nested:
+        - "{{ list_one }}"
+        - "{{ list_two }}"
+
+    - name: with_nested -> loop
+      debug:
+        msg: "{{ item.0 }} - {{ item.1 }}"
+      loop: "{{ list_one|product(list_two)|list }}"
+
+with_random_choice
+------------------
+
+``with_random_choice`` is replaced by just use of the ``random`` filter, without need of ``loop``.
+
+.. code-block:: yaml+jinja
+
+    - name: with_random_choice
+      debug:
+        msg: "{{ item }}"
+      with_random_choice: "{{ my_list }}"
+
+    - name: with_random_choice -> loop (No loop is needed here)
+      debug:
+        msg: "{{ my_list|random }}"
+      tags: random

docs/docsite/rst/user_guide/windows_dsc.rst

@@ -128,7 +128,7 @@ a custom class defined by that resource. Defining a value that takes in a
 For example, to define a ``[CimInstance]`` value in Ansible::
 
     # [CimInstance]AuthenticationInfo == MSFT_xWebAuthenticationInformation
-    AuthenticationInfo
+    AuthenticationInfo:
       Anonymous: no
       Basic: yes
       Digest: no

docs/templates/plugin.rst.j2

@@ -88,104 +88,107 @@ Parameters
 .. raw:: html
 
     <table  border=0 cellpadding=0 class="documentation-table">
-        {# Header of the documentation #}
+        {# Pre-compute the nesting depth to allocate columns -#}
+        @{ to_kludge_ns('maxdepth', 1) -}@
+        {% for key, value in options|dictsort recursive -%}
+            @{ to_kludge_ns('maxdepth', [loop.depth, from_kludge_ns('maxdepth')] | max) -}@
+            {% if value.suboptions -%}
+                {% if value.suboptions.items -%}
+                    @{ loop(value.suboptions.items()) -}@
+                {% elif value.suboptions[0].items -%}
+                    @{ loop(value.suboptions[0].items()) -}@
+                {% endif -%}
+            {% endif -%}
+        {% endfor -%}
+        {# Header of the documentation -#}
         <tr>
-            <th class="head"><div class="cell-border">Parameter</div></th>
-            <th class="head"><div class="cell-border">Choices/<font color="blue">Defaults</font></div></th>
+            <th colspan="@{ from_kludge_ns('maxdepth') }@">Parameter</th>
+            <th>Choices/<font color="blue">Defaults</font></th>
             {% if plugin_type != 'module' %}
-                <th class="head"><div class="cell-border">Configuration</div></th>
+                <th>Configuration</th>
             {% endif %}
-            <th class="head" width="100%"><div class="cell-border">Comments</div></th>
+            <th width="100%">Comments</th>
         </tr>
         {% for key, value in options|dictsort recursive %}
-            <tr class="return-value-column">
+            <tr>
+                {# indentation based on nesting level #}
+                {% for i in range(1, loop.depth) %}
+                    <td class="elbow-placeholder"></td>
+                {% endfor %}
                 {# parameter name with required and/or introduced label #}
-                <td>
-                    <div class="outer-elbow-container">
-                        {% for i in range(1, loop.depth) %}
-                            <div class="elbow-placeholder">&nbsp;</div>
-                        {% endfor %}
-                        <div class="elbow-key">
-                            <b>@{ key }@</b>
-                            {% if value.get('required', False) %}<br/><div style="font-size: small; color: red">required</div>{% endif %}
-                            {% if value.version_added %}<br/><div style="font-size: small; color: darkgreen">(added in @{value.version_added}@)</div>{% endif %}
-                        </div>
-                    </div>
+                <td colspan="@{ from_kludge_ns('maxdepth') - loop.depth0 }@">
+                    <b>@{ key }@</b>
+                    {% if value.get('required', False) %}<br/><div style="font-size: small; color: red">required</div>{% endif %}
+                    {% if value.version_added %}<br/><div style="font-size: small; color: darkgreen">(added in @{value.version_added}@)</div>{% endif %}
                 </td>
                 {# default / choices #}
                 <td>
-                    <div class="cell-border">
-                        {# Turn boolean values in 'yes' and 'no' values #}
-                        {% if value.default is sameas true %}
-                            {% set _x = value.update({'default': 'yes'}) %}
-                        {% elif value.default is sameas false %}
-                            {% set _x = value.update({'default': 'no'}) %}
-                        {% endif %}
-                        {% if value.type == 'bool' %}
-                            {% set _x = value.update({'choices': ['no', 'yes']}) %}
-                        {% endif %}
-                        {# Show possible choices and highlight details #}
-                        {% if value.choices %}
-                            <ul><b>Choices:</b>
-                                {% for choice in value.choices %}
-                                    {# Turn boolean values in 'yes' and 'no' values #}
-                                    {% if choice is sameas true %}
-                                        {% set choice = 'yes' %}
-                                    {% elif choice is sameas false %}
-                                        {% set choice = 'no' %}
-                                    {% endif %}
-                                    {% if (value.default is string and value.default == choice) or (value.default is iterable and value.default is not string and choice in value.default) %}
-                                        <li><div style="color: blue"><b>@{ choice | escape }@</b>&nbsp;&larr;</div></li>
-                                    {% else %}
-                                        <li>@{ choice | escape }@</li>
-                                    {% endif %}
-                                {% endfor %}
-                            </ul>
-                        {% endif %}
-                        {# Show default value, when multiple choice or no choices #}
-                        {% if value.default is defined and value.default not in value.choices %}
-                            <b>Default:</b><br/><div style="color: blue">@{ value.default | escape }@</div>
-                        {% endif %}
-                    </div>
+                    {# Turn boolean values in 'yes' and 'no' values #}
+                    {% if value.default is sameas true %}
+                        {% set _x = value.update({'default': 'yes'}) %}
+                    {% elif value.default is sameas false %}
+                        {% set _x = value.update({'default': 'no'}) %}
+                    {% endif %}
+                    {% if value.type == 'bool' %}
+                        {% set _x = value.update({'choices': ['no', 'yes']}) %}
+                    {% endif %}
+                    {# Show possible choices and highlight details #}
+                    {% if value.choices %}
+                        <ul><b>Choices:</b>
+                            {% for choice in value.choices %}
+                                {# Turn boolean values in 'yes' and 'no' values #}
+                                {% if choice is sameas true %}
+                                    {% set choice = 'yes' %}
+                                {% elif choice is sameas false %}
+                                    {% set choice = 'no' %}
+                                {% endif %}
+                                {% if (value.default is string and value.default == choice) or (value.default is iterable and value.default is not string and choice in value.default) %}
+                                    <li><div style="color: blue"><b>@{ choice | escape }@</b>&nbsp;&larr;</div></li>
+                                {% else %}
+                                    <li>@{ choice | escape }@</li>
+                                {% endif %}
+                            {% endfor %}
+                        </ul>
+                    {% endif %}
+                    {# Show default value, when multiple choice or no choices #}
+                    {% if value.default is defined and value.default not in value.choices %}
+                        <b>Default:</b><br/><div style="color: blue">@{ value.default | escape }@</div>
+                    {% endif %}
                 </td>
                 {# configuration #}
                 {% if plugin_type != 'module' %}
                     <td>
-                        <div class="cell-border">
-                            {% if 'ini' in value %}
-                                <div> ini entries:
-                                    {% for ini in value.ini %}
-                                        <p>[@{ ini.section }@ ]<br>@{ ini.key }@ = @{ value.default | default('VALUE') }@</p>
-                                    {% endfor %}
-                                </div>
-                            {% endif %}
-                            {% if 'env' in value %}
-                                {% for env in value.env %}
-                                    <div>env:@{ env.name }@</div>
+                        {% if 'ini' in value %}
+                            <div> ini entries:
+                                {% for ini in value.ini %}
+                                    <p>[@{ ini.section }@ ]<br>@{ ini.key }@ = @{ value.default | default('VALUE') }@</p>
                                 {% endfor %}
-                            {% endif %}
-                            {% if 'vars' in value %}
-                                {% for myvar in value.vars %}
-                                    <div>var: @{ myvar.name }@</div>
-                                {% endfor %}
-                            {% endif %}
-                        </div>
+                            </div>
+                        {% endif %}
+                        {% if 'env' in value %}
+                            {% for env in value.env %}
+                                <div>env:@{ env.name }@</div>
+                            {% endfor %}
+                        {% endif %}
+                        {% if 'vars' in value %}
+                            {% for myvar in value.vars %}
+                                <div>var: @{ myvar.name }@</div>
+                            {% endfor %}
+                        {% endif %}
                     </td>
                 {% endif %}
                 {# description #}
                 <td>
-                    <div class="cell-border">
-                        {% if value.description is string %}
-                            <div>@{ value.description | replace('\n', '\n    ') | html_ify }@</div>
-                        {% else %}
-                            {% for desc in value.description %}
-                                <div>@{ desc | replace('\n', '\n    ') | html_ify }@</div>
-                            {% endfor %}
-                        {% endif %}
-                        {% if 'aliases' in value and value.aliases %}
-                            <div style="font-size: small; color: darkgreen"><br/>aliases: @{ value.aliases|join(', ') }@</div>
-                        {% endif %}
-                    </div>
+                    {% if value.description is string %}
+                        <div>@{ value.description | replace('\n', '\n    ') | html_ify }@</div>
+                    {% else %}
+                        {% for desc in value.description %}
+                            <div>@{ desc | replace('\n', '\n    ') | html_ify }@</div>
+                        {% endfor %}
+                    {% endif %}
+                    {% if 'aliases' in value and value.aliases %}
+                        <div style="font-size: small; color: darkgreen"><br/>aliases: @{ value.aliases|join(', ') }@</div>
+                    {% endif %}
                 </td>
             </tr>
             {% if value.suboptions %}
@@ -242,43 +245,49 @@ Facts returned by this module are added/updated in the ``hostvars`` host facts a
 .. raw:: html
 
     <table border=0 cellpadding=0 class="documentation-table">
+        {# Pre-compute the nesting depth to allocate columns #}
+        @{ to_kludge_ns('maxdepth', 1) -}@
+        {% for key, value in returnfacts|dictsort recursive %}
+            @{ to_kludge_ns('maxdepth', [loop.depth, from_kludge_ns('maxdepth')] | max) -}@
+            {% if value.contains -%}
+                {% if value.contains.items -%}
+                    @{ loop(value.contains.items()) -}@
+                {% elif value.contains[0].items -%}
+                    @{ loop(value.contains[0].items()) -}@
+                {% endif -%}
+            {% endif -%}
+        {% endfor -%}
         <tr>
-            <th class="head"><div class="cell-border">Fact</div></th>
-            <th class="head"><div class="cell-border">Returned</div></th>
-            <th class="head" width="100%"><div class="cell-border">Description</div></th>
+            <th colspan="@{ from_kludge_ns('maxdepth') }@">Fact</th>
+            <th>Returned</th>
+            <th width="100%">Description</th>
         </tr>
         {% for key, value in returnfacts|dictsort recursive %}
-            <tr class="return-value-column">
-                <td>
-                    <div class="outer-elbow-container">
-                        {% for i in range(1, loop.depth) %}
-                            <div class="elbow-placeholder">&nbsp;</div>
-                        {% endfor %}
-                        <div class="elbow-key">
-                            <b>@{ key }@</b>
-                            <br/><div style="font-size: small; color: red">@{ value.type }@</div>
-                        </div>
-                    </div>
+            <tr>
+                {% for i in range(1, loop.depth) %}
+                    <td class="elbow-placeholder"></td>
+                {% endfor %}
+                <td colspan="@{ from_kludge_ns('maxdepth') - loop.depth0 }@" colspan="@{ from_kludge_ns('maxdepth') - loop.depth0 }@">
+                    <b>@{ key }@</b>
+                    <br/><div style="font-size: small; color: red">@{ value.type }@</div>
                 </td>
-                <td><div class="cell-border">@{ value.returned | html_ify }@</div></td>
+                <td>@{ value.returned | html_ify }@</td>
                 <td>
-                    <div class="cell-border">
-                        {% if value.description is string %}
-                            <div>@{ value.description | html_ify }@
+                    {% if value.description is string %}
+                        <div>@{ value.description | html_ify }@
+                        </div>
+                    {% else %}
+                        {% for desc in value.description %}
+                            <div>@{ desc | html_ify }@
                             </div>
-                        {% else %}
-                            {% for desc in value.description %}
-                                <div>@{ desc | html_ify }@
-                                </div>
-                            {% endfor %}
-                        {% endif %}
-                        <br/>
-                        {% if value.sample is defined and value.sample %}
-                            <div style="font-size: smaller"><b>Sample:</b></div>
-                            {# TODO: The sample should be escaped, using | escape or | htmlify, but both mess things up beyond repair with dicts #}
-                            <div style="font-size: smaller; color: blue; word-wrap: break-word; word-break: break-all;">@{ value.sample | replace('\n', '\n    ') | html_ify }@</div>
-                        {% endif %}
-                    </div>
+                        {% endfor %}
+                    {% endif %}
+                    <br/>
+                    {% if value.sample is defined and value.sample %}
+                        <div style="font-size: smaller"><b>Sample:</b></div>
+                        {# TODO: The sample should be escaped, using | escape or | htmlify, but both mess things up beyond repair with dicts #}
+                        <div style="font-size: smaller; color: blue; word-wrap: break-word; word-break: break-all;">@{ value.sample | replace('\n', '\n    ') | html_ify }@</div>
+                    {% endif %}
                 </td>
             </tr>
             {# ---------------------------------------------------------
@@ -308,41 +317,46 @@ Common return values are documented :ref:`here <common_return_values>`, the foll
 .. raw:: html
 
     <table border=0 cellpadding=0 class="documentation-table">
+        @{ to_kludge_ns('maxdepth', 1) -}@
+        {% for key, value in returndocs|dictsort recursive -%}
+            @{ to_kludge_ns('maxdepth', [loop.depth, from_kludge_ns('maxdepth')] | max) -}@
+            {% if value.contains -%}
+                {% if value.contains.items -%}
+                    @{ loop(value.contains.items()) -}@
+                {% elif value.contains[0].items -%}
+                    @{ loop(value.contains[0].items()) -}@
+                {% endif -%}
+            {% endif -%}
+        {% endfor -%}
         <tr>
-            <th class="head"><div class="cell-border">Key</div></th>
-            <th class="head"><div class="cell-border">Returned</div></th>
-            <th class="head" width="100%"><div class="cell-border">Description</div></th>
+            <th colspan="@{ from_kludge_ns('maxdepth') }@">Key</th>
+            <th>Returned</th>
+            <th width="100%">Description</th>
         </tr>
         {% for key, value in returndocs|dictsort recursive %}
-            <tr class="return-value-column">
-                <td>
-                    <div class="outer-elbow-container">
-                        {% for i in range(1, loop.depth) %}
-                            <div class="elbow-placeholder">&nbsp;</div>
-                        {% endfor %}
-                        <div class="elbow-key">
-                            <b>@{ key }@</b>
-                            <br/><div style="font-size: small; color: red">@{ value.type }@</div>
-                        </div>
-                    </div>
+            <tr>
+                {% for i in range(1, loop.depth) %}
+                    <td class="elbow-placeholder">&nbsp;</td>
+                {% endfor %}
+                <td colspan="@{ from_kludge_ns('maxdepth') - loop.depth0 }@">
+                    <b>@{ key }@</b>
+                    <br/><div style="font-size: small; color: red">@{ value.type }@</div>
                 </td>
-                <td><div class="cell-border">@{ value.returned | html_ify }@</div></td>
+                <td>@{ value.returned | html_ify }@</td>
                 <td>
-                    <div class="cell-border">
-                        {% if value.description is string %}
-                            <div>@{ value.description | html_ify |indent(4)}@</div>
-                        {% else %}
-                            {% for desc in value.description %}
-                                <div>@{ desc | html_ify |indent(4)}@</div>
-                            {% endfor %}
-                        {% endif %}
-                        <br/>
-                        {% if value.sample is defined and value.sample %}
-                            <div style="font-size: smaller"><b>Sample:</b></div>
-                            {# TODO: The sample should be escaped, using |escape or |htmlify, but both mess things up beyond repair with dicts #}
-                            <div style="font-size: smaller; color: blue; word-wrap: break-word; word-break: break-all;">@{ value.sample | replace('\n', '\n    ') | html_ify }@</div>
-                        {% endif %}
-                    </div>
+                    {% if value.description is string %}
+                        <div>@{ value.description | html_ify |indent(4)}@</div>
+                    {% else %}
+                        {% for desc in value.description %}
+                            <div>@{ desc | html_ify |indent(4)}@</div>
+                        {% endfor %}
+                    {% endif %}
+                    <br/>
+                    {% if value.sample is defined and value.sample %}
+                        <div style="font-size: smaller"><b>Sample:</b></div>
+                        {# TODO: The sample should be escaped, using |escape or |htmlify, but both mess things up beyond repair with dicts #}
+                        <div style="font-size: smaller; color: blue; word-wrap: break-word; word-break: break-all;">@{ value.sample | replace('\n', '\n    ') | html_ify }@</div>
+                    {% endif %}
                 </td>
             </tr>
             {# ---------------------------------------------------------

hacking/test-module

@@ -125,7 +125,7 @@ def boilerplate_module(modfile, args, interpreters, check, destfile):
 
     # default selinux fs list is pass in as _ansible_selinux_special_fs arg
     complex_args['_ansible_selinux_special_fs'] = C.DEFAULT_SELINUX_SPECIAL_FS
-    complex_args['_ansible_tmp'] = C.DEFAULT_LOCAL_TMP
+    complex_args['_ansible_tmpdir'] = C.DEFAULT_LOCAL_TMP
     complex_args['_ansible_keep_remote_files'] = C.DEFAULT_KEEP_REMOTE_FILES
 
     if args.startswith("@"):

lib/ansible/cli/__init__.py

@@ -180,9 +180,11 @@ class CLI(with_metaclass(ABCMeta, object)):
             ver = deprecated[1]['version']
             display.deprecated("%s option, %s %s" % (name, why, alt), version=ver)
 
-        # warn about typing issues with configuration entries
-        for unable in C.config.UNABLE:
-            display.warning("Unable to set correct type for configuration entry: %s" % unable)
+        # Errors with configuration entries
+        if C.config.UNABLE:
+            for unable in C.config.UNABLE:
+                display.error("Unable to set correct type for configuration entry for %s: %s" % (unable, C.config.UNABLE[unable]))
+            raise AnsibleError("Invalid configuration settings")
 
     @staticmethod
     def split_vault_id(vault_id):

lib/ansible/cli/doc.py

@@ -17,6 +17,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import datetime
+import json
 import os
 import textwrap
 import traceback
@@ -55,6 +56,18 @@ class DocCLI(CLI):
         super(DocCLI, self).__init__(args)
         self.plugin_list = set()
 
+        self.loader_map = {
+            'cache': cache_loader,
+            'callback': callback_loader,
+            'connection': connection_loader,
+            'lookup': lookup_loader,
+            'strategy': strategy_loader,
+            'vars': vars_loader,
+            'inventory': inventory_loader,
+            'shell': shell_loader,
+            'module': module_loader,
+        }
+
     def parse(self):
 
         self.parser = CLI.base_parser(
@@ -72,13 +85,15 @@ class DocCLI(CLI):
                                help='Show playbook snippet for specified plugin(s)')
         self.parser.add_option("-a", "--all", action="store_true", default=False, dest='all_plugins',
                                help='**For internal testing only** Show documentation for all plugins.')
+        self.parser.add_option("-j", "--json", action="store_true", default=False, dest='json_dump',
+                               help='**For internal testing only** Dump json metadata for all plugins.')
         self.parser.add_option("-t", "--type", action="store", default='module', dest='type', type='choice',
                                help='Choose which plugin type (defaults to "module")',
                                choices=C.DOCUMENTABLE_PLUGINS)
         super(DocCLI, self).parse()
 
-        if [self.options.all_plugins, self.options.list_dir, self.options.list_files, self.options.show_snippet].count(True) > 1:
-            raise AnsibleOptionsError("Only one of -l, -F, -s or -a can be used at the same time.")
+        if [self.options.all_plugins, self.options.json_dump, self.options.list_dir, self.options.list_files, self.options.show_snippet].count(True) > 1:
+            raise AnsibleOptionsError("Only one of -l, -F, -s, -j or -a can be used at the same time.")
 
         display.verbosity = self.options.verbosity
 
@@ -87,26 +102,7 @@ class DocCLI(CLI):
         super(DocCLI, self).run()
 
         plugin_type = self.options.type
-
-        # choose plugin type
-        if plugin_type == 'cache':
-            loader = cache_loader
-        elif plugin_type == 'callback':
-            loader = callback_loader
-        elif plugin_type == 'connection':
-            loader = connection_loader
-        elif plugin_type == 'lookup':
-            loader = lookup_loader
-        elif plugin_type == 'strategy':
-            loader = strategy_loader
-        elif plugin_type == 'vars':
-            loader = vars_loader
-        elif plugin_type == 'inventory':
-            loader = inventory_loader
-        elif plugin_type == 'shell':
-            loader = shell_loader
-        else:
-            loader = module_loader
+        loader = self.loader_map.get(plugin_type, self.loader_map['module'])
 
         # add to plugin path from command line
         if self.options.module_path:
@@ -122,7 +118,7 @@ class DocCLI(CLI):
         if self.options.list_files:
             paths = loader._get_paths()
             for path in paths:
-                self.find_plugins(path, plugin_type)
+                self.plugin_list.update(self.find_plugins(path, plugin_type))
 
             list_text = self.get_plugin_list_filenames(loader)
             self.pager(list_text)
@@ -132,17 +128,27 @@ class DocCLI(CLI):
         if self.options.list_dir:
             paths = loader._get_paths()
             for path in paths:
-                self.find_plugins(path, plugin_type)
+                self.plugin_list.update(self.find_plugins(path, plugin_type))
 
             self.pager(self.get_plugin_list_text(loader))
             return 0
 
         # process all plugins of type
         if self.options.all_plugins:
-            paths = loader._get_paths()
-            for path in paths:
-                self.find_plugins(path, plugin_type)
-            self.args = sorted(set(self.plugin_list))
+            self.args = self.get_all_plugins_of_type(plugin_type)
+
+        # dump plugin metadata as JSON
+        if self.options.json_dump:
+            plugin_data = {}
+            for plugin_type in self.loader_map.keys():
+                plugin_data[plugin_type] = dict()
+                plugin_names = self.get_all_plugins_of_type(plugin_type)
+                for plugin_name in plugin_names:
+                    plugin_data[plugin_type][plugin_name] = self.get_plugin_metadata(plugin_type, plugin_name)
+
+            self.pager(json.dumps(plugin_data, sort_keys=True, indent=4))
+
+            return 0
 
         if len(self.args) == 0:
             raise AnsibleOptionsError("Incorrect options passed")
@@ -150,65 +156,125 @@ class DocCLI(CLI):
         # process command line list
         text = ''
         for plugin in self.args:
-            try:
-                # if the plugin lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
-                filename = loader.find_plugin(plugin, mod_type='.py', ignore_deprecated=True, check_aliases=True)
-                if filename is None:
-                    display.warning("%s %s not found in:\n%s\n" % (plugin_type, plugin, search_paths))
-                    continue
+            textret = self.format_plugin_doc(plugin, loader, plugin_type, search_paths)
 
-                if any(filename.endswith(x) for x in C.BLACKLIST_EXTS):
-                    continue
-
-                try:
-                    doc, plainexamples, returndocs, metadata = get_docstring(filename, fragment_loader, verbose=(self.options.verbosity > 0))
-                except Exception:
-                    display.vvv(traceback.format_exc())
-                    display.error("%s %s has a documentation error formatting or is missing documentation." % (plugin_type, plugin), wrap_text=False)
-                    continue
-
-                if doc is not None:
-
-                    # assign from other sections
-                    doc['plainexamples'] = plainexamples
-                    doc['returndocs'] = returndocs
-                    doc['metadata'] = metadata
-
-                    # generate extra data
-                    if plugin_type == 'module':
-                        # is there corresponding action plugin?
-                        if plugin in action_loader:
-                            doc['action'] = True
-                        else:
-                            doc['action'] = False
-                    doc['filename'] = filename
-                    doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
-                    if 'docuri' in doc:
-                        doc['docuri'] = doc[plugin_type].replace('_', '-')
-
-                    if self.options.show_snippet and plugin_type == 'module':
-                        text += self.get_snippet_text(doc)
-                    else:
-                        text += self.get_man_text(doc)
-                else:
-                    # this typically means we couldn't even parse the docstring, not just that the YAML is busted,
-                    # probably a quoting issue.
-                    raise AnsibleError("Parsing produced an empty object.")
-            except Exception as e:
-                display.vvv(traceback.format_exc())
-                raise AnsibleError("%s %s missing documentation (or could not parse documentation): %s\n" % (plugin_type, plugin, str(e)))
+            if textret:
+                text += textret
 
         if text:
             self.pager(text)
+
         return 0
 
+    def get_all_plugins_of_type(self, plugin_type):
+        loader = self.loader_map[plugin_type]
+        plugin_list = set()
+        paths = loader._get_paths()
+        for path in paths:
+            plugins_to_add = self.find_plugins(path, plugin_type)
+            plugin_list.update(plugins_to_add)
+        return sorted(set(plugin_list))
+
+    def get_plugin_metadata(self, plugin_type, plugin_name):
+        # if the plugin lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
+        loader = self.loader_map[plugin_type]
+        filename = loader.find_plugin(plugin_name, mod_type='.py', ignore_deprecated=True, check_aliases=True)
+        if filename is None:
+            raise AnsibleError("unable to load {0} plugin named {1} ".format(plugin_type, plugin_name))
+
+        try:
+            doc, __, __, __ = get_docstring(filename, fragment_loader, verbose=(self.options.verbosity > 0))
+        except Exception:
+            display.vvv(traceback.format_exc())
+            raise AnsibleError(
+                "%s %s at %s has a documentation error formatting or is missing documentation." %
+                (plugin_type, plugin_name, filename), wrap_text=False)
+
+        return dict(
+            name=plugin_name,
+            namespace=self.namespace_from_plugin_filepath(filename, plugin_name, loader.package_path),
+            description=doc.get('short_description', "UNKNOWN"),
+            version_added=doc.get('version_added', "UNKNOWN")
+        )
+
+    def namespace_from_plugin_filepath(self, filepath, plugin_name, basedir):
+        if not basedir.endswith('/'):
+            basedir += '/'
+        rel_path = filepath.replace(basedir, '')
+        extension_free = os.path.splitext(rel_path)[0]
+        namespace_only = extension_free.rsplit(plugin_name, 1)[0].strip('/_')
+        clean_ns = namespace_only.replace('/', '.')
+        if clean_ns == '':
+            clean_ns = None
+
+        return clean_ns
+
+    def format_plugin_doc(self, plugin, loader, plugin_type, search_paths):
+        text = ''
+
+        try:
+            # if the plugin lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
+            filename = loader.find_plugin(plugin, mod_type='.py', ignore_deprecated=True, check_aliases=True)
+            if filename is None:
+                display.warning("%s %s not found in:\n%s\n" % (plugin_type, plugin, search_paths))
+                return
+
+            if any(filename.endswith(x) for x in C.BLACKLIST_EXTS):
+                return
+
+            try:
+                doc, plainexamples, returndocs, metadata = get_docstring(filename, fragment_loader,
+                                                                         verbose=(self.options.verbosity > 0))
+            except Exception:
+                display.vvv(traceback.format_exc())
+                display.error(
+                    "%s %s has a documentation error formatting or is missing documentation." % (plugin_type, plugin),
+                    wrap_text=False)
+                return
+
+            if doc is not None:
+
+                # assign from other sections
+                doc['plainexamples'] = plainexamples
+                doc['returndocs'] = returndocs
+                doc['metadata'] = metadata
+
+                # generate extra data
+                if plugin_type == 'module':
+                    # is there corresponding action plugin?
+                    if plugin in action_loader:
+                        doc['action'] = True
+                    else:
+                        doc['action'] = False
+                doc['filename'] = filename
+                doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
+                if 'docuri' in doc:
+                    doc['docuri'] = doc[plugin_type].replace('_', '-')
+
+                if self.options.show_snippet and plugin_type == 'module':
+                    text += self.get_snippet_text(doc)
+                else:
+                    text += self.get_man_text(doc)
+
+                return text
+            else:
+                # this typically means we couldn't even parse the docstring, not just that the YAML is busted,
+                # probably a quoting issue.
+                raise AnsibleError("Parsing produced an empty object.")
+        except Exception as e:
+            display.vvv(traceback.format_exc())
+            raise AnsibleError(
+                "%s %s missing documentation (or could not parse documentation): %s\n" % (plugin_type, plugin, str(e)))
+
     def find_plugins(self, path, ptype):
 
         display.vvvv("Searching %s for plugins" % path)
 
+        plugin_list = set()
+
         if not os.path.exists(path):
             display.vvvv("%s does not exist" % path)
-            return
+            return plugin_list
 
         bkey = ptype.upper()
         for plugin in os.listdir(path):
@@ -233,9 +299,11 @@ class DocCLI(CLI):
             plugin = plugin.lstrip('_')  # remove underscore from deprecated plugins
 
             if plugin not in BLACKLIST.get(bkey, ()):
-                self.plugin_list.add(plugin)
+                plugin_list.add(plugin)
                 display.vvvv("Added %s" % plugin)
 
+        return plugin_list
+
     def get_plugin_list_text(self, loader):
         columns = display.columns
         displace = max(len(x) for x in self.plugin_list)

lib/ansible/cli/inventory.py

@@ -42,6 +42,7 @@ INTERNAL_VARS = frozenset(['ansible_diff_mode',
                            'ansible_playbook_python',
                            'ansible_run_tags',
                            'ansible_skip_tags',
+                           'ansible_verbosity',
                            'ansible_version',
                            'inventory_dir',
                            'inventory_file',

lib/ansible/config/base.yml

@@ -736,7 +736,7 @@ DEFAULT_LOCAL_TMP:
   type: tmppath
 DEFAULT_LOG_PATH:
   name: Ansible log file path
-  default: ''
+  default: ~
   description: File to which Ansible will log on the controller. When empty logging is disabled.
   env: [{name: ANSIBLE_LOG_PATH}]
   ini:
@@ -931,9 +931,9 @@ DEFAULT_SCP_IF_SSH:
   # TODO: move to ssh plugin
   default: smart
   description:
-    - "Prefered method to use when transfering files over ssh"
-    - When set to smart, Ansible will try them until one succeeds or they all fail
-    - If set to True, it will force 'scp', if False it will use 'sftp'
+    - "Prefered method to use when transfering files over ssh."
+    - When set to smart, Ansible will try them until one succeeds or they all fail.
+    - If set to True, it will force 'scp', if False it will use 'sftp'.
   env: [{name: ANSIBLE_SCP_IF_SSH}]
   ini:
   - {key: scp_if_ssh, section: ssh_connection}
@@ -963,7 +963,7 @@ DEFAULT_SQUASH_ACTIONS:
   description:
     - Ansible can optimise actions that call modules that support list parameters when using ``with_`` looping.
       Instead of calling the module once for each item, the module is called once with the full list.
-    - The default value for this setting is only for certain package managers, but it can be used for any module
+    - The default value for this setting is only for certain package managers, but it can be used for any module.
     - Currently, this is only supported for modules that have a name or pkg parameter, and only when the item is the only thing being passed to the parameter.
   env: [{name: ANSIBLE_SQUASH_ACTIONS}]
   ini:

lib/ansible/config/manager.py

@@ -30,6 +30,8 @@ from ansible.utils.path import makedirs_safe
 Plugin = namedtuple('Plugin', 'name type')
 Setting = namedtuple('Setting', 'name value origin type')
 
+INTERNAL_DEFS = {'lookup': ('_terms',)}
+
 
 # FIXME: see if we can unify in module_utils with similar function used by argspec
 def ensure_type(value, value_type, origin=None):
@@ -60,7 +62,7 @@ def ensure_type(value, value_type, origin=None):
     if value_type in ('boolean', 'bool'):
         value = boolean(value, strict=False)
 
-    elif value:
+    elif value is not None:
         if value_type in ('integer', 'int'):
             value = int(value)
 
@@ -166,7 +168,7 @@ def find_ini_config_file():
 
 class ConfigManager(object):
 
-    UNABLE = []
+    UNABLE = {}
     DEPRECATED = []
 
     def __init__(self, conf_file=None, defs_file=None):
@@ -215,6 +217,11 @@ class ConfigManager(object):
         if cfile is not None:
             if ftype == 'ini':
                 self._parsers[cfile] = configparser.ConfigParser()
+                with open(cfile, 'rb') as f:
+                    try:
+                        cfg_text = to_text(f.read(), errors='surrogate_or_strict')
+                    except UnicodeError as e:
+                        raise AnsibleOptionsError("Error reading config file(%s) because the config file was not utf8 encoded: %s" % (cfile, to_native(e)))
                 try:
                     self._parsers[cfile].read(cfile)
                 except configparser.Error as e:
@@ -230,12 +237,12 @@ class ConfigManager(object):
         ''' Load YAML Config Files in order, check merge flags, keep origin of settings'''
         pass
 
-    def get_plugin_options(self, plugin_type, name, keys=None, variables=None):
+    def get_plugin_options(self, plugin_type, name, keys=None, variables=None, direct=None):
 
         options = {}
         defs = self.get_configuration_definitions(plugin_type, name)
         for option in defs:
-            options[option] = self.get_config_value(option, plugin_type=plugin_type, plugin_name=name, keys=keys, variables=variables)
+            options[option] = self.get_config_value(option, plugin_type=plugin_type, plugin_name=name, keys=keys, variables=variables, direct=direct)
 
         return options
 
@@ -279,12 +286,17 @@ class ConfigManager(object):
 
         return value, origin
 
-    def get_config_value(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None):
+    def get_config_value(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None, direct=None):
         ''' wrapper '''
-        value, _drop = self.get_config_value_and_origin(config, cfile=cfile, plugin_type=plugin_type, plugin_name=plugin_name, keys=keys, variables=variables)
+
+        try:
+            value, _drop = self.get_config_value_and_origin(config, cfile=cfile, plugin_type=plugin_type, plugin_name=plugin_name,
+                                                            keys=keys, variables=variables, direct=direct)
+        except Exception as e:
+            raise AnsibleError("Invalid settings supplied for %s: %s" % (config, to_native(e)))
         return value
 
-    def get_config_value_and_origin(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None):
+    def get_config_value_and_origin(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None, direct=None):
         ''' Given a config key figure out the actual value and report on the origin of the settings '''
 
         if cfile is None:
@@ -303,57 +315,71 @@ class ConfigManager(object):
             defs = self._plugins[plugin_type][plugin_name]
 
         if config in defs:
-            # Use 'variable overrides' if present, highest precedence, but only present when querying running play
-            if variables and defs[config].get('vars'):
-                value, origin = self._loop_entries(variables, defs[config]['vars'])
-                origin = 'var: %s' % origin
 
-            # use playbook keywords if you have em
-            if value is None and keys:
-                value, origin = self._loop_entries(keys, defs[config]['keywords'])
-                origin = 'keyword: %s' % origin
+            # direct setting via plugin arguments, can set to None so we bypass rest of processing/defaults
+            if direct and config in direct:
+                value = direct[config]
+                origin = 'Direct'
 
-            # env vars are next precedence
-            if value is None and defs[config].get('env'):
-                value, origin = self._loop_entries(os.environ, defs[config]['env'])
-                origin = 'env: %s' % origin
+            else:
+                # Use 'variable overrides' if present, highest precedence, but only present when querying running play
+                if variables and defs[config].get('vars'):
+                    value, origin = self._loop_entries(variables, defs[config]['vars'])
+                    origin = 'var: %s' % origin
 
-            # try config file entries next, if we have one
-            if self._parsers.get(cfile, None) is None:
-                self._parse_config_file(cfile)
+                # use playbook keywords if you have em
+                if value is None and keys and defs[config].get('keywords'):
+                    value, origin = self._loop_entries(keys, defs[config]['keywords'])
+                    origin = 'keyword: %s' % origin
 
-            if value is None and cfile is not None:
-                ftype = get_config_type(cfile)
-                if ftype and defs[config].get(ftype):
-                    if ftype == 'ini':
-                        # load from ini config
-                        try:  # FIXME: generalize _loop_entries to allow for files also, most of this code is dupe
-                            for ini_entry in defs[config]['ini']:
-                                temp_value = get_ini_config_value(self._parsers[cfile], ini_entry)
-                                if temp_value is not None:
-                                    value = temp_value
-                                    origin = cfile
-                                    if 'deprecated' in ini_entry:
-                                        self.DEPRECATED.append(('[%s]%s' % (ini_entry['section'], ini_entry['key']), ini_entry['deprecated']))
-                        except Exception as e:
-                            sys.stderr.write("Error while loading ini config %s: %s" % (cfile, to_native(e)))
-                    elif ftype == 'yaml':
-                        # FIXME: implement, also , break down key from defs (. notation???)
-                        origin = cfile
+                # env vars are next precedence
+                if value is None and defs[config].get('env'):
+                    value, origin = self._loop_entries(os.environ, defs[config]['env'])
+                    origin = 'env: %s' % origin
 
-            # set default if we got here w/o a value
-            if value is None:
-                value = defs[config].get('default')
-                origin = 'default'
-                # skip typing as this is a temlated default that will be resolved later in constants, which has needed vars
-                if plugin_type is None and isinstance(value, string_types) and (value.startswith('{{') and value.endswith('}}')):
-                    return value, origin
+                # try config file entries next, if we have one
+                if self._parsers.get(cfile, None) is None:
+                    self._parse_config_file(cfile)
 
-            # ensure correct type
-            try:
-                value = ensure_type(value, defs[config].get('type'), origin=origin)
-            except Exception as e:
-                self.UNABLE.append(config)
+                if value is None and cfile is not None:
+                    ftype = get_config_type(cfile)
+                    if ftype and defs[config].get(ftype):
+                        if ftype == 'ini':
+                            # load from ini config
+                            try:  # FIXME: generalize _loop_entries to allow for files also, most of this code is dupe
+                                for ini_entry in defs[config]['ini']:
+                                    temp_value = get_ini_config_value(self._parsers[cfile], ini_entry)
+                                    if temp_value is not None:
+                                        value = temp_value
+                                        origin = cfile
+                                        if 'deprecated' in ini_entry:
+                                            self.DEPRECATED.append(('[%s]%s' % (ini_entry['section'], ini_entry['key']), ini_entry['deprecated']))
+                            except Exception as e:
+                                sys.stderr.write("Error while loading ini config %s: %s" % (cfile, to_native(e)))
+                        elif ftype == 'yaml':
+                            # FIXME: implement, also , break down key from defs (. notation???)
+                            origin = cfile
+
+                # set default if we got here w/o a value
+                if value is None:
+                    if defs[config].get('required', False):
+                        entry = ''
+                        if plugin_type:
+                            entry += 'plugin_type: %s ' % plugin_type
+                            if plugin_name:
+                                entry += 'plugin: %s ' % plugin_name
+                        entry += 'setting: %s ' % config
+                        if not plugin_type or config not in INTERNAL_DEFS.get(plugin_type, {}):
+                            raise AnsibleError("No setting was provided for required configuration %s" % (entry))
+                    else:
+                        value = defs[config].get('default')
+                        origin = 'default'
+                        # skip typing as this is a temlated default that will be resolved later in constants, which has needed vars
+                        if plugin_type is None and isinstance(value, string_types) and (value.startswith('{{') and value.endswith('}}')):
+                            return value, origin
+
+            # ensure correct type, can raise exceptoins on mismatched types
+            value = ensure_type(value, defs[config].get('type'), origin=origin)
 
             # deal with deprecation of the setting
             if 'deprecated' in defs[config] and origin != 'default':
@@ -392,7 +418,13 @@ class ConfigManager(object):
                 raise AnsibleOptionsError("Invalid configuration definition '%s': type is %s" % (to_native(config), type(defs[config])))
 
             # get value and origin
-            value, origin = self.get_config_value_and_origin(config, configfile)
+            try:
+                value, origin = self.get_config_value_and_origin(config, configfile)
+            except Exception as e:
+                # when building constants.py we ignore invalid configs
+                # CLI takes care of warnings once 'display' is loaded
+                self.UNABLE[config] = to_text(e)
+                continue
 
             # set the constant
             self.data.update_setting(Setting(config, value, origin, defs[config].get('type', 'string')))

lib/ansible/constants.py

@@ -84,7 +84,7 @@ BECOME_MISSING_STRINGS = {
     'enable': '',
     'machinectl': '',
 }  # FIXME: deal with i18n
-BLACKLIST_EXTS = ('.pyc', '.pyo', '.swp', '.bak', '~', '.rpm', '.md', '.txt')
+BLACKLIST_EXTS = ('.pyc', '.pyo', '.swp', '.bak', '~', '.rpm', '.md', '.txt', '.rst')
 BOOL_TRUE = BOOLEANS_TRUE
 CONTROLER_LANG = os.getenv('LANG', 'en_US.UTF-8')
 DEFAULT_BECOME_PASS = None
@@ -185,6 +185,7 @@ for setting in config.data.get_settings():
                 pass  # not a python data structure
         except:
             pass  # not templatable
-        value = ensure_type(value, setting.name)
+
+        value = ensure_type(value, setting.type)
 
     set_constant(setting.name, value)

lib/ansible/executor/task_executor.py

@@ -278,14 +278,19 @@ class TaskExecutor:
         label = None
         loop_pause = 0
         templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars)
+
+        # FIXME: move this to the object itself to allow post_validate to take care of templating (loop_control.post_validate)
         if self._task.loop_control:
-            # FIXME: move this to the object itself to allow post_validate to take care of templating
             loop_var = templar.template(self._task.loop_control.loop_var)
             index_var = templar.template(self._task.loop_control.index_var)
             loop_pause = templar.template(self._task.loop_control.pause)
-            # the these may be 'None', so we still need to default to something useful
-            # this is tempalted below after an item is assigned
-            label = (self._task.loop_control.label or ('{{' + loop_var + '}}'))
+
+            # This may be 'None',so it is tempalted below after we ensure a value and an item is assigned
+            label = self._task.loop_control.label
+
+        # ensure we always have a label
+        if label is None:
+            label = '{{' + loop_var + '}}'
 
         if loop_var in task_vars:
             display.warning(u"The loop variable '%s' is already in use. "
@@ -339,8 +344,8 @@ class TaskExecutor:
             res['_ansible_item_result'] = True
             res['_ansible_ignore_errors'] = task_fields.get('ignore_errors')
 
-            if label is not None:
-                res['_ansible_item_label'] = templar.template(label, cache=False)
+            # gets templated here unlike rest of loop_control fields, depends on loop_var above
+            res['_ansible_item_label'] = templar.template(label, cache=False)
 
             self._rslt_q.put(
                 TaskResult(

lib/ansible/executor/task_result.py

@@ -110,7 +110,7 @@ class TaskResult:
         else:
             ignore = _IGNORE
 
-        if self._result.get('_ansible_no_log', False):
+        if self._task.no_log or self._result.get('_ansible_no_log', False):
             x = {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result"}
             for preserve in _PRESERVE:
                 if preserve in self._result:

lib/ansible/inventory/manager.py

@@ -375,7 +375,7 @@ class InventoryManager(object):
                     from random import shuffle
                     shuffle(hosts)
                 elif order not in [None, 'inventory']:
-                    AnsibleOptionsError("Invalid 'order' specified for inventory hosts: %s" % order)
+                    raise AnsibleOptionsError("Invalid 'order' specified for inventory hosts: %s" % order)
 
         return hosts
 

lib/ansible/module_utils/aws/core.py

@@ -133,6 +133,9 @@ class AnsibleAWSModule(object):
     def warn(self, *args, **kwargs):
         return self._module.warn(*args, **kwargs)
 
+    def md5(self, *args, **kwargs):
+        return self._module.md5(*args, **kwargs)
+
     def client(self, service, retry_decorator=None):
         region, ec2_url, aws_connect_kwargs = get_aws_connection_info(self, boto3=True)
         conn = boto3_conn(self, conn_type='client', resource=service,

lib/ansible/module_utils/azure_rm_common.py

@@ -195,23 +195,23 @@ AZURE_PKG_VERSIONS = {
     },
     'ComputeManagementClient': {
         'package_name': 'compute',
-        'expected_version': '2.0.0'
+        'expected_version': '2.1.0'
     },
     'ContainerInstanceManagementClient': {
         'package_name': 'containerinstance',
-        'expected_version': '0.3.1'
+        'expected_version': '0.4.0'
     },
     'NetworkManagementClient': {
         'package_name': 'network',
-        'expected_version': '1.3.0'
+        'expected_version': '1.7.1'
     },
     'ResourceManagementClient': {
         'package_name': 'resource',
-        'expected_version': '1.1.0'
+        'expected_version': '1.2.2'
     },
     'DnsManagementClient': {
         'package_name': 'dns',
-        'expected_version': '1.0.1'
+        'expected_version': '1.2.0'
     },
     'WebSiteManagementClient': {
         'package_name': 'web',
@@ -408,8 +408,11 @@ class AzureRMModuleBase(object):
                 return
             expected_version = package_version.get('expected_version')
             if Version(client_version) < Version(expected_version):
-                self.fail("Installed azure-mgmt-{0} client version is {1}. The supported version is {2}. Try "
+                self.fail("Installed azure-mgmt-{0} client version is {1}. The minimum supported version is {2}. Try "
                           "`pip install ansible[azure]`".format(client_name, client_version, expected_version))
+            if Version(client_version) != Version(expected_version):
+                self.module.warn("Installed azure-mgmt-{0} client version is {1}. The expected version is {2}. Try "
+                                 "`pip install ansible[azure]`".format(client_name, client_version, expected_version))
 
     def exec_module(self, **kwargs):
         self.fail("Error: {0} failed to implement exec_module method.".format(self.__class__.__name__))
@@ -945,6 +948,9 @@ class AzureRMModuleBase(object):
             profile_default_version = api_profile_dict.get('default_api_version', None)
             if api_version or profile_default_version:
                 client_kwargs['api_version'] = api_version or profile_default_version
+                if 'profile' in client_kwargs:
+                    # remove profile; only pass API version if specified
+                    client_kwargs.pop('profile')
 
         client = client_type(**client_kwargs)
 

lib/ansible/module_utils/facts/system/distribution.py

@@ -256,6 +256,7 @@ class DistributionFiles:
                 distribution_version = re.search(r'^VERSION_ID="?([0-9]+\.?[0-9]*)"?', line)
                 if distribution_version:
                     suse_facts['distribution_version'] = distribution_version.group(1)
+                    suse_facts['distribution_major_version'] = distribution_version.group(1).split('.')[0]
                 if 'open' in data.lower():
                     release = re.search(r'^VERSION_ID="?[0-9]+\.?([0-9]*)"?', line)
                     if release:

lib/ansible/module_utils/gcp_utils.py

@@ -106,6 +106,9 @@ class GcpSession(object):
         if not HAS_GOOGLE_LIBRARIES:
             self.module.fail_json(msg="Please install the google-auth library")
 
+        if 'auth_kind' not in self.module.params:
+            self.module.fail_json(msg="Auth kind parameter is missing")
+
         if self.module.params.get('service_account_email') is not None and self.module.params['auth_kind'] != 'machineaccount':
             self.module.fail_json(
                 msg="Service Acccount Email only works with Machine Account-based authentication"

lib/ansible/module_utils/k8s/common.py

@@ -28,6 +28,7 @@ from ansible.module_utils.basic import AnsibleModule
 try:
     import kubernetes
     from openshift.dynamic import DynamicClient
+    from openshift.dynamic.exceptions import ResourceNotFoundError, ResourceNotUniqueError
     HAS_K8S_MODULE_HELPER = True
 except ImportError:
     HAS_K8S_MODULE_HELPER = False
@@ -121,13 +122,14 @@ class K8sAnsibleMixin(object):
         self._argspec_cache = argument_spec
         return self._argspec_cache
 
-    def get_api_client(self, **auth):
+    def get_api_client(self, **auth_params):
         auth_args = AUTH_ARG_SPEC.keys()
 
-        auth = auth or getattr(self, 'params', {})
+        auth_params = auth_params or getattr(self, 'params', {})
+        auth = copy.deepcopy(auth_params)
 
         configuration = kubernetes.client.Configuration()
-        for key, value in iteritems(auth):
+        for key, value in iteritems(auth_params):
             if key in auth_args and value is not None:
                 if key == 'api_key':
                     setattr(configuration, key, {'authorization': "Bearer {0}".format(value)})
@@ -137,6 +139,7 @@ class K8sAnsibleMixin(object):
                 env_value = os.getenv('K8S_AUTH_{0}'.format(key.upper()), None)
                 if env_value is not None:
                     setattr(configuration, key, env_value)
+                    auth[key] = env_value
 
         kubernetes.client.Configuration.set_default(configuration)
 
@@ -174,6 +177,18 @@ class K8sAnsibleMixin(object):
                 return kubernetes.client.ApiClient()
             raise
 
+    def find_resource(self, kind, api_version, fail=False):
+        for attribute in ['kind', 'name', 'singular_name']:
+            try:
+                return self.client.resources.get(**{'api_version': api_version, attribute: kind})
+            except (ResourceNotFoundError, ResourceNotUniqueError):
+                pass
+        try:
+            return self.client.resources.get(api_version=api_version, short_names=[kind])
+        except (ResourceNotFoundError, ResourceNotUniqueError):
+            if fail:
+                self.fail(msg='Failed to find exact match for {0}.{1} by [kind, name, singularName, shortNames]'.format(api_version, kind))
+
     def remove_aliases(self):
         """
         The helper doesn't know what to do with aliased keys
@@ -189,12 +204,12 @@ class K8sAnsibleMixin(object):
         result = None
         path = os.path.normpath(src)
         if not os.path.exists(path):
-            self.fail_json(msg="Error accessing {0}. Does the file exist?".format(path))
+            self.fail(msg="Error accessing {0}. Does the file exist?".format(path))
         try:
             with open(path, 'r') as f:
                 result = list(yaml.safe_load_all(f))
         except (IOError, yaml.YAMLError) as exc:
-            self.fail_json(msg="Error loading resource_definition: {0}".format(exc))
+            self.fail(msg="Error loading resource_definition: {0}".format(exc))
         return result
 
     @staticmethod
@@ -202,16 +217,7 @@ class K8sAnsibleMixin(object):
         if not HAS_DICTDIFFER:
             return False, []
 
-        def get_shared_attrs(o1, o2):
-            shared_attrs = {}
-            for k, v in o2.items():
-                if isinstance(v, dict):
-                    shared_attrs[k] = get_shared_attrs(o1.get(k, {}), v)
-                else:
-                    shared_attrs[k] = o1.get(k)
-            return shared_attrs
-
-        diffs = list(dictdiffer.diff(new, get_shared_attrs(existing, new)))
+        diffs = list(dictdiffer.diff(new, existing))
         match = len(diffs) == 0
         return match, diffs
 
@@ -234,3 +240,6 @@ class KubernetesAnsibleModule(AnsibleModule, K8sAnsibleMixin):
 
     def execute_module(self):
         raise NotImplementedError()
+
+    def fail(self, msg=None):
+        self.fail_json(msg=msg)

lib/ansible/module_utils/k8s/raw.py

@@ -74,12 +74,9 @@ class KubernetesRawModule(KubernetesAnsibleModule):
             if kind.lower().endswith('list'):
                 search_kind = kind[:-4]
             api_version = definition.get('apiVersion')
-            try:
-                resource = self.client.resources.get(kind=search_kind, api_version=api_version)
-            except Exception as e:
-                self.fail_json(msg='Failed to find resource {0}.{1}: {2}'.format(
-                    api_version, search_kind, e
-                ))
+            resource = self.find_resource(search_kind, api_version, fail=True)
+            definition['kind'] = resource.kind
+            definition['apiVersion'] = resource.group_version
             result = self.perform_action(resource, definition)
             changed = changed or result['changed']
             results.append(result)
@@ -151,32 +148,33 @@ class KubernetesRawModule(KubernetesAnsibleModule):
                 result['method'] = 'create'
                 return result
 
+            match = False
+            diffs = []
+
             if existing and force:
                 if not self.check_mode:
                     try:
-                        k8s_obj = resource.replace(definition, name=name, namespace=namespace)
-                        result['result'] = k8s_obj.to_dict()
+                        k8s_obj = resource.replace(definition, name=name, namespace=namespace).to_dict()
+                        match, diffs = self.diff_objects(existing.to_dict(), k8s_obj)
+                        result['result'] = k8s_obj
                     except DynamicApiError as exc:
                         self.fail_json(msg="Failed to replace object: {0}".format(exc.body),
                                        error=exc.status, status=exc.status, reason=exc.reason)
-                result['changed'] = True
+                result['changed'] = not match
                 result['method'] = 'replace'
+                result['diff'] = diffs
                 return result
 
-            match, diffs = self.diff_objects(existing.to_dict(), definition)
-
-            if match:
-                result['result'] = existing.to_dict()
-                return result
             # Differences exist between the existing obj and requested params
             if not self.check_mode:
                 try:
-                    k8s_obj = resource.patch(definition, name=name, namespace=namespace)
-                    result['result'] = k8s_obj.to_dict()
+                    k8s_obj = resource.patch(definition, name=name, namespace=namespace).to_dict()
+                    match, diffs = self.diff_objects(existing.to_dict(), k8s_obj)
+                    result['result'] = k8s_obj
                 except DynamicApiError as exc:
                     self.fail_json(msg="Failed to patch object: {0}".format(exc.body),
                                    error=exc.status, status=exc.status, reason=exc.reason)
-            result['changed'] = True
+            result['changed'] = not match
             result['method'] = 'patch'
             result['diff'] = diffs
             return result

lib/ansible/module_utils/k8s/scale.py

@@ -52,6 +52,8 @@ class KubernetesAnsibleScaleModule(KubernetesRawModule):
 
         name = definition['metadata']['name']
         namespace = definition['metadata'].get('namespace')
+        api_version = definition['apiVersion']
+        kind = definition['kind']
         current_replicas = self.params.get('current_replicas')
         replicas = self.params.get('replicas')
         resource_version = self.params.get('resource_version')
@@ -62,7 +64,7 @@ class KubernetesAnsibleScaleModule(KubernetesRawModule):
         existing_count = None
         return_attributes = dict(changed=False, result=dict())
 
-        resource = self.client.resources.get(api_version=definition['apiVersion'], kind=definition['kind'])
+        resource = self.find_resource(kind, api_version, fail=True)
 
         try:
             existing = resource.get(name=name, namespace=namespace)

lib/ansible/module_utils/net_tools/nios/api.py

@@ -26,10 +26,9 @@
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
-
 import os
 from functools import partial
-
+from ansible.module_utils._text import to_native
 from ansible.module_utils.six import iteritems
 from ansible.module_utils._text import to_text
 
@@ -40,7 +39,15 @@ try:
 except ImportError:
     HAS_INFOBLOX_CLIENT = False
 
-nios_provider_spec = {
+# defining nios constants
+NIOS_DNS_VIEW = 'view'
+NIOS_NETWORK_VIEW = 'networkview'
+NIOS_HOST_RECORD = 'record:host'
+NIOS_IPV4_NETWORK = 'network'
+NIOS_IPV6_NETWORK = 'ipv6network'
+NIOS_ZONE = 'zone_auth'
+
+NIOS_PROVIDER_SPEC = {
     'host': dict(),
     'username': dict(),
     'password': dict(no_log=True),
@@ -57,10 +64,8 @@ nios_provider_spec = {
 
 def get_connector(*args, **kwargs):
     ''' Returns an instance of infoblox_client.connector.Connector
-
     :params args: positional arguments are silently ignored
     :params kwargs: dict that is passed to Connector init
-
     :returns: Connector
     '''
     if not HAS_INFOBLOX_CLIENT:
@@ -68,12 +73,11 @@ def get_connector(*args, **kwargs):
                         'to be installed.  It can be installed using the '
                         'command `pip install infoblox-client`')
 
-    if not set(kwargs.keys()).issubset(nios_provider_spec.keys()):
+    if not set(kwargs.keys()).issubset(NIOS_PROVIDER_SPEC.keys()):
         raise Exception('invalid or unsupported keyword argument for connector')
-
-    for key, value in iteritems(nios_provider_spec):
+    for key, value in iteritems(NIOS_PROVIDER_SPEC):
         if key not in kwargs:
-            # apply default values from nios_provider_spec since we cannot just
+            # apply default values from NIOS_PROVIDER_SPEC since we cannot just
             # assume the provider values are coming from AnsibleModule
             if 'default' in value:
                 kwargs[key] = value['default']
@@ -89,11 +93,9 @@ def get_connector(*args, **kwargs):
 
 def normalize_extattrs(value):
     ''' Normalize extattrs field to expected format
-
     The module accepts extattrs as key/value pairs.  This method will
     transform the key/value pairs into a structure suitable for
     sending across WAPI in the format of:
-
         extattrs: {
             key: {
                 value: <value>
@@ -105,29 +107,23 @@ def normalize_extattrs(value):
 
 def flatten_extattrs(value):
     ''' Flatten the key/value struct for extattrs
-
     WAPI returns extattrs field as a dict in form of:
-
         extattrs: {
             key: {
                 value: <value>
             }
         }
-
     This method will flatten the structure to:
-
         extattrs: {
             key: value
         }
-
     '''
     return dict([(k, v['value']) for k, v in iteritems(value)])
 
 
 class WapiBase(object):
     ''' Base class for implementing Infoblox WAPI API '''
-
-    provider_spec = {'provider': dict(type='dict', options=nios_provider_spec)}
+    provider_spec = {'provider': dict(type='dict', options=NIOS_PROVIDER_SPEC)}
 
     def __init__(self, provider):
         self.connector = get_connector(**provider)
@@ -163,7 +159,6 @@ class WapiInventory(WapiBase):
 
 class WapiModule(WapiBase):
     ''' Implements WapiBase for executing a NIOS module '''
-
     def __init__(self, module):
         self.module = module
         provider = module.params['provider']
@@ -174,29 +169,29 @@ class WapiModule(WapiBase):
 
     def handle_exception(self, method_name, exc):
         ''' Handles any exceptions raised
-
         This method will be called if an InfobloxException is raised for
-        any call to the instance of Connector.  This method will then
-        gracefully fail the module.
-
+        any call to the instance of Connector and also, in case of generic
+        exception. This method will then gracefully fail the module.
         :args exc: instance of InfobloxException
         '''
-        self.module.fail_json(
-            msg=exc.response['text'],
-            type=exc.response['Error'].split(':')[0],
-            code=exc.response.get('code'),
-            operation=method_name
-        )
+        if ('text' in exc.response):
+            self.module.fail_json(
+                msg=exc.response['text'],
+                type=exc.response['Error'].split(':')[0],
+                code=exc.response.get('code'),
+                operation=method_name
+            )
+        else:
+            self.module.fail_json(msg=to_native(exc))
 
     def run(self, ib_obj_type, ib_spec):
         ''' Runs the module and performans configuration tasks
-
         :args ib_obj_type: the WAPI object type to operate against
         :args ib_spec: the specification for the WAPI object as a dict
-
         :returns: a results dict
         '''
 
+        update = new_name = None
         state = self.module.params['state']
         if state not in ('present', 'absent'):
             self.module.fail_json(msg='state must be one of `present`, `absent`, got `%s`' % state)
@@ -204,10 +199,14 @@ class WapiModule(WapiBase):
         result = {'changed': False}
 
         obj_filter = dict([(k, self.module.params[k]) for k, v in iteritems(ib_spec) if v.get('ib_req')])
-        ib_obj = self.get_object(ib_obj_type, obj_filter.copy(), return_fields=ib_spec.keys())
 
-        if ib_obj:
-            current_object = ib_obj[0]
+        if('name' in obj_filter):
+            ib_obj_ref, update, new_name = self.get_object_ref(ib_obj_type, obj_filter, ib_spec)
+        else:
+            ib_obj_ref = self.get_object(ib_obj_type, obj_filter.copy(), return_fields=ib_spec.keys())
+
+        if ib_obj_ref:
+            current_object = ib_obj_ref[0]
             if 'extattrs' in current_object:
                 current_object['extattrs'] = flatten_extattrs(current_object['extattrs'])
             ref = current_object.pop('_ref')
@@ -223,25 +222,28 @@ class WapiModule(WapiBase):
                 else:
                     proposed_object[key] = self.module.params[key]
 
-        modified = not self.compare_objects(current_object, proposed_object)
+        # checks if the name's field has been updated
+        if update and new_name:
+            proposed_object['name'] = new_name
 
+        res = None
+        modified = not self.compare_objects(current_object, proposed_object)
         if 'extattrs' in proposed_object:
             proposed_object['extattrs'] = normalize_extattrs(proposed_object['extattrs'])
-
         if state == 'present':
             if ref is None:
                 if not self.module.check_mode:
                     self.create_object(ib_obj_type, proposed_object)
                 result['changed'] = True
             elif modified:
-                if 'network_view' in proposed_object:
-                    self.check_if_network_view_exists(proposed_object['network_view'])
-                    proposed_object.pop('network_view')
-                elif 'view' in proposed_object:
-                    self.check_if_dns_view_exists(proposed_object['view'])
-                if not self.module.check_mode:
+                if (ib_obj_type in (NIOS_HOST_RECORD, NIOS_NETWORK_VIEW, NIOS_DNS_VIEW)):
                     proposed_object = self.on_update(proposed_object, ib_spec)
                     res = self.update_object(ref, proposed_object)
+                elif 'network_view' in proposed_object:
+                    proposed_object.pop('network_view')
+                if not self.module.check_mode and res is None:
+                    proposed_object = self.on_update(proposed_object, ib_spec)
+                    self.update_object(ref, proposed_object)
                 result['changed'] = True
 
         elif state == 'absent':
@@ -252,42 +254,10 @@ class WapiModule(WapiBase):
 
         return result
 
-    def check_if_dns_view_exists(self, name, fail_on_missing=True):
-        ''' Checks if the specified DNS view is already configured
-
-        :args name: the name of the  DNS view to check
-        :args fail_on_missing: fail the module if the DNS view does not exist
-
-        :returns: True if the network_view exists and False if the  DNS view
-            does not exist and fail_on_missing is False
-        '''
-        res = self.get_object('view', {'name': name}) is not None
-        if not res and fail_on_missing:
-            self.module.fail_json(msg='DNS view %s does not exist, please create '
-                                      'it using nios_dns_view first' % name)
-        return res
-
-    def check_if_network_view_exists(self, name, fail_on_missing=True):
-        ''' Checks if the specified network_view is already configured
-
-        :args name: the name of the network view to check
-        :args fail_on_missing: fail the module if the network_view does not exist
-
-        :returns: True if the network_view exists and False if the network_view
-            does not exist and fail_on_missing is False
-        '''
-        res = self.get_object('networkview', {'name': name}) is not None
-        if not res and fail_on_missing:
-            self.module.fail_json(msg='Network view %s does not exist, please create '
-                                      'it using nios_network_view first' % name)
-        return res
-
     def issubset(self, item, objects):
         ''' Checks if item is a subset of objects
-
         :args item: the subset item to validate
         :args objects: superset list of objects to validate against
-
         :returns: True if item is a subset of one entry in objects otherwise
             this method will return None
         '''
@@ -322,16 +292,45 @@ class WapiModule(WapiBase):
 
         return True
 
+    def get_object_ref(self, ib_obj_type, obj_filter, ib_spec):
+        ''' this function gets and returns the current object based on name/old_name passed'''
+        update = False
+        old_name = new_name = None
+        try:
+            name_obj = self.module._check_type_dict(obj_filter['name'])
+            old_name = name_obj['old_name']
+            new_name = name_obj['new_name']
+        except TypeError:
+            name = obj_filter['name']
+
+        if old_name and new_name:
+            if (ib_obj_type == NIOS_HOST_RECORD):
+                test_obj_filter = dict([('name', old_name), ('view', obj_filter['view'])])
+            else:
+                test_obj_filter = dict([('name', old_name)])
+            # get the object reference
+            ib_obj = self.get_object(ib_obj_type, test_obj_filter, return_fields=ib_spec.keys())
+            if ib_obj:
+                obj_filter['name'] = new_name
+            else:
+                test_obj_filter['name'] = new_name
+                ib_obj = self.get_object(ib_obj_type, test_obj_filter, return_fields=ib_spec.keys())
+            update = True
+            return ib_obj, update, new_name
+        if (ib_obj_type == NIOS_HOST_RECORD):
+            test_obj_filter = dict([('name', name), ('view', obj_filter['view'])])
+        else:
+            test_obj_filter = dict([('name', name)])
+        ib_obj = self.get_object(ib_obj_type, test_obj_filter.copy(), return_fields=ib_spec.keys())
+        return ib_obj, update, new_name
+
     def on_update(self, proposed_object, ib_spec):
         ''' Event called before the update is sent to the API endpoing
-
         This method will allow the final proposed object to be changed
         and/or keys filtered before it is sent to the API endpoint to
         be processed.
-
         :args proposed_object: A dict item that will be encoded and sent
             the the API endpoint with the updated data structure
-
         :returns: updated object to be sent to API endpoint
         '''
         keys = set()
@@ -339,5 +338,4 @@ class WapiModule(WapiBase):
             update = ib_spec[key].get('update', True)
             if not update:
                 keys.add(key)
-
         return dict([(k, v) for k, v in iteritems(proposed_object) if k not in keys])

lib/ansible/module_utils/network/f5/bigip.py

@@ -37,7 +37,7 @@ class F5Client(F5BaseClient):
         if self._client:
             return self._client
 
-        for x in range(0, 60):
+        for x in range(0, 10):
             try:
                 result = ManagementRoot(
                     self.provider['server'],
@@ -53,7 +53,7 @@ class F5Client(F5BaseClient):
                 exc = ex
                 time.sleep(1)
         error = 'Unable to connect to {0} on port {1}.'.format(
-            self.params['server'], self.params['server_port']
+            self.provider['server'], self.provider['server_port']
         )
 
         if exc is not None:
@@ -71,7 +71,6 @@ class F5RestClient(F5BaseClient):
         exc = None
         if self._client:
             return self._client
-
         for x in range(0, 10):
             try:
                 url = "https://{0}:{1}/mgmt/shared/authn/login".format(

lib/ansible/module_utils/network/f5/bigiq.py

@@ -29,32 +29,33 @@ except ImportError:
 
 
 class F5Client(F5BaseClient):
+    def __init__(self, *args, **kwargs):
+        super(F5Client, self).__init__(*args, **kwargs)
+        self.provider = self.merge_provider_params()
+
     @property
     def api(self):
         exc = None
         if self._client:
             return self._client
-        for x in range(0, 3):
+        for x in range(0, 10):
             try:
-                server = self.params['provider']['server'] or self.params['server']
-                user = self.params['provider']['user'] or self.params['user']
-                password = self.params['provider']['password'] or self.params['password']
-                server_port = self.params['provider']['server_port'] or self.params['server_port'] or 443
-                validate_certs = self.params['provider']['validate_certs'] or self.params['validate_certs']
-
                 result = ManagementRoot(
-                    server,
-                    user,
-                    password,
-                    port=server_port,
-                    verify=validate_certs
+                    self.provider['server'],
+                    self.provider['user'],
+                    self.provider['password'],
+                    port=self.provider['server_port'],
+                    verify=self.provider['validate_certs']
                 )
                 self._client = result
                 return self._client
             except Exception as ex:
                 exc = ex
                 time.sleep(1)
-        error = 'Unable to connect to {0} on port {1}.'.format(self.params['server'], self.params['server_port'])
+        error = 'Unable to connect to {0} on port {1}.'.format(
+            self.provider['server'], self.provider['server_port']
+        )
+
         if exc is not None:
             error += ' The reported error was "{0}".'.format(str(exc))
         raise F5ModuleError(error)

lib/ansible/module_utils/network/meraki/meraki.py

@@ -151,7 +151,7 @@ class MerakiModule(object):
 
     def get_orgs(self):
         ''' Downloads all organizations '''
-        return json.loads(self.request('/organizations', method='GET'))
+        return self.request('/organizations', method='GET')
 
     def is_org_valid(self, data, org_name=None, org_id=None):
         ''' Checks whether a specific org exists and is duplicated '''
@@ -172,6 +172,7 @@ class MerakiModule(object):
             If org_id is specified as parameter, return that instead of a lookup
         '''
         orgs = self.get_orgs()
+        # self.fail_json(msg='ogs', orgs=orgs)
         if self.params['org_id'] is not None:
             if self.is_org_valid(orgs, org_id=self.params['org_id']) is True:
                 return self.params['org_id']
@@ -191,7 +192,7 @@ class MerakiModule(object):
             org_id = self.get_org_id(org_name)
         path = self.construct_path('get_all', org_id=org_id)
         r = self.request(path, method='GET')
-        return json.loads(r)
+        return r
 
     def get_net(self, org_name, net_name, data=None):
         ''' Return network information '''
@@ -213,7 +214,7 @@ class MerakiModule(object):
 
     def get_net_id(self, org_name=None, net_name=None, data=None):
         ''' Return network id from lookup or existing data '''
-        if not data:
+        if data is None:
             self.fail_json(msg='Must implement lookup')
         for n in data:
             if n['name'] == net_name:
@@ -225,7 +226,6 @@ class MerakiModule(object):
         if function is None:
             built_path = self.url_catalog[action][self.function]
         else:
-            self.function = function
             built_path = self.url_catalog[action][function]
         if org_name:
             org_id = self.get_org_id(org_name)
@@ -253,7 +253,10 @@ class MerakiModule(object):
 
         if self.status >= 300:
             self.fail_json(msg='Request failed for {url}: {status} - {msg}'.format(**info))
-        return to_native(resp.read())
+        try:
+            return json.loads(to_native(resp.read()))
+        except:
+            pass
 
     def exit_json(self, **kwargs):
         self.result['response'] = self.response

lib/ansible/module_utils/network/nxos/nxos.py

@@ -160,8 +160,7 @@ class Cli:
         msgs = []
         try:
             responses = connection.edit_config(config)
-            out = json.loads(responses)[1:-1]
-            msg = out
+            msg = json.loads(responses)
         except ConnectionError as e:
             code = getattr(e, 'code', 1)
             message = getattr(e, 'err', e)
@@ -479,3 +478,24 @@ def normalize_interface(name):
         proper_interface = name
 
     return proper_interface
+
+
+def get_interface_type(interface):
+    """Gets the type of interface
+    """
+    if interface.upper().startswith('ET'):
+        return 'ethernet'
+    elif interface.upper().startswith('VL'):
+        return 'svi'
+    elif interface.upper().startswith('LO'):
+        return 'loopback'
+    elif interface.upper().startswith('MG'):
+        return 'management'
+    elif interface.upper().startswith('MA'):
+        return 'management'
+    elif interface.upper().startswith('PO'):
+        return 'portchannel'
+    elif interface.upper().startswith('NV'):
+        return 'nve'
+    else:
+        return 'unknown'

lib/ansible/modules/cloud/amazon/aws_s3.py

@@ -278,16 +278,18 @@ s3_keys:
 import hashlib
 import mimetypes
 import os
-import traceback
 from ansible.module_utils.six.moves.urllib.parse import urlparse
 from ssl import SSLError
-from ansible.module_utils.basic import AnsibleModule, to_text, to_native
-from ansible.module_utils.ec2 import ec2_argument_spec, camel_dict_to_snake_dict, get_aws_connection_info, boto3_conn, HAS_BOTO3
+from ansible.module_utils.basic import to_text, to_native
+from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.ec2 import ec2_argument_spec, get_aws_connection_info, boto3_conn
 
 try:
     import botocore
 except ImportError:
-    pass  # will be detected by imported HAS_BOTO3
+    pass  # will be detected by imported AnsibleAWSModule
+
+IGNORE_S3_DROP_IN_EXCEPTIONS = ['XNotImplemented', 'NotImplemented']
 
 
 def key_check(module, s3, bucket, obj, version=None, validate=True):
@@ -306,8 +308,9 @@ def key_check(module, s3, bucket, obj, version=None, validate=True):
         elif error_code == 403 and validate is False:
             pass
         else:
-            module.fail_json(msg="Failed while looking up object (during key check) %s." % obj,
-                             exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+            module.fail_json_aws(e, msg="Failed while looking up object (during key check) %s." % obj)
+    except botocore.exceptions.BotoCoreError as e:
+        module.fail_json_aws(e, msg="Failed while looking up object (during key check) %s." % obj)
     return exists
 
 
@@ -362,16 +365,17 @@ def bucket_check(module, s3, bucket, validate=True):
         elif error_code == 403 and validate is False:
             pass
         else:
-            module.fail_json(msg="Failed while looking up bucket (during bucket_check) %s." % bucket,
-                             exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+            module.fail_json_aws(e, msg="Failed while looking up bucket (during bucket_check) %s." % bucket)
     except botocore.exceptions.EndpointConnectionError as e:
-        module.fail_json(msg="Invalid endpoint provided: %s" % to_text(e), exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+        module.fail_json_aws(e, msg="Invalid endpoint provided")
+    except botocore.exceptions.BotoCoreError as e:
+        module.fail_json_aws(e, msg="Failed while looking up bucket (during bucket_check) %s." % bucket)
     return exists
 
 
 def create_bucket(module, s3, bucket, location=None):
     if module.check_mode:
-        module.exit_json(msg="PUT operation skipped - running in check mode", changed=True)
+        module.exit_json(msg="CREATE operation skipped - running in check mode", changed=True)
     configuration = {}
     if location not in ('us-east-1', None):
         configuration['LocationConstraint'] = location
@@ -383,8 +387,12 @@ def create_bucket(module, s3, bucket, location=None):
         for acl in module.params.get('permission'):
             s3.put_bucket_acl(ACL=acl, Bucket=bucket)
     except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed while creating bucket or setting acl (check that you have CreateBucket and PutBucketAcl permission).",
-                         exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+        if e.response['Error']['Code'] in IGNORE_S3_DROP_IN_EXCEPTIONS:
+            module.warn("PutBucketAcl is not implemented by your storage provider. Set the permission parameters to the empty list to avoid this warning")
+        else:
+            module.fail_json_aws(e, msg="Failed while creating bucket or setting acl (check that you have CreateBucket and PutBucketAcl permission).")
+    except botocore.exceptions.BotoCoreError as e:
+        module.fail_json_aws(e, msg="Failed while creating bucket or setting acl (check that you have CreateBucket and PutBucketAcl permission).")
 
     if bucket:
         return True
@@ -403,10 +411,8 @@ def list_keys(module, s3, bucket, prefix, marker, max_keys):
     try:
         keys = sum(paginated_list(s3, **pagination_params), [])
         module.exit_json(msg="LIST operation complete", s3_keys=keys)
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed while listing the keys in the bucket {0}".format(bucket),
-                         exception=traceback.format_exc(),
-                         **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed while listing the keys in the bucket {0}".format(bucket))
 
 
 def delete_bucket(module, s3, bucket):
@@ -423,8 +429,8 @@ def delete_bucket(module, s3, bucket):
                 s3.delete_objects(Bucket=bucket, Delete={'Objects': formatted_keys})
         s3.delete_bucket(Bucket=bucket)
         return True
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed while deleting bucket %s.", exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed while deleting bucket %s." % bucket)
 
 
 def delete_key(module, s3, bucket, obj):
@@ -433,8 +439,8 @@ def delete_key(module, s3, bucket, obj):
     try:
         s3.delete_object(Bucket=bucket, Key=obj)
         module.exit_json(msg="Object deleted from bucket %s." % (bucket), changed=True)
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed while trying to delete %s." % obj, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed while trying to delete %s." % obj)
 
 
 def create_dirkey(module, s3, bucket, obj, encrypt):
@@ -448,9 +454,14 @@ def create_dirkey(module, s3, bucket, obj, encrypt):
         s3.put_object(**params)
         for acl in module.params.get('permission'):
             s3.put_object_acl(ACL=acl, Bucket=bucket, Key=obj)
-        module.exit_json(msg="Virtual directory %s created in bucket %s" % (obj, bucket), changed=True)
     except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed while creating object %s." % obj, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+        if e.response['Error']['Code'] in IGNORE_S3_DROP_IN_EXCEPTIONS:
+            module.warn("PutObjectAcl is not implemented by your storage provider. Set the permissions parameters to the empty list to avoid this warning")
+        else:
+            module.fail_json_aws(e, msg="Failed while creating object %s." % obj)
+    except botocore.exceptions.BotoCoreError as e:
+        module.fail_json_aws(e, msg="Failed while creating object %s." % obj)
+    module.exit_json(msg="Virtual directory %s created in bucket %s" % (obj, bucket), changed=True)
 
 
 def path_check(path):
@@ -501,14 +512,25 @@ def upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, heade
             extra['ContentType'] = content_type
 
         s3.upload_file(Filename=src, Bucket=bucket, Key=obj, ExtraArgs=extra)
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Unable to complete PUT operation.")
+    try:
         for acl in module.params.get('permission'):
             s3.put_object_acl(ACL=acl, Bucket=bucket, Key=obj)
+    except botocore.exceptions.ClientError as e:
+        if e.response['Error']['Code'] in IGNORE_S3_DROP_IN_EXCEPTIONS:
+            module.warn("PutObjectAcl is not implemented by your storage provider. Set the permission parameters to the empty list to avoid this warning")
+        else:
+            module.fail_json_aws(e, msg="Unable to set object ACL")
+    except botocore.exceptions.BotoCoreError as e:
+        module.fail_json_aws(e, msg="Unable to set object ACL")
+    try:
         url = s3.generate_presigned_url(ClientMethod='put_object',
                                         Params={'Bucket': bucket, 'Key': obj},
                                         ExpiresIn=expiry)
-        module.exit_json(msg="PUT operation complete", url=url, changed=True)
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Unable to complete PUT operation.", exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Unable to generate presigned URL")
+    module.exit_json(msg="PUT operation complete", url=url, changed=True)
 
 
 def download_s3file(module, s3, bucket, obj, dest, retries, version=None):
@@ -522,22 +544,26 @@ def download_s3file(module, s3, bucket, obj, dest, retries, version=None):
         else:
             key = s3.get_object(Bucket=bucket, Key=obj)
     except botocore.exceptions.ClientError as e:
-        if e.response['Error']['Code'] != "404":
-            module.fail_json(msg="Could not find the key %s." % obj, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+        if e.response['Error']['Code'] not in ("403", "404"):
+            # AccessDenied errors may be triggered if 1) file does not exist or 2) file exists but
+            # user does not have the s3:GetObject permission. 404 errors are handled by download_file().
+            module.fail_json_aws(e, msg="Could not find the key %s." % obj)
+    except botocore.exceptions.BotoCoreError as e:
+        module.fail_json_aws(e, msg="Could not find the key %s." % obj)
 
     for x in range(0, retries + 1):
         try:
             s3.download_file(bucket, obj, dest)
             module.exit_json(msg="GET operation complete", changed=True)
-        except botocore.exceptions.ClientError as e:
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             # actually fail on last pass through the loop.
             if x >= retries:
-                module.fail_json(msg="Failed while downloading %s." % obj, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+                module.fail_json_aws(e, msg="Failed while downloading %s." % obj)
             # otherwise, try again, this may be a transient timeout.
         except SSLError as e:  # will ClientError catch SSLError?
             # actually fail on last pass through the loop.
             if x >= retries:
-                module.fail_json(msg="s3 download failed: %s." % e, exception=traceback.format_exc())
+                module.fail_json_aws(e, msg="s3 download failed")
             # otherwise, try again, this may be a transient timeout.
 
 
@@ -550,9 +576,8 @@ def download_s3str(module, s3, bucket, obj, version=None, validate=True):
         else:
             contents = to_native(s3.get_object(Bucket=bucket, Key=obj)["Body"].read())
         module.exit_json(msg="GET operation complete", contents=contents, changed=True)
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed while getting contents of object %s as a string." % obj,
-                         exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed while getting contents of object %s as a string." % obj)
 
 
 def get_download_url(module, s3, bucket, obj, expiry, changed=True):
@@ -561,8 +586,8 @@ def get_download_url(module, s3, bucket, obj, expiry, changed=True):
                                         Params={'Bucket': bucket, 'Key': obj},
                                         ExpiresIn=expiry)
         module.exit_json(msg="Download url:", url=url, expiry=expiry, changed=changed)
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed while getting download url.", exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed while getting download url.")
 
 
 def is_fakes3(s3_url):
@@ -573,17 +598,6 @@ def is_fakes3(s3_url):
         return False
 
 
-def is_walrus(s3_url):
-    """ Return True if it's Walrus endpoint, not S3
-
-    We assume anything other than *.amazonaws.com is Walrus"""
-    if s3_url is not None:
-        o = urlparse(s3_url)
-        return not o.netloc.endswith('amazonaws.com')
-    else:
-        return False
-
-
 def get_s3_connection(module, aws_connect_kwargs, location, rgw, s3_url):
     if s3_url and rgw:  # TODO - test this
         rgw = urlparse(s3_url)
@@ -602,9 +616,6 @@ def get_s3_connection(module, aws_connect_kwargs, location, rgw, s3_url):
         params = dict(module=module, conn_type='client', resource='s3', region=location,
                       endpoint="%s://%s:%s" % (protocol, fakes3.hostname, to_text(port)),
                       use_ssl=fakes3.scheme == 'fakes3s', **aws_connect_kwargs)
-    elif is_walrus(s3_url):
-        walrus = urlparse(s3_url).hostname
-        params = dict(module=module, conn_type='client', resource='s3', region=location, endpoint=walrus, **aws_connect_kwargs)
     else:
         params = dict(module=module, conn_type='client', resource='s3', region=location, endpoint=s3_url, **aws_connect_kwargs)
     return boto3_conn(**params)
@@ -635,7 +646,7 @@ def main():
             ignore_nonexistent_bucket=dict(default=False, type='bool')
         ),
     )
-    module = AnsibleModule(
+    module = AnsibleAWSModule(
         argument_spec=argument_spec,
         supports_check_mode=True,
         required_if=[['mode', 'put', ['src', 'object']],
@@ -647,9 +658,6 @@ def main():
     if module._name == 's3':
         module.deprecate("The 's3' module is being renamed 'aws_s3'", version=2.7)
 
-    if not HAS_BOTO3:
-        module.fail_json(msg='boto3 and botocore required for this module')
-
     bucket = module.params.get('bucket')
     encrypt = module.params.get('encrypt')
     expiry = module.params.get('expiry')

lib/ansible/modules/cloud/amazon/ec2_ami.py

@@ -420,8 +420,8 @@ def create_image(module, connection):
                 device = rename_item_if_exists(device, 'volume_type', 'VolumeType', 'Ebs')
                 device = rename_item_if_exists(device, 'snapshot_id', 'SnapshotId', 'Ebs')
                 device = rename_item_if_exists(device, 'delete_on_termination', 'DeleteOnTermination', 'Ebs')
-                device = rename_item_if_exists(device, 'size', 'VolumeSize', 'Ebs')
-                device = rename_item_if_exists(device, 'volume_size', 'VolumeSize', 'Ebs')
+                device = rename_item_if_exists(device, 'size', 'VolumeSize', 'Ebs', attribute_type=int)
+                device = rename_item_if_exists(device, 'volume_size', 'VolumeSize', 'Ebs', attribute_type=int)
                 device = rename_item_if_exists(device, 'iops', 'Iops', 'Ebs')
                 device = rename_item_if_exists(device, 'encrypted', 'Encrypted', 'Ebs')
                 block_device_mapping.append(device)
@@ -626,13 +626,15 @@ def get_image_by_id(module, connection, image_id):
         module.fail_json_aws(e, msg="Error retrieving image by image_id")
 
 
-def rename_item_if_exists(dict_object, attribute, new_attribute, child_node=None):
+def rename_item_if_exists(dict_object, attribute, new_attribute, child_node=None, attribute_type=None):
     new_item = dict_object.get(attribute)
     if new_item is not None:
+        if attribute_type is not None:
+            new_item = attribute_type(new_item)
         if child_node is None:
-            dict_object[new_attribute] = dict_object.get(attribute)
+            dict_object[new_attribute] = new_item
         else:
-            dict_object[child_node][new_attribute] = dict_object.get(attribute)
+            dict_object[child_node][new_attribute] = new_item
         dict_object.pop(attribute)
     return dict_object
 

lib/ansible/modules/cloud/azure/azure_rm_containerinstance.py

@@ -50,6 +50,7 @@ options:
         choices:
             - public
             - none
+        default: 'none'
     ports:
         description:
             - List of ports exposed within the container group.
@@ -144,6 +145,7 @@ from ansible.module_utils.azure_rm_common import AzureRMModuleBase
 
 try:
     from msrestazure.azure_exceptions import CloudError
+    from msrestazure.azure_operation import AzureOperationPoller
     from azure.mgmt.containerinstance import ContainerInstanceManagementClient
 except ImportError:
     # This is handled in azure_rm_common
@@ -382,6 +384,9 @@ class AzureRMContainerInstance(AzureRMModuleBase):
                                                                  container_group_name=self.name,
                                                                  container_group=parameters)
 
+        if isinstance(response, AzureOperationPoller):
+            response = self.get_poller_result(response)
+
         return response.as_dict()
 
     def delete_containerinstance(self):

lib/ansible/modules/cloud/azure/azure_rm_mysqldatabase.py

@@ -96,9 +96,9 @@ import time
 from ansible.module_utils.azure_rm_common import AzureRMModuleBase
 
 try:
-    from msrestazure.azure_exceptions import CloudError
-    from msrestazure.azure_operation import AzureOperationPoller
     from azure.mgmt.rdbms.mysql import MySQLManagementClient
+    from msrestazure.azure_exceptions import CloudError
+    from msrest.polling import LROPoller
     from msrest.serialization import Model
 except ImportError:
     # This is handled in azure_rm_common
@@ -248,7 +248,7 @@ class AzureRMDatabases(AzureRMModuleBase):
                                                                    server_name=self.server_name,
                                                                    database_name=self.name,
                                                                    parameters=self.parameters)
-            if isinstance(response, AzureOperationPoller):
+            if isinstance(response, LROPoller):
                 response = self.get_poller_result(response)
 
         except CloudError as exc:

lib/ansible/modules/cloud/azure/azure_rm_mysqlserver.py

@@ -137,9 +137,9 @@ import time
 from ansible.module_utils.azure_rm_common import AzureRMModuleBase
 
 try:
-    from msrestazure.azure_exceptions import CloudError
-    from msrestazure.azure_operation import AzureOperationPoller
     from azure.mgmt.rdbms.mysql import MySQLManagementClient
+    from msrestazure.azure_exceptions import CloudError
+    from msrest.polling import LROPoller
     from msrest.serialization import Model
 except ImportError:
     # This is handled in azure_rm_common
@@ -316,10 +316,15 @@ class AzureRMServers(AzureRMModuleBase):
         self.log("Creating / Updating the MySQL Server instance {0}".format(self.name))
 
         try:
-            response = self.mgmt_client.servers.create_or_update(resource_group_name=self.resource_group,
-                                                                 server_name=self.name,
-                                                                 parameters=self.parameters)
-            if isinstance(response, AzureOperationPoller):
+            if (self.to_do == Actions.Create):
+                response = self.mgmt_client.servers.create(resource_group_name=self.resource_group,
+                                                           server_name=self.name,
+                                                           parameters=self.parameters)
+            else:
+                response = self.mgmt_client.servers.update(resource_group_name=self.resource_group,
+                                                           server_name=self.name,
+                                                           parameters=self.parameters)
+            if isinstance(response, LROPoller):
                 response = self.get_poller_result(response)
 
         except CloudError as exc:

lib/ansible/modules/cloud/azure/azure_rm_networkinterface.py

@@ -311,7 +311,7 @@ except ImportError:
     # This is handled in azure_rm_common
     pass
 
-from ansible.module_utils.azure_rm_common import AzureRMModuleBase, azure_id_to_dict
+from ansible.module_utils.azure_rm_common import AzureRMModuleBase, azure_id_to_dict, normalize_location_name
 from ansible.module_utils._text import to_native
 
 
@@ -446,6 +446,7 @@ class AzureRMNetworkInterface(AzureRMModuleBase):
         if not self.location:
             # Set default location
             self.location = resource_group.location
+        self.location = normalize_location_name(self.location)
 
         # parse the virtual network resource group and name
         self.virtual_network = self.parse_resource_to_dict(self.virtual_network)

lib/ansible/modules/cloud/azure/azure_rm_postgresqldatabase.py

@@ -96,9 +96,9 @@ import time
 from ansible.module_utils.azure_rm_common import AzureRMModuleBase
 
 try:
-    from msrestazure.azure_exceptions import CloudError
-    from msrestazure.azure_operation import AzureOperationPoller
     from azure.mgmt.rdbms.postgresql import PostgreSQLManagementClient
+    from msrestazure.azure_exceptions import CloudError
+    from msrest.polling import LROPoller
     from msrest.serialization import Model
 except ImportError:
     # This is handled in azure_rm_common
@@ -248,7 +248,7 @@ class AzureRMDatabases(AzureRMModuleBase):
                                                                    server_name=self.server_name,
                                                                    database_name=self.name,
                                                                    parameters=self.parameters)
-            if isinstance(response, AzureOperationPoller):
+            if isinstance(response, LROPoller):
                 response = self.get_poller_result(response)
 
         except CloudError as exc: