New release v2.16.2 (#82399)

* Allow older pickle protocols to pickle unsafe classes. Fixes #82356

* Address issues when iterating or getting single index from AnsibleUnsafeBytes. Fixes #82375

* clog frag

changelogs/CHANGELOG-v2.16.rst

@@ -5,6 +5,22 @@ ansible-core 2.16 "All My Love" Release Notes
 .. contents:: Topics
 
 
+v2.16.2
+=======
+
+Release Summary
+---------------
+
+| Release Date: 2023-12-11
+| `Porting Guide <https://docs.ansible.com/ansible-core/2.16/porting_guides/porting_guide_core_2.16.html>`__
+
+
+Bugfixes
+--------
+
+- unsafe data - Address an incompatibility when iterating or getting a single index from ``AnsibleUnsafeBytes``
+- unsafe data - Address an incompatibility with ``AnsibleUnsafeText`` and ``AnsibleUnsafeBytes`` when pickling with ``protocol=0``
+
 v2.16.1
 =======
 

changelogs/changelog.yaml

@@ -822,3 +822,20 @@ releases:
     - restore_role_param_precedence.yml
     - wait_for_mmap.yml
     release_date: '2023-11-27'
+  2.16.2:
+    changes:
+      bugfixes:
+      - unsafe data - Address an incompatibility when iterating or getting a single
+        index from ``AnsibleUnsafeBytes``
+      - unsafe data - Address an incompatibility with ``AnsibleUnsafeText`` and ``AnsibleUnsafeBytes``
+        when pickling with ``protocol=0``
+      release_summary: '| Release Date: 2023-12-11
+
+        | `Porting Guide <https://docs.ansible.com/ansible-core/2.16/porting_guides/porting_guide_core_2.16.html>`__
+
+        '
+    codename: All My Love
+    fragments:
+    - 2.16.2_summary.yaml
+    - unsafe-fixes-2.yml
+    release_date: '2023-12-11'

changelogs/fragments/2.16.2_summary.yaml

@@ -0,0 +1,3 @@
+release_summary: |
+   | Release Date: 2023-12-11
+   | `Porting Guide <https://docs.ansible.com/ansible-core/2.16/porting_guides/porting_guide_core_2.16.html>`__

changelogs/fragments/unsafe-fixes-2.yml

@@ -0,0 +1,3 @@
+bugfixes:
+- unsafe data - Address an incompatibility with ``AnsibleUnsafeText`` and ``AnsibleUnsafeBytes`` when pickling with ``protocol=0``
+- unsafe data - Address an incompatibility when iterating or getting a single index from ``AnsibleUnsafeBytes``

lib/ansible/release.py

@@ -19,6 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-__version__ = '2.16.1'
+__version__ = '2.16.2'
 __author__ = 'Ansible, Inc.'
 __codename__ = "All My Love"

lib/ansible/utils/unsafe_proxy.py

@@ -71,6 +71,9 @@ class AnsibleUnsafeBytes(bytes, AnsibleUnsafe):
     def _strip_unsafe(self):
         return super().__bytes__()
 
+    def __reduce__(self, /):
+        return (self.__class__, (self._strip_unsafe(),))
+
     def __str__(self, /):  # pylint: disable=invalid-str-returned
         return self.decode()
 
@@ -84,12 +87,10 @@ class AnsibleUnsafeBytes(bytes, AnsibleUnsafe):
         return AnsibleUnsafeText(super().__format__(format_spec))
 
     def __getitem__(self, key, /):
+        if isinstance(key, int):
+            return super().__getitem__(key)
         return self.__class__(super().__getitem__(key))
 
-    def __iter__(self, /):
-        cls = self.__class__
-        return (cls(c) for c in super().__iter__())
-
     def __reversed__(self, /):
         return self[::-1]
 
@@ -192,6 +193,9 @@ class AnsibleUnsafeText(str, AnsibleUnsafe):
     def _strip_unsafe(self, /):
         return super().__str__()
 
+    def __reduce__(self, /):
+        return (self.__class__, (self._strip_unsafe(),))
+
     def __str__(self, /):  # pylint: disable=invalid-str-returned
         return self
 

test/integration/targets/filter_urls/tasks/main.yml

@@ -19,6 +19,13 @@
       - "{'foo': 'bar', 'baz': 'buz'}|urlencode == 'foo=bar&baz=buz'"
       - "()|urlencode == ''"
 
+- name: verify urlencode works for unsafe strings
+  assert:
+    that:
+      - thing|urlencode == 'foo%3Abar'
+  vars:
+    thing: !unsafe foo:bar
+
 # Needed (temporarily) due to coverage reports not including the last task.
 - assert:
     that: true

test/integration/targets/subversion/roles/subversion/defaults/main.yml

@@ -3,8 +3,7 @@ apache_port: 11386  # cannot use 80 as httptester overrides this
 subversion_test_dir: /tmp/ansible-svn-test-dir
 subversion_server_dir: /tmp/ansible-svn  # cannot use a path in the home dir without userdir or granting exec permission to the apache user
 subversion_repo_name: ansible-test-repo
-subversion_repo_url: http://127.0.0.1:{{ apache_port }}/svn/{{ subversion_repo_name }}
-subversion_repo_auth_url: http://127.0.0.1:{{ apache_port }}/svnauth/{{ subversion_repo_name }}
+subversion_repo_url: https://localhost:{{ apache_port }}/svn/{{ subversion_repo_name }}  # svn can't verify TLS certificates against IP addresses
+subversion_repo_auth_url: https://localhost:{{ apache_port }}/svnauth/{{ subversion_repo_name }}
 subversion_username: subsvn_user'''
 subversion_password: Password123!
-subversion_external_repo_url: https://github.com/ansible/ansible.github.com  # GitHub serves SVN

test/integration/targets/subversion/roles/subversion/tasks/setup.yml

@@ -33,6 +33,60 @@
   include_tasks: setup_selinux.yml
   when: ansible_selinux.status == "enabled"
 
+- name: Generate CA and TLS certificates via trustme
+  vars:
+    venv_path: >-
+      {{ subversion_server_dir }}/.venv
+    venv_python: >-
+      {{ subversion_server_dir }}/.venv/bin/python
+  block:
+  - name: trustme -- provision a venv
+    command: >-
+      {{ ansible_python_interpreter }}
+      -{% if ansible_python.version.major != 2 %}I{% endif %}m
+      {% if ansible_python.version.major != 2 %}venv{%
+         else %}virtualenv{% endif %}
+
+      {{ venv_path }}
+  - name: trustme -- upgrade pip in venv | RHEL 7.9 & 8.8+py36
+    when: >-  # these don't know how to notice `cryptography` wheels
+      ansible_distribution == 'RedHat'
+      and ansible_distribution_major_version | int < 9
+    pip:
+      name: pip
+      state: latest
+      virtualenv: >-
+        {{ venv_path }}
+  - name: trustme -- install tool
+    pip:
+      name: trustme
+      virtualenv: >-
+        {{ venv_path }}
+  - name: trustme -- generate CA and TLS certs
+    command:
+      argv:
+      - >-
+        {{ venv_python }}
+      - -{%- if ansible_python.version.major != 2 -%}I{%- endif -%}m
+      - trustme
+      - --dir={{ subversion_server_dir }}
+
+- name: symlink trustme certificates into apache config dir - Red Hat
+  when: ansible_os_family in ['RedHat']
+  # when: ansible_distribution in ['Fedora', 'RedHat']
+  file:
+    src: /tmp/ansible-svn/server.{{ item.trustme_filetype }}
+    dest: /etc/pki/tls/{{ item.apache_target_path }}
+    state: link
+    force: yes  # Othewise Apache on CentOS 7 uses its own fake certificate
+  loop:
+  - apache_target_path: certs/localhost.crt
+    trustme_filetype: pem
+  - apache_target_path: certs/server-chain.crt
+    trustme_filetype: pem
+  - apache_target_path: private/localhost.key
+    trustme_filetype: key
+
 - name: template out configuration file
   template:
     src: subversion.conf.j2
@@ -66,3 +120,14 @@
   async: 3600  # We kill apache manually in the clean up phase
   poll: 0
   when: ansible_os_family in ['RedHat']
+
+- lineinfile:
+    dest: >-
+      {{ ansible_env.HOME }}/.subversion/servers
+    regexp: >-
+      ^#\s*ssl-authority-files\s*=\s*
+    line: >-
+      ssl-authority-files = {{ subversion_server_dir }}/client.pem
+    insertafter: >-
+      ^\[global\]
+    state: present

test/integration/targets/subversion/roles/subversion/tasks/tests.yml

@@ -18,10 +18,11 @@
 
 # checks out every branch so using a small repo
 
-- name: initial checkout
+- name: initial checkout with validate_certs=true
   subversion:
     repo: '{{ subversion_repo_url }}'
     dest: '{{ subversion_test_dir }}/svn'
+    validate_certs: yes
   register: subverted
 
 - name: check if dir was checked out
@@ -130,16 +131,27 @@
       - "export_branches.stat.isdir"
       - "subverted4.changed"
 
-- name: clone a small external repo with validate_certs=true
-  subversion:
-    repo: "{{ subversion_external_repo_url }}"
-    dest: "{{ subversion_test_dir }}/svn-external1"
-    validate_certs: yes
+- name: unconfigure client-side TLS trust
+  block:
+  - name: remove TLS CA chain file path from the SVN config
+    lineinfile:
+      dest: >-
+        {{ ansible_env.HOME }}/.subversion/servers
+      regexp: >-
+        ^(?:#)?\s*ssl-authority-files\s*=\s*
+      state: absent
+  - name: drop the client TLS CA chain file
+    file:
+      path: >-
+        {{ subversion_server_dir }}/client.pem
+      state: absent
 
-- name: clone a small external repo with validate_certs=false
+- name: >-
+    clone a HTTPS-accessible repo with validate_certs=false
+    and untrusted CA over TLS
   subversion:
-    repo: "{{ subversion_external_repo_url }}"
-    dest: "{{ subversion_test_dir }}/svn-external2"
+    repo: '{{ subversion_repo_url }}'
+    dest: '{{ subversion_test_dir }}/svn-untrusted-tls'
     validate_certs: no
 
 # TBA: test for additional options or URL variants welcome

test/integration/targets/subversion/roles/subversion/templates/subversion.conf.j2

@@ -19,6 +19,7 @@ LogFormat "%h %l %u %t \"%r\" %>s %O" common
 LogFormat "%{Referer}i -> %U" referer
 LogFormat "%{User-agent}i" agent
 
+Include mods-available/ssl.load
 IncludeOptional mods-enabled/*.load
 IncludeOptional mods-enabled/*.conf
 IncludeOptional conf-enabled/*.conf
@@ -32,6 +33,7 @@ IncludeOptional sites-enabled/*conf
 
 {% elif ansible_os_family == "FreeBSD" %}
 Include /usr/local/etc/apache24/httpd.conf
+LoadModule ssl_module        libexec/apache24/mod_ssl.so
 LoadModule dav_module        libexec/apache24/mod_dav.so
 LoadModule dav_svn_module    libexec/apache24/mod_dav_svn.so
 LoadModule authz_svn_module  libexec/apache24/mod_authz_svn.so
@@ -48,7 +50,14 @@ Include /etc/httpd/conf/httpd.conf
 {% endif %}
 
 PidFile {{ subversion_server_dir }}/apache.pid
-Listen 127.0.0.1:{{ apache_port }}
+Listen 127.0.0.1:{{ apache_port }} https
+{% if ansible_distribution not in ["Alpine", "CentOS", "Fedora", "openSUSE Leap", "Ubuntu"] %}
+Listen [::1]:{{ apache_port }} https
+{% endif %}
+SSLEngine on
+SSLCertificateFile	{{ subversion_server_dir }}/server.pem
+SSLCertificateKeyFile {{ subversion_server_dir }}/server.key
+SSLCertificateChainFile {{ subversion_server_dir }}/server.pem
 ErrorLog {{ subversion_server_dir }}/apache2-error.log
 
 <Location /svn>

test/integration/targets/subversion/vars/Alpine.yml

@@ -5,5 +5,6 @@ subversion_packages:
 - apache2-webdav
 - apache2-utils
 - apache2-ctl
+- apache2-ssl
 apache_user: apache
 apache_group: apache

test/integration/targets/subversion/vars/RedHat.yml

@@ -1,6 +1,7 @@
 ---
 subversion_packages:
 - mod_dav_svn
+- mod_ssl
 - subversion
 upgrade_packages:
 # prevent sqlite from being out-of-sync with the version subversion was compiled with

test/integration/targets/subversion/vars/Ubuntu-20.yml

@@ -1,5 +1,7 @@
 ---
 subversion_packages:
+- apache2  # /usr/sbin/apachectl
+- apache2-utils  # htpasswd
 - subversion
 - libapache2-mod-svn
 apache_user: www-data