habsi/ansible
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix KeyError for ansible-galaxy when caching paginated responses from v3 (#78325)

Browse Source 
* Fix KeyError for ansible-galaxy when caching paginated responses from v3

* changelog

* generate responses in loop for test

Co-authored-by: Matt Martz <matt@sivel.net>
This commit is contained in:
Sloane Hertel
2022-07-21 17:18:07 -04:00
committed by GitHub
parent 1429672213
commit 5728d72cda
3 changed files with 95 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
changelogs/fragments/78325-ansible-galaxy-fix-caching-paginated-responses-from-v3-servers.yml Normal file
View File

@ -0,0 +1,2 @@
bugfixes:
- ansible-galaxy - fix setting the cache for paginated responses from Galaxy NG/AH (https://github.com/ansible/ansible/issues/77911).

23
lib/ansible/galaxy/api.py
View File

@ -330,25 +330,27 @@ class GalaxyAPI:
should_retry_error=is_rate_limit_exception
)
def _call_galaxy(self, url, args=None, headers=None, method=None, auth_required=False, error_context_msg=None,
cache=False):
cache=False, cache_key=None):
url_info = urlparse(url)
cache_id = get_cache_id(url)
if not cache_key:
cache_key = url_info.path
query = parse_qs(url_info.query)
if cache and self._cache:
server_cache = self._cache.setdefault(cache_id, {})
iso_datetime_format = '%Y-%m-%dT%H:%M:%SZ'
valid = False
if url_info.path in server_cache:
expires = datetime.datetime.strptime(server_cache[url_info.path]['expires'], iso_datetime_format)
if cache_key in server_cache:
expires = datetime.datetime.strptime(server_cache[cache_key]['expires'], iso_datetime_format)
valid = datetime.datetime.utcnow() < expires
is_paginated_url = 'page' in query or 'offset' in query
if valid and not is_paginated_url:
# Got a hit on the cache and we aren't getting a paginated response
path_cache = server_cache[url_info.path]
path_cache = server_cache[cache_key]
if path_cache.get('paginated'):
if '/v3/' in url_info.path:
if '/v3/' in cache_key:
res = {'links': {'next': None}}
else:
res = {'next': None}
@ -368,7 +370,7 @@ class GalaxyAPI:
# The cache entry had expired or does not exist, start a new blank entry to be filled later.
expires = datetime.datetime.utcnow()
expires += datetime.timedelta(days=1)
server_cache[url_info.path] = {
server_cache[cache_key] = {
'expires': expires.strftime(iso_datetime_format),
'paginated': False,
}
@ -393,7 +395,7 @@ class GalaxyAPI:
% (resp.url, to_native(resp_data)))
if cache and self._cache:
path_cache = self._cache[cache_id][url_info.path]
path_cache = self._cache[cache_id][cache_key]
# v3 can return data or results for paginated results. Scan the result so we can determine what to cache.
paginated_key = None
@ -815,6 +817,7 @@ class GalaxyAPI:
page_size_name = 'limit' if 'v3' in self.available_api_versions else 'page_size'
versions_url = _urljoin(self.api_server, api_path, 'collections', namespace, name, 'versions', '/?%s=%d' % (page_size_name, COLLECTION_PAGE_SIZE))
versions_url_info = urlparse(versions_url)
cache_key = versions_url_info.path
# We should only rely on the cache if the collection has not changed. This may slow things down but it ensures
# we are not waiting a day before finding any new collections that have been published.
@ -834,7 +837,7 @@ class GalaxyAPI:
if cached_modified_date != modified_date:
modified_cache['%s.%s' % (namespace, name)] = modified_date
if versions_url_info.path in server_cache:
del server_cache[versions_url_info.path]
del server_cache[cache_key]
self._set_cache()
@ -842,7 +845,7 @@ class GalaxyAPI:
% (namespace, name, self.name, self.api_server)
try:
data = self._call_galaxy(versions_url, error_context_msg=error_context_msg, cache=True)
data = self._call_galaxy(versions_url, error_context_msg=error_context_msg, cache=True, cache_key=cache_key)
except GalaxyError as err:
if err.http_code != 404:
raise
@ -876,7 +879,7 @@ class GalaxyAPI:
next_link = versions_url.replace(versions_url_info.path, next_link)
data = self._call_galaxy(to_native(next_link, errors='surrogate_or_strict'),
error_context_msg=error_context_msg, cache=True)
error_context_msg=error_context_msg, cache=True, cache_key=cache_key)
self._set_cache()
return versions

80
test/units/galaxy/test_api.py
View File

@ -75,6 +75,57 @@ def get_test_galaxy_api(url, version, token_ins=None, token_value=None, no_cache
return api
def get_v3_collection_versions(namespace='namespace', name='collection'):
pagination_path = f"/api/galaxy/content/community/v3/plugin/{namespace}/content/community/collections/index/{namespace}/{name}/versions"
page_versions = (('1.0.0', '1.0.1',), ('1.0.2', '1.0.3',), ('1.0.4', '1.0.5'),)
responses = [
{}, # TODO: initial response
]
first = f"{pagination_path}/?limit=100"
last = f"{pagination_path}/?limit=100&offset=200"
page_versions = [
{
"versions": ('1.0.0', '1.0.1',),
"url": first,
},
{
"versions": ('1.0.2', '1.0.3',),
"url": f"{pagination_path}/?limit=100&offset=100",
},
{
"versions": ('1.0.4', '1.0.5'),
"url": last,
},
]
previous = None
for page in range(0, len(page_versions)):
data = []
if page_versions[page]["url"] == last:
next_page = None
else:
next_page = page_versions[page + 1]["url"]
links = {"first": first, "last": last, "next": next_page, "previous": previous}
for version in page_versions[page]["versions"]:
data.append(
{
"version": f"{version}",
"href": f"{pagination_path}/{version}/",
"created_at": "2022-05-13T15:55:58.913107Z",
"updated_at": "2022-05-13T15:55:58.913121Z",
"requires_ansible": ">=2.9.10"
}
)
responses.append({"meta": {"count": 6}, "links": links, "data": data})
previous = page_versions[page]["url"]
return responses
def get_collection_versions(namespace='namespace', name='collection'):
base_url = 'https://galaxy.server.com/api/v2/collections/{0}/{1}/'.format(namespace, name)
versions_url = base_url + 'versions/'
@ -1149,6 +1200,35 @@ def test_cache_complete_pagination(cache_dir, monkeypatch):
assert cached_versions == actual_versions
def test_cache_complete_pagination_v3(cache_dir, monkeypatch):
responses = get_v3_collection_versions()
cache_file = os.path.join(cache_dir, 'api.json')
api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v3', no_cache=False)
mock_open = MagicMock(
side_effect=[
StringIO(to_text(json.dumps(r)))
for r in responses
]
)
monkeypatch.setattr(galaxy_api, 'open_url', mock_open)
actual_versions = api.get_collection_versions('namespace', 'collection')
assert actual_versions == [u'1.0.0', u'1.0.1', u'1.0.2', u'1.0.3', u'1.0.4', u'1.0.5']
with open(cache_file) as fd:
final_cache = json.loads(fd.read())
cached_server = final_cache['galaxy.server.com:']
cached_collection = cached_server['/api/v3/collections/namespace/collection/versions/']
cached_versions = [r['version'] for r in cached_collection['results']]
assert final_cache == api._cache
assert cached_versions == actual_versions
def test_cache_flaky_pagination(cache_dir, monkeypatch):
responses = get_collection_versions()