From bc53e20a5f4a62011ddce47e6c16b0b416ccdb0f Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Mon, 8 Jun 2020 21:57:51 -0700 Subject: [PATCH] Allocate more workers to parse module docs. There are vastly more modules than other types of plugins so we want to allocate the bulk of our threads to processing documentation from them. This is an intermediate fix for #89 We may want to fix this in a better way later on. --- antsibull/docs_parsing/ansible_doc.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/antsibull/docs_parsing/ansible_doc.py b/antsibull/docs_parsing/ansible_doc.py index 394780bae..b22beede6 100644 --- a/antsibull/docs_parsing/ansible_doc.py +++ b/antsibull/docs_parsing/ansible_doc.py @@ -162,12 +162,19 @@ async def get_ansible_plugin_info(venv: Union['VenvRunner', 'FakeVenvRunner'], # Why use THREAD_MAX instead of process max? Even though this ultimately invokes separate # ansible-doc processes, the limiting factor is IO as ansible-doc reads from disk. So it makes # sense to scale up to THREAD_MAX instead of PROCESS_MAX. - max_workers = int(THREAD_MAX / len(DOCUMENTABLE_PLUGINS)) - if max_workers < 1: - max_workers = 1 + + # Allocate more for modules because the vast majority of plugins are modules + module_workers = max(int(.7 * THREAD_MAX), 1) + other_workers = int((THREAD_MAX - module_workers) / (len(DOCUMENTABLE_PLUGINS) - 1)) + if other_workers < 1: + other_workers = 1 extractors = {} for plugin_type in DOCUMENTABLE_PLUGINS: + if plugin_type == 'module': + max_workers = module_workers + else: + max_workers = other_workers extractors[plugin_type] = asyncio.create_task( _get_plugin_info(plugin_type, venv_ansible_doc, max_workers))