From a9c05a3d7086cc913dd5385df583fdf3b85f6952 Mon Sep 17 00:00:00 2001 From: abrand Date: Wed, 14 Aug 2024 13:33:17 -0400 Subject: [PATCH 1/7] fix cores detection on Windows OS --- rqd/rqd/rqmachine.py | 75 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 6874bd25b..9187b6440 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -631,20 +631,7 @@ def __initMachineStats(self, pathCpuInfo=None): hyperthreadingMultiplier = 1 if platform.system() == 'Windows': - # Windows memory information - stat = self.getWindowsMemory() - TEMP_DEFAULT = 1048576 - self.__renderHost.total_mcp = TEMP_DEFAULT - self.__renderHost.total_mem = int(stat.ullTotalPhys / 1024) - self.__renderHost.total_swap = int(stat.ullTotalPageFile / 1024) - - # Windows CPU information - logical_core_count = psutil.cpu_count(logical=True) - actual_core_count = psutil.cpu_count(logical=False) - hyperthreadingMultiplier = logical_core_count // actual_core_count - - __totalCores = logical_core_count * rqd.rqconstants.CORE_VALUE - __numProcs = 1 # TODO: figure out how to count sockets in Python + self.__init_stats_from_windows() # All other systems will just have one proc/core if not __numProcs or not __totalCores: @@ -674,6 +661,66 @@ def __initMachineStats(self, pathCpuInfo=None): if hyperthreadingMultiplier >= 1: self.__renderHost.attributes['hyperthreadingMultiplier'] = str(hyperthreadingMultiplier) + def __init_stats_from_windows(self): + """Init machine stats for Windows platforms. + + @rtype: tuple + @return: A 3-items tuple containing: + - the number of logical cores + - the number of physical processors + - the hyper-threading multiplier + + Implementation detail. + """ + # Windows memory information + stat = self.getWindowsMemory() + TEMP_DEFAULT = 1048576 + self.__renderHost.total_mcp = TEMP_DEFAULT + self.__renderHost.total_mem = int(stat.ullTotalPhys / 1024) + self.__renderHost.total_swap = int(stat.ullTotalPageFile / 1024) + + # Windows CPU information + self.__update_procs_mappings_from_windows() + + logical_core_count = psutil.cpu_count(logical=True) + actual_core_count = psutil.cpu_count(logical=False) + hyper_threading_multiplier = logical_core_count // actual_core_count + + physical_processor_count = len(self.__procs_by_physid_and_coreid) + + return logical_core_count, physical_processor_count, hyper_threading_multiplier + + def __update_procs_mappings_from_windows(self): + """Update `__procs_by_physid_and_coreid` and `__physid_and_coreid_by_proc` mappings for Windows platforms. + + Implementation detail. + """ + import wmi # Windows-specific + + # Reset mappings + self.__procs_by_physid_and_coreid = {} + self.__physid_and_coreid_by_proc = {} + + # Connect to the Windows Management Instrumentation (WMI) interface + wmi_instance = wmi.WMI() + + # Retrieve CPU information using WMI + for physical_id, processor in enumerate(wmi_instance.Win32_Processor()): + + thread_per_core = processor.NumberOfLogicalProcessors // processor.NumberOfCores + proc_id = 0 + + for core_id in range(processor.NumberOfCores): + for _ in range(thread_per_core): + self.__procs_by_physid_and_coreid.setdefault( + str(physical_id), {} + ).setdefault(str(core_id), set()).add(str(proc_id)) + self.__physid_and_coreid_by_proc[str(proc_id)] = ( + str(physical_id), + str(core_id), + ) + proc_id += 1 + def getWindowsMemory(self): """Gets information on system memory, Windows compatible version.""" # From From 0e7b590892f0193ecb54454de0489f37c5566ae0 Mon Sep 17 00:00:00 2001 From: abrand Date: Wed, 14 Aug 2024 13:35:05 -0400 Subject: [PATCH 2/7] add wmi requirement --- rqd/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rqd/setup.py b/rqd/setup.py index c1f7b9cc4..cc55dc098 100644 --- a/rqd/setup.py +++ b/rqd/setup.py @@ -61,7 +61,8 @@ 'grpcio', 'grpcio-tools', 'psutil', - 'pywin32; platform_system == "Windows"' + 'pywin32; platform_system == "Windows"', + 'wmi==1.5.1; platform_system == "Windows"' ] ) From 31c22c3ebd55708e6cc91138f8f49c1b88575397 Mon Sep 17 00:00:00 2001 From: abrand Date: Wed, 14 Aug 2024 13:48:27 -0400 Subject: [PATCH 3/7] reassign missing variables on windows stats init --- rqd/rqd/rqmachine.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 9187b6440..76ef58bde 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -631,7 +631,8 @@ def __initMachineStats(self, pathCpuInfo=None): hyperthreadingMultiplier = 1 if platform.system() == 'Windows': - self.__init_stats_from_windows() + logicalCoreCount, __numProcs, hyperthreadingMultiplier = self.__init_stats_from_windows() + __totalCores = logicalCoreCount * rqd.rqconstants.CORE_VALUE # All other systems will just have one proc/core if not __numProcs or not __totalCores: From ddeb921dc4f71191876ccd91604d0f9d3b85a9d5 Mon Sep 17 00:00:00 2001 From: abrand Date: Wed, 14 Aug 2024 13:52:15 -0400 Subject: [PATCH 4/7] rename variables and methods using camelcase for consistency with historical code --- rqd/rqd/rqmachine.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 76ef58bde..0000f8c85 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -631,7 +631,7 @@ def __initMachineStats(self, pathCpuInfo=None): hyperthreadingMultiplier = 1 if platform.system() == 'Windows': - logicalCoreCount, __numProcs, hyperthreadingMultiplier = self.__init_stats_from_windows() + logicalCoreCount, __numProcs, hyperthreadingMultiplier = self.__initStatsFromWindows() __totalCores = logicalCoreCount * rqd.rqconstants.CORE_VALUE # All other systems will just have one proc/core @@ -662,7 +662,7 @@ def __initMachineStats(self, pathCpuInfo=None): if hyperthreadingMultiplier >= 1: self.__renderHost.attributes['hyperthreadingMultiplier'] = str(hyperthreadingMultiplier) - def __init_stats_from_windows(self): + def __initStatsFromWindows(self): """Init machine stats for Windows platforms. @rtype: tuple @@ -681,17 +681,17 @@ def __init_stats_from_windows(self): self.__renderHost.total_swap = int(stat.ullTotalPageFile / 1024) # Windows CPU information - self.__update_procs_mappings_from_windows() + self.__updateProcsMappingsFromWindows() - logical_core_count = psutil.cpu_count(logical=True) - actual_core_count = psutil.cpu_count(logical=False) - hyper_threading_multiplier = logical_core_count // actual_core_count + logicalCoreCount = psutil.cpu_count(logical=True) + actualCoreCount = psutil.cpu_count(logical=False) + hyperThreadingMultiplier = logicalCoreCount // actualCoreCount - physical_processor_count = len(self.__procs_by_physid_and_coreid) + physicalProcessorCount = len(self.__procs_by_physid_and_coreid) - return logical_core_count, physical_processor_count, hyper_threading_multiplier + return logicalCoreCount, physicalProcessorCount, hyperThreadingMultiplier - def __update_procs_mappings_from_windows(self): + def __updateProcsMappingsFromWindows(self): """Update `__procs_by_physid_and_coreid` and `__physid_and_coreid_by_proc` mappings for Windows platforms. Implementation detail. @@ -703,24 +703,24 @@ def __update_procs_mappings_from_windows(self): self.__physid_and_coreid_by_proc = {} # Connect to the Windows Management Instrumentation (WMI) interface - wmi_instance = wmi.WMI() + wmiInstance = wmi.WMI() # Retrieve CPU information using WMI - for physical_id, processor in enumerate(wmi_instance.Win32_Processor()): + for physicalId, processor in enumerate(wmiInstance.Win32_Processor()): - thread_per_core = processor.NumberOfLogicalProcessors // processor.NumberOfCores - proc_id = 0 + threadPerCore = processor.NumberOfLogicalProcessors // processor.NumberOfCores + procId = 0 - for core_id in range(processor.NumberOfCores): - for _ in range(thread_per_core): + for coreId in range(processor.NumberOfCores): + for _ in range(threadPerCore): self.__procs_by_physid_and_coreid.setdefault( - str(physical_id), {} - ).setdefault(str(core_id), set()).add(str(proc_id)) - self.__physid_and_coreid_by_proc[str(proc_id)] = ( - str(physical_id), - str(core_id), + str(physicalId), {} + ).setdefault(str(coreId), set()).add(str(procId)) + self.__physid_and_coreid_by_proc[str(procId)] = ( + str(physicalId), + str(coreId), ) - proc_id += 1 + procId += 1 def getWindowsMemory(self): """Gets information on system memory, Windows compatible version.""" From 16493f1000188cd59642a1c090b249535eac801b Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 9 Oct 2024 11:13:56 -0700 Subject: [PATCH 5/7] Update rqd/setup.py Co-authored-by: Kern Attila GERMAIN <5556461+KernAttila@users.noreply.github.com> Signed-off-by: Diego Tavares --- rqd/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rqd/setup.py b/rqd/setup.py index cc55dc098..8e773372b 100644 --- a/rqd/setup.py +++ b/rqd/setup.py @@ -61,7 +61,7 @@ 'grpcio', 'grpcio-tools', 'psutil', - 'pywin32; platform_system == "Windows"', + 'pywin32==224; platform_system == "Windows"', 'wmi==1.5.1; platform_system == "Windows"' ] ) From 4fa8c13bd71a4b81d962798fd6b0cafdf91960b4 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 9 Oct 2024 11:14:06 -0700 Subject: [PATCH 6/7] Update rqd/rqd/rqmachine.py Signed-off-by: Diego Tavares --- rqd/rqd/rqmachine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 0000f8c85..a20a570aa 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -692,7 +692,9 @@ def __initStatsFromWindows(self): return logicalCoreCount, physicalProcessorCount, hyperThreadingMultiplier def __updateProcsMappingsFromWindows(self): - """Update `__procs_by_physid_and_coreid` and `__physid_and_coreid_by_proc` mappings for Windows platforms. + """ + Update `__procs_by_physid_and_coreid` and `__physid_and_coreid_by_proc` mappings + for Windows platforms. Implementation detail. """ From df4517e95f4e49bb1a6961c2d992f6b63142cada Mon Sep 17 00:00:00 2001 From: abrand Date: Wed, 23 Oct 2024 13:06:08 -0400 Subject: [PATCH 7/7] lint --- rqd/rqd/rqmachine.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 806c841d6..554d071ec 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -693,8 +693,6 @@ def __initStatsFromWindows(self): - the number of logical cores - the number of physical processors - the hyper-threading multiplier - - Implementation detail. """ # Windows memory information stat = self.getWindowsMemory() @@ -718,10 +716,9 @@ def __updateProcsMappingsFromWindows(self): """ Update `__procs_by_physid_and_coreid` and `__physid_and_coreid_by_proc` mappings for Windows platforms. - - Implementation detail. """ - import wmi # Windows-specific + # Windows-specific + import wmi # pylint:disable=import-outside-toplevel,import-error # Reset mappings self.__procs_by_physid_and_coreid = {}