From ffbaf89c73560f5a75b7f3d29960343d21c55821 Mon Sep 17 00:00:00 2001 From: Simon Humpohl Date: Fri, 25 Aug 2023 14:34:33 +0200 Subject: [PATCH 1/6] Add automated mask shrinking --- qupulse/hardware/dacs/alazar.py | 5 ++- qupulse/utils/performance.py | 62 ++++++++++++++++++++++++++++++-- tests/utils/performance_tests.py | 17 ++++++++- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/qupulse/hardware/dacs/alazar.py b/qupulse/hardware/dacs/alazar.py index 7398e211..c694c4be 100644 --- a/qupulse/hardware/dacs/alazar.py +++ b/qupulse/hardware/dacs/alazar.py @@ -16,7 +16,7 @@ from qupulse.utils.types import TimeType from qupulse.hardware.dacs.dac_base import DAC from qupulse.hardware.util import traced -from qupulse.utils.performance import time_windows_to_samples +from qupulse.utils.performance import time_windows_to_samples, shrink_overlapping_windows logger = logging.getLogger(__name__) @@ -283,8 +283,7 @@ def _make_mask(self, mask_id: str, begins, lengths) -> Mask: if mask_type not in ('auto', 'cross_buffer', None): warnings.warn("Currently only CrossBufferMask is implemented.") - if np.any(begins[:-1]+lengths[:-1] > begins[1:]): - raise ValueError('Found overlapping windows in begins') + begins, lengths = shrink_overlapping_windows(begins, lengths) mask = CrossBufferMask() mask.identifier = mask_id diff --git a/qupulse/utils/performance.py b/qupulse/utils/performance.py index 4076b664..b68bc292 100644 --- a/qupulse/utils/performance.py +++ b/qupulse/utils/performance.py @@ -1,3 +1,4 @@ +import warnings from typing import Tuple import numpy as np @@ -24,6 +25,64 @@ def _is_monotonic_numpy(arr: np.ndarray) -> bool: return np.all(arr[1:] >= arr[:-1]) +def _shrink_overlapping_windows_numpy(begins, lengths) -> bool: + ends = begins + lengths + + overlaps = np.zeros_like(ends) + np.maximum(ends[:-1] - begins[1:], 0, out=overlaps[1:]) + + if np.any(overlaps >= lengths): + raise ValueError("Overlap is bigger than measurement window") + if np.any(overlaps > 0): + begins += overlaps + lengths -= overlaps + return True + return False + + +@njit +def _shrink_overlapping_windows_numba(begins, lengths) -> bool: + shrank = False + for idx in range(len(begins) - 1): + end = begins[idx] + lengths[idx] + next_begin = begins[idx + 1] + overlap = end - next_begin + + if overlap > 0: + shrank = True + if lengths[idx + 1] > overlap: + begins[idx + 1] += overlap + lengths[idx + 1] -= overlap + else: + raise ValueError("Overlap is bigger than measurement window") + return shrank + + +class WindowOverlapWarning(RuntimeWarning): + pass + + +def shrink_overlapping_windows(begins, lengths, use_numba: bool = numba is not None) -> Tuple[np.array, np.array]: + """Shrink windows in place if they overlap. Emits WindowOverlapWarning if a window was shrunk. + + Raises: + ValueError: if the overlap is bigger than a window. + + Warnings: + WindowOverlapWarning + """ + if use_numba: + backend = _shrink_overlapping_windows_numba + else: + backend = _shrink_overlapping_windows_numpy + begins = begins.copy() + lengths = lengths.copy() + if backend(begins, lengths): + warnings.warn("Found overlapping measurement windows which are automatically shrunken if possible.", + category=WindowOverlapWarning) + return begins, lengths + + @njit def _time_windows_to_samples_sorted_numba(begins, lengths, sample_rate: float) -> Tuple[np.ndarray, np.ndarray]: @@ -79,6 +138,3 @@ def time_windows_to_samples(begins: np.ndarray, lengths: np.ndarray, is_monotonic = _is_monotonic_numpy else: is_monotonic = _is_monotonic_numba - - - diff --git a/tests/utils/performance_tests.py b/tests/utils/performance_tests.py index d158dce5..ea07574b 100644 --- a/tests/utils/performance_tests.py +++ b/tests/utils/performance_tests.py @@ -2,7 +2,8 @@ import numpy as np -from qupulse.utils.performance import _time_windows_to_samples_numba, _time_windows_to_samples_numpy +from qupulse.utils.performance import (_time_windows_to_samples_numba, _time_windows_to_samples_numpy, + shrink_overlapping_windows) class TimeWindowsToSamplesTest(unittest.TestCase): @@ -28,3 +29,17 @@ def test_unsorted(self): self.assert_implementations_equal(begins, lengths, sr) +class TestOverlappingWindowReduction(unittest.TestCase): + def test_shrink_overlapping_windows_numba(self): + np.testing.assert_equal( + (np.array([1, 4, 8]), np.array([3, 4, 4])), + shrink_overlapping_windows(np.array([1, 4, 7]), + np.array([3, 4, 5]), use_numba=True) + ) + + def test_shrink_overlapping_windows_numpy(self): + np.testing.assert_equal( + (np.array([1, 4, 8]), np.array([3, 4, 4])), + shrink_overlapping_windows(np.array([1, 4, 7]), + np.array([3, 4, 5]), use_numba=False) + ) \ No newline at end of file From 607099147121d0182bdd7b0b799d87ad094b8896 Mon Sep 17 00:00:00 2001 From: Simon Humpohl Date: Fri, 25 Aug 2023 15:19:34 +0200 Subject: [PATCH 2/6] Fix unsigned integer overflow error... --- qupulse/utils/performance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qupulse/utils/performance.py b/qupulse/utils/performance.py index b68bc292..d1d669f8 100644 --- a/qupulse/utils/performance.py +++ b/qupulse/utils/performance.py @@ -46,9 +46,9 @@ def _shrink_overlapping_windows_numba(begins, lengths) -> bool: for idx in range(len(begins) - 1): end = begins[idx] + lengths[idx] next_begin = begins[idx + 1] - overlap = end - next_begin - if overlap > 0: + if end > next_begin: + overlap = end - next_begin shrank = True if lengths[idx + 1] > overlap: begins[idx + 1] += overlap From dba662cb67716186c8554218de723203f57a3802 Mon Sep 17 00:00:00 2001 From: Simon Humpohl Date: Fri, 25 Aug 2023 15:20:41 +0200 Subject: [PATCH 3/6] Test for warning --- tests/hardware/alazar_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/hardware/alazar_tests.py b/tests/hardware/alazar_tests.py index c6e7d032..034f0525 100644 --- a/tests/hardware/alazar_tests.py +++ b/tests/hardware/alazar_tests.py @@ -6,7 +6,7 @@ from ..hardware import * from qupulse.hardware.dacs.alazar import AlazarCard, AlazarProgram from qupulse.utils.types import TimeType - +from qupulse.utils.performance import WindowOverlapWarning class AlazarProgramTest(unittest.TestCase): def setUp(self) -> None: @@ -112,7 +112,7 @@ def test_make_mask(self): with self.assertRaises(KeyError): card._make_mask('N', begins, lengths) - with self.assertRaises(ValueError): + with self.assertWarns(WindowOverlapWarning): card._make_mask('M', begins, lengths*3) mask = card._make_mask('M', begins, lengths) From 108c28d021d3726fe06650748f648d967f96702b Mon Sep 17 00:00:00 2001 From: Simon Humpohl Date: Thu, 4 Apr 2024 11:07:58 +0200 Subject: [PATCH 4/6] Make overlapping windows a default error upon import and extend tests --- qupulse/utils/performance.py | 12 +++++-- tests/utils/performance_tests.py | 55 ++++++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/qupulse/utils/performance.py b/qupulse/utils/performance.py index 6159b21a..f68737a0 100644 --- a/qupulse/utils/performance.py +++ b/qupulse/utils/performance.py @@ -59,7 +59,15 @@ def _shrink_overlapping_windows_numba(begins, lengths) -> bool: class WindowOverlapWarning(RuntimeWarning): - pass + COMMENT = (" This warning is an error by default. " + "Call 'warnings.simplefilter(WindowOverlapWarning, \"always\")' " + "to demote it to a regular warning.") + + def __str__(self): + return super().__str__() + self.COMMENT + + +warnings.simplefilter(category=WindowOverlapWarning, action='error') def shrink_overlapping_windows(begins, lengths, use_numba: bool = numba is not None) -> Tuple[np.array, np.array]: @@ -78,7 +86,7 @@ def shrink_overlapping_windows(begins, lengths, use_numba: bool = numba is not N begins = begins.copy() lengths = lengths.copy() if backend(begins, lengths): - warnings.warn("Found overlapping measurement windows which are automatically shrunken if possible.", + warnings.warn("Found overlapping measurement windows which can be automatically shrunken if possible.", category=WindowOverlapWarning) return begins, lengths diff --git a/tests/utils/performance_tests.py b/tests/utils/performance_tests.py index 9a75485f..736bb322 100644 --- a/tests/utils/performance_tests.py +++ b/tests/utils/performance_tests.py @@ -1,11 +1,12 @@ import unittest +import warnings import numpy as np -from qupulse.utils.performance import (_time_windows_to_samples_numba, _time_windows_to_samples_numpy, - shrink_overlapping_windows) -from qupulse.utils.performance import (_time_windows_to_samples_numba, _time_windows_to_samples_numpy, - _average_windows_numba, _average_windows_numpy, average_windows) +from qupulse.utils.performance import ( + _time_windows_to_samples_numba, _time_windows_to_samples_numpy, + _average_windows_numba, _average_windows_numpy, average_windows, + shrink_overlapping_windows, WindowOverlapWarning) class TimeWindowsToSamplesTest(unittest.TestCase): @@ -57,17 +58,43 @@ def test_single_channel(self): def test_dual_channel(self): self.assert_implementations_equal(self.time, self.values, self.begins, self.ends) + + class TestOverlappingWindowReduction(unittest.TestCase): + def setUp(self): + self.shrank = np.array([1, 4, 8]), np.array([3, 4, 4]) + self.to_shrink = np.array([1, 4, 7]), np.array([3, 4, 5]) + + def assert_noop(self, shrink_fn): + begins = np.array([1, 3, 5]) + lengths = np.array([2, 1, 6]) + result = shrink_fn(begins, lengths) + np.testing.assert_equal((begins, lengths), result) + + def assert_shrinks(self, shrink_fn): + with warnings.catch_warnings(): + warnings.simplefilter("always", WindowOverlapWarning) + with self.assertWarns(WindowOverlapWarning): + shrank = shrink_fn(*self.to_shrink) + np.testing.assert_equal(self.shrank, shrank) + + def assert_empty_window_error(self, shrink_fn): + invalid = np.array([1, 2]), np.array([5, 1]) + with self.assertRaisesRegex(ValueError, "Overlap is bigger than measurement window"): + shrink_fn(*invalid) + def test_shrink_overlapping_windows_numba(self): - np.testing.assert_equal( - (np.array([1, 4, 8]), np.array([3, 4, 4])), - shrink_overlapping_windows(np.array([1, 4, 7]), - np.array([3, 4, 5]), use_numba=True) - ) + def shrink_fn(begins, lengths): + return shrink_overlapping_windows(begins, lengths, use_numba=True) + + self.assert_noop(shrink_fn) + self.assert_shrinks(shrink_fn) + self.assert_empty_window_error(shrink_fn) def test_shrink_overlapping_windows_numpy(self): - np.testing.assert_equal( - (np.array([1, 4, 8]), np.array([3, 4, 4])), - shrink_overlapping_windows(np.array([1, 4, 7]), - np.array([3, 4, 5]), use_numba=False) - ) \ No newline at end of file + def shrink_fn(begins, lengths): + return shrink_overlapping_windows(begins, lengths, use_numba=False) + + self.assert_noop(shrink_fn) + self.assert_shrinks(shrink_fn) + self.assert_empty_window_error(shrink_fn) From a5092dad629db95a029d9e099d38dafd6cbf5ee1 Mon Sep 17 00:00:00 2001 From: Simon Humpohl Date: Thu, 4 Apr 2024 11:53:11 +0200 Subject: [PATCH 5/6] Fix most tests --- qupulse/utils/performance.py | 12 ++++++++---- tests/utils/performance_tests.py | 20 +++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/qupulse/utils/performance.py b/qupulse/utils/performance.py index f68737a0..ebdb3b0a 100644 --- a/qupulse/utils/performance.py +++ b/qupulse/utils/performance.py @@ -26,16 +26,20 @@ def _is_monotonic_numpy(arr: np.ndarray) -> bool: def _shrink_overlapping_windows_numpy(begins, lengths) -> bool: + supported_dtypes = ('int64', 'uint64') + if begins.dtype.name not in supported_dtypes or lengths.dtype.name not in supported_dtypes: + raise NotImplementedError("This function only supports 64 bit integer types yet.") + ends = begins + lengths - overlaps = np.zeros_like(ends) - np.maximum(ends[:-1] - begins[1:], 0, out=overlaps[1:]) + overlaps = np.zeros_like(ends, dtype=np.int64) + np.maximum(ends[:-1].view(np.int64) - begins[1:].view(np.int64), 0, out=overlaps[1:]) if np.any(overlaps >= lengths): raise ValueError("Overlap is bigger than measurement window") if np.any(overlaps > 0): - begins += overlaps - lengths -= overlaps + begins += overlaps.view(begins.dtype) + lengths -= overlaps.view(lengths.dtype) return True return False diff --git a/tests/utils/performance_tests.py b/tests/utils/performance_tests.py index 736bb322..bc31960a 100644 --- a/tests/utils/performance_tests.py +++ b/tests/utils/performance_tests.py @@ -62,12 +62,22 @@ def test_dual_channel(self): class TestOverlappingWindowReduction(unittest.TestCase): def setUp(self): - self.shrank = np.array([1, 4, 8]), np.array([3, 4, 4]) - self.to_shrink = np.array([1, 4, 7]), np.array([3, 4, 5]) + self.shrank = np.array([1, 4, 8], dtype=np.uint64), np.array([3, 4, 4], dtype=np.uint64) + self.to_shrink = np.array([1, 4, 7], dtype=np.uint64), np.array([3, 4, 5], dtype=np.uint64) def assert_noop(self, shrink_fn): - begins = np.array([1, 3, 5]) - lengths = np.array([2, 1, 6]) + begins = np.array([1, 3, 5], dtype=np.uint64) + lengths = np.array([2, 1, 6], dtype=np.uint64) + result = shrink_fn(begins, lengths) + np.testing.assert_equal((begins, lengths), result) + + begins = (np.arange(100) * 176.5).astype(dtype=np.uint64) + lengths = (np.ones(100) * 10 * np.pi).astype(dtype=np.uint64) + result = shrink_fn(begins, lengths) + np.testing.assert_equal((begins, lengths), result) + + begins = np.arange(15, dtype=np.uint64)*16 + lengths = 1+np.arange(15, dtype=np.uint64) result = shrink_fn(begins, lengths) np.testing.assert_equal((begins, lengths), result) @@ -79,7 +89,7 @@ def assert_shrinks(self, shrink_fn): np.testing.assert_equal(self.shrank, shrank) def assert_empty_window_error(self, shrink_fn): - invalid = np.array([1, 2]), np.array([5, 1]) + invalid = np.array([1, 2], dtype=np.uint64), np.array([5, 1], dtype=np.uint64) with self.assertRaisesRegex(ValueError, "Overlap is bigger than measurement window"): shrink_fn(*invalid) From 9c4507e7b39e268d3284d85b220c12a47b6389fe Mon Sep 17 00:00:00 2001 From: Simon Humpohl Date: Thu, 4 Apr 2024 12:11:24 +0200 Subject: [PATCH 6/6] Add newspiece --- changes.d/791.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes.d/791.feature diff --git a/changes.d/791.feature b/changes.d/791.feature new file mode 100644 index 00000000..7c64960f --- /dev/null +++ b/changes.d/791.feature @@ -0,0 +1 @@ +Measurement windows can now automatically shrank in case of overlap to counteract small numeric errors. \ No newline at end of file