Merge pull request #65 from DUNE-DAQ/jtena/MPDTest

Configurable send partial fragments in daq conf
DUNE-DAQ · Dec 13, 2023 · c9cb1e8 · c9cb1e8
2 parents ea3557a + e14bc85
commit c9cb1e8
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 45 deletions.
diff --git a/integtest/README.md b/integtest/README.md
@@ -0,0 +1,8 @@
+"integtests" are intended to be automated integration and/or system tests that make use of the
+"pytest" framework to validate the operation of the DAQ system in various scenarios.
+
+Here is a sample command for invoking a test (feel free to keep or drop the options in brackets, as you prefer):
+
+```
+pytest -s test_mpd-raw.py [--nanorc-option partition-number 2] [--nanorc-option timeout 300]
+```
diff --git a/integtest/frames.bin b/integtest/frames.bin
diff --git a/integtest/test_mpd-raw.py b/integtest/test_mpd-raw.py
@@ -6,26 +6,24 @@
 
 # Values that help determine the running conditions
 number_of_data_producers=1
-rate = 1.0
 sleep_time = 0 
-sent_data = 100
+sent_data = 20
+delay=15
+rate = 0.1
 
-#delay sending
-delay=0
-
-run_duration=int(sent_data*rate) + sleep_time # seconds
+run_duration=int(sent_data*rate) + sleep_time + delay# seconds
 
 # Default values for validation parameters
 expected_number_of_data_files=1
 check_for_logfile_errors=True
 expected_event_count_tolerance=1
-
+test_packet_size=sent_data*3796
 mpd_frag_hsi_trig_params={"fragment_type_description": "MPD",
                           "fragment_type": "MPD",
                           "hdf5_source_subsystem": "Detector_Readout",
                           "expected_fragment_count": number_of_data_producers,
                           "min_size_bytes": 448, 
-                          "max_size_bytes": 3796}
+                          "max_size_bytes": test_packet_size}
 
 # The next three variable declarations *must* be present as globals in the test
 # file. They're read by the "fixtures" in conftest.py to determine how
@@ -42,9 +40,10 @@
 conf_dict = config_file_gen.get_default_config_dict()
 conf_dict["detector"]["op_env"] = "integtest"
 conf_dict["detector"]["clock_speed_hz"] = 62500000
-conf_dict["trigger"]["trigger_window_before_ticks"] = 30000
-conf_dict["trigger"]["trigger_window_after_ticks"] = 30000
+conf_dict["trigger"]["trigger_window_before_ticks"] = 61500000
+conf_dict["trigger"]["trigger_window_after_ticks"] = 1000000
 conf_dict["trigger"]["mlt_merge_overlapping_tcs"] = False
+conf_dict["readout"]["send_partial_fragments"] = True
 
 confgen_arguments={"MPDSystem": conf_dict}
 
@@ -89,6 +88,8 @@ def test_data_file(run_nanorc):
 data = 'tcp://127.0.0.1:5556'
 
 def send_mpd(packets, n_packets):
+    time.sleep(1)
+
     try:
         # Set up sockets
         print("Setting up ZMQ sockets...")
@@ -102,6 +103,7 @@ def send_mpd(packets, n_packets):
         print("Parsing socket options...")
         for opt in socket_opts:
             data_socket.setsockopt(*opt)
+
         print("Connecting sockets...")
         id = 0
         while id == 0:
@@ -117,7 +119,7 @@ def send_mpd(packets, n_packets):
                 continue
 
         print('Initialising...')
-        time.sleep(1+delay)
+        time.sleep(delay)
 
         print('Sending ', n_packets, ' MPD messages.')
 
@@ -129,7 +131,6 @@ def send_mpd(packets, n_packets):
             message_count += 1
             print("Total messages sent:",message_count)        
             time.sleep(rate);
-
         print("Sleeping for 10 seconds before exiting...")
         time.sleep(sleep_time)
     except:
@@ -144,7 +145,7 @@ def send_mpd(packets, n_packets):
 use_random_size = False
 mpd_data = mpd.mpd(f"{lbrulibs_dir}/test/example-mpd-data-100events-noise.data", 1, sent_data, use_random_size) 
 
-run_duration=int(mpd_data.num_packets()*rate) + sleep_time  # seconds
+run_duration=int(sent_data*rate) + sleep_time # seconds
 expected_event_count=run_duration
 
 print( ' Sending ', mpd_data.num_packets() , ' packets' )

diff --git a/integtest/test_pacman-raw.py b/integtest/test_pacman-raw.py
@@ -45,6 +45,7 @@
 conf_dict["trigger"]["trigger_window_before_ticks"] = "2500000"
 conf_dict["trigger"]["trigger_window_after_ticks"]  = "2500000"
 conf_dict["trigger"]["mlt_merge_overlapping_tcs"] = False
+conf_dict["readout"]["send_partial_fragments"] = True
 
 confgen_arguments={"PACMANSystem": conf_dict}
 

diff --git a/test/mpd-hdf5decoder-RAW.py b/test/mpd-hdf5decoder-RAW.py
@@ -35,40 +35,45 @@ def main(filename):
             frag_hdr = frag.get_header()
             frag_ts = frag.get_trigger_timestamp()
 
-            print(f'\tTrigger timestamp for fragment is {frag_ts}')
-            mpd_f = nddetdataformats.MPDFrame(frag.get_data())
+            fragment_data_size = frag.get_data_size()
+            frame_size_i = 0
+            while fragment_data_size > frame_size_i : 
+                print(f'\tTrigger timestamp for fragment is {frag_ts}')
+                data = frag.get_data(frame_size_i) 
+                mpd_f = nddetdataformats.MPDFrame(data)
 
-            #print header info
-            print('\n\t==== MPD HEADER (First Frame) ====')
-            OSheader = mpd_f.get_OSheader()
-            event_header = mpd_f.get_event_header()
-            device_header = mpd_f.get_device_header()
-            trigger_header = mpd_f.get_trigger_header()
-            trigger_data_header = mpd_f.get_trigger_data_header()
-            frame_size = mpd_f.get_frame_size() 
-
-            #Check if Timestamp Sync number is correct
-            prefix = '\t\t'
-            print(f'{prefix} Timestamp Sync: {hex(OSheader.timestamp_sync)}')
-            print(f'{prefix} Timestamp size: {OSheader.timestamp_length}')
-            print(f'{prefix} TimeStamp OS: {OSheader.timestamp_OS}')
-            print(f'{prefix} SyncMagic: {hex(event_header.SyncMagic)}')
-            print(f'{prefix} Length: {event_header.length}')
-            print(f'{prefix} \033[1mEvent number: {event_header.event_num} \033[0m ')
-            print(f'{prefix} Device serial number: {device_header.device_serial_num}')
-            print(f'{prefix} Device length: {device_header.device_length}')
-            print(f'{prefix} Device model ID: {device_header.device_model_id}')
-            print(f'{prefix} Trigger type: {trigger_header.trigger_type}')
-            print(f'{prefix} Trigger length: {trigger_header.trigger_length}')
-            print(f'{prefix} Trigger channel number: {trigger_header.trigger_channel_number}')
-            print(f'{prefix} Event timestamp 1: {trigger_data_header.event_timestamp_1}')
-            print(f'{prefix} Event timestamp 2: {trigger_data_header.event_timestamp_2}')
-            print(f'{prefix} Flags: {trigger_data_header.flags}')
-            print(f'{prefix} Channel bit mask: {trigger_data_header.channel_bit_mask}')
-            print(f'{prefix} \033[1mDUNE Clock tick Time Stamp: {mpd_f.get_timestamp()}\033[0m')
-            print(f'{prefix} Number enabled channels : {mpd_f.get_nchannels()}')
-            print(f'{prefix} Number of samples per channel : {mpd_f.get_nsamples()}') 
+                #print header info
+                print('\n\t==== MPD HEADER (First Frame) ====')
+                OSheader = mpd_f.get_OSheader()
+                event_header = mpd_f.get_event_header()
+                device_header = mpd_f.get_device_header()
+                trigger_header = mpd_f.get_trigger_header()
+                trigger_data_header = mpd_f.get_trigger_data_header()
+                frame_size = mpd_f.get_frame_size()
+                frame_size_i += frame_size
 
+                #Check if Timestamp Sync number is correct
+                prefix = '\t\t'
+                print(f'{prefix} Timestamp Sync: {hex(OSheader.timestamp_sync)}')
+                print(f'{prefix} Timestamp size: {OSheader.timestamp_length}')
+                print(f'{prefix} TimeStamp OS: {OSheader.timestamp_OS}')
+                print(f'{prefix} SyncMagic: {hex(event_header.SyncMagic)}')
+                print(f'{prefix} Length: {event_header.length}')
+                print(f'{prefix} \033[1mEvent number: {event_header.event_num} \033[0m ')
+                print(f'{prefix} Device serial number: {device_header.device_serial_num}')
+                print(f'{prefix} Device length: {device_header.device_length}')
+                print(f'{prefix} Device model ID: {device_header.device_model_id}')
+                print(f'{prefix} Trigger type: {trigger_header.trigger_type}')
+                print(f'{prefix} Trigger length: {trigger_header.trigger_length}')
+                print(f'{prefix} Trigger channel number: {trigger_header.trigger_channel_number}')
+                print(f'{prefix} Event timestamp 1: {trigger_data_header.event_timestamp_1}')
+                print(f'{prefix} Event timestamp 2: {trigger_data_header.event_timestamp_2}')
+                print(f'{prefix} Flags: {trigger_data_header.flags}')
+                print(f'{prefix} Channel bit mask: {trigger_data_header.channel_bit_mask}')
+                print(f'{prefix} \033[1mDUNE Clock tick Time Stamp: {mpd_f.get_timestamp()}\033[0m')
+                print(f'{prefix} Number enabled channels : {mpd_f.get_nchannels()}')
+                print(f'{prefix} Number of samples per channel : {mpd_f.get_nsamples()}') 
+
     print(f'Processed all requested records')
     print(f'Valid processed: {len(records_to_process)-count_invalid}')
     print(f'Invalid processed: {count_invalid}')