diff --git a/integtest/3ru_1df_multirun_test.py b/integtest/3ru_1df_multirun_test.py index a0c25fa..440ea2b 100644 --- a/integtest/3ru_1df_multirun_test.py +++ b/integtest/3ru_1df_multirun_test.py @@ -229,10 +229,18 @@ def test_data_files(run_nanorc): fragment_check_list = [triggercandidate_frag_params, hsi_frag_params] if run_nanorc.confgen_config.tpg_enabled: local_expected_event_count += ( - (6250 / ta_prescale) * number_of_data_producers * number_of_readout_apps * run_duration / 100 + (6250 / ta_prescale) + * number_of_data_producers + * number_of_readout_apps + * run_duration + / 100 ) local_event_count_tolerance += ( - (250 / ta_prescale) * number_of_data_producers * number_of_readout_apps * run_duration / 100 + (250 / ta_prescale) + * number_of_data_producers + * number_of_readout_apps + * run_duration + / 100 ) # fragment_check_list.append(wib1_frag_multi_trig_params) # ProtoWIB # fragment_check_list.append(wib2_frag_multi_trig_params) # DuneWIB diff --git a/integtest/fake_data_producer_test.py b/integtest/fake_data_producer_test.py index ea6063c..a7ae342 100644 --- a/integtest/fake_data_producer_test.py +++ b/integtest/fake_data_producer_test.py @@ -9,44 +9,59 @@ import integrationtest.log_file_checks as log_file_checks import integrationtest.data_classes as data_classes -pytest_plugins="integrationtest.integrationtest_drunc" +pytest_plugins = "integrationtest.integrationtest_drunc" # Values that help determine the running conditions -run_duration=20 # seconds +run_duration = 20 # seconds # baseline_fragment_size_bytes=72+(464*81) # 81 frames of 464 bytes each with 72-byte Fragment header # ProtoWIB -#baseline_fragment_size_bytes=72+(472*math.ceil(2001/32)) # 63 frames of 472 bytes each with 72-byte Fragment header # DuneWIB -baseline_fragment_size_bytes=72+(7200*math.ceil(2001/2048)) # 1 frame of 7200 bytes with 72-byte Fragment header # WIBEth -baseline_fragment_size_bytes_max=72+(7200*(1+math.ceil(2001/2048))) # 1 frame of 7200 bytes with 72-byte Fragment header # WIBEth -data_rate_slowdown_factor=10 +# baseline_fragment_size_bytes=72+(472*math.ceil(2001/32)) # 63 frames of 472 bytes each with 72-byte Fragment header # DuneWIB +baseline_fragment_size_bytes = 72 + ( + 7200 * math.ceil(2001 / 2048) +) # 1 frame of 7200 bytes with 72-byte Fragment header # WIBEth +baseline_fragment_size_bytes_max = 72 + ( + 7200 * (1 + math.ceil(2001 / 2048)) +) # 1 frame of 7200 bytes with 72-byte Fragment header # WIBEth +data_rate_slowdown_factor = 10 number_of_data_producers = 2 # Default values for validation parameters -expected_number_of_data_files=3 -check_for_logfile_errors=True -expected_event_count=run_duration -expected_event_count_tolerance=2 -wib1_frag_hsi_trig_params={"fragment_type_description": "WIB", - "fragment_type": "ProtoWIB", - "hdf5_source_subsystem": "Detector_Readout", - "expected_fragment_count": number_of_data_producers, - "min_size_bytes": baseline_fragment_size_bytes, - "max_size_bytes": baseline_fragment_size_bytes_max} -wib2_frag_params={"fragment_type_description": "WIB2", - "fragment_type": "WIB", - "hdf5_source_subsystem": "Detector_Readout", - "expected_fragment_count": number_of_data_producers, - "min_size_bytes": baseline_fragment_size_bytes, "max_size_bytes": baseline_fragment_size_bytes_max} -wibeth_frag_params={"fragment_type_description": "WIBEth", - "fragment_type": "WIBEth", - "hdf5_source_subsystem": "Detector_Readout", - "expected_fragment_count": number_of_data_producers, - "min_size_bytes": baseline_fragment_size_bytes, "max_size_bytes": baseline_fragment_size_bytes_max} -hsi_frag_params ={"fragment_type_description": "HSI", - "fragment_type": "Hardware_Signal", - "hdf5_source_subsystem": "HW_Signals_Interface", - "expected_fragment_count": 1, - "min_size_bytes": 72, "max_size_bytes": 100} -ignored_logfile_problems={} +expected_number_of_data_files = 3 +check_for_logfile_errors = True +expected_event_count = run_duration +expected_event_count_tolerance = 2 +wib1_frag_hsi_trig_params = { + "fragment_type_description": "WIB", + "fragment_type": "ProtoWIB", + "hdf5_source_subsystem": "Detector_Readout", + "expected_fragment_count": number_of_data_producers, + "min_size_bytes": baseline_fragment_size_bytes, + "max_size_bytes": baseline_fragment_size_bytes_max, +} +wib2_frag_params = { + "fragment_type_description": "WIB2", + "fragment_type": "WIB", + "hdf5_source_subsystem": "Detector_Readout", + "expected_fragment_count": number_of_data_producers, + "min_size_bytes": baseline_fragment_size_bytes, + "max_size_bytes": baseline_fragment_size_bytes_max, +} +wibeth_frag_params = { + "fragment_type_description": "WIBEth", + "fragment_type": "WIBEth", + "hdf5_source_subsystem": "Detector_Readout", + "expected_fragment_count": number_of_data_producers, + "min_size_bytes": baseline_fragment_size_bytes, + "max_size_bytes": baseline_fragment_size_bytes_max, +} +hsi_frag_params = { + "fragment_type_description": "HSI", + "fragment_type": "Hardware_Signal", + "hdf5_source_subsystem": "HW_Signals_Interface", + "expected_fragment_count": 1, + "min_size_bytes": 72, + "max_size_bytes": 100, +} +ignored_logfile_problems = {} # The next three variable declarations *must* be present as globals in the test # file. They're read by the "fixtures" in conftest.py to determine how @@ -57,39 +72,41 @@ conf_dict = data_classes.drunc_config() -#conf_dict["daq_common"]["data_rate_slowdown_factor"] = data_rate_slowdown_factor -#conf_dict["readout"]["use_fake_data_producers"] = True -#conf_dict["readout"]["clock_speed_hz"] = 50000000 # ProtoWIB -#conf_dict["detector"]["clock_speed_hz"] = 62500000 # DuneWIB/WIBEth -#ttcm_conf = [{'signal': 1, 'tc_type_name': 'kTiming', 'time_before': 1000, 'time_after': 1001}] -#conf_dict["trigger"]["ttcm_input_map"] = ttcm_conf -#conf_dict["hsi"]["use_hsi"] = True -#conf_dict["trigger"]["segment_config"] = "INTEGTEST_CONFDIR/trigger-segment-fakehsi.data.xml" - -#doublewindow_conf = copy.deepcopy(conf_dict) -#ttcm_conf2 = [{'signal': 1, 'tc_type_name': 'kTiming', 'time_before': 2000, 'time_after': 2001}] -#doublewindow_conf["trigger"]["ttcm_input_map"] = ttcm_conf2 - -confgen_arguments={"Baseline_Window_Size": conf_dict, - #"Double_Window_Size": doublewindow_conf, - } +# conf_dict["daq_common"]["data_rate_slowdown_factor"] = data_rate_slowdown_factor +# conf_dict["readout"]["use_fake_data_producers"] = True +# conf_dict["readout"]["clock_speed_hz"] = 50000000 # ProtoWIB +# conf_dict["detector"]["clock_speed_hz"] = 62500000 # DuneWIB/WIBEth +# ttcm_conf = [{'signal': 1, 'tc_type_name': 'kTiming', 'time_before': 1000, 'time_after': 1001}] +# conf_dict["trigger"]["ttcm_input_map"] = ttcm_conf +# conf_dict["hsi"]["use_hsi"] = True +# conf_dict["trigger"]["segment_config"] = "INTEGTEST_CONFDIR/trigger-segment-fakehsi.data.xml" + +# doublewindow_conf = copy.deepcopy(conf_dict) +# ttcm_conf2 = [{'signal': 1, 'tc_type_name': 'kTiming', 'time_before': 2000, 'time_after': 2001}] +# doublewindow_conf["trigger"]["ttcm_input_map"] = ttcm_conf2 + +confgen_arguments = { + "Baseline_Window_Size": conf_dict, + # "Double_Window_Size": doublewindow_conf, +} # The commands to run in nanorc, as a list # (the first run [#100] is included to warm up the DAQ processes and avoid warnings and errors caused by # startup sluggishness seen on slower test computers) -#nanorc_command_list="boot conf".split() -#nanorc_command_list+="start_run 101 wait ".split() + [str(run_duration)] + "stop_run --wait 2 wait 2".split() -#nanorc_command_list+="start_run --wait 1 102 wait ".split() + [str(run_duration)] + "stop_run wait 2".split() -#nanorc_command_list+="start_run --wait 2 103 wait ".split() + [str(run_duration)] + "stop_run --wait 1 wait 2".split() -#nanorc_command_list+="scrap terminate".split() -nanorc_command_list=["boot", "terminate"] +# nanorc_command_list="boot conf".split() +# nanorc_command_list+="start_run 101 wait ".split() + [str(run_duration)] + "stop_run --wait 2 wait 2".split() +# nanorc_command_list+="start_run --wait 1 102 wait ".split() + [str(run_duration)] + "stop_run wait 2".split() +# nanorc_command_list+="start_run --wait 2 103 wait ".split() + [str(run_duration)] + "stop_run --wait 1 wait 2".split() +# nanorc_command_list+="scrap terminate".split() +nanorc_command_list = ["boot", "terminate"] # Don't require the --frame-file option since we don't need it -frame_file_required=False +frame_file_required = False # The tests themselves + def test_nanorc_success(run_nanorc): pytest.skip("FakeDataApplication is not currently implemented.") - current_test=os.environ.get('PYTEST_CURRENT_TEST') + current_test = os.environ.get("PYTEST_CURRENT_TEST") match_obj = re.search(r".*\[(.+)\].*", current_test) if match_obj: current_test = match_obj.group(1) @@ -98,40 +115,52 @@ def test_nanorc_success(run_nanorc): print(current_test) print(banner_line) # Check that nanorc completed correctly - assert run_nanorc.completed_process.returncode==0 + assert run_nanorc.completed_process.returncode == 0 + def test_log_files(run_nanorc): pytest.skip("FakeDataApplication is not currently implemented.") - local_check_flag=check_for_logfile_errors + local_check_flag = check_for_logfile_errors if local_check_flag: # Check that there are no warnings or errors in the log files - assert log_file_checks.logs_are_error_free(run_nanorc.log_files, True, True, ignored_logfile_problems) + assert log_file_checks.logs_are_error_free( + run_nanorc.log_files, True, True, ignored_logfile_problems + ) + def test_data_files(run_nanorc): pytest.skip("FakeDataApplication is not currently implemented.") - local_expected_event_count=expected_event_count - local_event_count_tolerance=expected_event_count_tolerance - #frag_params=wib1_frag_hsi_trig_params # ProtoWIB - #frag_params=wib2_frag_params # DuneWIB - frag_params=wibeth_frag_params + local_expected_event_count = expected_event_count + local_event_count_tolerance = expected_event_count_tolerance + # frag_params=wib1_frag_hsi_trig_params # ProtoWIB + # frag_params=wib2_frag_params # DuneWIB + frag_params = wibeth_frag_params if run_nanorc.confgen_config["trigger"]["ttcm_input_map"][0]["time_before"] == 2000: - #frag_params["min_size_bytes"]=72+(464*161) # 161 frames of 464 bytes each with 72-byte Fragment header # ProtoWIB - #frag_params["max_size_bytes"]=72+(464*161) - #frag_params["min_size_bytes"]=72+(472*math.ceil(4001/32)) # 126 frames of 472 bytes each with 72-byte Fragment header # DuneWIB - #frag_params["max_size_bytes"]=72+(472*math.ceil(4001/32)) - frag_params["min_size_bytes"]=72+(7200*math.ceil(4001/2048)) # 2 frames of 7200 bytes each with 72-byte Fragment header # WIBEth - frag_params["max_size_bytes"]=72+(7200*(1+math.ceil(4001/2048))) - fragment_check_list=[frag_params, hsi_frag_params] + # frag_params["min_size_bytes"]=72+(464*161) # 161 frames of 464 bytes each with 72-byte Fragment header # ProtoWIB + # frag_params["max_size_bytes"]=72+(464*161) + # frag_params["min_size_bytes"]=72+(472*math.ceil(4001/32)) # 126 frames of 472 bytes each with 72-byte Fragment header # DuneWIB + # frag_params["max_size_bytes"]=72+(472*math.ceil(4001/32)) + frag_params["min_size_bytes"] = 72 + ( + 7200 * math.ceil(4001 / 2048) + ) # 2 frames of 7200 bytes each with 72-byte Fragment header # WIBEth + frag_params["max_size_bytes"] = 72 + (7200 * (1 + math.ceil(4001 / 2048))) + fragment_check_list = [frag_params, hsi_frag_params] # Run some tests on the output data file - assert len(run_nanorc.data_files)==expected_number_of_data_files + assert len(run_nanorc.data_files) == expected_number_of_data_files for idx in range(len(run_nanorc.data_files)): - data_file=data_file_checks.DataFile(run_nanorc.data_files[idx]) + data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) assert data_file_checks.sanity_check(data_file) assert data_file_checks.check_file_attributes(data_file) - assert data_file_checks.check_event_count(data_file, local_expected_event_count, local_event_count_tolerance) + assert data_file_checks.check_event_count( + data_file, local_expected_event_count, local_event_count_tolerance + ) for jdx in range(len(fragment_check_list)): - assert data_file_checks.check_fragment_count(data_file, fragment_check_list[jdx]) - assert data_file_checks.check_fragment_sizes(data_file, fragment_check_list[jdx]) + assert data_file_checks.check_fragment_count( + data_file, fragment_check_list[jdx] + ) + assert data_file_checks.check_fragment_sizes( + data_file, fragment_check_list[jdx] + ) diff --git a/integtest/long_window_readout_test.py b/integtest/long_window_readout_test.py index a9390d2..ea82ae2 100644 --- a/integtest/long_window_readout_test.py +++ b/integtest/long_window_readout_test.py @@ -4,7 +4,6 @@ import copy import shutil import psutil -import urllib.request import integrationtest.data_file_checks as data_file_checks import integrationtest.log_file_checks as log_file_checks @@ -33,24 +32,7 @@ # Default values for validation parameters expected_number_of_data_files = 4 * number_of_dataflow_apps check_for_logfile_errors = True -expected_event_count = 202 -expected_event_count_tolerance = 9 -wib1_frag_hsi_trig_params = { - "fragment_type_description": "WIB", - "fragment_type": "ProtoWIB", - "hdf5_source_subsystem": "Detector_Readout", - "expected_fragment_count": (number_of_data_producers * number_of_readout_apps), - "min_size_bytes": 3712072, - "max_size_bytes": 3712536, -} -wib2_frag_params = { - "fragment_type_description": "WIB2", - "fragment_type": "WIB", - "hdf5_source_subsystem": "Detector_Readout", - "expected_fragment_count": number_of_data_producers * number_of_readout_apps, - "min_size_bytes": 29808, - "max_size_bytes": 30280, -} + wibeth_frag_params = { "fragment_type_description": "WIBEth", "fragment_type": "WIBEth", @@ -71,7 +53,7 @@ "fragment_type_description": "HSI", "fragment_type": "Hardware_Signal", "hdf5_source_subsystem": "HW_Signals_Interface", - "expected_fragment_count": 0, + "expected_fragment_count": 1, "min_size_bytes": 72, "max_size_bytes": 100, } @@ -130,6 +112,7 @@ conf_dict.session = "longwindow" conf_dict.tpg_enabled = False conf_dict.n_df_apps = number_of_dataflow_apps +conf_dict.fake_hsi_enabled = True # FakeHSI must be enabled to set trigger window width! conf_dict.config_substitutions.append( data_classes.config_substitution( @@ -138,12 +121,6 @@ updates={"data_rate_slowdown_factor": data_rate_slowdown_factor}, ) ) -conf_dict.config_substitutions.append( - data_classes.config_substitution( - obj_class="RandomTCMakerConf", - updates={"trigger_interval_ticks": 62500000 / trigger_rate}, - ) -) conf_dict.config_substitutions.append( data_classes.config_substitution( obj_class="LatencyBuffer", updates={"size": latency_buffer_size} @@ -153,17 +130,13 @@ conf_dict.config_substitutions.append( data_classes.config_substitution( - obj_class="TimingTriggerOffsetMap", - obj_id="ttcm-off-0", - updates={ - "time_before": readout_window_time_before, - "time_after": readout_window_time_after, - }, + obj_class="FakeHSIEventGeneratorConf", + updates={"trigger_rate": trigger_rate}, ) ) conf_dict.config_substitutions.append( data_classes.config_substitution( - obj_class="TCReadoutMap", + obj_class="HSISignalWindow", updates={ "time_before": readout_window_time_before, "time_after": readout_window_time_after, @@ -186,7 +159,7 @@ ) ) -confgen_arguments = { # "No_TR_Splitting": conf_dict, +confgen_arguments = { "No_TR_Splitting": conf_dict, "With_TR_Splitting": trsplit_conf, } @@ -278,30 +251,40 @@ def test_data_files(run_nanorc): f"The raw data output path ({actual_output_path}) does not have enough space to run this test." ) - local_expected_event_count = expected_event_count - local_event_count_tolerance = expected_event_count_tolerance + current_test = os.environ.get("PYTEST_CURRENT_TEST") + + local_expected_event_count = run_duration * trigger_rate / expected_number_of_data_files + local_wibeth_frag_params = copy.deepcopy(wibeth_frag_params) + + if "With_TR_Splitting" in current_test: + local_expected_event_count = local_expected_event_count * (readout_window_time_before + readout_window_time_after) / trigger_record_max_window + else: + local_wibeth_frag_params["min_size_bytes"] = 352800000 + local_wibeth_frag_params["max_size_bytes"] = 357782472 + + local_event_count_tolerance = local_expected_event_count // 10 fragment_check_list = [triggercandidate_frag_params, hsi_frag_params] - # fragment_check_list.append(wib1_frag_hsi_trig_params) # ProtoWIB - # fragment_check_list.append(wib2_frag_params) # DuneWIB - fragment_check_list.append(wibeth_frag_params) # WIBEth + fragment_check_list.append(local_wibeth_frag_params) # WIBEth + all_ok = True # Run some tests on the output data file - assert len(run_nanorc.data_files) == expected_number_of_data_files + all_ok &= len(run_nanorc.data_files) == expected_number_of_data_files for idx in range(len(run_nanorc.data_files)): data_file = data_file_checks.DataFile(run_nanorc.data_files[idx]) - assert data_file_checks.sanity_check(data_file) - assert data_file_checks.check_file_attributes(data_file) - assert data_file_checks.check_event_count( + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + all_ok &= data_file_checks.check_event_count( data_file, local_expected_event_count, local_event_count_tolerance ) for jdx in range(len(fragment_check_list)): - assert data_file_checks.check_fragment_count( + all_ok &= data_file_checks.check_fragment_count( data_file, fragment_check_list[jdx] ) - assert data_file_checks.check_fragment_sizes( + all_ok &= data_file_checks.check_fragment_sizes( data_file, fragment_check_list[jdx] ) + assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" def test_cleanup(run_nanorc): diff --git a/integtest/readout_type_scan.py b/integtest/readout_type_scan.py index fbde66f..1e251fc 100644 --- a/integtest/readout_type_scan.py +++ b/integtest/readout_type_scan.py @@ -143,6 +143,12 @@ updates={"trigger_interval_ticks": 62500000}, ) ) +conf_dict.config_substitutions.append( + data_classes.config_substitution( + obj_class="FakeHSIEventGeneratorConf", + updates={"trigger_rate": 1.0}, + ) +) swtpg_conf = copy.deepcopy(conf_dict) swtpg_conf.tpg_enabled = True @@ -158,37 +164,27 @@ tde_conf.frame_file = "asset://?checksum=759e5351436bead208cf4963932d6327" pds_stream_conf = copy.deepcopy(conf_dict) +pds_stream_conf.fake_hsi_enabled = ( + True # FakeHSI must be enabled to set trigger window width! +) pds_stream_conf.dro_map_config.det_id = 2 # det_id = 2 for HD_PDS pds_stream_conf.frame_file = "asset://?label=DAPHNEStream&subsystem=readout" pds_stream_conf.config_substitutions.append( data_classes.config_substitution( - obj_class="TimingTriggerOffsetMap", - obj_id="ttcm-off-0", - updates={"time_before": 62000, "time_after": 500}, - ) -) -pds_stream_conf.config_substitutions.append( - data_classes.config_substitution( - obj_class="TCReadoutMap", + obj_class="HSISignalWindow", updates={"time_before": 62000, "time_after": 500}, ) ) pds_conf = copy.deepcopy(conf_dict) +pds_conf.fake_hsi_enabled = True # FakeHSI must be enabled to set trigger window width! pds_conf.dro_map_config.det_id = 2 # det_id = 2 for HD_PDS pds_conf.frame_file = "asset://?label=DAPHNE&subsystem=readout" pds_conf.config_substitutions.append( data_classes.config_substitution( - obj_class="TimingTriggerOffsetMap", - obj_id="ttcm-off-0", - updates={"time_before": 62000, "time_after": 500}, - ) -) -pds_conf.config_substitutions.append( - data_classes.config_substitution( - obj_class="TCReadoutMap", + obj_class="HSISignalWindow", updates={"time_before": 62000, "time_after": 500}, ) )