Skip to content

Commit

Permalink
Merge pull request #124 from DUNE-DAQ/eflumerf/UseHSIForWindow
Browse files Browse the repository at this point in the history 
RandomTCMaker does not support changing window size. Use FakeHSI when long windows are needed
  • Loading branch information
@eflumerf
eflumerf authored Oct 4, 2024
2 parents 10a8b34 + 4bc585c commit 485b2e7
Show file tree
Hide file tree
Showing 4 changed files with 155 additions and 139 deletions.
12 changes: 10 additions & 2 deletions integtest/3ru_1df_multirun_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,10 +229,18 @@ def test_data_files(run_nanorc):
fragment_check_list = [triggercandidate_frag_params, hsi_frag_params]
if run_nanorc.confgen_config.tpg_enabled:
local_expected_event_count += (
(6250 / ta_prescale) * number_of_data_producers * number_of_readout_apps * run_duration / 100
(6250 / ta_prescale)
* number_of_data_producers
* number_of_readout_apps
* run_duration
/ 100
)
local_event_count_tolerance += (
(250 / ta_prescale) * number_of_data_producers * number_of_readout_apps * run_duration / 100
(250 / ta_prescale)
* number_of_data_producers
* number_of_readout_apps
* run_duration
/ 100
)
# fragment_check_list.append(wib1_frag_multi_trig_params) # ProtoWIB
# fragment_check_list.append(wib2_frag_multi_trig_params) # DuneWIB
Expand Down
181 changes: 105 additions & 76 deletions integtest/fake_data_producer_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,44 +9,59 @@
import integrationtest.log_file_checks as log_file_checks
import integrationtest.data_classes as data_classes

pytest_plugins="integrationtest.integrationtest_drunc"
pytest_plugins = "integrationtest.integrationtest_drunc"

# Values that help determine the running conditions
run_duration=20 # seconds
run_duration = 20 # seconds
# baseline_fragment_size_bytes=72+(464*81) # 81 frames of 464 bytes each with 72-byte Fragment header # ProtoWIB
#baseline_fragment_size_bytes=72+(472*math.ceil(2001/32)) # 63 frames of 472 bytes each with 72-byte Fragment header # DuneWIB
baseline_fragment_size_bytes=72+(7200*math.ceil(2001/2048)) # 1 frame of 7200 bytes with 72-byte Fragment header # WIBEth
baseline_fragment_size_bytes_max=72+(7200*(1+math.ceil(2001/2048))) # 1 frame of 7200 bytes with 72-byte Fragment header # WIBEth
data_rate_slowdown_factor=10
# baseline_fragment_size_bytes=72+(472*math.ceil(2001/32)) # 63 frames of 472 bytes each with 72-byte Fragment header # DuneWIB
baseline_fragment_size_bytes = 72 + (
7200 * math.ceil(2001 / 2048)
) # 1 frame of 7200 bytes with 72-byte Fragment header # WIBEth
baseline_fragment_size_bytes_max = 72 + (
7200 * (1 + math.ceil(2001 / 2048))
) # 1 frame of 7200 bytes with 72-byte Fragment header # WIBEth
data_rate_slowdown_factor = 10
number_of_data_producers = 2

# Default values for validation parameters
expected_number_of_data_files=3
check_for_logfile_errors=True
expected_event_count=run_duration
expected_event_count_tolerance=2
wib1_frag_hsi_trig_params={"fragment_type_description": "WIB",
"fragment_type": "ProtoWIB",
"hdf5_source_subsystem": "Detector_Readout",
"expected_fragment_count": number_of_data_producers,
"min_size_bytes": baseline_fragment_size_bytes,
"max_size_bytes": baseline_fragment_size_bytes_max}
wib2_frag_params={"fragment_type_description": "WIB2",
"fragment_type": "WIB",
"hdf5_source_subsystem": "Detector_Readout",
"expected_fragment_count": number_of_data_producers,
"min_size_bytes": baseline_fragment_size_bytes, "max_size_bytes": baseline_fragment_size_bytes_max}
wibeth_frag_params={"fragment_type_description": "WIBEth",
"fragment_type": "WIBEth",
"hdf5_source_subsystem": "Detector_Readout",
"expected_fragment_count": number_of_data_producers,
"min_size_bytes": baseline_fragment_size_bytes, "max_size_bytes": baseline_fragment_size_bytes_max}
hsi_frag_params ={"fragment_type_description": "HSI",
"fragment_type": "Hardware_Signal",
"hdf5_source_subsystem": "HW_Signals_Interface",
"expected_fragment_count": 1,
"min_size_bytes": 72, "max_size_bytes": 100}
ignored_logfile_problems={}
expected_number_of_data_files = 3
check_for_logfile_errors = True
expected_event_count = run_duration
expected_event_count_tolerance = 2
wib1_frag_hsi_trig_params = {
"fragment_type_description": "WIB",
"fragment_type": "ProtoWIB",
"hdf5_source_subsystem": "Detector_Readout",
"expected_fragment_count": number_of_data_producers,
"min_size_bytes": baseline_fragment_size_bytes,
"max_size_bytes": baseline_fragment_size_bytes_max,
}
wib2_frag_params = {
"fragment_type_description": "WIB2",
"fragment_type": "WIB",
"hdf5_source_subsystem": "Detector_Readout",
"expected_fragment_count": number_of_data_producers,
"min_size_bytes": baseline_fragment_size_bytes,
"max_size_bytes": baseline_fragment_size_bytes_max,
}
wibeth_frag_params = {
"fragment_type_description": "WIBEth",
"fragment_type": "WIBEth",
"hdf5_source_subsystem": "Detector_Readout",
"expected_fragment_count": number_of_data_producers,
"min_size_bytes": baseline_fragment_size_bytes,
"max_size_bytes": baseline_fragment_size_bytes_max,
}
hsi_frag_params = {
"fragment_type_description": "HSI",
"fragment_type": "Hardware_Signal",
"hdf5_source_subsystem": "HW_Signals_Interface",
"expected_fragment_count": 1,
"min_size_bytes": 72,
"max_size_bytes": 100,
}
ignored_logfile_problems = {}

# The next three variable declarations *must* be present as globals in the test
# file. They're read by the "fixtures" in conftest.py to determine how
Expand All @@ -57,39 +72,41 @@

conf_dict = data_classes.drunc_config()

#conf_dict["daq_common"]["data_rate_slowdown_factor"] = data_rate_slowdown_factor
#conf_dict["readout"]["use_fake_data_producers"] = True
#conf_dict["readout"]["clock_speed_hz"] = 50000000 # ProtoWIB
#conf_dict["detector"]["clock_speed_hz"] = 62500000 # DuneWIB/WIBEth
#ttcm_conf = [{'signal': 1, 'tc_type_name': 'kTiming', 'time_before': 1000, 'time_after': 1001}]
#conf_dict["trigger"]["ttcm_input_map"] = ttcm_conf
#conf_dict["hsi"]["use_hsi"] = True
#conf_dict["trigger"]["segment_config"] = "INTEGTEST_CONFDIR/trigger-segment-fakehsi.data.xml"

#doublewindow_conf = copy.deepcopy(conf_dict)
#ttcm_conf2 = [{'signal': 1, 'tc_type_name': 'kTiming', 'time_before': 2000, 'time_after': 2001}]
#doublewindow_conf["trigger"]["ttcm_input_map"] = ttcm_conf2

confgen_arguments={"Baseline_Window_Size": conf_dict,
#"Double_Window_Size": doublewindow_conf,
}
# conf_dict["daq_common"]["data_rate_slowdown_factor"] = data_rate_slowdown_factor
# conf_dict["readout"]["use_fake_data_producers"] = True
# conf_dict["readout"]["clock_speed_hz"] = 50000000 # ProtoWIB
# conf_dict["detector"]["clock_speed_hz"] = 62500000 # DuneWIB/WIBEth
# ttcm_conf = [{'signal': 1, 'tc_type_name': 'kTiming', 'time_before': 1000, 'time_after': 1001}]
# conf_dict["trigger"]["ttcm_input_map"] = ttcm_conf
# conf_dict["hsi"]["use_hsi"] = True
# conf_dict["trigger"]["segment_config"] = "INTEGTEST_CONFDIR/trigger-segment-fakehsi.data.xml"

# doublewindow_conf = copy.deepcopy(conf_dict)
# ttcm_conf2 = [{'signal': 1, 'tc_type_name': 'kTiming', 'time_before': 2000, 'time_after': 2001}]
# doublewindow_conf["trigger"]["ttcm_input_map"] = ttcm_conf2

confgen_arguments = {
"Baseline_Window_Size": conf_dict,
# "Double_Window_Size": doublewindow_conf,
}
# The commands to run in nanorc, as a list
# (the first run [#100] is included to warm up the DAQ processes and avoid warnings and errors caused by
# startup sluggishness seen on slower test computers)
#nanorc_command_list="boot conf".split()
#nanorc_command_list+="start_run 101 wait ".split() + [str(run_duration)] + "stop_run --wait 2 wait 2".split()
#nanorc_command_list+="start_run --wait 1 102 wait ".split() + [str(run_duration)] + "stop_run wait 2".split()
#nanorc_command_list+="start_run --wait 2 103 wait ".split() + [str(run_duration)] + "stop_run --wait 1 wait 2".split()
#nanorc_command_list+="scrap terminate".split()
nanorc_command_list=["boot", "terminate"]
# nanorc_command_list="boot conf".split()
# nanorc_command_list+="start_run 101 wait ".split() + [str(run_duration)] + "stop_run --wait 2 wait 2".split()
# nanorc_command_list+="start_run --wait 1 102 wait ".split() + [str(run_duration)] + "stop_run wait 2".split()
# nanorc_command_list+="start_run --wait 2 103 wait ".split() + [str(run_duration)] + "stop_run --wait 1 wait 2".split()
# nanorc_command_list+="scrap terminate".split()
nanorc_command_list = ["boot", "terminate"]
# Don't require the --frame-file option since we don't need it
frame_file_required=False
frame_file_required = False

# The tests themselves


def test_nanorc_success(run_nanorc):
pytest.skip("FakeDataApplication is not currently implemented.")
current_test=os.environ.get('PYTEST_CURRENT_TEST')
current_test = os.environ.get("PYTEST_CURRENT_TEST")
match_obj = re.search(r".*\[(.+)\].*", current_test)
if match_obj:
current_test = match_obj.group(1)
Expand All @@ -98,40 +115,52 @@ def test_nanorc_success(run_nanorc):
print(current_test)
print(banner_line)
# Check that nanorc completed correctly
assert run_nanorc.completed_process.returncode==0
assert run_nanorc.completed_process.returncode == 0


def test_log_files(run_nanorc):
pytest.skip("FakeDataApplication is not currently implemented.")
local_check_flag=check_for_logfile_errors
local_check_flag = check_for_logfile_errors

if local_check_flag:
# Check that there are no warnings or errors in the log files
assert log_file_checks.logs_are_error_free(run_nanorc.log_files, True, True, ignored_logfile_problems)
assert log_file_checks.logs_are_error_free(
run_nanorc.log_files, True, True, ignored_logfile_problems
)


def test_data_files(run_nanorc):
pytest.skip("FakeDataApplication is not currently implemented.")
local_expected_event_count=expected_event_count
local_event_count_tolerance=expected_event_count_tolerance
#frag_params=wib1_frag_hsi_trig_params # ProtoWIB
#frag_params=wib2_frag_params # DuneWIB
frag_params=wibeth_frag_params
local_expected_event_count = expected_event_count
local_event_count_tolerance = expected_event_count_tolerance
# frag_params=wib1_frag_hsi_trig_params # ProtoWIB
# frag_params=wib2_frag_params # DuneWIB
frag_params = wibeth_frag_params
if run_nanorc.confgen_config["trigger"]["ttcm_input_map"][0]["time_before"] == 2000:
#frag_params["min_size_bytes"]=72+(464*161) # 161 frames of 464 bytes each with 72-byte Fragment header # ProtoWIB
#frag_params["max_size_bytes"]=72+(464*161)
#frag_params["min_size_bytes"]=72+(472*math.ceil(4001/32)) # 126 frames of 472 bytes each with 72-byte Fragment header # DuneWIB
#frag_params["max_size_bytes"]=72+(472*math.ceil(4001/32))
frag_params["min_size_bytes"]=72+(7200*math.ceil(4001/2048)) # 2 frames of 7200 bytes each with 72-byte Fragment header # WIBEth
frag_params["max_size_bytes"]=72+(7200*(1+math.ceil(4001/2048)))
fragment_check_list=[frag_params, hsi_frag_params]
# frag_params["min_size_bytes"]=72+(464*161) # 161 frames of 464 bytes each with 72-byte Fragment header # ProtoWIB
# frag_params["max_size_bytes"]=72+(464*161)
# frag_params["min_size_bytes"]=72+(472*math.ceil(4001/32)) # 126 frames of 472 bytes each with 72-byte Fragment header # DuneWIB
# frag_params["max_size_bytes"]=72+(472*math.ceil(4001/32))
frag_params["min_size_bytes"] = 72 + (
7200 * math.ceil(4001 / 2048)
) # 2 frames of 7200 bytes each with 72-byte Fragment header # WIBEth
frag_params["max_size_bytes"] = 72 + (7200 * (1 + math.ceil(4001 / 2048)))
fragment_check_list = [frag_params, hsi_frag_params]

# Run some tests on the output data file
assert len(run_nanorc.data_files)==expected_number_of_data_files
assert len(run_nanorc.data_files) == expected_number_of_data_files

for idx in range(len(run_nanorc.data_files)):
data_file=data_file_checks.DataFile(run_nanorc.data_files[idx])
data_file = data_file_checks.DataFile(run_nanorc.data_files[idx])
assert data_file_checks.sanity_check(data_file)
assert data_file_checks.check_file_attributes(data_file)
assert data_file_checks.check_event_count(data_file, local_expected_event_count, local_event_count_tolerance)
assert data_file_checks.check_event_count(
data_file, local_expected_event_count, local_event_count_tolerance
)
for jdx in range(len(fragment_check_list)):
assert data_file_checks.check_fragment_count(data_file, fragment_check_list[jdx])
assert data_file_checks.check_fragment_sizes(data_file, fragment_check_list[jdx])
assert data_file_checks.check_fragment_count(
data_file, fragment_check_list[jdx]
)
assert data_file_checks.check_fragment_sizes(
data_file, fragment_check_list[jdx]
)
73 changes: 28 additions & 45 deletions integtest/long_window_readout_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import copy
import shutil
import psutil
import urllib.request

import integrationtest.data_file_checks as data_file_checks
import integrationtest.log_file_checks as log_file_checks
Expand Down Expand Up @@ -33,24 +32,7 @@
# Default values for validation parameters
expected_number_of_data_files = 4 * number_of_dataflow_apps
check_for_logfile_errors = True
expected_event_count = 202
expected_event_count_tolerance = 9
wib1_frag_hsi_trig_params = {
"fragment_type_description": "WIB",
"fragment_type": "ProtoWIB",
"hdf5_source_subsystem": "Detector_Readout",
"expected_fragment_count": (number_of_data_producers * number_of_readout_apps),
"min_size_bytes": 3712072,
"max_size_bytes": 3712536,
}
wib2_frag_params = {
"fragment_type_description": "WIB2",
"fragment_type": "WIB",
"hdf5_source_subsystem": "Detector_Readout",
"expected_fragment_count": number_of_data_producers * number_of_readout_apps,
"min_size_bytes": 29808,
"max_size_bytes": 30280,
}

wibeth_frag_params = {
"fragment_type_description": "WIBEth",
"fragment_type": "WIBEth",
Expand All @@ -71,7 +53,7 @@
"fragment_type_description": "HSI",
"fragment_type": "Hardware_Signal",
"hdf5_source_subsystem": "HW_Signals_Interface",
"expected_fragment_count": 0,
"expected_fragment_count": 1,
"min_size_bytes": 72,
"max_size_bytes": 100,
}
Expand Down Expand Up @@ -130,6 +112,7 @@
conf_dict.session = "longwindow"
conf_dict.tpg_enabled = False
conf_dict.n_df_apps = number_of_dataflow_apps
conf_dict.fake_hsi_enabled = True # FakeHSI must be enabled to set trigger window width!

conf_dict.config_substitutions.append(
data_classes.config_substitution(
Expand All @@ -138,12 +121,6 @@
updates={"data_rate_slowdown_factor": data_rate_slowdown_factor},
)
)
conf_dict.config_substitutions.append(
data_classes.config_substitution(
obj_class="RandomTCMakerConf",
updates={"trigger_interval_ticks": 62500000 / trigger_rate},
)
)
conf_dict.config_substitutions.append(
data_classes.config_substitution(
obj_class="LatencyBuffer", updates={"size": latency_buffer_size}
Expand All @@ -153,17 +130,13 @@

conf_dict.config_substitutions.append(
data_classes.config_substitution(
obj_class="TimingTriggerOffsetMap",
obj_id="ttcm-off-0",
updates={
"time_before": readout_window_time_before,
"time_after": readout_window_time_after,
},
obj_class="FakeHSIEventGeneratorConf",
updates={"trigger_rate": trigger_rate},
)
)
conf_dict.config_substitutions.append(
data_classes.config_substitution(
obj_class="TCReadoutMap",
obj_class="HSISignalWindow",
updates={
"time_before": readout_window_time_before,
"time_after": readout_window_time_after,
Expand All @@ -186,7 +159,7 @@
)
)

confgen_arguments = { # "No_TR_Splitting": conf_dict,
confgen_arguments = { "No_TR_Splitting": conf_dict,
"With_TR_Splitting": trsplit_conf,
}

Expand Down Expand Up @@ -278,30 +251,40 @@ def test_data_files(run_nanorc):
f"The raw data output path ({actual_output_path}) does not have enough space to run this test."
)

local_expected_event_count = expected_event_count
local_event_count_tolerance = expected_event_count_tolerance
current_test = os.environ.get("PYTEST_CURRENT_TEST")

local_expected_event_count = run_duration * trigger_rate / expected_number_of_data_files
local_wibeth_frag_params = copy.deepcopy(wibeth_frag_params)

if "With_TR_Splitting" in current_test:
local_expected_event_count = local_expected_event_count * (readout_window_time_before + readout_window_time_after) / trigger_record_max_window
else:
local_wibeth_frag_params["min_size_bytes"] = 352800000
local_wibeth_frag_params["max_size_bytes"] = 357782472

local_event_count_tolerance = local_expected_event_count // 10
fragment_check_list = [triggercandidate_frag_params, hsi_frag_params]
# fragment_check_list.append(wib1_frag_hsi_trig_params) # ProtoWIB
# fragment_check_list.append(wib2_frag_params) # DuneWIB
fragment_check_list.append(wibeth_frag_params) # WIBEth
fragment_check_list.append(local_wibeth_frag_params) # WIBEth

all_ok = True
# Run some tests on the output data file
assert len(run_nanorc.data_files) == expected_number_of_data_files
all_ok &= len(run_nanorc.data_files) == expected_number_of_data_files

for idx in range(len(run_nanorc.data_files)):
data_file = data_file_checks.DataFile(run_nanorc.data_files[idx])
assert data_file_checks.sanity_check(data_file)
assert data_file_checks.check_file_attributes(data_file)
assert data_file_checks.check_event_count(
all_ok &= data_file_checks.sanity_check(data_file)
all_ok &= data_file_checks.check_file_attributes(data_file)
all_ok &= data_file_checks.check_event_count(
data_file, local_expected_event_count, local_event_count_tolerance
)
for jdx in range(len(fragment_check_list)):
assert data_file_checks.check_fragment_count(
all_ok &= data_file_checks.check_fragment_count(
data_file, fragment_check_list[jdx]
)
assert data_file_checks.check_fragment_sizes(
all_ok &= data_file_checks.check_fragment_sizes(
data_file, fragment_check_list[jdx]
)
assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more data file checks failed! \N{POLICE CARS REVOLVING LIGHT}"


def test_cleanup(run_nanorc):
Expand Down
Loading

0 comments on commit 485b2e7

Please sign in to comment.