Merge pull request #505 from DUNE-DAQ/eflumerf/IntegtestUpdates

Use tpg_enabled to determine whether to generate tc-maker-1 app in ge…

docs/README.md

@@ -1,21 +1,66 @@
 # OKS Configuration Generation
-This repository contains scripts for generating OKS database files.
+This repository contains scripts for generating and manipulating OKS database files.
 
-## createOKSdb
+## Manipulation Tools
 
+### oks_enable
+  Add Resource objects to or remove from the `disabled` relationship of a Session
+
+### consolidate
+  Merge the contents of several database files, putting all objects into a single output file. Output file will only include schemas.
+
+### consolidate_files
+  Merge the contents of several database files, preserving included databases. Output file will contain only objects defined in files given on command line.
+
+### copy_configuration
+  Copy the input file(s) to the specified directory, also moving any included files and updating include paths, to create a clone of the configuration databases.
+
+### get_apps
+  Retrieve the DAQ applications defined in the given configuration
+
+### oks-format
+  Ensure that database files are in the "DBE format", alphabetized and with correct spacing
+
+### oks_enable_tpg
+  Enable or disable TPG for a Session's ReadoutApplications
+
+### validate
+  Attempt to determine if a given Session configuration is valid and does not contain common errors
+
+## Generation Tools
+
+### createOKSdb
    A script that generates an 'empty' OKS database, just containging
 the include files for the core schema and any other schema/data files
 you specify on the commad line.
 
-## oks_enable
-
-  Add Resource objects to or remove from the `disabled` relationship
-of a Session
+### dromap2oks
+  Convert a JSON readout map file from dunedaq v4 to an OKS file.
 
-## dromap2oks
-  Convert a JSON readout map file to an OKS file.
-
-## generate_readoutOKS
+### generate_readoutOKS
 
   Create an OKS configuration file defining ReadoutApplications for
   all readout groups defined in a readout map.
+
+## Additional Python Utilities
+
+### assets.py
+  Read the DUNE-DAQ asset file database and return a path to a referenced asset file
+
+### generate_dataflowOKS.py
+  Create a basic Dataflow Segment (DFO, DF application(s) and optionally a TPStream writer), using pre-defined objects.
+
+### generate_hsiOKS.py
+  Create a basic FakeHSI Segment (FakeHSI app, HSI-to-TC app), using pre-defined objects
+
+### generate_hwmap.py
+  Create a set of DetectorToDaqConnection objects, GeoIDs, and streams for the given number of links and applications.
+
+### generate_sessionOKS.py
+  Create a Session using a number of input Segment databases.
+
+### generate_triggerOKS.py
+  Create a basic Trigger Segment (mlt, optionally TC maker), using pre-defined objects.
+
+### utils.py
+  Utilities for parsing OKS databases. Currently contains an include file search routine.

python/daqconf/consolidate.py

@@ -1,5 +1,7 @@
+from pathlib import Path
 import conffwk
 import sys
+import os
 
 
 def get_all_includes(db, file):
@@ -43,6 +45,49 @@ def consolidate_db(oksfile, output_file):
     new_db.commit()
     print("DONE")
 
+def copy_configuration(dest_dir : Path, input_files: list):
+    if len(input_files) == 0:
+        return []
+
+    print(f"Copying configuration represented by databases: {input_files} to {dest_dir}")
+    dest_dir = dest_dir.resolve() # Always include by absolute path when copying
+    sys.setrecursionlimit(10000)  # for example
+
+    output_dbs = []
+
+    for input_file in input_files:
+        db = conffwk.Configuration("oksconflibs:" + input_file)
+        includes = db.get_includes(None)
+        schemas = [i for i in includes if "schema.xml" in i]
+        dbs = [i for i in includes if "data.xml" in i]
+        newdbs = copy_configuration(dest_dir, dbs)
+
+        #print("Creating new database")
+        output_file = dest_dir / os.path.basename(input_file)
+
+        new_db = conffwk.Configuration("oksconflibs")
+        new_db.create_db(str(output_file), schemas + newdbs)
+        new_db.commit()
+
+        #print("Reading dal objects from old db")
+        dals = db.get_all_dals()
+
+        #print(f"Copying objects to new db")
+        for dal in dals:
+
+            # print(f"Loading object {dal} into cache")
+            db.get_dal(dals[dal].className(), dals[dal].id)
+
+            # print(f"Copying object: {dal}")
+            new_db.add_dal(dals[dal])
+
+        #print("Saving database")
+        new_db.commit()
+        output_dbs.append(str(output_file))
+    print("DONE")
+
+    return output_dbs
+
 
 def consolidate_files(oksfile, *input_files):
     includes = []

python/daqconf/generate_dataflowOKS.py

@@ -14,6 +14,7 @@ def generate_dataflow(
     tpwriting_enabled,
     segment,
     session: str = "",
+    n_data_writers = 1,
 ):
     """Simple script to create an OKS configuration file for a dataflow segment.
 
@@ -165,7 +166,7 @@ def generate_dataflow(
             network_rules=dfapp_netrules,
             opmon_conf=opmon_conf,
             trb=trb_conf,
-            data_writers=[dw_conf],
+            data_writers=[dw_conf] * n_data_writers,
             uses=dfhw,
         )
         db.update_dal(dfapp)

python/daqconf/generate_hsiOKS.py

@@ -130,8 +130,9 @@ def generate_hsi(
         source_id=hsi_tc_source_id,
         network_rules=tc_netrules,
         opmon_conf=opmon_conf,
-        hsievent_conf=hsi_to_tc_conf,
+        hsevent_to_tc_conf=hsi_to_tc_conf,
     )
+    db.update_dal(hsi_to_tc)
 
     if segment or session != "":
         fsm = db.get_dal(class_name="FSMconfiguration", uid="FSMconfiguration_noAction")

python/daqconf/generate_triggerOKS.py

@@ -12,6 +12,8 @@ def generate_trigger(
     include,
     segment,
     session="",
+    tpg_enabled=True,
+    hsi_enabled=False,
 ):
     """Simple script to create an OKS configuration file for a trigger segment.
 
@@ -120,7 +122,7 @@ def generate_trigger(
     random_tc_generator = db.get_dal(
         class_name="RandomTCMakerConf", uid="random-tc-generator"
     )
-    tc_confs = [random_tc_generator]
+    tc_confs = [] if hsi_enabled else [random_tc_generator]
 
     mlt = dal.MLTApplication(
         "mlt",
@@ -138,22 +140,23 @@ def generate_trigger(
     )
     db.update_dal(mlt)
 
-    ta_subscriber = db.get_dal(class_name="DataReaderConf", uid="ta-subscriber-1")
-    ta_handler = db.get_dal(class_name="DataHandlerConf", uid="def-ta-handler")
+    if tpg_enabled:
+        ta_subscriber = db.get_dal(class_name="DataReaderConf", uid="ta-subscriber-1")
+        ta_handler = db.get_dal(class_name="DataHandlerConf", uid="def-ta-handler")
 
-    tcmaker = dal.TriggerApplication(
-        "tc-maker-1",
-        runs_on=host,
-        application_name="daq_application",
-        exposes_service=[tc_maker_control, triggerActivities, dataRequests],
-        source_id=tc_source_id,
-        queue_rules=tapp_qrules,
-        network_rules=tapp_netrules,
-        opmon_conf=opmon_conf,
-        data_subscriber=ta_subscriber,
-        trigger_inputs_handler=ta_handler,
-    )
-    db.update_dal(tcmaker)
+        tcmaker = dal.TriggerApplication(
+            "tc-maker-1",
+            runs_on=host,
+            application_name="daq_application",
+            exposes_service=[tc_maker_control, triggerActivities, dataRequests],
+            source_id=tc_source_id,
+            queue_rules=tapp_qrules,
+            network_rules=tapp_netrules,
+            opmon_conf=opmon_conf,
+            data_subscriber=ta_subscriber,
+            trigger_inputs_handler=ta_handler,
+        )
+        db.update_dal(tcmaker)
 
     if segment or session != "":
         fsm = db.get_dal(class_name="FSMconfiguration", uid="FSMconfiguration_noAction")
@@ -172,7 +175,7 @@ def generate_trigger(
         db.update_dal(controller)
 
         seg = dal.Segment(
-            f"trg-segment", controller=controller, applications=[mlt, tcmaker]
+            f"trg-segment", controller=controller, applications=[mlt] + ([tcmaker] if tpg_enabled else [])
         )
         db.update_dal(seg)
 

scripts/copy_configuration

@@ -0,0 +1,14 @@
+#!/bin/env python3
+import click
+import pathlib
+from daqconf.consolidate import copy_configuration
+
+@click.command()
+@click.argument('output_directory', type=click.Path(exists=True), nargs=1)
+@click.argument('databases', nargs=-1)
+def copy_config(output_directory, databases):
+    """Copy to OUTPUT_DIRECTORY configuration represented by DATABASES"""
+    copy_configuration(pathlib.Path(output_directory), databases)  
+
+if __name__ == '__main__':
+    copy_config()