Updating --cds-items option in epicsqarch-qs

.github/workflows/standard.yml

@@ -25,7 +25,7 @@ jobs:
       # System packages to be installed only for conda-based testing:
       conda-system-packages: ""
       # System packages to be installed only for pip-based testing:
-      pip-system-packages: ""
+      pip-system-packages: "libkrb5-dev"
       # System packages to be installed only for documentation:
       docs-system-packages: "pandoc"
       # Set if using setuptools-scm for the conda-build workflow

conda-recipe/meta.yaml

@@ -39,6 +39,7 @@ requirements:
   - psdaq-control-minimal >=3.3.19
   - psdm_qs_cli >=0.3.1
   - pyfiglet >=0.8.0
+  - pykerberos
   - pyqt =5
   - pyyaml >=5.4.0
   - requests >=2.26.0

hutch_python/epics_arch.py

@@ -2,9 +2,11 @@
 import argparse
 import logging
 import os
+import os.path
 import re
 import subprocess
 import sys
+from configparser import ConfigParser, NoOptionError
 from dataclasses import dataclass
 
 from prettytable import PrettyTable
@@ -19,6 +21,8 @@
     psdm_qs_cli = None
     QuestionnaireClient = None
 
+logger = logging.getLogger(__name__)
+
 
 # Annotation with dataclass, making struct to help organize cds objects in prettytable
 @dataclass
@@ -28,9 +32,6 @@ class QStruct:
     pvtype: str
 
 
-logger = logging.getLogger(__name__)
-
-
 def _create_parser():
     """Argument Parser Setup. Define shell commands."""
     parser = argparse.ArgumentParser(description='Create an epicsArch file '
@@ -54,15 +55,14 @@ def _create_parser():
     parser.add_argument('--level', '-l', required=False, type=str, default="INFO",
                         help='Change the logging level, e.g. DEBUG to show the debug logging stream')
 
-    parser.add_argument('--cds-items', nargs=2, action='store', default=None,
-                        help="Pulls all data from CDS tab in the form of a "
-                        "dictionary. E.g.: xppx1003221 --cds-items run_xx "
-                        "experiment_name, where xx is the run number. This option "
-                        "will not automatically update the archfile.")
+    parser.add_argument('--cds-items', action='store_true', default=None,
+                        help="Pulls all data from CDS tab. E.g.: epicsarch-qs xppx1003221 --cds-items"
+                        "This option will not automatically update the archfile.")
 
     parser.add_argument('--softlink', '-sl', action='store_true', default=None,
                         help="create softlink for experiment. This is run after "
                         "the archfile has been updated/created.")
+
     parser.add_argument('--link-path', '-sl_path', action='store',
                         default=EPICS_ARCH_FILE_PATH,
                         help="Provide user with option to supply custom path for "
@@ -81,13 +81,14 @@ def main():
 
 def logger_setup(args):
     # Setting up the logger, to show the level when enabled
-    logging.getLogger().addHandler(logging.NullHandler())
-    logging.basicConfig(level="INFO")
+    # logging.getLogger().addHandler(logging.NullHandler())
+    logging.basicConfig()
     logger.setLevel(args.level)
 
 
 def create_arch_file(experiment, level=None, hutch=None, path=None, dry_run=False,
                      cds_items=None, softlink=None, link_path=None):
+
     """
     Create an epicsArch file for the experiment.
 
@@ -139,7 +140,7 @@ def create_arch_file(experiment, level=None, hutch=None, path=None, dry_run=Fals
     if experiment and not dry_run:
         # set the path to write the epicsArch file to.
         if cds_items:
-            pull_cds_data(experiment, cds_items)
+            pull_cds_items(experiment)
             return
         if path:
             if path and not os.path.exists(path):
@@ -156,7 +157,8 @@ def create_arch_file(experiment, level=None, hutch=None, path=None, dry_run=Fals
         print_dry_run(experiment)
 
 
-def pull_cds_data(exp, run):
+def pull_cds_items(exp):
+
     """
     Gather all user objects from the CDS tab in the questionnaire.
     Parse objects and separate them based on type.
@@ -173,16 +175,91 @@ def pull_cds_data(exp, run):
     -------
         PrettyTable visualization of cds objects
     """
-    # pull run data from questionnaire api, then take the data and sort it
-    # create PrettyTable instance and if the values from the run data contain
-    # pcdssetup then put them into a seperate dictionary as they are cds items
-    logger.debug('pull_cds_items(%s)', exp)
-    client = QuestionnaireClient()
-    logger.debug("in cds items, run numb:", str(run[1]))
-    runDetails_Dict = client.getProposalDetailsForRun(str(run[0]), str(run[1]))
+    """
+    pull run data from questionnaire api, then take the data and sort it
+    create Pretty Table instance and if the values from the run data contain pcdssetup
+    then put them into a seperate dictionary as they are cds items
+    """
+    logger.debug("in client")
+    logger.debug('pull_cds_items: %s', exp)
+
+    cfg = ConfigParser()
+    cfgs = cfg.read(['qs.cfg', '.qs.cfg',
+                     os.path.expanduser('~/.qs.cfg'),
+                     'web.cfg', '.web.cfg',
+                     os.path.expanduser('~/.web.cfg')])
+    # Ws-auth
+    client = None
+    if cfgs:
+        user = cfg.get('DEFAULT', 'user', fallback=None)
+        try:
+            print("Trying to authenticate with cfg file.")
+            pw = cfg.get('DEFAULT', 'pw')
+            client = QuestionnaireClient(use_kerberos=False, user=user, pw=pw)
+        except NoOptionError:
+            logger.error("Could not find valid username and password in qs.cfg or web.cfg, attempting to autheticate with kerberos instead.")
+    else:
+        logger.debug("Cfgs could not be found. ")
+    if client is None:
+        try:
+            client = QuestionnaireClient(use_kerberos=True)
+        except Exception as exc:
+            logger.error("Not able to authenticate with kerberos, please make sure you have an active token.")
+            raise exc
+
+    formatted_run_id = ''
+    run_num = ''
+
+    # handle formatting of proposal id, want: e.g. X-10032
+    # Case -  Expected Format: e.g. xppx1003221
+    if re.match('^[a-zA-Z]{4}[0-9]+[0-9]{2}$', exp):
+        logger.debug("experiment name format")
+        run_num = 'run'+exp[-2:]
+        logger.debug('run num: %s', run_num)
+        run_id = str(exp[3:-2])
+        logger.debug('run_id: %s', run_id)
+        formatted_run_id = run_id.capitalize()
+        formatted_run_id = formatted_run_id[:1] + '-' + formatted_run_id[1:]
+    # Case - X-10032 or LY45
+    elif re.match('[A-Z]{1}-[0-9]{5}$', exp) or re.match('^[A-Z]{2}[0-9]{2}$', exp):
+        # Case - Proposal ID Format, have user enter run num manually
+        logger.debug("run_id format")
+        formatted_run_id = exp
+        run_num = 'run' + str(input('Please enter run number: '))
+    else:
+        print('Unrecognized format, please follow the prompts to find experiment data.')
+
+        run_num = 'run' + input('Please enter run number: ')
+        formatted_run_id = input('Please enter proposal ID: ')
+
+    logger.debug('formatted run_id: %s', formatted_run_id)
+    logger.debug('run_num: %s', run_num)
+
+    matchedKey = ''
+
+    try:
+        runDetails_Dict = client.getProposalsListForRun(run_num)
+        for key, vals in runDetails_Dict.items():
+            logger.debug("%s, %s", formatted_run_id, key)
+            if str(key) == str(formatted_run_id):
+                logger.debug('matched key: %s', key)
+                matchedKey = key
+
+    except Exception as e:
+        logger.error("An invalid https request, please check the run number, proposal id and experiment number: %s", e)
+        return
+
+    try:
+        runDetails_Dict = client.getProposalDetailsForRun(run_num, matchedKey)
+        # questionnaireFlag = True
+    except (Exception, UnboundLocalError) as e:
+        logger.error('Could not find experiment, please check to make sure information is correct: %s', e)
+        return
+
     sorted_runDetails_Dict = dict(sorted(runDetails_Dict.items()))
     cds_dict = {}
     myTable = PrettyTable(["Alias", "PV Base", "Type"])
+
     for keys, vals in sorted_runDetails_Dict.items():
         if "pcdssetup" in keys:
             cds_dict[keys] = vals
@@ -193,6 +270,7 @@ def pull_cds_data(exp, run):
 
     # iterate through all cds items and label them based on their type
     # use the struct members to identify
+
     displayList = []
     for k, v in cds_dict.items():
         if re.match('pcdssetup-motors.*-name', k):
@@ -219,6 +297,7 @@ def pull_cds_data(exp, run):
         elif re.match('pcdssetup-temp.*-name', k):
             pv = cds_dict.get(re.sub('name', 'pvbase', k), '')
             displayList.append(QStruct(v, pv, "temperature"))
+    # logger.debug("displayList: %s", displayList)
 
     for struct in displayList:
         myTable.add_row([struct.alias, struct.pvbase, struct.pvtype])
@@ -297,7 +376,7 @@ def check_for_duplicates(qs_data, af_data):
     # Looking for duplicates of PVs in the questionaire
     # also print out the alias for PV, change removing to warning operator to remove dup then rerun
     for dup in pvDuplicate:
-        logger.debug("!Duplicate PV in questionnaire!:" + str(dup))
+        logger.debug("!Duplicate PV in questionnaire!: " + str(dup))
         for value in rev_keyDict[dup][1:]:
             logger.debug("Found PV duplicate(s) from questionnaire: " + value
                          + ", " + sorted_qsDict[value])
@@ -329,8 +408,7 @@ def check_for_duplicates(qs_data, af_data):
 
     sorted_afDict = dict(sorted(sorted_afDict.items()))
     updated_arch_list = [x for item in sorted_afDict.items() for x in item]
-    logger.debug("\nUpdated Arch List:\n")
-    logger.debug(updated_arch_list)
+    logger.debug("\nUpdated Arch List: %s\n", updated_arch_list)
     return updated_arch_list
 
 

requirements.txt

@@ -14,11 +14,13 @@ pcdsdevices>=7.0.0
 pcdsutils>=0.5.0
 psdaq-control-minimal
 pyfiglet>=0.8.0
+pykerberos
 pyqt5
 pyyaml>=5.4.0
 requests>=2.26.0
 simplejson>=3.17.0
 
+
 # Some are unavailable on pypi:
 # elog