Merge branch 'dl2_dir' into 'master'

DL2 subdir in running analysis

See merge request cta-array-elements/lst/analysis/lstosa!58

cfg/sequencer.cfg

@@ -100,6 +100,8 @@ NIGHTSUMMARYSCRIPT = %(PYTHONDIR)s/create_nightsummary.py
 # Amount of tries each datasequence job is allowed to run before being vetoed
 MAXTRYFAILED = 2
 
+merge_dl1_datacheck: True
+
 [PROGRAM]
 PEDESTAL = lstchain_data_create_drs4_pedestal_file 
 CALIBRATION = lstchain_create_calibration_file

cfg/sequencer_test.cfg

@@ -94,6 +94,8 @@ NIGHTSUMMARYSCRIPT = %(PYTHONDIR)s/create_nightsummary.py
 # Amount of tries each datasequence job is allowed to run before being vetoed
 MAXTRYFAILED = 2
 
+merge_dl1_datacheck: True
+
 [PROGRAM]
 PEDESTAL = lstchain_data_create_drs4_pedestal_file 
 CALIBRATION = lstchain_create_calibration_file

osa/scripts/closer.py

@@ -221,7 +221,8 @@ def post_process(seq_tuple):
     post_process_files(seq_list)
 
     # First merge DL1 datacheck files and produce PDFs
-    merge_dl1datacheck(seq_list)
+    if cfg.getboolean("LSTOSA", "merge_dl1_datacheck"):
+        merge_dl1datacheck(seq_list)
 
     # Extract the provenance info
     extract_provenance(seq_list)
@@ -245,7 +246,7 @@ def post_process_files(seq_list):
 
     DL1_RE = re.compile(fr"{options.directory}/dl1(?:.*).(?:h5|hdf5|hdf)")
     DL1AB_RE = re.compile(fr"{options.dl1_prod_id}(?:.*)/dl1(?:.*).(?:h5|hdf5|hdf)")
-    DL2_RE = re.compile(r"dl2(?:.*).(?:h5|hdf5|hdf)")
+    DL2_RE = re.compile(fr"{options.dl2_prod_id}(?:.*)/dl2(?:.*).(?:h5|hdf5|hdf)")
     MUONS_RE = re.compile(r"muons(?:.*).fits")
     DATACHECK_RE = re.compile(r"datacheck_dl1(?:.*).(?:h5|hdf5|hdf)")
     CALIB_RE = re.compile(r"/calibration(?:.*).(?:h5|hdf5|hdf)")

osa/scripts/datasequence.py

@@ -320,6 +320,7 @@ def dl1_to_dl2(run_str, historyfile):
         return 0
 
     dl1ab_subdirectory = os.path.join(options.directory, options.dl1_prod_id)
+    dl2_subdirectory = os.path.join(options.directory, options.dl2_prod_id)
 
     configfile = cfg.get("LSTOSA", "DL2CONFIGFILE")
     rf_models_directory = cfg.get("LSTOSA", "RF-MODELS-DIR")
@@ -332,7 +333,7 @@ def dl1_to_dl2(run_str, historyfile):
     commandargs = [
         command,
         "--input-file=" + datafile,
-        "--output-dir=" + options.directory,
+        "--output-dir=" + dl2_subdirectory,
         "--path-models=" + rf_models_directory,
         "--config=" + configfile,
     ]

osa/scripts/provprocess.py

@@ -254,9 +254,9 @@ def produce_provenance():
         processed_lines = []
         # derive destination folder
         if fold == pathDL2:
-            step_path = Path(fold) / options.date / options.dl2_prod_id
+            step_path = Path(fold) / options.date / options.prod_id / options.dl2_prod_id
         else:
-            step_path = Path(fold) / options.date / options.prod_id
+            step_path = Path(fold) / options.date / options.prod_id / options.dl1_prod_id
 
         # check destination folder exists
         if not step_path.exists():

osa/utils/register.py

@@ -112,7 +112,8 @@ def register_files(type, run_str, inputdir, prefix, suffix, outputdir):
 def register_run_concept_files(run_string, concept):
     """
     Prepare files to be moved to final destination directories
-    from the running_analysis original directory.
+    from the running_analysis original directory. DL1ab, datacheck
+    and DL2 are firstly stored in the corresponding subdirectory.
 
     Parameters
     ----------
@@ -120,9 +121,12 @@ def register_run_concept_files(run_string, concept):
     concept
     """
 
-    if concept in ["MUON", "DL2", "PEDESTAL", "CALIB", "TIMECALIB"]:
+    if concept in ["MUON", "PEDESTAL", "CALIB", "TIMECALIB"]:
         inputdir = options.directory
 
+    elif concept == "DL2":
+        inputdir = join(options.directory, options.dl2_prod_id)
+
     elif concept in ["DL1AB", "DATACHECK"]:
         inputdir = join(options.directory, options.dl1_prod_id)