Commit 77535607 authored by Daniel Morcuende's avatar Daniel Morcuende
Browse files

Merge branch 'dl2_dir' into 'master'

DL2 subdir in running analysis

See merge request cta-array-elements/lst/analysis/lstosa!58
parents b2e17b6c e2380ea8
......@@ -100,6 +100,8 @@ NIGHTSUMMARYSCRIPT = %(PYTHONDIR)s/create_nightsummary.py
# Amount of tries each datasequence job is allowed to run before being vetoed
MAXTRYFAILED = 2
merge_dl1_datacheck: True
[PROGRAM]
PEDESTAL = lstchain_data_create_drs4_pedestal_file
CALIBRATION = lstchain_create_calibration_file
......
......@@ -94,6 +94,8 @@ NIGHTSUMMARYSCRIPT = %(PYTHONDIR)s/create_nightsummary.py
# Amount of tries each datasequence job is allowed to run before being vetoed
MAXTRYFAILED = 2
merge_dl1_datacheck: True
[PROGRAM]
PEDESTAL = lstchain_data_create_drs4_pedestal_file
CALIBRATION = lstchain_create_calibration_file
......
......@@ -221,7 +221,8 @@ def post_process(seq_tuple):
post_process_files(seq_list)
# First merge DL1 datacheck files and produce PDFs
merge_dl1datacheck(seq_list)
if cfg.getboolean("LSTOSA", "merge_dl1_datacheck"):
merge_dl1datacheck(seq_list)
# Extract the provenance info
extract_provenance(seq_list)
......@@ -245,7 +246,7 @@ def post_process_files(seq_list):
DL1_RE = re.compile(fr"{options.directory}/dl1(?:.*).(?:h5|hdf5|hdf)")
DL1AB_RE = re.compile(fr"{options.dl1_prod_id}(?:.*)/dl1(?:.*).(?:h5|hdf5|hdf)")
DL2_RE = re.compile(r"dl2(?:.*).(?:h5|hdf5|hdf)")
DL2_RE = re.compile(fr"{options.dl2_prod_id}(?:.*)/dl2(?:.*).(?:h5|hdf5|hdf)")
MUONS_RE = re.compile(r"muons(?:.*).fits")
DATACHECK_RE = re.compile(r"datacheck_dl1(?:.*).(?:h5|hdf5|hdf)")
CALIB_RE = re.compile(r"/calibration(?:.*).(?:h5|hdf5|hdf)")
......
......@@ -320,6 +320,7 @@ def dl1_to_dl2(run_str, historyfile):
return 0
dl1ab_subdirectory = os.path.join(options.directory, options.dl1_prod_id)
dl2_subdirectory = os.path.join(options.directory, options.dl2_prod_id)
configfile = cfg.get("LSTOSA", "DL2CONFIGFILE")
rf_models_directory = cfg.get("LSTOSA", "RF-MODELS-DIR")
......@@ -332,7 +333,7 @@ def dl1_to_dl2(run_str, historyfile):
commandargs = [
command,
"--input-file=" + datafile,
"--output-dir=" + options.directory,
"--output-dir=" + dl2_subdirectory,
"--path-models=" + rf_models_directory,
"--config=" + configfile,
]
......
......@@ -254,9 +254,9 @@ def produce_provenance():
processed_lines = []
# derive destination folder
if fold == pathDL2:
step_path = Path(fold) / options.date / options.dl2_prod_id
step_path = Path(fold) / options.date / options.prod_id / options.dl2_prod_id
else:
step_path = Path(fold) / options.date / options.prod_id
step_path = Path(fold) / options.date / options.prod_id / options.dl1_prod_id
# check destination folder exists
if not step_path.exists():
......
......@@ -112,7 +112,8 @@ def register_files(type, run_str, inputdir, prefix, suffix, outputdir):
def register_run_concept_files(run_string, concept):
"""
Prepare files to be moved to final destination directories
from the running_analysis original directory.
from the running_analysis original directory. DL1ab, datacheck
and DL2 are firstly stored in the corresponding subdirectory.
Parameters
----------
......@@ -120,9 +121,12 @@ def register_run_concept_files(run_string, concept):
concept
"""
if concept in ["MUON", "DL2", "PEDESTAL", "CALIB", "TIMECALIB"]:
if concept in ["MUON", "PEDESTAL", "CALIB", "TIMECALIB"]:
inputdir = options.directory
elif concept == "DL2":
inputdir = join(options.directory, options.dl2_prod_id)
elif concept in ["DL1AB", "DATACHECK"]:
inputdir = join(options.directory, options.dl1_prod_id)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment