Skip to content
GitLab
Commit 72e8538b authored by Lab's avatar Lab
Browse files

Calibration sequence ready to test

parent 4f55d148
Hide whitespace changes
Inline Side-by-side
calibrationsequence.py
View file @ 72e8538b
......@@ -14,27 +14,30 @@ def calibrationsequence(args):
from osa.reports import report
from osa.jobs.job import historylevel
ped_output_file = args[0]
run_ped = args[1]
run_cal = args[2]
pedestal_output_file = args[0]
calibration_output_file = args[1]
run_ped = args[2]
run_cal = args[3]
print("DEBUG")
historyfile = join(options.directory, 'sequence_' + options.tel_id + '_' + run_cal + '.history')
level, rc = historylevel(historyfile, 'CALIBRATION')
verbose(tag, "Going to level {0}".format(level))
print("historyfile:",historyfile, run_ped)
print("DEBUG2")
print("PEDESTAL directory:",options.directory, options.tel_id)
print("level & rc:",level, rc)
# exit()
if level == 2:
rc = drs4_pedestal(run_ped, pedestal_output_file, historyfile)
level -=1
verbose(tag, "Going to level {0}".format(level))
if level == 1:
rc = drs4_pedestal(run_ped, ped_output_file, historyfile)
rc = calibrate(run_cal, pedestal_output_file, calibration_output_file, historyfile)
level -=1
verbose(tag, "Going to level {0}".format(level))
# if level == 1:
# rc = calibrate(run_cal, historyfile)
# level -=1
# verbose(tag, "Going to level {0}".format(level))
if level == 0:
verbose(tag, "Job for sequence {0} finished without fatal errors".format(run))
verbose(tag, "Job for sequence {0} finished without fatal errors".format(run_ped))
return rc
##############################################################################
......@@ -42,7 +45,7 @@ def calibrationsequence(args):
# DRS4 pedestal
#
##############################################################################
def drs4_pedestal(run_ped, ped_output_file, historyfile):
def drs4_pedestal(run_ped, pedestal_output_file, historyfile):
tag = gettag()
from sys import exit
from os.path import join
......@@ -63,15 +66,15 @@ def drs4_pedestal(run_ped, ped_output_file, historyfile):
nightdir = lstdate_to_dir(options.date)
input_file = join(cfg.get('LST1','RAWDIR'),nightdir,
'LST-1.1.Run{0}.'.format(run_ped)+'0000{1}{2}'.format(run_ped,cfg.get('LSTOSA','FITSSUFFIX'),cfg.get('LSTOSA','COMPRESSEDSUFFIX')))
'LST-1.1.Run{0}.'.format(run_ped)+'0000{0}{1}'.format(cfg.get('LSTOSA','FITSSUFFIX'),cfg.get('LSTOSA','COMPRESSEDSUFFIX')))
max_events = cfg.get('LSTOSA', 'MAX_PED_EVENTS')
commandargs.append('--input_file=' + input_file)
# commandargs.append( input_file)
commandargs.append('--output_file=' + ped_output_file )
# commandargs.append( ped_output_file )
commandargs.append('--output_file=' + pedestal_output_file )
# commandargs.append( pedestal_output_file )
commandargs.append('--max_events='+ max_events)
print("COMAND for pedestal:",commandargs)
......@@ -82,12 +85,12 @@ def drs4_pedestal(run_ped, ped_output_file, historyfile):
rc = subprocess.call(commandargs)
#except OSError as (ValueError, NameError):
except OSError as ValueError:
history(run_ped, commandconcept, ped_output_file, inputcard, ValueError, historyfile)
history(run_ped, commandconcept, pedestal_output_file, inputcard, ValueError, historyfile)
error(tag, "Could not execute \"{0}\", {1}".format(stringify(commandargs), NameError), ValueError)
except subprocess.CalledProcessError as Error:
error(tag, Error, rc)
else:
history(run_ped, commandconcept, pedestalfile, inputcard, rc, historyfile)
history(run_ped, commandconcept, pedestal_output_file, inputcard, rc, historyfile)
""" Error handling, for now no nonfatal errors are implemented for CALIBRATION """
......@@ -100,7 +103,7 @@ def drs4_pedestal(run_ped, ped_output_file, historyfile):
# calibrate
#
##############################################################################
def calibrate(run, historyfile):
def calibrate(calibration_run_id, pedestal_file, calibration_output_file, historyfile):
tag = gettag()
from sys import exit
from os.path import join
......@@ -108,32 +111,46 @@ def calibrate(run, historyfile):
from osa.configs.config import cfg
from osa.reports.report import history
from register import register_run_concept_files
from osa.utils.utils import lstdate_to_dir
sequencetextfile = join(options.directory, 'sequence_' + options.tel_id + '_' + run + '.txt')
#sequencetextfile = join(options.directory, 'sequence_' + options.tel_id + '_' + run + '.txt')
bindir = cfg.get('LSTOSA', 'LSTCHAINDIR')
daqdir = cfg.get(options.tel_id, 'RAWDIR')
#carddir = cfg.get('LSTOSA', 'CARDDIR')
inputcard = cfg.get(options.tel_id, 'CALIBRATIONCONFIGCARD')
#configcard = join(carddir, inputcard)
commandargs = [join(bindir, cfg.get('PROGRAM', 'CALIBRATION'))]
commandargs = [cfg.get('PROGRAM', 'CALIBRATION')]
nightdir = lstdate_to_dir(options.date)
calibration_data_file = join(cfg.get('LST1','RAWDIR'),nightdir,
'LST-1.1.Run{0}.'.format(calibration_run_id)+'0000{0}{1}'.format(cfg.get('LSTOSA','FITSSUFFIX'),cfg.get('LSTOSA','COMPRESSEDSUFFIX')))
calib_config_file = cfg.get('LSTOSA','CALIBCONFIGFILE')
flat_field_sample_size = cfg.get('LSTOSA', 'FLATFIELDCALCULATORSAMPLESIZE')
pedestal_cal_sample_size = cfg.get('LSTOSA', 'PEDESTALCALCULATORSAMPLESIZE')
event_source_max_events = cfg.get('LSTOSA', 'EVENTSOURCEMAXEVENTS')
calibration_version = cfg.get('LSTOSA', 'CALIBRATION_VERSION') #def: 0
n_events_statistics = cfg.get('LSTOSA', 'STATISTICS') #def: 10000
calibration_base_dir = cfg.get('LSTOSA', 'CALIB_BASE_DIRECTORY') #def: '/fefs/aswg/data/real'
calculate_time_run = cfg.get('LSTOSA', 'CALCULATE_TIME_RUN') #def: '1625'
# calibration_version = cfg.get('LSTOSA', 'CALIBRATION_VERSION') #def: 0
# n_events_statistics = cfg.get('LSTOSA', 'STATISTICS') #def: 10000
# calibration_base_dir = cfg.get('LSTOSA', 'CALIB_BASE_DIRECTORY') #def: '/fefs/aswg/data/real'
# calculate_time_run = cfg.get('LSTOSA', 'CALCULATE_TIME_RUN') #def: '1625'
commandargs.append('--run_number' + calibration_run_id)
commandargs.append('--pedestal_run' + predestal_run_id)
commandargs.append('--input_file=' + calibration_data_file)
commandargs.append('--output_file=' + calibration_output_file)
commandargs.append('--version' + calibration_version)
commandargs.append('--statistics' + n_events_statistics)
commandargs.append('--base_dir' + calibration_base_dir)
commandargs.append('--default_time_run' + calculate_time_run)
commandargs.append('--pedestal_file=' + pedestal_file)
commandargs.append('--FlatFieldCalculator.sample_size=' + flat_field_sample_size)
commandargs.append('--PedestalCalculator.sample_size=' + pedestal_cal_sample_size)
commandargs.append('--EventSource.max_events=' + event_source_max_events)
commandargs.append('--config=' + calib_config_file)
'''
optional.add_argument('--ff_calibration', help="Perform the charge calibration (yes/no)",type=str, default='yes')
optional.add_argument('--tel_id', help="telescope id. Default = 1", type=int, default=1)
'''
'''
optional.add_argument('--ff_calibration', help="Perform the charge calibration (yes/no)",type=str, default='yes')
optional.add_argument('--tel_id', help="telescope id. Default = 1", type=int, default=1)
'''
print("COMAND for calib:",commandargs)
commandconcept = 'calibration'
......@@ -143,12 +160,12 @@ optional.add_argument('--tel_id', help="telescope id. Default = 1", type=int, de
rc = subprocess.call(commandargs)
#except OSError as (ValueError, NameError):
except OSError as ValueError:
history(run, commandconcept, calibrationfile, inputcard, ValueError, historyfile)
history(calibration_run_id, commandconcept, calibrationfile, inputcard, ValueError, historyfile)
error(tag, "Could not execute \"{0}\", {1}".format(stringify(commandargs), NameError), ValueError)
except subprocess.CalledProcessError as Error:
error(tag, Error, rc)
else:
history(run, commandconcept, calibrationfile, inputcard, rc, historyfile)
history(calibration_run_id, commandconcept, calibrationfile, inputcard, rc, historyfile)
""" Error handling, for now no nonfatal errors are implemented for CALIBRATION """
if rc != 0:
......
cfg/sequencer.cfg
View file @ 72e8538b
......@@ -7,7 +7,7 @@
# The %()s variables are wildcards for string substitutions
#
[ENV]
SBATCHBIN = srun
SBATCHBIN =srun
#SBATCHBIN =
PYTHONBIN = /usr/bin
......@@ -16,6 +16,7 @@ PYTHONBIN = /usr/bin
HOMEDIR = /home/lab.saha
LSTCHAINDIR = /home/lab.saha
CONFIGFILE = /fefs/aswg/software/virtual_env/ctasoft/cta-lstchain/lstchain/data/lstchain_standard_config.json
CALIBCONFIGFILE = /fefs/aswg/software/virtual_env/ctasoft/cta-lstchain/lstchain/data/onsite_camera_calibration_param.json
PYTHONDIR = %(HOMEDIR)s/lstosa
#CALIBDIR = %(HOMEDIR)s/data/real/calibration/v000
#PEDESTALDIR =%(CALIBDIR)s
......@@ -30,10 +31,21 @@ DL1-DL2 = hello.py
# Applications
CALIBRATION = calibration
MAX_PED_EVENTS = 8000
CALIBRATION_VERSION = v1
STATISTICS = 10000
CALIB_BASE_DIRECTORY = %(HOMEDIR)s/data/real/calibration/v00
CALCULATE_TIME_RUN = 1625
FLATFIELDCALCULATORSAMPLESIZE = 10000
PEDESTALCALCULATORSAMPLESIZE = 10000
EVENTSOURCEMAXEVENTS = 1000000
#COMBINEMUON = combinemuons
GRAPH = dot
# File type pattern for DB
# Prefixes
SCALIBPREFIX = calibration.Run
SSIGNALPREFIX = drs4_pedestal.Run
......@@ -46,6 +58,9 @@ WORKFLOWPREFIX = Workflow
MUONPREFIX = Muon
INCIDENCESPREFIX = Incidences
# HDF5 file patterns
SCALIBPATTERN =
SSIGNALPATTERN =
......@@ -80,7 +95,7 @@ MAXTRYFAILED = 2
[PROGRAM]
PEDESTAL = lstchain_data_create_drs4_pedestal_file
CALIBRATION = onsite_create_calibration_file
CALIBRATION = lstchain_data_create_calibration_file
[LST]
DATESEPARATOR = _
......
osa/jobs/job.py
View file @ 72e8538b
......@@ -33,14 +33,17 @@ def historylevel(historyfile, type):
from osa.configs.config import cfg
level = 3
exit_status = 0
if type == 'PEDESTAL':
level -= 2
if type == 'CALIBRATION':
level -= 2
level -= 1
if exists(historyfile):
for line in readfromfile(historyfile).splitlines():
words = line.split()
try:
program = words[1]
exit_status = int(words[10])
print("DEBUG:",program, exit_status)
except IndexError as e:
error(tag, "Malformed history file {0}, e".format(historyfile), 3)
except ValueError as e:
......@@ -63,10 +66,16 @@ def historylevel(historyfile, type):
level = 0
else:
level = 1
elif program == 'drs4_pedestal':
if exit_status == 0:
level = 1
else:
level = 0
else:
error(tag, 'Programme name not identified {0}'.format(program), 6)
print(level,exit_status)
return level, exit_status
##############################################################################
#
......@@ -207,7 +216,11 @@ def setsequencecalibfilenames(sequence_list):
drive_suffix = cfg.get('LSTOSA', 'DRIVESUFFIX')
for s in sequence_list:
if len(s.parent_list) == 0:
calfile = '666calib.root'
#calfile = '666calib.root'
cal_run_string = str(s.run).zfill(5)
calfile = "calibration.Run{0}.0000{1}".\
format(cal_run_string, scalib_suffix)
#pedfile = '666ped.root'
ped_run_string = str(s.previousrun).zfill(5)
pedfile = "drs4_pedestal.Run{0}.0000{1}".\
......@@ -337,6 +350,7 @@ def createjobtemplate(s):
if s.type == 'CALI':
commandargs.append(os.path.join(pedestaldir, nightdir, version, s.pedestal))
commandargs.append(os.path.join(calibdir, nightdir, version, s.calibration))
ped_run = str(s.previousrun).zfill(5)
commandargs.append(ped_run)
......@@ -352,14 +366,16 @@ def createjobtemplate(s):
#commandargs.append(str(s.run).zfill(5))
# if s.type != 'STEREO':
# commandargs.append(options.tel_id)
for sub in s.subrun_list:
n_subruns = int(sub.subrun)
content = "#!/bin/env python\n"
# SLURM assignments
content += "#SBATCH -p compute\n"
content += "#SBATCH --tasks=2\n"
if s.type == 'DATA':
content += "#SBATCH --array=1-{0}\n".format(len(s.subrun_list))
content += "#SBATCH --array=0-{0}\n".format(int(n_subruns)-1)
content += "#SBATCH --cpus-per-task=1\n"
content += "#SBATCH --mem-per-cpu=1600\n"
content += "#SBATCH -t 0-48:00\n"
......@@ -367,24 +383,32 @@ def createjobtemplate(s):
content += "#SBATCH -e ./log/slurm.%j.%N.err\n"
#
content +="import subprocess\n"
subruns = []
content +="import os\n"
content +="subruns=os.getenv('SLURM_ARRAY_TASK_ID')\n"
dat = ''
for sub in s.subrun_list:
dat += formatrunsubrun(s.run, sub.subrun) + ' '
if s.type == 'DATA':
for i in range(int(sub.subrun)):
srun = str(i).zfill(4)
subruns.append(srun)
else:
subruns.append(str(0).zfill(4))
content += "subruns={0}\n".format(subruns)
content += "for subrun in subruns:\n"
#for sub in s.subrun_list:
# dat += formatrunsubrun(s.run, sub.subrun) + ' '
# if s.type == 'DATA':
# for i in range(int(sub.subrun)):
# srun = str(i).zfill(4)
# subruns.append(srun)
# # content += "subruns={0}\n".format(subruns)
#dat += formatrunsubrun(s.run, sub.subrun) + ' '
#else:
# #subruns.append(str(0).zfill(4))
#content += "subruns={0}\n".format(subruns)
# content += "for subrun in subruns:\n"
# content +="subprocess.call({0})\n".format(commandargs)
content += " subprocess.call(["
content += "subprocess.call(["
for i in commandargs:
content += " '{0}',\n".format(i)
content += " '{0}".format(str(s.run).zfill(5))+".{0}'"+'.format(subrun)'+','
if s.type == 'DATA':
content += " '{0}".format(str(s.run).zfill(5))+".{0}'"+'.format(str(subruns).zfill(4))'+','
else:
content += " '{0}'".format(str(s.run).zfill(5)) + ','
content += " '{0}'".format(options.tel_id)
content +=" ])"
......
osa/utils/cliopts.py
View file @ 72e8538b
......@@ -138,7 +138,7 @@ def pedestalsequencecliparsing(command):
##############################################################################
def calibrationsequencecliparsing(command):
tag = standardhandle.gettag()
message = "usage: %prog [-vw] [-c CONFIGFILE] [-d DATE] [-o OUTPUTDIR] [-z] <CAL_RUN_ID> <PED_RUN_ID> <TEL_ID>"
message = "usage: %prog [-vw] [-c CONFIGFILE] [-d DATE] [-o OUTPUTDIR] [-z] <pedoutfile> <caloutfile> <CAL_RUN_ID> <PED_RUN_ID> <TEL_ID>"
parser = OptionParser(usage = message)
parser.add_option("-c", "--config", action = "store", dest = "configfile", default = None,
help = "use specific config file [default cfg/osa.cfg]")
......@@ -175,14 +175,14 @@ def calibrationsequencecliparsing(command):
standardhandle.verbose(tag, "the argument is {0}".format(args))
# Mapping the telescope argument to an option parameter (it might become an option in the future)
if len(args) != 4:
if len(args) != 5:
standardhandle.error(tag, "incorrect number of arguments, type -h for help", 2)
elif args[3] == 'ST':
elif args[4] == 'ST':
standardhandle.error(tag, "not yet ready for telescope {0}".format(options.options.tel_id), 2)
elif args[3] != 'LST1' and args[3] != 'LST2':
elif args[4] != 'LST1' and args[4] != 'LST2':
standardhandle.error(tag, "wrong telescope id, use 'LST1', 'LST2' or 'ST'", 2)
options.tel_id = args[3]
options.tel_id = args[4]
# Setting the default date and directory if needed
options.configfile = set_default_configfile_if_needed(command)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment