Commit 28bd3249 authored by Lab Saha's avatar Lab Saha
Browse files

Merge branch 'master' into 'master'

Master

See merge request contrera/lstosa!4
parents 7088598f 372bae1f
This diff is collapsed.
from standardhandle import error, verbose, gettag, warning
import options
##############################################################################
#
# readconf
#
##############################################################################
def readconf(file):
tag = gettag()
from os.path import exists
conf = None
try:
# Python 2.7
import ConfigParser
except ImportError as Error:
warning(tag, "Increasing to python 3 ConfigParser")
import configparser
conf = configparser.SafeConfigParser(allow_no_value=True)
else:
conf = ConfigParser.SafeConfigParser(allow_no_value=True)
try:
conf.read(file)
except ConfigParser.Error as NameError:
error(tag, NameError, 3)
verbose(tag, "sections of the config file are {0}".format(conf.sections()))
return conf
##############################################################################
#
# read_properties
#
##############################################################################
def read_properties(file):
tag = gettag()
""" To be used when config file has no header, creating a DUMMY header"""
import tempfile
from os import unlink
fname = None
with tempfile.NamedTemporaryFile(delete=False) as tf:
tf.write('[DUMMY]\n')
fname = tf.name
with open(file) as f:
tf.write(f.read())
tf.seek(0)
conf = readconf(tf.name)
unlink(tf.name)
return conf
##############################################################################
#
# cfg
#
##############################################################################
cfg = readconf(options.configfile)
class Period(object):
def __init__(self):
self.period = None
class Night(Period):
def __init__(self):
super(Night, self).__init__()
self.night = None
class Telescope(Night):
def __init__(self):
super(Telescope, self).__init__()
self.telescope = None
class Source(Telescope):
def __init__(self):
super(Source, self).__init__()
self.source = None
class Wobble(Source):
def __init__(self):
super(Wobble, self).__init__()
self.sourcewobble = None
self.wobble = None
class RunObj(Wobble):
def __init__(self):
super(RunObj, self).__init__()
self.run_str = None
self.run = None
self.type = None
self.subrun_list = []
self.subruns = 0
class SubrunObj(RunObj):
def __init__(self):
self.runobj = None
self.subrun_str = None
self.subrun = None
self.kind = None
self.timestamp = None
self.time = None
self.date = None
class Sequence(RunObj):
def __init__(self):
super(Sequence, self).__init__()
self.seq = None
self.parent_list = []
self.parent = None
self.parentjobid = None
self.previousrun = None
self.script = None
self.veto = None
self.closed = None
self.history = None
self.queue = None
self.jobname = None
self.jobid = None
self.action = None
self.tries = None
self.state = None
self.jobhost = None
self.cputime = None
self.walltime = None
self.exit = None
def associate(self, r):
for a in r.__dict__.keys():
self.__dict__.update({a: r.__dict__[a]})
class SequenceCalibration(Sequence):
def __init__(self, r):
super(SequenceCalibration, self).__init__()
super(SequenceCalibration, self).associate(r)
self.scalibstatus = None
class SequenceData(Sequence):
def __init__(self, r):
super(SequenceData, self).__init__()
super(SequenceData, self).associate(r)
self.calibration = None
self.pedestal = None
self.drive = None
self.ssignalstatus = None
self.sorcererstatus = None
self.merppstatus = None
self.starstatus = None
self.starhistogramstatus = None
class SequenceStereo(Sequence):
def __init__(self, v, w):
super(SequenceStereo, self).__init__()
attr_list = ['run', 'subrun_list' , 'subruns',\
'wobble', 'sourcewobble', 'source', 'night']
for a in attr_list:
""" This copies the unique attrs of both sequences """
self.__dict__.update({a: self.\
set_unique(v.__dict__[a], w.__dict__[a])})
self.type = 'STEREO'
self.telescope = 'ST'
self.subruns = v.subruns + w.subruns
self.parent_list = [v, w]
self.parent = "{0},{1}".format(v.seq, w.seq)
self.parentjobid = "{0}:{1}".format(v.jobid, w.jobid)
self.superstarstatus = None
self.superstarhistogramstatus = None
self.melibeastatus = None
self.melibeahistogramstatus = None
self.odiestatus = None
def set_unique(self, v_attr, w_attr):
if v_attr == w_attr:
return v_attr
else:
return None
from standardhandle import output, warning, verbose, error, gettag
import options, cliopts
##############################################################################
#
# writeworkflow
#
##############################################################################
def writeworkflow(sequence_list):
tag = gettag()
from iofile import writetofile
from os.path import join, exists
from config import cfg
from utils import magicdate_to_number
replaced = None
dot_basename = "{0}_{1}_{2}{3}".\
format(cfg.get('OSA', 'WORKFLOWPREFIX'),\
magicdate_to_number(options.date),\
options.tel_id, cfg.get('OSA', 'GRAPHSUFFIX'))
dot_path = join(options.directory, dot_basename)
# We could think of using the pydot interface as well, but this is relatively simple anyway
content = "strict digraph {\n"
content += "label=\"OSA workflow for " + options.tel_id + " on " + options.date + "\";"
content += "labelloc=t;\n"
content += "rankdir=LR;\n"
content += "node [shape=box];\n"
content += "edge [headport=w];\n"
content += "start [label=\"\", style=invisible];\n"
# Now we reversely-loop over the requirements (just for make ordered the drawing in Left-Right mode)
for i in sequence_list:
content += 'n{1} [label="{0} {1}|{2} [{3}]"\nshape="record"];\n'.format(i.telescope, i.seq, i.run, i.subruns)
if len(i.parent_list) == 0:
content += "start -> n{0} [style=invis];\n".format(i.seq)
elif len(i.parent_list) == 1:
""" One parent, typical single process mode, id different"""
content += "n{0} -> n{1};\n".format(i.parent_list[0].seq, i.seq)
else:
""" Two parents, stereo mode, assign indexes for each range,
this means 0 - m-1 (for the st), m - 2m-1 (for M1), 2m - 3m-1 (for M2)"""
m = len(sequence_list)
index = i.seq
for j in i.parent_list:
index += m
content += 'n{0} [label="{1} {2}|{3} [{4}]"\nshape="record"];\n'.format(index, j.telescope, j.seq, j.run, j.subruns)
content += "start -> n{0} [style=invis];\n".format(index)
content += "n{0} -> n{1};\n".format(index, i.seq)
# Closing the content
content += "}\n"
if not options.simulate:
replaced = writetofile(dot_path, content)
verbose(tag, "Workflow updated? {0} in {1}".format(replaced, dot_path))
svg_path = dot_path.rsplit('.', 1)[0] + cfg.get('OSA', 'SVGSUFFIX')
if replaced or not exists(svg_path):
verbose(tag, "Updating workflow file: {0}".format(dot_path))
convert_dot_into_svg(dot_path, svg_path)
##############################################################################
#
# convert_dot_into_svg
#
##############################################################################
def convert_dot_into_svg(dotfile, svgfile):
tag = gettag()
# Pretty clear what it does, isn't it?
import subprocess
from config import cfg
command = cfg.get('OSA', 'GRAPH')
svgflag = '-' + cfg.get('OSA', 'SVGSUFFIX').replace('.', 'T')
try:
commandoutput = subprocess.check_output(['which', command])
except subprocess.CalledProcessError as Error:
error(tag, Error, 3)
else:
commandargs = [command, svgflag, '-o' + svgfile, dotfile]
try:
subprocess.call(commandargs)
#except OSError as (ValueError, NameError):
except OSError as NameError:
warning(tag, "svg file could not be created from dot file {0}, {1}".format(dotfile, NameError))
else:
verbose(tag, "Workflow sketched in file {0} ".format(dotfile))
from standardhandle import output, verbose, error, errornonfatal, gettag
import options
##############################################################################
#
# readfromfile
#
##############################################################################
def readfromfile(file):
tag = gettag()
from os.path import exists, isfile
if exists(file):
if isfile(file):
try:
with open(file, 'r') as f:
return f.read()
except (IOError, OSError) as e:
error(tag, "{0} {1}".format(e.strerror, e.filename), e.errno)
else:
error(tag, "{0} is not a file".format(file), 1)
else:
error(tag, "File does not exists {0}".format(file), 1)
##############################################################################
#
# writetofile
#
##############################################################################
def writetofile(f, content):
tag = gettag()
from os.path import exists
from os import remove, rename
import subprocess
ftemp= f + '.tmp'
try:
with open(ftemp, 'w') as filehandle:
filehandle.write("{0}".format(content))
except (IOError, OSError) as e:
error(tag, "{0} {1}".format(e.strerror, e.filename), e.errno)
if exists(f):
import filecmp
if filecmp.cmp(f, ftemp):
remove(ftemp)
return False
else:
if options.simulate:
remove(ftemp)
verbose(tag, "SIMULATE File {0} would replace {1}. Deleting {0}".format(ftemp, f))
else:
try:
rename(ftemp, f)
except (IOError, OSError) as e:
error(tag, "{0} {1}".\
format(e.strerror, e.filename), e.errno)
else:
if options.simulate:
verbose(tag, "SIMULATE File {0} would be written as {1}. Deleting {0}".format(ftemp, f))
else:
rename(ftemp, f)
return True
##############################################################################
#
# appendtofile
#
##############################################################################
def appendtofile(f, content):
tag = gettag()
from os.path import exists, isfile
if exists(f) and isfile(f):
if options.simulate:
verbose(tag, "SIMULATE File {0} would be appended".format(f))
else:
with open(f, 'a') as filehandle:
try:
filehandle.write(content)
except IOError as NameError:
error(tag, "{0} {1}".\
format(e.strerror, e.filename), e.errno)
else:
writetofile(f, content)
return True
##############################################################################
#
# sedsi (an equivalent to sed s///g -i)
#
##############################################################################
def sedsi(pattern, replace, file):
tag = gettag()
old_content = readfromfile(file)
new_content = old_content.replace(pattern, replace)
writetofile(file, new_content)
This diff is collapsed.
#!/usr/bin/env python2.7
from subprocess import call, check_output
import xml.dom.minidom
import xmlhandle
import sys
""" Read the argument """
if len(sys.argv) != 2 or (sys.argv[1] != 'suspend' and sys.argv[1] != 'resume' and sys.argv[1] != 'kill'):
sys.stderr.write("Error. Usage osa_jobs.py <action>\n where action: suspend, resume, kill\n")
sys.exit(1)
argument = sys.argv[1]
""" Check the jobs in queue """
commandargs = ['qstat', '-x']
xml_output = check_output(commandargs)
""" We have to parse the xml """
document = xml.dom.minidom.parseString(xml_output)
queue_list = xmlhandle.xmlhandleData(document)
"""
The queue list contains element like
{'name': u'STDIN', 'jobhost': u'fcana2/0', 'cputime': u'00:00:00', 'jobid': 9412, 'state': u'C', 'exit': 0, 'walltime': u'00:00:17'}
"""
for job in queue_list:
if job['state']:
if job['state'] != 'C':
if (argument == 'suspend' and job['state'] != 'S') or (argument == 'resume' and job['state'] == 'S'):
print "{0}ing jobid {1}, {2}...".format(argument, job['jobid'], job['name']),
commandargs = ['qsig', '-s', argument, str(job['jobid'])]
rc = call(commandargs)
if rc == 0:
print "Success!"
else:
print "Failed!"
else:
print "Warning, no state for jobid {0}".format(job['jobid'])
from standardhandle import output, verbose, error, gettag, errornonfatal,\
stringify
import options
##############################################################################
#
# send_command_file
#
##############################################################################
def send_stream(stream):
tag = gettag()
""" TODO: use the email package from python """
import subprocess
commandargs = ['/usr/lib/sendmail', '-t', '-O', 'ErrorMode=m']
if not options.simulate:
try:
subprocess.check_call(commandargs, stdin=stream)
except subprocess.CalledProcessError as e:
errornonfatal(tag, e.output)
else:
verbose(tag, "Simulating: {0}".stringify(commandargs))
##############################################################################
#
# send_file
#
##############################################################################
def send_file(file):
tag = gettag()
with open(file, 'r') as i:
send_command_stream(i)
##############################################################################
#
# send_content
#
##############################################################################
def send_assignments(assignments):
tag = gettag()
import tempfile
content = set_content(assignments)
if content:
i = tempfile.SpooledTemporaryFile('w+')
i.write(content)
i.seek(0, 0)
send_stream(i)
else:
errornonfatal(tag, "Empty assignments {0}".format(assignments))
##############################################################################
#
# send_email
#
##############################################################################
def send_email(assignments):
import smtplib
from email.mime.text import MIMEText
msg = MIMEText(assignments['Content'])
msg['Subject'] = assignments['Subject']
msg['From'] = 'OSA Daemon <analysis@ana7.magic.iac.es>'
msg['To'] = '<magic-onsite@gae.ucm.es>'
try:
s = smtplib.SMTP('localhost')
s.sendmail('analysis@ana7.magic.iac.es', ['magic-onsite@gae.ucm.es'],
msg.as_string())
return True
except smtplib.SMTPException:
return False
##############################################################################
#
# set_headers
#
##############################################################################
def set_content(assignments):
tag = gettag()
""" TODO: use the email package from python """
content = "MIME-Version: 1.0\n"
content += "Content-type: text/plain; charset=utf-8\n"
content += "Content-Transfer-Encoding: quoted-printable\n"
keys = ['From', 'To', 'Cc', 'Bcc', 'Reply-to', 'Subject', 'Content', 'Signature']
if len(assignments) != 0:
for key in keys:
if key in assignments.keys():
if key == 'Content' or key == 'Signature':
content += "\n{1}".format(key, assignments[key])
else:
content += "{0}: {1}\n".format(key, assignments[key])
return content
else:
return None
#!/usr/bin/env python2.7
from standardhandle import output, warning, error, stringify
##############################################################################
#
# monolith
#
##############################################################################
def monolith():
tag = monolith.__name__
import datetime
import report
import utils
import options, cliopts
# Initiating report
now = datetime.datetime.utcnow()
report.header("Starting {0} at {1} UTC for MAGIC, Telescope: {2}".format(tag, now.strftime("%Y-%m-%d %H:%M:%S"), options.tel_id))
# Building the set of days
datadays = None
if options.tel_id == 'M1' or options.tel_id == 'M2':
datadays = utils.getrawdatadays()
else:
""" The days in which we have stereo data sequences """
datadays = utils.getstereodatadays()
finisheddays = utils.getfinisheddays()
unfinisheddays = utils.sorted_nicely(datadays - finisheddays)
# Finalizing report
report.rule()
loopaskingsequencer(unfinisheddays)
##############################################################################
#
# loopaskingsequencer
#
##############################################################################
def loopaskingsequencer(unfinisheddays):
tag = loopaskingsequencer.__name__
for day in unfinisheddays:
if options.noninteractive:
launchsequencer(day)
else:
answercheck = True
while (answercheck):
try:
answer = raw_input("OSA {0} {1} is not finished. Run sequencer? (y/n): ".format(options.tel_id, day))
except KeyboardInterrupt:
import sys
print ''
warning(tag, "Program quitted by user not willing to answer")
sys.exit(1)
except EOFError as e:
error(tag, "End of file not expected", e)
else:
answercheck = False
if answer == 'y' or answer == 'Y':
launchsequencer(day)
elif answer == 'n' or answer == 'N':
continue
else:
warning(tag, "Answer not understood, please type y or n")
answercheck = True
##############################################################################
#
# launchsequencer
#
##############################################################################
def launchsequencer(day):
tag = launchsequencer.__name__
import subprocess
from os.path import join
import config
commandargs = [join(config.cfg.get('OSA', 'PYTHONDIR'), 'sequencer.py')]
if options.configfile:
commandargs.append('-c')
commandargs.append(options.configfile)
if options.simulate:
commandargs.append('-s')
if options.warning:
commandargs.append('-w')
if options.verbose:
commandargs.append('-v')
commandargs.append('-d')
commandargs.append(day)
commandargs.append(options.tel_id)
if options.simulate:
output(tag, "{0}".format(stringify(commandargs)))
else:
try:
stdout = subprocess.check_output(commandargs)
except OSError as (ValueError, NameError):
error(tag, "Calling {0} failed, {1}".format(stringify(commandargs), NameError), ValueError)
else:
output(tag, stdout)
##############################################################################
#
# MAIN
#
##############################################################################
if __name__ == '__main__':
import sys
import os.path
tag = os.path.basename(__file__)
import options, cliopts
# Set the options through cli parsing
cliopts.monolithcliparsing(sys.argv[0])
# Define the telescope array
tel_array = []
if options.tel_id:
tel_array.append(options.tel_id)
else:
tel_array = ['M1', 'M2', 'ST']
# Run the routine as many times as required in the telescope array
for tel in tel_array:
options.tel_id = tel
monolith()
#!/usr/bin/env python2.6
from standardhandle import output, warning, verbose, error, stringify, gettag
import options, cliopts
import config
##############################################################################
#
# select_db
#
##############################################################################