Merge branch 'master' into 'master'

Master

See merge request contrera/lstosa!4

config/config.py

+from standardhandle import error, verbose, gettag, warning
+import options
+##############################################################################
+#
+# readconf
+#
+##############################################################################
+def readconf(file):
+    tag = gettag()
+    from os.path import exists
+
+    conf = None
+
+    try:
+        # Python 2.7
+        import ConfigParser
+    except ImportError as Error:
+        warning(tag, "Increasing to python 3 ConfigParser")
+        import configparser
+        conf = configparser.SafeConfigParser(allow_no_value=True)
+    else:
+        conf = ConfigParser.SafeConfigParser(allow_no_value=True)
+
+    try:
+        conf.read(file)
+    except ConfigParser.Error as NameError:
+        error(tag, NameError, 3)
+
+    verbose(tag, "sections of the config file are {0}".format(conf.sections()))
+    return conf
+##############################################################################
+#
+# read_properties
+#
+##############################################################################
+def read_properties(file):
+    tag = gettag()
+
+    """ To be used when config file has no header, creating a DUMMY header""" 
+    import tempfile
+    from os import unlink
+    fname = None
+    with tempfile.NamedTemporaryFile(delete=False) as tf:
+        tf.write('[DUMMY]\n')
+        fname = tf.name
+        with open(file) as f:
+            tf.write(f.read())
+        tf.seek(0)
+        conf = readconf(tf.name)
+    unlink(tf.name)
+    return conf
+##############################################################################
+#
+# cfg
+#
+##############################################################################
+cfg = readconf(options.configfile)

datamodel.py

+class Period(object):
+    def __init__(self):
+        self.period = None
+
+class Night(Period):
+    def __init__(self):
+        super(Night, self).__init__()
+        self.night = None
+
+class Telescope(Night):
+    def __init__(self):
+        super(Telescope, self).__init__()
+        self.telescope = None
+
+class Source(Telescope):
+    def __init__(self):
+        super(Source, self).__init__()    
+        self.source = None
+
+class Wobble(Source):
+    def __init__(self):
+        super(Wobble, self).__init__()
+        self.sourcewobble = None
+        self.wobble = None
+
+class RunObj(Wobble):
+    def __init__(self):
+        super(RunObj, self).__init__()
+        self.run_str = None
+        self.run = None
+        self.type = None
+        self.subrun_list = []
+        self.subruns = 0 
+
+class SubrunObj(RunObj):
+    def __init__(self):
+        self.runobj = None 
+        self.subrun_str = None
+        self.subrun = None
+        self.kind = None
+        self.timestamp = None
+        self.time = None
+        self.date = None
+
+class Sequence(RunObj):
+    def __init__(self):
+        super(Sequence, self).__init__()
+        self.seq = None
+        self.parent_list = []
+        self.parent = None
+        self.parentjobid = None
+        self.previousrun = None
+        self.script = None
+        self.veto = None
+        self.closed = None
+        self.history = None
+        self.queue = None
+        self.jobname = None
+        self.jobid = None
+        self.action = None
+        self.tries = None
+        self.state = None
+        self.jobhost = None
+        self.cputime = None
+        self.walltime = None
+        self.exit = None
+
+    def associate(self, r):
+        for a in r.__dict__.keys():
+            self.__dict__.update({a: r.__dict__[a]})
+
+class SequenceCalibration(Sequence):
+    def __init__(self, r):
+        super(SequenceCalibration, self).__init__()
+        super(SequenceCalibration, self).associate(r)        
+        self.scalibstatus = None
+
+class SequenceData(Sequence):
+    def __init__(self, r):
+        super(SequenceData, self).__init__()
+        super(SequenceData, self).associate(r)
+        self.calibration = None
+        self.pedestal = None
+        self.drive = None
+        self.ssignalstatus = None
+        self.sorcererstatus = None
+        self.merppstatus = None
+        self.starstatus = None
+        self.starhistogramstatus = None
+
+class SequenceStereo(Sequence):
+    def __init__(self, v, w):
+        super(SequenceStereo, self).__init__()
+        attr_list = ['run', 'subrun_list' , 'subruns',\
+         'wobble', 'sourcewobble', 'source', 'night']
+        for a in attr_list:
+            """ This copies the unique attrs of both sequences """
+            self.__dict__.update({a: self.\
+             set_unique(v.__dict__[a], w.__dict__[a])})
+        self.type = 'STEREO'
+        self.telescope = 'ST'
+        self.subruns = v.subruns + w.subruns
+        self.parent_list = [v, w]
+        self.parent = "{0},{1}".format(v.seq, w.seq)
+        self.parentjobid = "{0}:{1}".format(v.jobid, w.jobid)
+        self.superstarstatus = None
+        self.superstarhistogramstatus = None
+        self.melibeastatus = None
+        self.melibeahistogramstatus = None
+        self.odiestatus = None
+    def set_unique(self, v_attr, w_attr):
+        if v_attr == w_attr:
+            return v_attr
+        else:
+            return None

dot.py

+from standardhandle import output, warning, verbose, error, gettag
+import options, cliopts
+##############################################################################
+#
+# writeworkflow
+#
+##############################################################################
+def writeworkflow(sequence_list):
+    tag = gettag()
+    from iofile import writetofile
+    from os.path import join, exists
+    from config import cfg
+    from utils import magicdate_to_number
+    replaced = None
+    dot_basename = "{0}_{1}_{2}{3}".\
+     format(cfg.get('OSA', 'WORKFLOWPREFIX'),\
+     magicdate_to_number(options.date),\
+     options.tel_id, cfg.get('OSA', 'GRAPHSUFFIX'))
+    dot_path = join(options.directory, dot_basename)
+
+    # We could think of using the pydot interface as well, but this is relatively simple anyway
+    content = "strict digraph {\n"
+    content += "label=\"OSA workflow for " + options.tel_id + " on " + options.date + "\";"
+    content += "labelloc=t;\n"
+    content += "rankdir=LR;\n"
+    content += "node [shape=box];\n"
+    content += "edge [headport=w];\n"
+    content += "start [label=\"\", style=invisible];\n"
+    # Now we reversely-loop over the requirements (just for make ordered the drawing in Left-Right mode)
+    for i in sequence_list:
+        content += 'n{1} [label="{0} {1}|{2} [{3}]"\nshape="record"];\n'.format(i.telescope, i.seq, i.run, i.subruns)
+        if len(i.parent_list) == 0:
+            content += "start -> n{0} [style=invis];\n".format(i.seq)
+        elif len(i.parent_list) == 1:
+            """ One parent, typical single process mode, id different"""
+            content += "n{0} -> n{1};\n".format(i.parent_list[0].seq, i.seq)
+        else:
+            """ Two parents, stereo mode, assign indexes for each range,
+            this means 0 - m-1 (for the st), m - 2m-1 (for M1), 2m - 3m-1 (for M2)"""
+            m = len(sequence_list)
+            index = i.seq
+            for j in i.parent_list:
+                index += m
+                content += 'n{0} [label="{1} {2}|{3} [{4}]"\nshape="record"];\n'.format(index, j.telescope, j.seq, j.run, j.subruns)
+                content += "start -> n{0} [style=invis];\n".format(index)
+                content += "n{0} -> n{1};\n".format(index, i.seq)
+    # Closing the content
+    content += "}\n"
+    if not options.simulate:
+        replaced = writetofile(dot_path, content)
+    verbose(tag, "Workflow updated? {0} in {1}".format(replaced, dot_path))
+    svg_path = dot_path.rsplit('.', 1)[0] + cfg.get('OSA', 'SVGSUFFIX')
+    if replaced or not exists(svg_path):
+        verbose(tag, "Updating workflow file: {0}".format(dot_path))
+        convert_dot_into_svg(dot_path, svg_path)
+##############################################################################
+#
+# convert_dot_into_svg
+#
+##############################################################################
+def convert_dot_into_svg(dotfile, svgfile):
+    tag = gettag()
+    # Pretty clear what it does, isn't it?
+    import subprocess
+    from config import cfg
+
+    command = cfg.get('OSA', 'GRAPH')
+    svgflag = '-' + cfg.get('OSA', 'SVGSUFFIX').replace('.', 'T')
+    try:
+        commandoutput = subprocess.check_output(['which', command])
+    except subprocess.CalledProcessError as Error:
+        error(tag, Error, 3)
+    else:
+        commandargs = [command, svgflag, '-o' + svgfile, dotfile]
+
+    try:
+        subprocess.call(commandargs)
+    #except OSError as (ValueError, NameError):
+    except OSError as NameError:
+        warning(tag, "svg file could not be created from dot file {0}, {1}".format(dotfile, NameError))
+    else:
+        verbose(tag, "Workflow sketched in file {0} ".format(dotfile))

iofile.py

+from standardhandle import output, verbose, error, errornonfatal, gettag
+import options
+##############################################################################
+#
+# readfromfile
+#
+##############################################################################
+def readfromfile(file):
+    tag = gettag()
+    from os.path import exists, isfile
+    if exists(file):
+        if isfile(file):
+            try:
+                with open(file, 'r') as f:
+                    return f.read()
+            except (IOError, OSError) as e:
+                error(tag, "{0} {1}".format(e.strerror, e.filename), e.errno)
+        else:
+           error(tag, "{0} is not a file".format(file), 1)
+    else:
+        error(tag, "File does not exists {0}".format(file), 1)
+##############################################################################
+#
+# writetofile
+#
+##############################################################################
+def writetofile(f, content):
+    tag = gettag()
+    from os.path import exists
+    from os import remove, rename
+    import subprocess
+    ftemp= f + '.tmp'
+    try:
+        with open(ftemp, 'w') as filehandle:
+            filehandle.write("{0}".format(content))
+    except (IOError, OSError) as e:
+        error(tag, "{0} {1}".format(e.strerror, e.filename), e.errno)
+    
+    if exists(f):
+        import filecmp
+        if filecmp.cmp(f, ftemp):
+            remove(ftemp)
+            return False
+        else:
+            if options.simulate:
+                remove(ftemp)
+                verbose(tag, "SIMULATE File {0} would replace {1}. Deleting {0}".format(ftemp, f))
+            else:
+                try:
+                    rename(ftemp, f)
+                except (IOError, OSError) as e:
+                    error(tag, "{0} {1}".\
+                     format(e.strerror, e.filename), e.errno)
+    else:
+        if options.simulate:
+            verbose(tag, "SIMULATE File {0} would be written as {1}. Deleting {0}".format(ftemp, f))
+        else:
+            rename(ftemp, f)
+    return True
+##############################################################################
+#
+# appendtofile
+#
+##############################################################################
+def appendtofile(f, content):
+    tag = gettag()
+    from os.path import exists, isfile
+    if exists(f) and isfile(f):
+        if options.simulate:
+            verbose(tag, "SIMULATE File {0} would be appended".format(f)) 
+        else:
+            with open(f, 'a') as filehandle:
+                try:
+                    filehandle.write(content)
+                except IOError as NameError:
+                    error(tag, "{0} {1}".\
+                     format(e.strerror, e.filename), e.errno)
+    else:
+        writetofile(f, content)
+    return True
+##############################################################################
+#
+# sedsi (an equivalent to sed s///g -i)
+#
+##############################################################################
+def sedsi(pattern, replace, file):
+    tag = gettag()
+    old_content = readfromfile(file)
+    new_content = old_content.replace(pattern, replace)
+    writetofile(file, new_content)

jobs/osa_jobs.py

+#!/usr/bin/env python2.7
+
+from subprocess import call, check_output
+import xml.dom.minidom
+import xmlhandle
+import sys
+
+""" Read the argument """
+
+if len(sys.argv) != 2 or (sys.argv[1] != 'suspend' and sys.argv[1] != 'resume' and sys.argv[1] != 'kill'):
+    sys.stderr.write("Error. Usage osa_jobs.py <action>\n   where action: suspend, resume, kill\n")
+    sys.exit(1)
+    
+argument = sys.argv[1]
+
+""" Check the jobs in queue """
+
+commandargs = ['qstat', '-x']
+xml_output = check_output(commandargs)
+
+""" We have to parse the xml """
+document = xml.dom.minidom.parseString(xml_output)
+queue_list = xmlhandle.xmlhandleData(document)
+
+""" 
+The queue list contains element like
+{'name': u'STDIN', 'jobhost': u'fcana2/0', 'cputime': u'00:00:00', 'jobid': 9412, 'state': u'C', 'exit': 0, 'walltime': u'00:00:17'}
+"""
+
+for job in queue_list:
+    if job['state']:
+        if job['state'] != 'C':
+            if (argument == 'suspend' and job['state'] != 'S') or (argument == 'resume' and job['state'] == 'S'):
+                print "{0}ing jobid {1}, {2}...".format(argument, job['jobid'], job['name']),
+                commandargs = ['qsig', '-s', argument, str(job['jobid'])]
+                rc = call(commandargs)
+                if rc == 0:
+                    print "Success!"
+                else:
+                    print "Failed!"
+    else:
+        print "Warning, no state for jobid {0}".format(job['jobid'])

mail.py

+from standardhandle import output, verbose, error, gettag, errornonfatal,\
+ stringify
+import options
+##############################################################################
+#
+# send_command_file
+#
+##############################################################################
+def send_stream(stream):
+    tag = gettag()
+    """ TODO: use the email package from python """
+
+    import subprocess
+    commandargs = ['/usr/lib/sendmail', '-t', '-O', 'ErrorMode=m']
+    if not options.simulate:
+        try:
+            subprocess.check_call(commandargs, stdin=stream)
+        except subprocess.CalledProcessError as e:
+            errornonfatal(tag, e.output)
+    else:
+        verbose(tag, "Simulating: {0}".stringify(commandargs))
+##############################################################################
+#
+# send_file
+#
+##############################################################################
+def send_file(file):
+    tag = gettag()
+    with open(file, 'r') as i:
+        send_command_stream(i)
+##############################################################################
+#
+# send_content
+#
+##############################################################################
+def send_assignments(assignments):
+    tag = gettag()
+    import tempfile
+    content = set_content(assignments)
+    if content:
+        i = tempfile.SpooledTemporaryFile('w+')
+        i.write(content)
+        i.seek(0, 0)
+        send_stream(i)
+    else:
+        errornonfatal(tag, "Empty assignments {0}".format(assignments))
+##############################################################################
+#
+# send_email
+#
+##############################################################################
+def send_email(assignments):
+    import smtplib
+    from email.mime.text import MIMEText
+    msg = MIMEText(assignments['Content'])
+    msg['Subject'] = assignments['Subject']
+    msg['From'] = 'OSA Daemon <analysis@ana7.magic.iac.es>'
+    msg['To']   = '<magic-onsite@gae.ucm.es>'
+    try:
+        s = smtplib.SMTP('localhost')
+        s.sendmail('analysis@ana7.magic.iac.es', ['magic-onsite@gae.ucm.es'],
+                   msg.as_string())
+        return True
+    except smtplib.SMTPException:
+        return False
+##############################################################################
+#
+# set_headers 
+#
+##############################################################################
+def set_content(assignments):
+    tag = gettag()
+    """ TODO: use the email package from python """
+
+    content = "MIME-Version: 1.0\n"
+    content += "Content-type: text/plain; charset=utf-8\n"
+    content += "Content-Transfer-Encoding: quoted-printable\n"
+    keys = ['From', 'To', 'Cc', 'Bcc', 'Reply-to', 'Subject', 'Content', 'Signature']
+    if len(assignments) != 0:
+        for key in keys:
+            if key in assignments.keys():
+                if key == 'Content' or key == 'Signature':
+                    content += "\n{1}".format(key, assignments[key])
+                else:
+                    content += "{0}: {1}\n".format(key, assignments[key])
+        return content
+    else:
+        return None

monolith.py

+#!/usr/bin/env python2.7
+from standardhandle import output, warning, error, stringify
+##############################################################################
+#
+# monolith
+#
+##############################################################################
+def monolith():
+    tag = monolith.__name__
+    import datetime
+    import report
+    import utils
+    import options, cliopts
+
+    # Initiating report
+    now = datetime.datetime.utcnow()
+    report.header("Starting {0} at {1} UTC for MAGIC, Telescope: {2}".format(tag, now.strftime("%Y-%m-%d %H:%M:%S"), options.tel_id))
+
+
+    # Building the set of days
+    datadays = None
+    if options.tel_id == 'M1' or options.tel_id == 'M2':
+        datadays = utils.getrawdatadays()
+    else:
+        """ The days in which we have stereo data sequences """
+        datadays = utils.getstereodatadays()
+    finisheddays = utils.getfinisheddays()
+    unfinisheddays = utils.sorted_nicely(datadays - finisheddays)
+
+    # Finalizing report
+    report.rule()
+    loopaskingsequencer(unfinisheddays)
+##############################################################################
+#
+# loopaskingsequencer
+#
+##############################################################################
+def loopaskingsequencer(unfinisheddays):
+    tag = loopaskingsequencer.__name__
+    for day in unfinisheddays:
+        if options.noninteractive:
+            launchsequencer(day)
+        else:
+            answercheck = True
+            while (answercheck):
+                try:
+                    answer = raw_input("OSA {0} {1} is not finished. Run sequencer? (y/n): ".format(options.tel_id, day))
+                except KeyboardInterrupt:
+                    import sys
+                    print ''
+                    warning(tag, "Program quitted by user not willing to answer")
+                    sys.exit(1)
+                except EOFError as e:
+                    error(tag, "End of file not expected", e)
+                else:           
+                    answercheck = False
+                    if answer == 'y' or answer == 'Y':
+                        launchsequencer(day)
+                    elif answer == 'n' or answer == 'N':
+                        continue
+                    else:
+                        warning(tag, "Answer not understood, please type y or n")
+                        answercheck = True
+##############################################################################
+#
+# launchsequencer
+#
+##############################################################################
+def launchsequencer(day):
+    tag = launchsequencer.__name__
+    import subprocess
+    from os.path import join
+    import config
+    commandargs = [join(config.cfg.get('OSA', 'PYTHONDIR'), 'sequencer.py')]
+    if options.configfile:
+        commandargs.append('-c')
+        commandargs.append(options.configfile)
+    if options.simulate:
+        commandargs.append('-s')
+    if options.warning:
+        commandargs.append('-w')
+    if options.verbose:
+        commandargs.append('-v')
+    commandargs.append('-d')
+    commandargs.append(day)
+    commandargs.append(options.tel_id)
+    if options.simulate:
+        output(tag, "{0}".format(stringify(commandargs)))
+    else:
+        try:
+            stdout = subprocess.check_output(commandargs)
+        except OSError as (ValueError, NameError):
+            error(tag, "Calling {0} failed, {1}".format(stringify(commandargs), NameError), ValueError)
+        else:
+            output(tag, stdout)
+##############################################################################
+#
+# MAIN
+#
+##############################################################################
+if __name__ == '__main__':
+
+    import sys
+    import os.path
+    tag = os.path.basename(__file__)
+    import options, cliopts
+    # Set the options through cli parsing
+    cliopts.monolithcliparsing(sys.argv[0])
+
+    # Define the telescope array
+    tel_array = []
+    if options.tel_id:
+        tel_array.append(options.tel_id)
+    else:
+        tel_array = ['M1', 'M2', 'ST']
+    # Run the routine as many times as required in the telescope array
+    for tel in tel_array:
+        options.tel_id = tel
+        monolith()

mysql.py

+#!/usr/bin/env python2.6
+from standardhandle import output, warning, verbose, error, stringify, gettag
+import options, cliopts 
+import config
+##############################################################################
+#
+# select_db
+#
+##############################################################################