Latest update

cfg/sequencer.cfg

@@ -99,7 +99,7 @@ CLOSERDIR = %(DIR)s/lst-osa/Closer
 ENDOFRAWTRANSFERDIR = %(DIR)s/Reports/Compress
 CALIBRATIONCONFIGCARD = onsite_camera_calibration_param.json 
 CONFIGCARD = 
-VERSION = v01
+VERSION = v00
 
 [LST2]
 DIR = /fefs/aswg/lst-osa

dot.py

@@ -9,18 +9,18 @@ def writeworkflow(sequence_list):
     tag = gettag()
     from iofile import writetofile
     from os.path import join, exists
-    from config import cfg
-    from utils import magicdate_to_number
+    from osa.configs.config import cfg
+    from osa.utils.utils import lstdate_to_number
     replaced = None
     dot_basename = "{0}_{1}_{2}{3}".\
-     format(cfg.get('OSA', 'WORKFLOWPREFIX'),\
-     magicdate_to_number(options.date),\
-     options.tel_id, cfg.get('OSA', 'GRAPHSUFFIX'))
+     format(cfg.get('LSTOSA', 'WORKFLOWPREFIX'),\
+     lstdate_to_number(options.date),\
+     options.tel_id, cfg.get('LSTOSA', 'GRAPHSUFFIX'))
     dot_path = join(options.directory, dot_basename)
 
     # We could think of using the pydot interface as well, but this is relatively simple anyway
     content = "strict digraph {\n"
-    content += "label=\"OSA workflow for " + options.tel_id + " on " + options.date + "\";"
+    content += "label=\"LSTOSA workflow for " + options.tel_id + " on " + options.date + "\";"
     content += "labelloc=t;\n"
     content += "rankdir=LR;\n"
     content += "node [shape=box];\n"
@@ -49,7 +49,7 @@ def writeworkflow(sequence_list):
     if not options.simulate:
         replaced = writetofile(dot_path, content)
     verbose(tag, "Workflow updated? {0} in {1}".format(replaced, dot_path))
-    svg_path = dot_path.rsplit('.', 1)[0] + cfg.get('OSA', 'SVGSUFFIX')
+    svg_path = dot_path.rsplit('.', 1)[0] + cfg.get('LSTOSA', 'SVGSUFFIX')
     if replaced or not exists(svg_path):
         verbose(tag, "Updating workflow file: {0}".format(dot_path))
         convert_dot_into_svg(dot_path, svg_path)
@@ -62,10 +62,10 @@ def convert_dot_into_svg(dotfile, svgfile):
     tag = gettag()
     # Pretty clear what it does, isn't it?
     import subprocess
-    from config import cfg
+    from osa.configs.config import cfg
 
-    command = cfg.get('OSA', 'GRAPH')
-    svgflag = '-' + cfg.get('OSA', 'SVGSUFFIX').replace('.', 'T')
+    command = cfg.get('LSTOSA', 'GRAPH')
+    svgflag = '-' + cfg.get('LSTOSA', 'SVGSUFFIX').replace('.', 'T')
     try:
         commandoutput = subprocess.check_output(['which', command])
     except subprocess.CalledProcessError as Error:

osa/jobs/job.py

@@ -343,8 +343,9 @@ def createjobtemplate(s):
     content = "#!/bin/env python\n"
     # SLURM assignments
     content += "#SBATCH -p compute\n"
-    content += "#SBATCH --tasks=60\n"
-    content += "#SBATCH --nodes=2\n"
+    content += "#SBATCH --tasks=2\n"
+    if s.type == DATA:
+       content += "#SBATCH --array=1-{0}\n".format(len(s.subrun_list))
     content += "#SBATCH --cpus-per-task=1\n"
     content += "#SBATCH --mem-per-cpu=1600\n"
     content += "#SBATCH -t 0-48:00\n"
@@ -377,75 +378,88 @@ def createjobtemplate(s):
 # submitjobs
 #
 ##############################################################################
-def submitjobs(sequence_list, queue_list, veto_list):
+#def submitjobs(sequence_list, queue_list, veto_list):
+def submitjobs(sequence_list):
     tag = gettag()
     import subprocess
     from os.path import join
     from osa.configs import config
     job_list = []
-    command = 'qsub'
+    command = 'sbatch'
     for s in sequence_list:
          commandargs = [command, s.script] # List of two elements
-        commandargs.append('-W')
-        commandargs.append('umask=0022')
-        """ Introduce the job dependencies """
-        if len(s.parent_list) != 0:
-            commandargs.append('-W')
-            depend_string = 'depend='
-            if s.type == 'DATA':
-                depend_string += 'afterok'
-            elif s.type == 'STEREO':
-                depend_string += 'afterany'
-            for pseq in s.parent_list:
-                if pseq.jobid > 0:
-                    depend_string += ":{0}".format(pseq.jobid)
-            commandargs.append(depend_string)
-        """ Skip vetoed """
-        if s.action == 'Veto':
-            verbose(tag, "job {0} has been vetoed".format(s.jobname))
-        elif s.action == 'Closed':
-            verbose(tag, "job {0} is already closed".format(s.jobname))
-        elif s.action == 'Check' and s.state != 'C':
-            verbose(tag, "job {0} checked to be dispatched but not completed yet".format(s.jobname))
-            if s.state == 'H' or s.state == 'R':
-                # Reset values
-                s.exit = None
-                if s.state == 'H':
-                    s.jobhost = None
-                    s.cputime = None
-                    s.walltime = None
-        elif s.action == 'Check' and s.state == 'C' and s.exit == 0:
-            verbose(tag, "job {0} checked to be successful".format(s.jobname))
-        else:
-            if options.simulate == True:
-                commandargs.insert(0, 'echo')
-                s.action = 'Simulate'
-                # This jobid is negative showing it belongs to a simulated environment (not real jobid)
-                s.jobid = -1 - s.seq
-            else:
-                s.action = 'Submit'
-                # Reset the values to avoid misleading info from previous jobs
-                s.jobhost = None
-                s.state = 'Q'
-                s.cputime = None
-                s.walltime = None
-                s.exit = None
+#        commandargs.append('-W')
+#        commandargs.append('umask=0022')
+#        """ Introduce the job dependencies """
+#        if len(s.parent_list) != 0:
+#            commandargs.append('-W')
+#            depend_string = 'depend='
+#            if s.type == 'DATA':
+#                depend_string += 'afterok'
+#            elif s.type == 'STEREO':
+#                depend_string += 'afterany'
+#            for pseq in s.parent_list:
+#                if pseq.jobid > 0:
+#                    depend_string += ":{0}".format(pseq.jobid)
+#            commandargs.append(depend_string)
+#        """ Skip vetoed """
+#        if s.action == 'Veto':
+#            verbose(tag, "job {0} has been vetoed".format(s.jobname))
+#        elif s.action == 'Closed':
+#            verbose(tag, "job {0} is already closed".format(s.jobname))
+#        elif s.action == 'Check' and s.state != 'C':
+#            verbose(tag, "job {0} checked to be dispatched but not completed yet".format(s.jobname))
+#            if s.state == 'H' or s.state == 'R':
+#                # Reset values
+#                s.exit = None
+#                if s.state == 'H':
+#                    s.jobhost = None
+#                    s.cputime = None
+#                    s.walltime = None
+#        elif s.action == 'Check' and s.state == 'C' and s.exit == 0:
+#            verbose(tag, "job {0} checked to be successful".format(s.jobname))
+#        else:
+#            if options.simulate == True:
+#                commandargs.insert(0, 'echo')
+#                s.action = 'Simulate'
+#                # This jobid is negative showing it belongs to a simulated environment (not real jobid)
+#                s.jobid = -1 - s.seq
+#            else:
+#                s.action = 'Submit'
+#                # Reset the values to avoid misleading info from previous jobs
+#                s.jobhost = None
+#                s.state = 'Q'
+#                s.cputime = None
+#                s.walltime = None
+#                s.exit = None
+#            try:
+#                stdout = subprocess.check_output(commandargs)
+#            except subprocess.CalledProcessError as Error:
+#                error(tag, Error, 2)
+#            except OSError (ValueError, NameError):
+#                error(tag, "Command {0}, {1}".format(stringify(commandargs), NameError), ValueError)
+#            else:
+#                if options.simulate == False:
+#                    try:
+#                        s.jobid = int(stdout.split('.', 1)[0])
+#                    except ValueError as e:
+#                        warning(tag, "Wrong parsing of jobid {0} not being an integer, {1}".format(stdout.split('.', 1)[0], e))
+#        job_list.append(s.jobid)
+#        verbose(tag, "{0} {1}".format(s.action, stringify(commandargs)))
+         print("Launching scripts {0} ".format(str(s.script)))
          try:
+             verbose(tag,"Launching scripts {0} ".format(str(s.script)))
              stdout = subprocess.check_output(commandargs)
          except subprocess.CalledProcessError as Error:
                 error(tag, Error, 2)
          except OSError (ValueError, NameError):
                 error(tag, "Command {0}, {1}".format(stringify(commandargs), NameError), ValueError)  
-            else:
-                if options.simulate == False:
-                    try:
-                        s.jobid = int(stdout.split('.', 1)[0])
-                    except ValueError as e:
-                        warning(tag, "Wrong parsing of jobid {0} not being an integer, {1}".format(stdout.split('.', 1)[0], e))
-        job_list.append(s.jobid)
-        verbose(tag, "{0} {1}".format(s.action, stringify(commandargs)))
+
+         print(commandargs)
+         job_list.append(s.script) 
+        
     return job_list
-##############################################################################
+ ##############################################################################
 #
 # getqueuejoblist
 #
@@ -459,7 +473,7 @@ def getqueuejoblist(sequence_list):
     command = config.cfg.get('ENV', 'SBATCHBIN')
     commandargs = [command]
     queue_list = []
-    print("DEBUG",command)
+    print("DEBUG",commandargs)
     try:
         xmloutput = subprocess.check_output(commandargs)
     except subprocess.CalledProcessError as Error:
@@ -474,6 +488,7 @@ def getqueuejoblist(sequence_list):
             document = xml.dom.minidom.parseString(xmloutput)
             queue_list = xmlhandle.xmlhandleData(document)
             setqueuevalues(queue_list, sequence_list)
+    print("DEBUG",command)
     return queue_list
 ##############################################################################
 #

osa/utils/cliopts.py

@@ -427,9 +427,9 @@ def set_default_date_if_needed():
     if is_defined(options.date):
         return options.date
     else:
-        from utils import getcurrentdate2
-        from config import cfg
-        return getcurrentdate2(cfg.get('MAGIC', 'DATESEPARATOR'))
+        from .utils import getcurrentdate2
+        from osa.configs.config import cfg
+        return getcurrentdate2(cfg.get('LST', 'DATESEPARATOR'))
 ##############################################################################
 #
 # set_default_directory_if_needed

sequencer.py

@@ -108,13 +108,15 @@ def single_process(telescope, process_mode):
 #    dot.writeworkflow(sequence_list)
     # Adds the scripts
     job.preparejobs(sequence_list, subrun_list)
-    queue_list = job.getqueuejoblist(sequence_list)
+
+#    queue_list = job.getqueuejoblist(sequence_list)
 #    veto_list = veto.getvetolist(sequence_list)
 #    closed_list = veto.getclosedlist(sequence_list)
 #    updatelstchainstatus(sequence_list)
 #    updatesequencedb(sequence_list)
     # actually, submitjobs does not need the queue_list nor veto_list
 #    job_list = job.submitjobs(sequence_list, queue_list, veto_list)
+    job_list = job.submitjobs(sequence_list)
 #    combine_muon(job_list)
 #    # Report
 #    if is_report_needed: