X-Git-Url: https://git.sthu.org/?p=paralleljobs.git;a=blobdiff_plain;f=paralleljobs.py;h=5041d408b3b32f74d05270ee400feab9ad24c9db;hp=c9e313d185b1fd5a7bccb2cd1a89bac53772faae;hb=HEAD;hpb=c0720bddfafeeba49eeb16524c15ab88fd960bc8

diff --git a/paralleljobs.py b/paralleljobs.py
index c9e313d..5041d40 100755
--- a/paralleljobs.py
+++ b/paralleljobs.py
@@ -1,4 +1,12 @@
-#!/usr/bin/python
+#!/usr/bin/env python
+""" A simple tool to run jobs from a database in parallel."""
+
+__author__ = "Stefan Huber"
+__copyright__ = "Copyright 2013"
+
+__version__ = "1.0"
+__license__ = "LGPL3"
+
 
 import sys, getopt, os
 import sqlite3
@@ -15,6 +23,9 @@ def printStatusInfo(conn):
     c.execute("SELECT count(id) FROM jobs WHERE done=1;")
     nodone, = c.fetchone()
 
+    c.execute("SELECT count(id) FROM jobs WHERE started=1;")
+    nostarted, = c.fetchone()
+
     c.execute("SELECT sum(workloadestm) FROM jobs WHERE done=1;")
     wldone, = c.fetchone()
     if wldone == None:
@@ -23,14 +34,23 @@ def printStatusInfo(conn):
     c.execute("SELECT sum(workloadestm) FROM jobs;")
     wltotal, = c.fetchone()
 
-    c.close()
 
-    print(nototal, nodone, wldone, wltotal)
-    perdone = 100.0*float(nodone)/float(nototal)
-    perwl = 100.0*float(wldone)/float(wltotal)
+    perdone = 0
+    perwl = 0
+    if nototal > 0:
+        perdone = 100.0*float(nodone)/float(nototal)
+    if wltotal > 0:
+        perwl = 100.0*float(wldone)/float(wltotal)
 
-    print("%d (%.1f%%) of %d jobs done. %.1f%% of the workload finished." % \
-            (nodone, perdone, nototal, perwl))
+    print("%d (%.1f%%) of %d jobs and %.1f%% of the workload done. %d jobs are running." % \
+            (nodone, perdone, nototal, perwl, nostarted-nodone))
+
+    print("Exit code stats:")
+    c.execute("SELECT exitcode, count(exitcode) AS cnt FROM jobs WHERE exitcode >= 0 GROUP BY exitcode ORDER BY exitcode ASC;")
+    for code, cnt in c.fetchall():
+        print("  %3s: %6s  (%5.1f%%)" % (code, cnt, 100.0*float(cnt)/nodone))
+
+    c.close()
 
 def createPropertiesTable(conn, propdef):
     conn.execute("BEGIN EXCLUSIVE")
@@ -54,18 +74,23 @@ def runCmd(cmd):
     exitcode = proc.wait()
 
     if verbose:
-        print(out, err)
+        print out, err
 
     return exitcode, out, err
 
 def processJob(conn, jobid):
-    print("Process job %d" % (jobid))
 
     c = conn.cursor()
     c.execute("SELECT cmd FROM jobs WHERE id=?", (jobid,))
     cmd, = c.fetchone()
 
+    print "Job %d: '%s'..." % (jobid, cmd),
     ec, out, err = runCmd(cmd)
+    if ec==0:
+        print " [OK]"
+    else:
+        print " [FAILED: %d]" % ec
+
     c.execute("UPDATE jobs SET exitcode=?, done=1 WHERE id=?;", (ec, jobid))
 
     propstr = []
@@ -109,7 +134,7 @@ def createSchema(conn):
     # Create table, if necessary
     c.execute("SELECT count(name) FROM sqlite_master WHERE name='jobs';")
     if c.fetchone() == (0,):
-        print("Creating jobs table.")
+        print "Creating jobs table."
         conn.execute("CREATE TABLE jobs ( \
                         id INTEGER PRIMARY KEY AUTOINCREMENT, \
                         cmd STRING NOT NULL, \
@@ -144,28 +169,29 @@ def usage():
     """Print usage text of this program"""
 
     print("""
-Take the jobs defined in jobs table of the given database and process one job
+Take the jobs defined in the jobs table of the given database and process one
 after the other. Multiple instances may be launched against the same database.
 
 Usage:
-  {0} [OPTIONS] [COMMANDS] -d database
+  {0} [OPTIONS] [COMMANDS] -d DB
   {0} -h
 
 COMMANDS:
-  -c cmdfn        add jobs from the file with list of commands
+  -c FILE         add each line as a job resp. job's command to DB
   -h              print this text
   -s              print status information
-  -w              work on the database
+  -w              do work and process jobs
 
 OPTIONS:
-  -d database     the database to process
-  -p cols-def     create properties table with SQL column spec
-  -v              print output of the job's command
+  -d DB           the database to process
+  -n NUM          in -w mode, only process num-many jobs
+  -p COL-DEF      create properties table with SQL column spec
+  -v              verbose output
 
 Commands may be combined in one call of {0}.
 
 A list of jobs may be importet line-by-line from a file using the -c option.
-Every job may output to stdout or stderr a string of the form
+Every job may output to stdout or stderr one or more strings of the form
     DB-PROPERTIES: {{ "key": "value", "key2": 1.23, "key3": True }}
 It is assumed that a table 'properties' exists with the columns jobid, key,
 key2, and key3. The corresponding values are inserted into this table. Using
@@ -174,7 +200,7 @@ column definitions in SQL style.
 
 The jobs table also contains a 'workloadestm' column that is used when
 estimating the finished workload so far. The entries default to 1 and may be
-set externally.
+adjusted.
 
 Examples:
   # create cmds.sh with jobs
@@ -199,9 +225,10 @@ if __name__ == "__main__":
     propdef = None
     work = False
     status = False
+    numjobs = None
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "hd:c:p:wsv")
+        opts, args = getopt.getopt(sys.argv[1:], "hd:c:p:wsvn:")
 
         for opt, arg in opts:
             if opt == "-h":
@@ -219,6 +246,8 @@ if __name__ == "__main__":
                 status = True
             elif opt == "-v":
                 verbose = True
+            elif opt == "-n":
+                numjobs = int(arg)
             else:
                 print("Unknown option '", opt, "'.")
 
@@ -231,7 +260,7 @@ if __name__ == "__main__":
         print("No database given.")
         sys.exit(os.EX_USAGE)
 
-    conn = sqlite3.connect(dbfn)
+    conn = sqlite3.connect(dbfn, timeout=60)
     createSchema(conn)
 
     if status:
@@ -247,12 +276,15 @@ if __name__ == "__main__":
         insertJobs(conn, cmds)
 
     if work:
-        while True:
+        n = 0
+        while not numjobs or n < numjobs:
+
             jobid = getNextJobId(conn)
             if jobid == None:
                 print("All jobs have been started.")
                 break
             processJob(conn, jobid)
+            n += 1
 
 
     conn.close()