Adding -v, -w and -s options

diff --git a/paralleljobs.py b/paralleljobs.py
index ba579a41ec05b6c8e0ba439ba5d77070dff9f1fe..1326bc3d9dff55d5435d510668f38ec8a1cfdae2 100755 (executable)
--- a/paralleljobs.py
+++ b/paralleljobs.py
@@ -4,6 +4,33 @@ import sys, getopt, os
 import sqlite3
 import subprocess
 
+verbose = False
+
+def printStatusInfo(conn):
+    c = conn.cursor()
+
+    c.execute("SELECT count(id) FROM jobs;")
+    nototal, = c.fetchone()
+
+    c.execute("SELECT count(id) FROM jobs WHERE done=1;")
+    nodone, = c.fetchone()
+
+    c.execute("SELECT sum(workloadestm) FROM jobs WHERE done=1;")
+    wldone, = c.fetchone()
+    if wldone == None:
+        wldone = 0.0
+
+    c.execute("SELECT sum(workloadestm) FROM jobs;")
+    wltotal, = c.fetchone()
+
+    c.close()
+
+    print(nototal, nodone, wldone, wltotal)
+    perdone = 100.0*float(nodone)/float(nototal)
+    perwl = 100.0*float(wldone)/float(wltotal)
+
+    print("%d (%.1f%%) of %d jobs done. %.1f%% of the workload finished." % \
+            (nodone, perdone, nototal, perwl))
 
 def createPropertiesTable(conn, propdef):
     conn.execute("BEGIN EXCLUSIVE")
@@ -12,9 +39,12 @@ def createPropertiesTable(conn, propdef):
     c.execute("SELECT count(name) FROM sqlite_master WHERE name='properties';")
     if c.fetchone() == (0,):
         print("Creating properties table.")
-        sqlstmt = "CREATE TABLE properties (jobid INTEGER PRIMARY KEY, %s, \
+        sqlstmt = "CREATE TABLE properties (\
+                jobid INTEGER PRIMARY KEY,\
+                %s, \
                 FOREIGN KEY (jobid) REFERENCES jobs (id));" % (propdef,)
         c.execute(sqlstmt)
+    c.close()
     conn.commit()
 
 def runCmd(cmd):
@@ -22,6 +52,10 @@ def runCmd(cmd):
             stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
     out, err = proc.communicate()
     exitcode = proc.wait()
+
+    if verbose:
+        print(out, err)
+
     return exitcode, out, err
 
 def processJob(conn, jobid):
@@ -59,6 +93,7 @@ def processJob(conn, jobid):
         sqlstmt = "INSERT INTO properties (%s) VALUES (%s);" % (collist,vallist)
         c.execute(sqlstmt, [jobid] + list(prop.values()))
 
+    c.close()
     conn.commit()
 
 def insertJobs(conn, cmds):
@@ -80,7 +115,9 @@ def createSchema(conn):
                         cmd STRING NOT NULL, \
                         started BOOL DEFAULT (0) NOT NULL, \
                         done BOOL DEFAULT (0) NOT NULL, \
-                        exitcode INTEGER )")
+                        exitcode INTEGER, \
+                        workloadestm REAL DEFAULT (1) NOT NULL)")
+    c.close()
     conn.commit()
 
 def getNextJobId(conn):
@@ -95,6 +132,8 @@ def getNextJobId(conn):
 
     jobid, = r
     conn.execute("UPDATE jobs SET started=1 WHERE id=?;", (jobid,))
+
+    c.close()
     conn.commit()
 
     return jobid
@@ -105,29 +144,46 @@ def usage():
     """Print usage text of this program"""
 
     print("""
-Take the jobs defined in jobs table of given database and process one job after
-the other. Multiple instances may be launched against the same database.
+Take the jobs defined in jobs table of the given database and process one job
+after the other. Multiple instances may be launched against the same database.
+
+Usage:
+  {0} [OPTIONS] [COMMANDS] -d database
+  {0} -h
+
+COMMANDS:
+  -c cmdfn        add jobs from the file with list of commands
+  -h              print this text
+  -s              print status information
+  -w              work on the database
+
+OPTIONS:
+  -d database     the database to process
+  -p cols-def     create properties table with SQL column spec
+  -v              print output of the job's command
+
+Commands may be combined in one call of {0}.
 
 A list of jobs may be importet line-by-line from a file using the -c option.
 Every job may output to stdout or stderr a string of the form
-  DB-PROPERTIES: {{ "key": "value", "key2": 1.23, "key3": True}}
+    DB-PROPERTIES: {{ "key": "value", "key2": 1.23, "key3": True }}
 It is assumed that a table 'properties' exists with the columns jobid, key,
 key2, and key3. The corresponding values are inserted into this table. Using
 the option -p such a properties table can be created by giving a list of
 column definitions in SQL style.
 
-Usage:
-  {0} [OPTIONS] -d database
-  {0} -h
-
-OPTIONS:
-  -c cmdfn        add jobs from the file with list of commands
-  -d database     the database to process
-  -h              print this text
-  -p cols-def     create properties table with SQL column spec
+The jobs table also contains a 'workloadestm' column that is used when
+estimating the finished workload so far. The entries default to 1 and may be
+set externally.
 
 Examples:
-  {0} -d stats.db -c cmds.sh -p 'time REAL, mem INTEGER'
+  # create an initial database, but do not work
+  {0} -d jobs.db -c cmds.sh -p 'time REAL, mem INTEGER'
+  # launch two workers
+  {0} -d jobs.db -w
+  {0} -d jobs.db -w
+  # print status info
+  {0} -d jobs.db -s
 """.format(sys.argv[0]))
 
 
@@ -137,9 +193,11 @@ if __name__ == "__main__":
     dbfn = None
     cmdfn = None
     propdef = None
+    work = False
+    status = False
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "hd:c:p:")
+        opts, args = getopt.getopt(sys.argv[1:], "hd:c:p:wsv")
 
         for opt, arg in opts:
             if opt == "-h":
@@ -151,6 +209,12 @@ if __name__ == "__main__":
                 cmdfn = arg
             elif opt == "-p":
                 propdef = arg
+            elif opt == "-w":
+                work = True
+            elif opt == "-s":
+                status = True
+            elif opt == "-v":
+                verbose = True
             else:
                 print("Unknown option '", opt, "'.")
 
@@ -163,10 +227,12 @@ if __name__ == "__main__":
         print("No database given.")
         sys.exit(os.EX_USAGE)
 
-    #try:
     conn = sqlite3.connect(dbfn)
     createSchema(conn)
 
+    if status:
+        printStatusInfo(conn)
+
     if propdef != None:
         createPropertiesTable(conn, propdef)
 
@@ -176,12 +242,13 @@ if __name__ == "__main__":
         cmds = [(c.strip(),) for c in cmds]
         insertJobs(conn, cmds)
 
-    while True:
-        jobid = getNextJobId(conn)
-        if jobid == None:
-            print("All jobs have been started.")
-            break
-        processJob(conn, jobid)
+    if work:
+        while True:
+            jobid = getNextJobId(conn)
+            if jobid == None:
+                print("All jobs have been started.")
+                break
+            processJob(conn, jobid)
 
 
     conn.close()