From: Stefan Huber Date: Sat, 27 Apr 2013 08:50:05 +0000 (+0200) Subject: Adding -v, -w and -s options X-Git-Url: https://git.sthu.org/?a=commitdiff_plain;h=af9ae9005e2b92d439b09de3a80773d2bbaaaef3;p=shutils.git Adding -v, -w and -s options --- diff --git a/paralleljobs.py b/paralleljobs.py index ba579a4..1326bc3 100755 --- a/paralleljobs.py +++ b/paralleljobs.py @@ -4,6 +4,33 @@ import sys, getopt, os import sqlite3 import subprocess +verbose = False + +def printStatusInfo(conn): + c = conn.cursor() + + c.execute("SELECT count(id) FROM jobs;") + nototal, = c.fetchone() + + c.execute("SELECT count(id) FROM jobs WHERE done=1;") + nodone, = c.fetchone() + + c.execute("SELECT sum(workloadestm) FROM jobs WHERE done=1;") + wldone, = c.fetchone() + if wldone == None: + wldone = 0.0 + + c.execute("SELECT sum(workloadestm) FROM jobs;") + wltotal, = c.fetchone() + + c.close() + + print(nototal, nodone, wldone, wltotal) + perdone = 100.0*float(nodone)/float(nototal) + perwl = 100.0*float(wldone)/float(wltotal) + + print("%d (%.1f%%) of %d jobs done. %.1f%% of the workload finished." % \ + (nodone, perdone, nototal, perwl)) def createPropertiesTable(conn, propdef): conn.execute("BEGIN EXCLUSIVE") @@ -12,9 +39,12 @@ def createPropertiesTable(conn, propdef): c.execute("SELECT count(name) FROM sqlite_master WHERE name='properties';") if c.fetchone() == (0,): print("Creating properties table.") - sqlstmt = "CREATE TABLE properties (jobid INTEGER PRIMARY KEY, %s, \ + sqlstmt = "CREATE TABLE properties (\ + jobid INTEGER PRIMARY KEY,\ + %s, \ FOREIGN KEY (jobid) REFERENCES jobs (id));" % (propdef,) c.execute(sqlstmt) + c.close() conn.commit() def runCmd(cmd): @@ -22,6 +52,10 @@ def runCmd(cmd): stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out, err = proc.communicate() exitcode = proc.wait() + + if verbose: + print(out, err) + return exitcode, out, err def processJob(conn, jobid): @@ -59,6 +93,7 @@ def processJob(conn, jobid): sqlstmt = "INSERT INTO properties (%s) VALUES (%s);" % (collist,vallist) c.execute(sqlstmt, [jobid] + list(prop.values())) + c.close() conn.commit() def insertJobs(conn, cmds): @@ -80,7 +115,9 @@ def createSchema(conn): cmd STRING NOT NULL, \ started BOOL DEFAULT (0) NOT NULL, \ done BOOL DEFAULT (0) NOT NULL, \ - exitcode INTEGER )") + exitcode INTEGER, \ + workloadestm REAL DEFAULT (1) NOT NULL)") + c.close() conn.commit() def getNextJobId(conn): @@ -95,6 +132,8 @@ def getNextJobId(conn): jobid, = r conn.execute("UPDATE jobs SET started=1 WHERE id=?;", (jobid,)) + + c.close() conn.commit() return jobid @@ -105,29 +144,46 @@ def usage(): """Print usage text of this program""" print(""" -Take the jobs defined in jobs table of given database and process one job after -the other. Multiple instances may be launched against the same database. +Take the jobs defined in jobs table of the given database and process one job +after the other. Multiple instances may be launched against the same database. + +Usage: + {0} [OPTIONS] [COMMANDS] -d database + {0} -h + +COMMANDS: + -c cmdfn add jobs from the file with list of commands + -h print this text + -s print status information + -w work on the database + +OPTIONS: + -d database the database to process + -p cols-def create properties table with SQL column spec + -v print output of the job's command + +Commands may be combined in one call of {0}. A list of jobs may be importet line-by-line from a file using the -c option. Every job may output to stdout or stderr a string of the form - DB-PROPERTIES: {{ "key": "value", "key2": 1.23, "key3": True}} + DB-PROPERTIES: {{ "key": "value", "key2": 1.23, "key3": True }} It is assumed that a table 'properties' exists with the columns jobid, key, key2, and key3. The corresponding values are inserted into this table. Using the option -p such a properties table can be created by giving a list of column definitions in SQL style. -Usage: - {0} [OPTIONS] -d database - {0} -h - -OPTIONS: - -c cmdfn add jobs from the file with list of commands - -d database the database to process - -h print this text - -p cols-def create properties table with SQL column spec +The jobs table also contains a 'workloadestm' column that is used when +estimating the finished workload so far. The entries default to 1 and may be +set externally. Examples: - {0} -d stats.db -c cmds.sh -p 'time REAL, mem INTEGER' + # create an initial database, but do not work + {0} -d jobs.db -c cmds.sh -p 'time REAL, mem INTEGER' + # launch two workers + {0} -d jobs.db -w + {0} -d jobs.db -w + # print status info + {0} -d jobs.db -s """.format(sys.argv[0])) @@ -137,9 +193,11 @@ if __name__ == "__main__": dbfn = None cmdfn = None propdef = None + work = False + status = False try: - opts, args = getopt.getopt(sys.argv[1:], "hd:c:p:") + opts, args = getopt.getopt(sys.argv[1:], "hd:c:p:wsv") for opt, arg in opts: if opt == "-h": @@ -151,6 +209,12 @@ if __name__ == "__main__": cmdfn = arg elif opt == "-p": propdef = arg + elif opt == "-w": + work = True + elif opt == "-s": + status = True + elif opt == "-v": + verbose = True else: print("Unknown option '", opt, "'.") @@ -163,10 +227,12 @@ if __name__ == "__main__": print("No database given.") sys.exit(os.EX_USAGE) - #try: conn = sqlite3.connect(dbfn) createSchema(conn) + if status: + printStatusInfo(conn) + if propdef != None: createPropertiesTable(conn, propdef) @@ -176,12 +242,13 @@ if __name__ == "__main__": cmds = [(c.strip(),) for c in cmds] insertJobs(conn, cmds) - while True: - jobid = getNextJobId(conn) - if jobid == None: - print("All jobs have been started.") - break - processJob(conn, jobid) + if work: + while True: + jobid = getNextJobId(conn) + if jobid == None: + print("All jobs have been started.") + break + processJob(conn, jobid) conn.close()