import sqlite3
import subprocess
+verbose = False
+
+def printStatusInfo(conn):
+ c = conn.cursor()
+
+ c.execute("SELECT count(id) FROM jobs;")
+ nototal, = c.fetchone()
+
+ c.execute("SELECT count(id) FROM jobs WHERE done=1;")
+ nodone, = c.fetchone()
+
+ c.execute("SELECT sum(workloadestm) FROM jobs WHERE done=1;")
+ wldone, = c.fetchone()
+ if wldone == None:
+ wldone = 0.0
+
+ c.execute("SELECT sum(workloadestm) FROM jobs;")
+ wltotal, = c.fetchone()
+
+ c.close()
+
+ print(nototal, nodone, wldone, wltotal)
+ perdone = 100.0*float(nodone)/float(nototal)
+ perwl = 100.0*float(wldone)/float(wltotal)
+
+ print("%d (%.1f%%) of %d jobs done. %.1f%% of the workload finished." % \
+ (nodone, perdone, nototal, perwl))
def createPropertiesTable(conn, propdef):
conn.execute("BEGIN EXCLUSIVE")
c.execute("SELECT count(name) FROM sqlite_master WHERE name='properties';")
if c.fetchone() == (0,):
print("Creating properties table.")
- sqlstmt = "CREATE TABLE properties (jobid INTEGER PRIMARY KEY, %s, \
+ sqlstmt = "CREATE TABLE properties (\
+ jobid INTEGER PRIMARY KEY,\
+ %s, \
FOREIGN KEY (jobid) REFERENCES jobs (id));" % (propdef,)
c.execute(sqlstmt)
+ c.close()
conn.commit()
def runCmd(cmd):
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = proc.communicate()
exitcode = proc.wait()
+
+ if verbose:
+ print(out, err)
+
return exitcode, out, err
def processJob(conn, jobid):
sqlstmt = "INSERT INTO properties (%s) VALUES (%s);" % (collist,vallist)
c.execute(sqlstmt, [jobid] + list(prop.values()))
+ c.close()
conn.commit()
def insertJobs(conn, cmds):
cmd STRING NOT NULL, \
started BOOL DEFAULT (0) NOT NULL, \
done BOOL DEFAULT (0) NOT NULL, \
- exitcode INTEGER )")
+ exitcode INTEGER, \
+ workloadestm REAL DEFAULT (1) NOT NULL)")
+ c.close()
conn.commit()
def getNextJobId(conn):
jobid, = r
conn.execute("UPDATE jobs SET started=1 WHERE id=?;", (jobid,))
+
+ c.close()
conn.commit()
return jobid
"""Print usage text of this program"""
print("""
-Take the jobs defined in jobs table of given database and process one job after
-the other. Multiple instances may be launched against the same database.
+Take the jobs defined in jobs table of the given database and process one job
+after the other. Multiple instances may be launched against the same database.
+
+Usage:
+ {0} [OPTIONS] [COMMANDS] -d database
+ {0} -h
+
+COMMANDS:
+ -c cmdfn add jobs from the file with list of commands
+ -h print this text
+ -s print status information
+ -w work on the database
+
+OPTIONS:
+ -d database the database to process
+ -p cols-def create properties table with SQL column spec
+ -v print output of the job's command
+
+Commands may be combined in one call of {0}.
A list of jobs may be importet line-by-line from a file using the -c option.
Every job may output to stdout or stderr a string of the form
- DB-PROPERTIES: {{ "key": "value", "key2": 1.23, "key3": True}}
+ DB-PROPERTIES: {{ "key": "value", "key2": 1.23, "key3": True }}
It is assumed that a table 'properties' exists with the columns jobid, key,
key2, and key3. The corresponding values are inserted into this table. Using
the option -p such a properties table can be created by giving a list of
column definitions in SQL style.
-Usage:
- {0} [OPTIONS] -d database
- {0} -h
-
-OPTIONS:
- -c cmdfn add jobs from the file with list of commands
- -d database the database to process
- -h print this text
- -p cols-def create properties table with SQL column spec
+The jobs table also contains a 'workloadestm' column that is used when
+estimating the finished workload so far. The entries default to 1 and may be
+set externally.
Examples:
- {0} -d stats.db -c cmds.sh -p 'time REAL, mem INTEGER'
+ # create an initial database, but do not work
+ {0} -d jobs.db -c cmds.sh -p 'time REAL, mem INTEGER'
+ # launch two workers
+ {0} -d jobs.db -w
+ {0} -d jobs.db -w
+ # print status info
+ {0} -d jobs.db -s
""".format(sys.argv[0]))
dbfn = None
cmdfn = None
propdef = None
+ work = False
+ status = False
try:
- opts, args = getopt.getopt(sys.argv[1:], "hd:c:p:")
+ opts, args = getopt.getopt(sys.argv[1:], "hd:c:p:wsv")
for opt, arg in opts:
if opt == "-h":
cmdfn = arg
elif opt == "-p":
propdef = arg
+ elif opt == "-w":
+ work = True
+ elif opt == "-s":
+ status = True
+ elif opt == "-v":
+ verbose = True
else:
print("Unknown option '", opt, "'.")
print("No database given.")
sys.exit(os.EX_USAGE)
- #try:
conn = sqlite3.connect(dbfn)
createSchema(conn)
+ if status:
+ printStatusInfo(conn)
+
if propdef != None:
createPropertiesTable(conn, propdef)
cmds = [(c.strip(),) for c in cmds]
insertJobs(conn, cmds)
- while True:
- jobid = getNextJobId(conn)
- if jobid == None:
- print("All jobs have been started.")
- break
- processJob(conn, jobid)
+ if work:
+ while True:
+ jobid = getNextJobId(conn)
+ if jobid == None:
+ print("All jobs have been started.")
+ break
+ processJob(conn, jobid)
conn.close()
projects / shutils.git / commitdiff