]>
git.sthu.org Git - shutils.git/blob - paralleljobs.py
9 def printStatusInfo(conn
):
12 c
.execute("SELECT count(id) FROM jobs;")
13 nototal
, = c
.fetchone()
15 c
.execute("SELECT count(id) FROM jobs WHERE done=1;")
16 nodone
, = c
.fetchone()
18 c
.execute("SELECT sum(workloadestm) FROM jobs WHERE done=1;")
19 wldone
, = c
.fetchone()
23 c
.execute("SELECT sum(workloadestm) FROM jobs;")
24 wltotal
, = c
.fetchone()
28 print(nototal
, nodone
, wldone
, wltotal
)
29 perdone
= 100.0*float(nodone
)/float(nototal
)
30 perwl
= 100.0*float(wldone
)/float(wltotal
)
32 print("%d (%.1f%%) of %d jobs done. %.1f%% of the workload finished." % \
33 (nodone
, perdone
, nototal
, perwl
))
35 def createPropertiesTable(conn
, propdef
):
36 conn
.execute("BEGIN EXCLUSIVE")
39 c
.execute("SELECT count(name) FROM sqlite_master WHERE name='properties';")
40 if c
.fetchone() == (0,):
41 print("Creating properties table.")
42 sqlstmt
= "CREATE TABLE properties (\
43 jobid INTEGER PRIMARY KEY,\
45 FOREIGN KEY (jobid) REFERENCES jobs (id));" % (propdef
,)
51 proc
= subprocess
.Popen(cmd
, \
52 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=True)
53 out
, err
= proc
.communicate()
54 exitcode
= proc
.wait()
59 return exitcode
, out
, err
61 def processJob(conn
, jobid
):
62 print("Process job %d" % (jobid
))
65 c
.execute("SELECT cmd FROM jobs WHERE id=?", (jobid
,))
68 ec
, out
, err
= runCmd(cmd
)
69 c
.execute("UPDATE jobs SET exitcode=?, done=1 WHERE id=?;", (ec
, jobid
))
72 for l
in out
.splitlines():
73 if l
.startswith("DB-PROPERTIES:"):
75 for l
in err
.splitlines():
76 if l
.startswith("DB-PROPERTIES:"):
82 for k
, v
in p
.iteritems():
86 collist
= ", ".join([str(k
) for k
in prop
.keys()])
87 collist
= "jobid, " + collist
89 vallist
= ", ".join(["?" for k
in prop
.keys()])
90 vallist
= "?, " + vallist
93 sqlstmt
= "INSERT INTO properties (%s) VALUES (%s);" % (collist
,vallist
)
94 c
.execute(sqlstmt
, [jobid
] + list(prop
.values()))
99 def insertJobs(conn
, cmds
):
100 conn
.execute("BEGIN EXCLUSIVE")
101 conn
.executemany("INSERT INTO jobs (cmd) VALUES (?);", cmds
)
104 def createSchema(conn
):
107 c
.execute("BEGIN EXCLUSIVE")
109 # Create table, if necessary
110 c
.execute("SELECT count(name) FROM sqlite_master WHERE name='jobs';")
111 if c
.fetchone() == (0,):
112 print("Creating jobs table.")
113 conn
.execute("CREATE TABLE jobs ( \
114 id INTEGER PRIMARY KEY AUTOINCREMENT, \
115 cmd STRING NOT NULL, \
116 started BOOL DEFAULT (0) NOT NULL, \
117 done BOOL DEFAULT (0) NOT NULL, \
119 workloadestm REAL DEFAULT (1) NOT NULL)")
123 def getNextJobId(conn
):
126 c
.execute("BEGIN EXCLUSIVE")
127 c
.execute("SELECT id FROM jobs WHERE NOT started=1 LIMIT 1;")
134 conn
.execute("UPDATE jobs SET started=1 WHERE id=?;", (jobid
,))
144 """Print usage text of this program"""
147 Take the jobs defined in jobs table of the given database and process one job
148 after the other. Multiple instances may be launched against the same database.
151 {0} [OPTIONS] [COMMANDS] -d database
155 -c cmdfn add jobs from the file with list of commands
157 -s print status information
158 -w work on the database
161 -d database the database to process
162 -p cols-def create properties table with SQL column spec
163 -v print output of the job's command
165 Commands may be combined in one call of {0}.
167 A list of jobs may be importet line-by-line from a file using the -c option.
168 Every job may output to stdout or stderr a string of the form
169 DB-PROPERTIES: {{ "key": "value", "key2": 1.23, "key3": True }}
170 It is assumed that a table 'properties' exists with the columns jobid, key,
171 key2, and key3. The corresponding values are inserted into this table. Using
172 the option -p such a properties table can be created by giving a list of
173 column definitions in SQL style.
175 The jobs table also contains a 'workloadestm' column that is used when
176 estimating the finished workload so far. The entries default to 1 and may be
180 # create cmds.sh with jobs
181 echo "ulimit -v 2000000 -t 1200; ./isprime 65535" > cmds.sh
182 echo "ulimit -v 2000000 -t 1200; ./isprime 65537" >> cmds.sh
183 # create an initial database, but do not work
184 {0} -d jobs.db -c cmds.sh -p 'number INTEGER, time REAL, mem INTEGER'
190 """.format(sys
.argv
[0]))
193 if __name__
== "__main__":
203 opts
, args
= getopt
.getopt(sys
.argv
[1:], "hd:c:p:wsv")
205 for opt
, arg
in opts
:
222 print("Unknown option '", opt
, "'.")
224 except getopt
.GetoptError
as e
:
225 print("Error parsing arguments:", e
)
227 sys
.exit(os
.EX_USAGE
)
230 print("No database given.")
231 sys
.exit(os
.EX_USAGE
)
233 conn
= sqlite3
.connect(dbfn
)
237 printStatusInfo(conn
)
240 createPropertiesTable(conn
, propdef
)
243 print("Adding jobs...")
244 cmds
= open(cmdfn
).readlines()
245 cmds
= [(c
.strip(),) for c
in cmds
]
246 insertJobs(conn
, cmds
)
250 jobid
= getNextJobId(conn
)
252 print("All jobs have been started.")
254 processJob(conn
, jobid
)