pep8/pyflakes on the bin folder
This commit is contained in:
Stephen M. McQuay
parent
1a4d635362
commit
0916023080
153
bin/gbench.py
153
bin/gbench.py
|
|
@ -10,99 +10,100 @@ from optparse import OptionParser
|
|||
import progressbar
|
||||
|
||||
import interp.bootstrap
|
||||
interp.bootstrap
|
||||
|
||||
from interp.grid.gmsh import ggrid
|
||||
from interp.grid.delaunay import dgrid
|
||||
from interp.grid.gmsh import ggrid
|
||||
from interp.grid.delaunay import dgrid
|
||||
from interp.tools import improved_answer
|
||||
from interp.tools import baker_exact_2D, baker_exact_3D
|
||||
from interp.tools import friendly_exact_2D, friendly_exact_3D
|
||||
from interp.tools import exact_me
|
||||
|
||||
from interp.tools import improved_answer
|
||||
from interp.tools import baker_exact_2D, baker_exact_3D
|
||||
from interp.tools import friendly_exact_2D, friendly_exact_3D
|
||||
|
||||
def get_right_exact_func(options):
|
||||
f = None
|
||||
if options.dimension == 3:
|
||||
if options.baker:
|
||||
f = baker_exact_3D
|
||||
else:
|
||||
f =friendly_exact_3D
|
||||
elif options.dimension == 2:
|
||||
if options.baker:
|
||||
f = baker_exact_2D
|
||||
else:
|
||||
f = friendly_exact_2D
|
||||
return f
|
||||
f = None
|
||||
if options.dimension == 3:
|
||||
if options.baker:
|
||||
f = baker_exact_3D
|
||||
else:
|
||||
f = friendly_exact_3D
|
||||
elif options.dimension == 2:
|
||||
if options.baker:
|
||||
f = baker_exact_2D
|
||||
else:
|
||||
f = friendly_exact_2D
|
||||
return f
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
parser = OptionParser(usage = "%prog [options] <input file (gmsh) or random grid count (delaunay)> <number of attempts>")
|
||||
parser = OptionParser(usage="%prog [options] <input file (gmsh) or random"
|
||||
" grid count (delaunay)> <number of attempts>")
|
||||
|
||||
parser.add_option("-v", "--verbose",
|
||||
action="store_true", dest="verbose", default=False,
|
||||
help="verbose flag")
|
||||
parser.add_option("-b", "--baker",
|
||||
action="store_true", dest="baker", default=False,
|
||||
help="use more impressive baker funcs")
|
||||
|
||||
parser.add_option("-b", "--baker",
|
||||
action="store_true", dest="baker", default=False,
|
||||
help="use more impressive baker funcs")
|
||||
meshtype_options = ('gmsh', 'delaunay')
|
||||
parser.add_option('-t', '--type',
|
||||
type="str", dest="meshtype", default='delaunay',
|
||||
help="specify mesh type (default: %default, options gmsh, delaunay)")
|
||||
|
||||
meshtype_options = ('gmsh', 'delaunay')
|
||||
parser.add_option('-t', '--type',
|
||||
type="str", dest="meshtype", default='delaunay',
|
||||
help="specify mesh type (default: %default, options gmsh, delaunay)")
|
||||
parser.add_option("-d", "--dimension",
|
||||
type="int", dest="dimension", default=3,
|
||||
help="use 3D or 2D gmsh lib (default: %default)")
|
||||
|
||||
parser.add_option("-d", "--dimension",
|
||||
type="int", dest="dimension", default=3,
|
||||
help="use 3D or 2D gmsh lib (default: %default)")
|
||||
parser.add_option("-o", "--order",
|
||||
type="int", dest="order", default=3,
|
||||
help="order of interpolation (default: %default)")
|
||||
|
||||
parser.add_option("-o", "--order",
|
||||
type="int", dest="order", default=3,
|
||||
help="order of interpolation (default: %default)")
|
||||
parser.add_option("-e", "--extra-points",
|
||||
type="int", dest="extra", default=3,
|
||||
help="number of extra points (default: %default)")
|
||||
|
||||
parser.add_option("-e", "--extra-points",
|
||||
type="int", dest="extra", default=3,
|
||||
help="number of extra points (default: %default)")
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if len(args) != 2:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
if len(args) != 2:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
input_file, count = args
|
||||
count = int(count)
|
||||
|
||||
input_file, count = args
|
||||
count = int(count)
|
||||
if options.meshtype == 'gmsh':
|
||||
g = ggrid(input_file, options.dimension)
|
||||
g.values = exact_me(g.points, get_right_exact_func(options))
|
||||
elif options.meshtype == 'delaunay':
|
||||
point_count = int(input_file)
|
||||
v = np.random.random((point_count, options.dimension))
|
||||
q = exact_me(v, get_right_exact_func(options))
|
||||
g = dgrid(v, q)
|
||||
else:
|
||||
raise Exception("improper mesh type specified. options: %s\n%s".format(
|
||||
str(meshtype_options), options.meshtype))
|
||||
|
||||
if options.meshtype == 'gmsh':
|
||||
g = ggrid(input_file, options.dimension)
|
||||
elif options.meshtype == 'delaunay':
|
||||
point_count = int(input_file)
|
||||
v = np.random.random((point_count, options.dimension))
|
||||
g = dgrid(v)
|
||||
else:
|
||||
raise Exception("improper mesh type specified. options: %s\n%s" % (str(meshtype_options), options.meshtype))
|
||||
results = {True: 0, False: 0}
|
||||
i = 0
|
||||
outside = 0
|
||||
pbar = progressbar.ProgressBar().start()
|
||||
start = time.time()
|
||||
while i < count:
|
||||
try:
|
||||
X = np.random.rand(1, options.dimension)[0]
|
||||
|
||||
g.q = np.array([get_right_exact_func(options)(x) for x in g.verts])
|
||||
a = g.interpolate(X, order=options.order,
|
||||
extra_points=options.extra)
|
||||
e = get_right_exact_func(options)(X)
|
||||
|
||||
results = {True:0, False:0}
|
||||
i = 0
|
||||
outside = 0
|
||||
pbar = progressbar.ProgressBar().start()
|
||||
start = time.time()
|
||||
while i < count:
|
||||
try:
|
||||
X = np.random.random((1,options.dimension))[0]
|
||||
results[improved_answer(a, e)] += 1
|
||||
update = i / float(count) * 100
|
||||
pbar.update(update)
|
||||
i += 1
|
||||
except Exception as e:
|
||||
# print e
|
||||
# print X, i, count
|
||||
outside += 1
|
||||
|
||||
a = g.run_baker(X, order = options.order, extra_points = options.extra)
|
||||
e = get_right_exact_func(options)(X)
|
||||
|
||||
results[improved_answer(a, e)] += 1
|
||||
update = i/float(count) * 100
|
||||
pbar.update(update)
|
||||
i += 1
|
||||
except Exception as e:
|
||||
# print e
|
||||
# print X, i, count
|
||||
outside += 1
|
||||
|
||||
pbar.finish()
|
||||
end = time.time()
|
||||
print "total skipped points: %d" % outside
|
||||
print results, end - start
|
||||
pbar.finish()
|
||||
end = time.time()
|
||||
print "total skipped points: %d" % outside
|
||||
print results, end - start
|
||||
|
|
|
|||
|
|
@ -1,58 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
import pickle
|
||||
import pdb
|
||||
|
||||
from interp.grid.DD import rect_grid, random_grid
|
||||
from interp.baker import run_baker
|
||||
from interp.tools import baker_exact_2D as exact_func
|
||||
|
||||
qfile = '/tmp/grid_regular.txt'
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
rx = int(sys.argv[1])
|
||||
ry = int(sys.argv[2])
|
||||
except (IndexError, ValueError) as e:
|
||||
print "problem with argv: %s" % e
|
||||
rx = 4
|
||||
ry = 4 * rx
|
||||
|
||||
source_mesh = rect_grid(rx, ry)
|
||||
|
||||
# print source_mesh
|
||||
|
||||
X = [0.5, 0.5]
|
||||
|
||||
try:
|
||||
print "trying to get simplex and nearest points using nearest points (kdtree)"
|
||||
(R, S) = source_mesh.get_simplex_and_nearest_points(X, extra_points=4)
|
||||
print "R for nearest-neighbor:\n", R
|
||||
print "S for nearest-neighbor:\n", S
|
||||
print "trying to run baker"
|
||||
print run_baker(X, R, S)
|
||||
except Exception as e:
|
||||
print "caught error: %s" % e
|
||||
|
||||
print "trying to get simplex and nearest points using connectivity scheme"
|
||||
(R, S) = source_mesh.get_points_conn(X)
|
||||
print "R for connectivity:\n", R
|
||||
print "S for connectivity:\n", S
|
||||
print run_baker(X, R, S)
|
||||
|
||||
|
||||
print "repeating the above just using the grid object:"
|
||||
r = source_mesh.run_baker(X)
|
||||
exact = exact_func(X)
|
||||
print r
|
||||
print 'exact', exact
|
||||
print 'qlin' , r['qlin']
|
||||
print 'error', r['error']
|
||||
print 'final', r['final']
|
||||
|
||||
if abs(r['final'] - exact) <= abs(r['qlin'] - exact):
|
||||
print "win"
|
||||
else:
|
||||
print "failure"
|
||||
open(qfile, 'w').write(source_mesh.for_qhull())
|
||||
142
bin/iqmgr.py
142
bin/iqmgr.py
|
|
@ -1,87 +1,93 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
from interp.cluster import QueueManager, get_qs
|
||||
from optparse import OptionParser
|
||||
from interp.cluster import QueueManager, get_qs
|
||||
from optparse import OptionParser
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = OptionParser(usage = "usage: %s [options] <status|watch|add|fresult #|slay|clear|clearall|clearresults>")
|
||||
parser = OptionParser(usage="usage: %s [options] <status|watch|add|"
|
||||
"result #|slay|clear|clearall|clearresults>")
|
||||
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
|
||||
parser.add_option('-a', '--auth-key',
|
||||
type="str", dest="authkey", default='asdf',
|
||||
help="authkey (default: %default)")
|
||||
parser.add_option('-a', '--auth-key',
|
||||
type="str", dest="authkey", default='asdf',
|
||||
help="authkey (default: %default)")
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if len(args) == 0:
|
||||
cmd = 'status'
|
||||
else:
|
||||
cmd = args[0]
|
||||
|
||||
m = QueueManager(address=('', options.port), authkey=options.authkey)
|
||||
m.connect()
|
||||
|
||||
tq,rq,mq,sq = get_qs(m)
|
||||
|
||||
if cmd.startswith("st"):
|
||||
print "interp queue status:"
|
||||
print " tasksq : %d" % tq.qsize()
|
||||
print " resultsq : %d" % rq.qsize()
|
||||
print " masterq : %d" % mq.qsize()
|
||||
print " minionsq : %d" % sq.qsize()
|
||||
|
||||
if cmd.startswith("wa"):
|
||||
if len(args) == 2:
|
||||
sleeptime = float(args[1])
|
||||
if len(args) == 0:
|
||||
cmd = 'status'
|
||||
else:
|
||||
sleeptime = 1
|
||||
cmd = args[0]
|
||||
|
||||
i = 0
|
||||
while True:
|
||||
time.sleep(sleeptime)
|
||||
if i % 20 == 0:
|
||||
print "tasksq resultsq masterq minionsq"
|
||||
print "%d %d %d %d" % \
|
||||
(tq.qsize(),
|
||||
rq.qsize(),
|
||||
mq.qsize(),
|
||||
sq.qsize(),)
|
||||
i += 1
|
||||
m = QueueManager(address=('', options.port), authkey=options.authkey)
|
||||
m.connect()
|
||||
|
||||
if cmd == 'add':
|
||||
for i in xrange(int(args[1])):
|
||||
mq.put('jane%d' % i)
|
||||
tq, rq, mq, sq = get_qs(m)
|
||||
|
||||
if cmd == 'fresult':
|
||||
for i in xrange(int(args[1])):
|
||||
rq.put('fake.%d' % i)
|
||||
if cmd.startswith("st"):
|
||||
print "interp queue status:"
|
||||
print " tasksq : %d" % tq.qsize()
|
||||
print " resultsq : %d" % rq.qsize()
|
||||
print " masterq : %d" % mq.qsize()
|
||||
print " minionsq : %d" % sq.qsize()
|
||||
|
||||
if cmd == 'slay':
|
||||
if len(args) == 1:
|
||||
for i in xrange(mq.qsize()):
|
||||
print i, "killing", mq.get()
|
||||
sq.put("slay")
|
||||
elif len(args) == 2:
|
||||
for i in xrange(int(args[1])):
|
||||
print i, "killing", mq.get()
|
||||
sq.put("slay")
|
||||
if cmd.startswith("wa"):
|
||||
if len(args) == 2:
|
||||
sleeptime = float(args[1])
|
||||
else:
|
||||
sleeptime = 1
|
||||
|
||||
i = 0
|
||||
while True:
|
||||
time.sleep(sleeptime)
|
||||
if i % 20 == 0:
|
||||
print "tasksq resultsq masterq minionsq"
|
||||
print "%d %d %d %d" % \
|
||||
(tq.qsize(),
|
||||
rq.qsize(),
|
||||
mq.qsize(),
|
||||
sq.qsize(),)
|
||||
i += 1
|
||||
|
||||
if cmd == 'clear':
|
||||
for i in xrange(tq.qsize()): print tq.get()
|
||||
for i in xrange(rq.qsize()): print rq.get()
|
||||
if cmd == 'add':
|
||||
for i in xrange(int(args[1])):
|
||||
mq.put('jane%d' % i)
|
||||
|
||||
if cmd == 'clearall':
|
||||
for i in xrange(tq.qsize()): print tq.get()
|
||||
for i in xrange(rq.qsize()): print rq.get()
|
||||
for i in xrange(mq.qsize()): print mq.get()
|
||||
for i in xrange(sq.qsize()): print sq.get()
|
||||
if cmd == 'fresult':
|
||||
for i in xrange(int(args[1])):
|
||||
rq.put('fake.%d' % i)
|
||||
|
||||
if cmd == 'clearresults':
|
||||
for i in xrange(rq.qsize()): print rq.get()
|
||||
if cmd == 'slay':
|
||||
if len(args) == 1:
|
||||
for i in xrange(mq.qsize()):
|
||||
print i, "killing", mq.get()
|
||||
sq.put("slay")
|
||||
elif len(args) == 2:
|
||||
for i in xrange(int(args[1])):
|
||||
print i, "killing", mq.get()
|
||||
sq.put("slay")
|
||||
|
||||
if cmd == 'clear':
|
||||
for i in xrange(tq.qsize()):
|
||||
print tq.get()
|
||||
for i in xrange(rq.qsize()):
|
||||
print rq.get()
|
||||
|
||||
if cmd == 'clearall':
|
||||
for i in xrange(tq.qsize()):
|
||||
print tq.get()
|
||||
for i in xrange(rq.qsize()):
|
||||
print rq.get()
|
||||
for i in xrange(mq.qsize()):
|
||||
print mq.get()
|
||||
for i in xrange(sq.qsize()):
|
||||
print sq.get()
|
||||
|
||||
if cmd == 'clearresults':
|
||||
for i in xrange(rq.qsize()):
|
||||
print rq.get()
|
||||
|
|
|
|||
223
bin/master.py
223
bin/master.py
|
|
@ -5,143 +5,146 @@ import os
|
|||
|
||||
import time
|
||||
import shelve
|
||||
from collections import defaultdict
|
||||
from optparse import OptionParser
|
||||
from collections import defaultdict
|
||||
from optparse import OptionParser
|
||||
|
||||
import logging
|
||||
log = logging.getLogger("interp")
|
||||
|
||||
import numpy as np
|
||||
|
||||
from interp.cluster import QueueManager, get_qs
|
||||
from interp.cluster import QueueManager, get_qs
|
||||
|
||||
from progressbar import *
|
||||
from progressbar import ProgressBar, Percentage, Bar, ETA
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = OptionParser(usage = "usage: %s [options] <server> <interp count>")
|
||||
parser = OptionParser(usage="usage: %s [options] <server> <interp count>")
|
||||
|
||||
parser.add_option("-l", "--last-time",
|
||||
action="store_true", dest="last", default=False,
|
||||
help="when finished, send shutdown signal to connected nodes (default: %default)")
|
||||
parser.add_option("-l", "--last-time",
|
||||
action="store_true", dest="last", default=False,
|
||||
help="when finished, send shutdown signal to connected nodes"
|
||||
"(default: %default)")
|
||||
|
||||
parser.add_option('-n', '--node-count',
|
||||
type="int", dest="participants", default=None,
|
||||
help="specify how many participants we should wait for (default: %default)")
|
||||
parser.add_option('-n', '--node-count',
|
||||
type="int", dest="participants", default=None,
|
||||
help="specify how many participants we should wait for"
|
||||
"(default: %default)")
|
||||
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
|
||||
parser.add_option("-o", "--order",
|
||||
type="int", dest="order", default=2,
|
||||
help="order of interpolation (default: %default)")
|
||||
parser.add_option("-o", "--order",
|
||||
type="int", dest="order", default=2,
|
||||
help="order of interpolation (default: %default)")
|
||||
|
||||
parser.add_option("-e", "--extra-points",
|
||||
type="int", dest="extra", default=3,
|
||||
help="number of extra points (default: %default)")
|
||||
parser.add_option("-e", "--extra-points",
|
||||
type="int", dest="extra", default=3,
|
||||
help="number of extra points (default: %default)")
|
||||
|
||||
parser.add_option('-s', '--shelve',
|
||||
type="str", dest="shelvename", default=os.path.expanduser('~/interp.shelve'),
|
||||
help="shelve output file (default: %default)")
|
||||
parser.add_option('-s', '--shelve',
|
||||
type="str", dest="shelvename",
|
||||
default=os.path.expanduser('~/interp.shelve'),
|
||||
help="shelve output file (default: %default)")
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
if len(args) != 2:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
(options, args) = parser.parse_args()
|
||||
if len(args) != 2:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
|
||||
server = args[0]
|
||||
count = int(float(args[1]))
|
||||
server = args[0]
|
||||
count = int(float(args[1]))
|
||||
|
||||
m = QueueManager(address=(server, options.port), authkey='asdf')
|
||||
m.connect()
|
||||
m = QueueManager(address=(server, options.port), authkey='asdf')
|
||||
m.connect()
|
||||
|
||||
tasksq, resultsq, masterq, minionsq = get_qs(m)
|
||||
tasksq, resultsq, masterq, minionsq = get_qs(m)
|
||||
|
||||
workers = []
|
||||
workers = []
|
||||
|
||||
if not options.participants:
|
||||
print "wait on all announced participants"
|
||||
participants = 0
|
||||
while not masterq.empty():
|
||||
participants += 1
|
||||
worker = masterq.get()
|
||||
workers.append(worker)
|
||||
print "%d: %s is ready" % (participants, worker)
|
||||
if participants == 0:
|
||||
print "nobody found"
|
||||
sys.exit(1)
|
||||
else:
|
||||
participants = options.participants
|
||||
print "wait on %d participants" % participants
|
||||
for i in xrange(participants):
|
||||
worker = masterq.get()
|
||||
workers.append(worker)
|
||||
print "%d of %d: %s is ready" % (i + 1, participants, worker)
|
||||
|
||||
if len(set(workers)) != len(workers):
|
||||
for i in workers:
|
||||
minionsq.put("slay")
|
||||
raise Exception("duplicate workers reported")
|
||||
|
||||
results = []
|
||||
|
||||
widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()]
|
||||
pbar = ProgressBar(widgets=widgets, maxval=count)
|
||||
pbar.start()
|
||||
submit_start = time.time()
|
||||
for i in xrange(count):
|
||||
X = np.random.random((1, 3))[0]
|
||||
tasksq.put((i, options.order, options.extra, X))
|
||||
pbar.update(i + 1)
|
||||
submit_end = time.time()
|
||||
pbar.finish()
|
||||
|
||||
if not options.participants:
|
||||
print "wait on all announced participants"
|
||||
participants = 0
|
||||
while not masterq.empty():
|
||||
participants += 1
|
||||
worker = masterq.get()
|
||||
workers.append(worker)
|
||||
print "%d: %s is ready" % (participants, worker)
|
||||
if participants == 0:
|
||||
print "nobody found"
|
||||
sys.exit(1)
|
||||
else:
|
||||
participants = options.participants
|
||||
print "wait on %d participants" % participants
|
||||
for i in xrange(participants):
|
||||
worker = masterq.get()
|
||||
workers.append(worker)
|
||||
print "%d of %d: %s is ready" % (i+1, participants, worker)
|
||||
print "sending worker %d start message" % (i + 1,)
|
||||
minionsq.put("start")
|
||||
|
||||
if len(set(workers)) != len(workers):
|
||||
for i in workers:
|
||||
minionsq.put("slay")
|
||||
raise Exception("duplicate workers reported")
|
||||
receive_start = time.time()
|
||||
widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()]
|
||||
pbar = ProgressBar(widgets=widgets, maxval=count)
|
||||
pbar.start()
|
||||
for i in xrange(count):
|
||||
cur_result = resultsq.get()
|
||||
results.append(cur_result)
|
||||
pbar.update(i + 1)
|
||||
receive_end = time.time()
|
||||
pbar.finish()
|
||||
|
||||
results = []
|
||||
submit = submit_end - submit_start
|
||||
receive = receive_end - receive_start
|
||||
|
||||
widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()]
|
||||
pbar = ProgressBar(widgets = widgets, maxval = count)
|
||||
pbar.start()
|
||||
submit_start = time.time()
|
||||
for i in xrange(count):
|
||||
X = np.random.random((1,3))[0]
|
||||
tasksq.put((i, options.order, options.extra, X))
|
||||
pbar.update(i+1)
|
||||
submit_end = time.time()
|
||||
pbar.finish()
|
||||
# shut down all participants
|
||||
for i in xrange(participants):
|
||||
if options.last:
|
||||
minionsq.put("teardown")
|
||||
|
||||
for i in xrange(participants):
|
||||
print "sending worker %d start message" % (i+1,)
|
||||
minionsq.put("start")
|
||||
# post processing
|
||||
stats = {}
|
||||
stats['submit'] = float(submit)
|
||||
stats['receive'] = float(receive)
|
||||
stats['count'] = count
|
||||
stats['participants'] = participants
|
||||
stats['extra'] = options.extra
|
||||
stats['order'] = options.order
|
||||
|
||||
receive_start = time.time()
|
||||
widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()]
|
||||
pbar = ProgressBar(widgets = widgets, maxval = count)
|
||||
pbar.start()
|
||||
for i in xrange(count):
|
||||
cur_result = resultsq.get()
|
||||
results.append(cur_result)
|
||||
pbar.update(i+1)
|
||||
receive_end = time.time()
|
||||
pbar.finish()
|
||||
print "%s" % stats
|
||||
log.error("stats: %s", stats)
|
||||
|
||||
submit = submit_end - submit_start
|
||||
receive = receive_end - receive_start
|
||||
tasks_accomplished_by = defaultdict(int)
|
||||
for i in results:
|
||||
tasks_accomplished_by[i[1]] += 1
|
||||
stats['tasks'] = tasks_accomplished_by
|
||||
|
||||
# shut down all participants
|
||||
for i in xrange(participants):
|
||||
if options.last:
|
||||
minionsq.put("teardown")
|
||||
npresults = np.array([(i[0], i[2], i[3], i[4], i[5]) for i in results])
|
||||
|
||||
# post processing
|
||||
stats = {}
|
||||
stats['submit' ] = float(submit)
|
||||
stats['receive' ] = float(receive)
|
||||
stats['count' ] = count
|
||||
stats['participants'] = participants
|
||||
stats['extra' ] = options.extra
|
||||
stats['order' ] = options.order
|
||||
|
||||
print "%s" % stats
|
||||
log.error("stats: %s", stats)
|
||||
|
||||
tasks_accomplished_by = defaultdict(int)
|
||||
for i in results:
|
||||
tasks_accomplished_by[i[1]] += 1
|
||||
stats['tasks'] = tasks_accomplished_by
|
||||
|
||||
# npresults = np.array([(i[0],i[2],i[3],i[4], i[5]) for i in results])
|
||||
|
||||
n = str(time.time())
|
||||
s = shelve.open(options.shelvename)
|
||||
s[n] = {
|
||||
'stats' : stats,
|
||||
# 'results' : npresults,
|
||||
}
|
||||
s.close()
|
||||
n = str(time.time())
|
||||
s = shelve.open(options.shelvename)
|
||||
s[n] = {
|
||||
'stats': stats,
|
||||
'results': npresults,
|
||||
}
|
||||
s.close()
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
import sys
|
||||
from interp.grid.gmsh import *
|
||||
import time
|
||||
s = time.time()
|
||||
g = ggrid(sys.argv[1])
|
||||
e = time.time()
|
||||
|
||||
print "time to parse mesh:", e-s
|
||||
|
||||
104
bin/minion.py
104
bin/minion.py
|
|
@ -2,79 +2,71 @@
|
|||
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
|
||||
from multiprocessing.managers import BaseManager
|
||||
|
||||
from optparse import OptionParser
|
||||
import datetime
|
||||
|
||||
import numpy as np
|
||||
|
||||
from interp.grid.gmsh import ggrid
|
||||
from interp.tools import baker_exact_3D as exact
|
||||
|
||||
from interp.grid.gmsh import ggrid
|
||||
from interp.tools import baker_exact_3D as exact
|
||||
from interp.cluster import QueueManager, get_qs
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = OptionParser(usage = "usage: %s [options] <server> <gmsh file>")
|
||||
parser = OptionParser(usage="usage: %s [options] <server> <gmsh file>")
|
||||
|
||||
parser.add_option("-v", "--verbose",
|
||||
action="store_true", dest="verbose", default=False,
|
||||
help="verbose flag (default: %default)")
|
||||
parser.add_option("-v", "--verbose",
|
||||
action="store_true", dest="verbose", default=False,
|
||||
help="verbose flag (default: %default)")
|
||||
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if len(args) != 2:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
if len(args) != 2:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
|
||||
server, input_file = args
|
||||
server, input_file = args
|
||||
|
||||
myname = "%s-%d" % (os.uname()[1], os.getpid())
|
||||
if options.verbose:
|
||||
print "%s: started" % myname
|
||||
|
||||
|
||||
m = QueueManager(address=(server, options.port), authkey='asdf')
|
||||
m.connect()
|
||||
|
||||
tasksq, resultsq, masterq, minionsq = get_qs(m)
|
||||
|
||||
if options.verbose:
|
||||
print "%s: starting parse input file" % myname
|
||||
g = ggrid(input_file)
|
||||
g.q = np.array([exact(x) for x in g.verts])
|
||||
if options.verbose:
|
||||
print "%s: done parsing input file" % myname
|
||||
|
||||
|
||||
while True:
|
||||
myname = "%s-%d" % (os.uname()[1], os.getpid())
|
||||
if options.verbose:
|
||||
print "%s: letting master know that I am ready" % myname
|
||||
masterq.put(myname)
|
||||
print "%s: started" % myname
|
||||
|
||||
m = QueueManager(address=(server, options.port), authkey='asdf')
|
||||
m.connect()
|
||||
|
||||
tasksq, resultsq, masterq, minionsq = get_qs(m)
|
||||
|
||||
if options.verbose:
|
||||
print "%s: waiting for master to tell me to start" % myname
|
||||
action = minionsq.get()
|
||||
print "%s: starting parse input file" % myname
|
||||
g = ggrid(input_file)
|
||||
g.q = np.array([exact(x) for x in g.verts])
|
||||
if options.verbose:
|
||||
print "%s: master said go!!" % myname
|
||||
print "%s: done parsing input file" % myname
|
||||
|
||||
if action in ('teardown', 'slay'):
|
||||
break
|
||||
while True:
|
||||
if options.verbose:
|
||||
print "%s: letting master know that I am ready" % myname
|
||||
masterq.put(myname)
|
||||
|
||||
while not tasksq.empty():
|
||||
i, o, e, X = tasksq.get()
|
||||
try:
|
||||
a = g.run_baker(X, order = o, extra_points = e)
|
||||
resultsq.put((i, myname, a['qlin'], a['error'], a['final'], exact(X)))
|
||||
except Exception as e:
|
||||
print X, e
|
||||
resultsq.put((i, myname, 0.0, 0.0, 0.0, 0.0))
|
||||
if options.verbose:
|
||||
print "%s: waiting for master to tell me to start" % myname
|
||||
action = minionsq.get()
|
||||
if options.verbose:
|
||||
print "%s: master said go!!" % myname
|
||||
|
||||
if options.verbose:
|
||||
print "%s: exiting" % myname
|
||||
if action in ('teardown', 'slay'):
|
||||
break
|
||||
|
||||
while not tasksq.empty():
|
||||
i, o, e, X = tasksq.get()
|
||||
try:
|
||||
a = g.run_baker(X, order=o, extra_points=e)
|
||||
resultsq.put((i, myname, a.qlin, a.error, a.final, exact(X)))
|
||||
except Exception as e:
|
||||
print X, e
|
||||
resultsq.put((i, myname, 0.0, 0.0, 0.0, 0.0))
|
||||
|
||||
if options.verbose:
|
||||
print "%s: exiting" % myname
|
||||
|
|
|
|||
|
|
@ -1,58 +0,0 @@
|
|||
import sys
|
||||
import pickle
|
||||
|
||||
import time
|
||||
|
||||
from progressbar import *
|
||||
|
||||
from interp.grid.DD import *
|
||||
from interp.tools import *
|
||||
from interp import bootstrap
|
||||
|
||||
EXTRA_POINTS = 64
|
||||
|
||||
RES_START = 10
|
||||
RES_END = 100
|
||||
RES_DELTA = 10
|
||||
|
||||
ORDER_START = 1
|
||||
ORDER_END = 8
|
||||
|
||||
dest_verts, output_filename = sys.argv[1:]
|
||||
dest_verts = pickle.load(open(dest_verts, 'r'))
|
||||
|
||||
output_file = open(output_filename, 'w')
|
||||
|
||||
count = len(dest_verts) * len(xrange(RES_START,RES_END + 1, RES_DELTA)) * len(xrange(ORDER_START, ORDER_END + 1))
|
||||
|
||||
widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()]
|
||||
pbar = ProgressBar(widgets = widgets, maxval = count)
|
||||
pbar.start()
|
||||
|
||||
i=0
|
||||
for resolution in xrange(RES_START,RES_END + 1, RES_DELTA):
|
||||
r = rect_grid(resolution, resolution)
|
||||
|
||||
for v in xrange(len(r.verts)):
|
||||
r.q[v] = baker_exact_2D(r.verts[v])
|
||||
|
||||
output_file.write("%f" % (1.0 / resolution))
|
||||
|
||||
for order in xrange(ORDER_START, ORDER_END + 1):
|
||||
errors = []
|
||||
|
||||
start = time.time()
|
||||
for X in dest_verts:
|
||||
a = r.run_baker(X, order, EXTRA_POINTS)
|
||||
cur_exact = baker_exact_2D(X)
|
||||
e = a['final'] - cur_exact
|
||||
errors.append(e)
|
||||
i+=1
|
||||
pbar.update(i)
|
||||
end = time.time()
|
||||
|
||||
rmserror = rms(errors)
|
||||
output_file.write(" %e" % rmserror)
|
||||
output_file.write("\n")
|
||||
output_file.flush()
|
||||
pbar.finish()
|
||||
|
|
@ -1,21 +1,21 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
from interp.cluster import QueueManager
|
||||
from optparse import OptionParser
|
||||
from interp.cluster import QueueManager
|
||||
from optparse import OptionParser
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = OptionParser(usage = "usage: %s [options] <server> <interp count>")
|
||||
parser = OptionParser(usage="usage: %s [options] <server> <interp count>")
|
||||
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
|
||||
parser.add_option('-a', '--auth-key',
|
||||
type="str", dest="authkey", default='asdf',
|
||||
help="authkey (default: %default)")
|
||||
parser.add_option('-a', '--auth-key',
|
||||
type="str", dest="authkey", default='asdf',
|
||||
help="authkey (default: %default)")
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
m = QueueManager(address=('', options.port), authkey=options.authkey)
|
||||
s = m.get_server()
|
||||
s.serve_forever()
|
||||
m = QueueManager(address=('', options.port), authkey=options.authkey)
|
||||
s = m.get_server()
|
||||
s.serve_forever()
|
||||
|
|
|
|||
160
bin/sheep.py
160
bin/sheep.py
|
|
@ -3,108 +3,102 @@
|
|||
import sys
|
||||
import os
|
||||
import time
|
||||
|
||||
from multiprocessing.managers import BaseManager
|
||||
|
||||
from optparse import OptionParser
|
||||
|
||||
import numpy as np
|
||||
|
||||
import interp.bootstrap
|
||||
from interp.grid.gmsh import ggrid
|
||||
from interp.tools import *
|
||||
|
||||
from interp.grid.gmsh import ggrid
|
||||
from interp.tools import (baker_exact_2D, baker_exact_3D,
|
||||
friendly_exact_2D, friendly_exact_3D)
|
||||
from interp.cluster import QueueManager, get_qs
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = OptionParser(usage = "usage: %s [options] <server> <gmsh file> <dimension> <baker|friendly>")
|
||||
parser = OptionParser(usage="usage: %s [options] <server>"
|
||||
" <gmsh file> <dimension> <baker|friendly>")
|
||||
|
||||
parser.add_option("-v", "--verbose",
|
||||
action="store_true", dest="verbose", default=False,
|
||||
help="verbose flag (default: %default)")
|
||||
parser.add_option("-v", "--verbose",
|
||||
action="store_true", dest="verbose", default=False,
|
||||
help="verbose flag (default: %default)")
|
||||
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if len(args) != 4:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
if len(args) != 4:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
|
||||
server, input_file, dimension, exact_type = args
|
||||
dimension = int(dimension)
|
||||
server, input_file, dimension, exact_type = args
|
||||
dimension = int(dimension)
|
||||
|
||||
if dimension == 2 and exact_type == 'baker':
|
||||
exact = baker_exact_2D
|
||||
elif dimension == 2 and exact_type == 'friendly':
|
||||
exact = friendly_exact_2D
|
||||
elif dimension == 3 and exact_type == 'baker':
|
||||
exact = baker_exact_3D
|
||||
elif dimension == 3 and exact_type == 'friendly':
|
||||
exact = friendly_exact_3D
|
||||
if dimension == 2 and exact_type == 'baker':
|
||||
exact = baker_exact_2D
|
||||
elif dimension == 2 and exact_type == 'friendly':
|
||||
exact = friendly_exact_2D
|
||||
elif dimension == 3 and exact_type == 'baker':
|
||||
exact = baker_exact_3D
|
||||
elif dimension == 3 and exact_type == 'friendly':
|
||||
exact = friendly_exact_3D
|
||||
|
||||
|
||||
myname = "%s-%d" % (os.uname()[1], os.getpid())
|
||||
if options.verbose:
|
||||
print "%s: started" % myname
|
||||
|
||||
|
||||
m = QueueManager(address=(server, options.port), authkey='asdf')
|
||||
m.connect()
|
||||
|
||||
tasksq, resultsq, masterq, minionsq = get_qs(m)
|
||||
|
||||
if options.verbose:
|
||||
print "%s: starting parse input file" % myname
|
||||
g = ggrid(input_file, dimension)
|
||||
g.q = np.array([exact(x) for x in g.verts])
|
||||
if options.verbose:
|
||||
print "%s: done parsing input file" % myname
|
||||
|
||||
|
||||
while True:
|
||||
myname = "%s-%d" % (os.uname()[1], os.getpid())
|
||||
if options.verbose:
|
||||
print "%s: letting master know that I am ready" % myname
|
||||
masterq.put(myname)
|
||||
print "%s: started" % myname
|
||||
|
||||
m = QueueManager(address=(server, options.port), authkey='asdf')
|
||||
m.connect()
|
||||
|
||||
tasksq, resultsq, masterq, minionsq = get_qs(m)
|
||||
|
||||
if options.verbose:
|
||||
print "%s: waiting for master to tell me to start" % myname
|
||||
action = minionsq.get()
|
||||
print "%s: starting parse input file" % myname
|
||||
g = ggrid(input_file, dimension)
|
||||
g.q = np.array([exact(x) for x in g.verts])
|
||||
if options.verbose:
|
||||
print "%s: got message from master: %s" % (myname, action)
|
||||
print "%s: done parsing input file" % myname
|
||||
|
||||
if action in ('teardown', 'slay'):
|
||||
masterq.get()
|
||||
break
|
||||
while True:
|
||||
if options.verbose:
|
||||
print "%s: letting master know that I am ready" % myname
|
||||
masterq.put(myname)
|
||||
|
||||
while not tasksq.empty():
|
||||
i, order, extra, X = tasksq.get()
|
||||
try:
|
||||
s = time.time()
|
||||
a = g.run_baker(X, order = order, extra_points = extra)
|
||||
cur_qlin = a['qlin' ]
|
||||
cur_error = a['error']
|
||||
cur_final = a['final']
|
||||
cur_exact = exact(X)
|
||||
e = time.time()
|
||||
duration = e-s
|
||||
except Exception as e:
|
||||
print >>sys.stderr, X, e
|
||||
cur_qlin = 0.0
|
||||
cur_error = 0.0
|
||||
cur_final = 0.0
|
||||
cur_exact = 0.0
|
||||
duration = 0
|
||||
if options.verbose:
|
||||
print "%s: waiting for master to tell me to start" % myname
|
||||
action = minionsq.get()
|
||||
if options.verbose:
|
||||
print "%s: got message from master: %s" % (myname, action)
|
||||
|
||||
resultsq.put((i,
|
||||
cur_qlin,
|
||||
cur_error,
|
||||
cur_final,
|
||||
cur_exact,
|
||||
duration,
|
||||
))
|
||||
if action in ('teardown', 'slay'):
|
||||
masterq.get()
|
||||
break
|
||||
|
||||
if options.verbose:
|
||||
print "%s: exiting" % myname
|
||||
while not tasksq.empty():
|
||||
i, order, extra, X = tasksq.get()
|
||||
try:
|
||||
s = time.time()
|
||||
a = g.run_baker(X, order=order, extra_points=extra)
|
||||
cur_qlin = a.qlin
|
||||
cur_error = a.error
|
||||
cur_final = a.final
|
||||
cur_exact = exact(X)
|
||||
e = time.time()
|
||||
duration = e - s
|
||||
except Exception as e:
|
||||
print >>sys.stderr, X, e
|
||||
cur_qlin = 0.0
|
||||
cur_error = 0.0
|
||||
cur_final = 0.0
|
||||
cur_exact = 0.0
|
||||
duration = 0
|
||||
|
||||
resultsq.put((i,
|
||||
cur_qlin,
|
||||
cur_error,
|
||||
cur_final,
|
||||
cur_exact,
|
||||
duration,
|
||||
))
|
||||
|
||||
if options.verbose:
|
||||
print "%s: exiting" % myname
|
||||
|
|
|
|||
206
bin/shepherd.py
206
bin/shepherd.py
|
|
@ -1,131 +1,133 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
from optparse import OptionParser
|
||||
import pickle
|
||||
import sys
|
||||
|
||||
import time
|
||||
|
||||
from progressbar import *
|
||||
from optparse import OptionParser
|
||||
import pickle
|
||||
|
||||
import numpy as np
|
||||
from progressbar import ProgressBar, ETA, Percentage, Bar
|
||||
|
||||
from interp.cluster import QueueManager, get_qs
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = OptionParser(usage = "%prog [options] <server> <destination vertexes file (pickle)> <results output file> <dimension>")
|
||||
parser = OptionParser(usage="%prog [options] <server>"
|
||||
" <destination vertexes file (pickle)>"
|
||||
" <results output file> <dimension>")
|
||||
|
||||
parser.add_option("-v", "--verbose",
|
||||
action="store_true", dest="verbose", default=False,
|
||||
help="verbose flag")
|
||||
parser.add_option("-v", "--verbose",
|
||||
action="store_true", dest="verbose", default=False,
|
||||
help="verbose flag")
|
||||
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
parser.add_option('-p', '--port',
|
||||
type="int", dest="port", default=6666,
|
||||
help="specify the port to use on the server (default: %default)")
|
||||
|
||||
parser.add_option('-n', '--node-count',
|
||||
type="int", dest="participants", default=None,
|
||||
help="specify how many participants we should wait for (default: %default)")
|
||||
parser.add_option('-n', '--node-count',
|
||||
type="int", dest="participants", default=None,
|
||||
help="specify how many participants we should"
|
||||
" wait for (default: %default)")
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
if len(args) != 4:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
(options, args) = parser.parse_args()
|
||||
if len(args) != 4:
|
||||
parser.print_usage()
|
||||
sys.exit(1)
|
||||
|
||||
server, dest_verts, results_p_name, dimension = args
|
||||
dimension = int(dimension)
|
||||
server, dest_verts, results_p_name, dimension = args
|
||||
dimension = int(dimension)
|
||||
|
||||
dest_verts = pickle.load(open(dest_verts, 'r'))
|
||||
dest_verts = pickle.load(open(dest_verts, 'r'))
|
||||
|
||||
orders = [2,3,4,5]
|
||||
extras = [4,6,8,12, 16,20,32,48, 64,96,128,192, 256]
|
||||
orders = [2, 3, 4, 5]
|
||||
extras = [4, 6, 8, 12, 16, 20, 32, 48, 64, 96, 128, 192, 256]
|
||||
|
||||
count = len(orders) * len(extras) * len(dest_verts)
|
||||
|
||||
count = len(orders) * len(extras) * len(dest_verts)
|
||||
results = [None] * count
|
||||
# results = [id, order, extra, X, qlin, err, final, exact, time]
|
||||
|
||||
results = [None] * count
|
||||
# results = [id, order, extra, X, qlin, err, final, exact, time]
|
||||
m = QueueManager(address=(server, options.port), authkey='asdf')
|
||||
m.connect()
|
||||
|
||||
m = QueueManager(address=(server, options.port), authkey='asdf')
|
||||
m.connect()
|
||||
tasksq, resultsq, masterq, minionsq = get_qs(m)
|
||||
|
||||
tasksq, resultsq, masterq, minionsq = get_qs(m)
|
||||
if not options.participants:
|
||||
print "wait on all announced participants"
|
||||
participants = 0
|
||||
while not masterq.empty():
|
||||
participants += 1
|
||||
worker = masterq.get()
|
||||
print "%s (%d)" % (worker, participants)
|
||||
if participants == 0:
|
||||
print "nobody found"
|
||||
sys.exit(1)
|
||||
else:
|
||||
participants = options.participants
|
||||
print "wait on %d participants" % participants
|
||||
for i in xrange(participants):
|
||||
worker = masterq.get()
|
||||
print "%s (%d/%d)" % (worker, i + 1, participants)
|
||||
|
||||
print "Submitting %d pieces of workload" % count
|
||||
|
||||
widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()]
|
||||
pbar = ProgressBar(widgets=widgets, maxval=count)
|
||||
pbar.start()
|
||||
|
||||
submit_start = time.time()
|
||||
i = 0
|
||||
for order in orders:
|
||||
for extra in extras:
|
||||
for X in dest_verts:
|
||||
cur_job = (i, order, extra, X)
|
||||
results[i] = [order, extra, X]
|
||||
tasksq.put(cur_job)
|
||||
i += 1
|
||||
pbar.update(i)
|
||||
submit_end = time.time()
|
||||
pbar.finish()
|
||||
|
||||
print("it took %0.2f seconds to submit"
|
||||
" the workload" % (submit_end - submit_start,))
|
||||
|
||||
print "len(results)", len(results)
|
||||
|
||||
if not options.participants:
|
||||
print "wait on all announced participants"
|
||||
participants = 0
|
||||
while not masterq.empty():
|
||||
participants += 1
|
||||
worker = masterq.get()
|
||||
print "%s (%d)" % (worker,participants)
|
||||
if participants == 0:
|
||||
print "nobody found"
|
||||
sys.exit(1)
|
||||
else:
|
||||
participants = options.participants
|
||||
print "wait on %d participants" % participants
|
||||
for i in xrange(participants):
|
||||
worker = masterq.get()
|
||||
print "%s (%d/%d)" % (worker, i+1, participants)
|
||||
print "sending worker %d start message" % (i + 1,)
|
||||
minionsq.put("start")
|
||||
|
||||
print "Submitting %d pieces of workload" % count
|
||||
receive_start = time.time()
|
||||
widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()]
|
||||
pbar = ProgressBar(widgets=widgets, maxval=count)
|
||||
pbar.start()
|
||||
for i in xrange(count):
|
||||
rid, qlin, err, final, exact, duration = resultsq.get()
|
||||
results[rid].extend((qlin, err, final, exact, duration))
|
||||
pbar.update(i + 1)
|
||||
receive_end = time.time()
|
||||
pbar.finish()
|
||||
|
||||
widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()]
|
||||
pbar = ProgressBar(widgets = widgets, maxval = count)
|
||||
pbar.start()
|
||||
# pickle.dump(results, open(results_p_name, 'w'))
|
||||
o = open(results_p_name, 'w')
|
||||
for i in results:
|
||||
if dimension == 2:
|
||||
o.write("%d %d %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0],
|
||||
i[2][1], i[3], i[4], i[5], i[6], i[7]))
|
||||
elif dimension == 3:
|
||||
o.write("%d %d %e %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0],
|
||||
i[2][1], i[2][2], i[3], i[4], i[5], i[6], i[7]))
|
||||
|
||||
submit_start = time.time()
|
||||
i = 0
|
||||
for order in orders:
|
||||
for extra in extras:
|
||||
for X in dest_verts:
|
||||
cur_job = (i, order, extra, X)
|
||||
results[i] = [ order, extra, X]
|
||||
tasksq.put(cur_job)
|
||||
i+=1
|
||||
pbar.update(i)
|
||||
submit_end = time.time()
|
||||
pbar.finish()
|
||||
submit = submit_end - submit_start
|
||||
receive = receive_end - receive_start
|
||||
|
||||
print "it took %0.2f seconds to submit the workload" % (submit_end - submit_start,)
|
||||
# shut down all participants
|
||||
for i in xrange(participants):
|
||||
minionsq.put("teardown")
|
||||
|
||||
print "len(results)", len(results)
|
||||
# post processing
|
||||
stats = {}
|
||||
stats['submit'] = float(submit)
|
||||
stats['receive'] = float(receive)
|
||||
stats['count'] = count
|
||||
stats['participants'] = participants
|
||||
|
||||
for i in xrange(participants):
|
||||
print "sending worker %d start message" % (i+1,)
|
||||
minionsq.put("start")
|
||||
|
||||
receive_start = time.time()
|
||||
widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()]
|
||||
pbar = ProgressBar(widgets = widgets, maxval = count)
|
||||
pbar.start()
|
||||
for i in xrange(count):
|
||||
rid, qlin, err, final, exact, duration = resultsq.get()
|
||||
results[rid].extend((qlin, err, final, exact, duration))
|
||||
pbar.update(i+1)
|
||||
receive_end = time.time()
|
||||
pbar.finish()
|
||||
|
||||
# pickle.dump(results, open(results_p_name, 'w'))
|
||||
o = open(results_p_name, 'w')
|
||||
for i in results:
|
||||
if dimension == 2:
|
||||
o.write("%d %d %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0], i[2][1], i[3], i[4], i[5], i[6], i[7]))
|
||||
elif dimension == 3:
|
||||
o.write("%d %d %e %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0], i[2][1], i[2][2], i[3], i[4], i[5], i[6], i[7]))
|
||||
|
||||
submit = submit_end - submit_start
|
||||
receive = receive_end - receive_start
|
||||
|
||||
# shut down all participants
|
||||
for i in xrange(participants):
|
||||
minionsq.put("teardown")
|
||||
|
||||
# post processing
|
||||
stats = {}
|
||||
stats['submit' ] = float(submit)
|
||||
stats['receive' ] = float(receive)
|
||||
stats['count' ] = count
|
||||
stats['participants'] = participants
|
||||
|
||||
print "%s" % stats
|
||||
print "%s" % stats
|
||||
|
|
|
|||
Loading…
Reference in New Issue