sm
/
smbinterp
1
0
Fork 0
Code Issues Pull Requests Activity

pep8/pyflakes on the bin folder

This commit is contained in:
Stephen M. McQuay 2011-09-27 21:54:04 -06:00
parent 1a4d635362
commit 0916023080
10 changed files with 506 additions and 633 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
bin/gbench.py
View File

@ -10,99 +10,100 @@ from optparse import OptionParser
import progressbar import progressbar
import interp.bootstrap import interp.bootstrap
interp.bootstrap
from interp.grid.gmsh import ggrid from interp.grid.gmsh import ggrid
from interp.grid.delaunay import dgrid from interp.grid.delaunay import dgrid
from interp.tools import improved_answer
from interp.tools import baker_exact_2D, baker_exact_3D
from interp.tools import friendly_exact_2D, friendly_exact_3D
from interp.tools import exact_me
from interp.tools import improved_answer
from interp.tools import baker_exact_2D, baker_exact_3D
from interp.tools import friendly_exact_2D, friendly_exact_3D
def get_right_exact_func(options): def get_right_exact_func(options):
f = None f = None
if options.dimension == 3: if options.dimension == 3:
if options.baker: if options.baker:
f = baker_exact_3D f = baker_exact_3D
else: else:
f =friendly_exact_3D f = friendly_exact_3D
elif options.dimension == 2: elif options.dimension == 2:
if options.baker: if options.baker:
f = baker_exact_2D f = baker_exact_2D
else: else:
f = friendly_exact_2D f = friendly_exact_2D
return f return f
if __name__ == '__main__': if __name__ == '__main__':
parser = OptionParser(usage = "%prog [options] <input file (gmsh) or random grid count (delaunay)> <number of attempts>") parser = OptionParser(usage="%prog [options] <input file (gmsh) or random"
" grid count (delaunay)> <number of attempts>")
parser.add_option("-v", "--verbose", parser.add_option("-b", "--baker",
action="store_true", dest="verbose", default=False, action="store_true", dest="baker", default=False,
help="verbose flag") help="use more impressive baker funcs")
parser.add_option("-b", "--baker", meshtype_options = ('gmsh', 'delaunay')
action="store_true", dest="baker", default=False, parser.add_option('-t', '--type',
help="use more impressive baker funcs") type="str", dest="meshtype", default='delaunay',
help="specify mesh type (default: %default, options gmsh, delaunay)")
meshtype_options = ('gmsh', 'delaunay') parser.add_option("-d", "--dimension",
parser.add_option('-t', '--type', type="int", dest="dimension", default=3,
type="str", dest="meshtype", default='delaunay', help="use 3D or 2D gmsh lib (default: %default)")
help="specify mesh type (default: %default, options gmsh, delaunay)")
parser.add_option("-d", "--dimension", parser.add_option("-o", "--order",
type="int", dest="dimension", default=3, type="int", dest="order", default=3,
help="use 3D or 2D gmsh lib (default: %default)") help="order of interpolation (default: %default)")
parser.add_option("-o", "--order", parser.add_option("-e", "--extra-points",
type="int", dest="order", default=3, type="int", dest="extra", default=3,
help="order of interpolation (default: %default)") help="number of extra points (default: %default)")
parser.add_option("-e", "--extra-points", (options, args) = parser.parse_args()
type="int", dest="extra", default=3,
help="number of extra points (default: %default)")
if len(args) != 2:
parser.print_usage()
sys.exit(1)
(options, args) = parser.parse_args() input_file, count = args
if len(args) != 2: count = int(count)
parser.print_usage()
sys.exit(1)
input_file, count = args if options.meshtype == 'gmsh':
count = int(count) g = ggrid(input_file, options.dimension)
g.values = exact_me(g.points, get_right_exact_func(options))
elif options.meshtype == 'delaunay':
point_count = int(input_file)
v = np.random.random((point_count, options.dimension))
q = exact_me(v, get_right_exact_func(options))
g = dgrid(v, q)
else:
raise Exception("improper mesh type specified. options: %s\n%s".format(
str(meshtype_options), options.meshtype))
if options.meshtype == 'gmsh': results = {True: 0, False: 0}
g = ggrid(input_file, options.dimension) i = 0
elif options.meshtype == 'delaunay': outside = 0
point_count = int(input_file) pbar = progressbar.ProgressBar().start()
v = np.random.random((point_count, options.dimension)) start = time.time()
g = dgrid(v) while i < count:
else: try:
raise Exception("improper mesh type specified. options: %s\n%s" % (str(meshtype_options), options.meshtype)) X = np.random.rand(1, options.dimension)[0]
g.q = np.array([get_right_exact_func(options)(x) for x in g.verts]) a = g.interpolate(X, order=options.order,
extra_points=options.extra)
e = get_right_exact_func(options)(X)
results = {True:0, False:0} results[improved_answer(a, e)] += 1
i = 0 update = i / float(count) * 100
outside = 0 pbar.update(update)
pbar = progressbar.ProgressBar().start() i += 1
start = time.time() except Exception as e:
while i < count: # print e
try: # print X, i, count
X = np.random.random((1,options.dimension))[0] outside += 1
a = g.run_baker(X, order = options.order, extra_points = options.extra) pbar.finish()
e = get_right_exact_func(options)(X) end = time.time()
print "total skipped points: %d" % outside
results[improved_answer(a, e)] += 1 print results, end - start
update = i/float(count) * 100
pbar.update(update)
i += 1
except Exception as e:
# print e
# print X, i, count
outside += 1
pbar.finish()
end = time.time()
print "total skipped points: %d" % outside
print results, end - start

58
bin/grid_regular.py
View File

@ -1,58 +0,0 @@
#!/usr/bin/env python
import sys
import pickle
import pdb
from interp.grid.DD import rect_grid, random_grid
from interp.baker import run_baker
from interp.tools import baker_exact_2D as exact_func
qfile = '/tmp/grid_regular.txt'
if __name__ == '__main__':
try:
rx = int(sys.argv[1])
ry = int(sys.argv[2])
except (IndexError, ValueError) as e:
print "problem with argv: %s" % e
rx = 4
ry = 4 * rx
source_mesh = rect_grid(rx, ry)
# print source_mesh
X = [0.5, 0.5]
try:
print "trying to get simplex and nearest points using nearest points (kdtree)"
(R, S) = source_mesh.get_simplex_and_nearest_points(X, extra_points=4)
print "R for nearest-neighbor:\n", R
print "S for nearest-neighbor:\n", S
print "trying to run baker"
print run_baker(X, R, S)
except Exception as e:
print "caught error: %s" % e
print "trying to get simplex and nearest points using connectivity scheme"
(R, S) = source_mesh.get_points_conn(X)
print "R for connectivity:\n", R
print "S for connectivity:\n", S
print run_baker(X, R, S)
print "repeating the above just using the grid object:"
r = source_mesh.run_baker(X)
exact = exact_func(X)
print r
print 'exact', exact
print 'qlin' , r['qlin']
print 'error', r['error']
print 'final', r['final']
if abs(r['final'] - exact) <= abs(r['qlin'] - exact):
print "win"
else:
print "failure"
open(qfile, 'w').write(source_mesh.for_qhull())

142
bin/iqmgr.py
View File

@ -1,87 +1,93 @@
#!/usr/bin/env python #!/usr/bin/env python
import sys
import time import time
from interp.cluster import QueueManager, get_qs from interp.cluster import QueueManager, get_qs
from optparse import OptionParser from optparse import OptionParser
if __name__ == '__main__': if __name__ == '__main__':
parser = OptionParser(usage = "usage: %s [options] <status|watch|add|fresult #|slay|clear|clearall|clearresults>") parser = OptionParser(usage="usage: %s [options] <status|watch|add|"
"result #|slay|clear|clearall|clearresults>")
parser.add_option('-p', '--port', parser.add_option('-p', '--port',
type="int", dest="port", default=6666, type="int", dest="port", default=6666,
help="specify the port to use on the server (default: %default)") help="specify the port to use on the server (default: %default)")
parser.add_option('-a', '--auth-key', parser.add_option('-a', '--auth-key',
type="str", dest="authkey", default='asdf', type="str", dest="authkey", default='asdf',
help="authkey (default: %default)") help="authkey (default: %default)")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
if len(args) == 0: if len(args) == 0:
cmd = 'status' cmd = 'status'
else:
cmd = args[0]
m = QueueManager(address=('', options.port), authkey=options.authkey)
m.connect()
tq,rq,mq,sq = get_qs(m)
if cmd.startswith("st"):
print "interp queue status:"
print " tasksq : %d" % tq.qsize()
print " resultsq : %d" % rq.qsize()
print " masterq : %d" % mq.qsize()
print " minionsq : %d" % sq.qsize()
if cmd.startswith("wa"):
if len(args) == 2:
sleeptime = float(args[1])
else: else:
sleeptime = 1 cmd = args[0]
i = 0 m = QueueManager(address=('', options.port), authkey=options.authkey)
while True: m.connect()
time.sleep(sleeptime)
if i % 20 == 0:
print "tasksq resultsq masterq minionsq"
print "%d %d %d %d" % \
(tq.qsize(),
rq.qsize(),
mq.qsize(),
sq.qsize(),)
i += 1
if cmd == 'add': tq, rq, mq, sq = get_qs(m)
for i in xrange(int(args[1])):
mq.put('jane%d' % i)
if cmd == 'fresult': if cmd.startswith("st"):
for i in xrange(int(args[1])): print "interp queue status:"
rq.put('fake.%d' % i) print " tasksq : %d" % tq.qsize()
print " resultsq : %d" % rq.qsize()
print " masterq : %d" % mq.qsize()
print " minionsq : %d" % sq.qsize()
if cmd == 'slay': if cmd.startswith("wa"):
if len(args) == 1: if len(args) == 2:
for i in xrange(mq.qsize()): sleeptime = float(args[1])
print i, "killing", mq.get() else:
sq.put("slay") sleeptime = 1
elif len(args) == 2:
for i in xrange(int(args[1])):
print i, "killing", mq.get()
sq.put("slay")
i = 0
while True:
time.sleep(sleeptime)
if i % 20 == 0:
print "tasksq resultsq masterq minionsq"
print "%d %d %d %d" % \
(tq.qsize(),
rq.qsize(),
mq.qsize(),
sq.qsize(),)
i += 1
if cmd == 'clear': if cmd == 'add':
for i in xrange(tq.qsize()): print tq.get() for i in xrange(int(args[1])):
for i in xrange(rq.qsize()): print rq.get() mq.put('jane%d' % i)
if cmd == 'clearall': if cmd == 'fresult':
for i in xrange(tq.qsize()): print tq.get() for i in xrange(int(args[1])):
for i in xrange(rq.qsize()): print rq.get() rq.put('fake.%d' % i)
for i in xrange(mq.qsize()): print mq.get()
for i in xrange(sq.qsize()): print sq.get()
if cmd == 'clearresults': if cmd == 'slay':
for i in xrange(rq.qsize()): print rq.get() if len(args) == 1:
for i in xrange(mq.qsize()):
print i, "killing", mq.get()
sq.put("slay")
elif len(args) == 2:
for i in xrange(int(args[1])):
print i, "killing", mq.get()
sq.put("slay")
if cmd == 'clear':
for i in xrange(tq.qsize()):
print tq.get()
for i in xrange(rq.qsize()):
print rq.get()
if cmd == 'clearall':
for i in xrange(tq.qsize()):
print tq.get()
for i in xrange(rq.qsize()):
print rq.get()
for i in xrange(mq.qsize()):
print mq.get()
for i in xrange(sq.qsize()):
print sq.get()
if cmd == 'clearresults':
for i in xrange(rq.qsize()):
print rq.get()

223
bin/master.py
View File

@ -5,143 +5,146 @@ import os
import time import time
import shelve import shelve
from collections import defaultdict from collections import defaultdict
from optparse import OptionParser from optparse import OptionParser
import logging import logging
log = logging.getLogger("interp") log = logging.getLogger("interp")
import numpy as np import numpy as np
from interp.cluster import QueueManager, get_qs from interp.cluster import QueueManager, get_qs
from progressbar import * from progressbar import ProgressBar, Percentage, Bar, ETA
if __name__ == '__main__': if __name__ == '__main__':
parser = OptionParser(usage = "usage: %s [options] <server> <interp count>") parser = OptionParser(usage="usage: %s [options] <server> <interp count>")
parser.add_option("-l", "--last-time", parser.add_option("-l", "--last-time",
action="store_true", dest="last", default=False, action="store_true", dest="last", default=False,
help="when finished, send shutdown signal to connected nodes (default: %default)") help="when finished, send shutdown signal to connected nodes"
"(default: %default)")
parser.add_option('-n', '--node-count', parser.add_option('-n', '--node-count',
type="int", dest="participants", default=None, type="int", dest="participants", default=None,
help="specify how many participants we should wait for (default: %default)") help="specify how many participants we should wait for"
"(default: %default)")
parser.add_option('-p', '--port', parser.add_option('-p', '--port',
type="int", dest="port", default=6666, type="int", dest="port", default=6666,
help="specify the port to use on the server (default: %default)") help="specify the port to use on the server (default: %default)")
parser.add_option("-o", "--order", parser.add_option("-o", "--order",
type="int", dest="order", default=2, type="int", dest="order", default=2,
help="order of interpolation (default: %default)") help="order of interpolation (default: %default)")
parser.add_option("-e", "--extra-points", parser.add_option("-e", "--extra-points",
type="int", dest="extra", default=3, type="int", dest="extra", default=3,
help="number of extra points (default: %default)") help="number of extra points (default: %default)")
parser.add_option('-s', '--shelve', parser.add_option('-s', '--shelve',
type="str", dest="shelvename", default=os.path.expanduser('~/interp.shelve'), type="str", dest="shelvename",
help="shelve output file (default: %default)") default=os.path.expanduser('~/interp.shelve'),
help="shelve output file (default: %default)")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
if len(args) != 2: if len(args) != 2:
parser.print_usage() parser.print_usage()
sys.exit(1) sys.exit(1)
server = args[0] server = args[0]
count = int(float(args[1])) count = int(float(args[1]))
m = QueueManager(address=(server, options.port), authkey='asdf') m = QueueManager(address=(server, options.port), authkey='asdf')
m.connect() m.connect()
tasksq, resultsq, masterq, minionsq = get_qs(m) tasksq, resultsq, masterq, minionsq = get_qs(m)
workers = [] workers = []
if not options.participants:
print "wait on all announced participants"
participants = 0
while not masterq.empty():
participants += 1
worker = masterq.get()
workers.append(worker)
print "%d: %s is ready" % (participants, worker)
if participants == 0:
print "nobody found"
sys.exit(1)
else:
participants = options.participants
print "wait on %d participants" % participants
for i in xrange(participants):
worker = masterq.get()
workers.append(worker)
print "%d of %d: %s is ready" % (i + 1, participants, worker)
if len(set(workers)) != len(workers):
for i in workers:
minionsq.put("slay")
raise Exception("duplicate workers reported")
results = []
widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()]
pbar = ProgressBar(widgets=widgets, maxval=count)
pbar.start()
submit_start = time.time()
for i in xrange(count):
X = np.random.random((1, 3))[0]
tasksq.put((i, options.order, options.extra, X))
pbar.update(i + 1)
submit_end = time.time()
pbar.finish()
if not options.participants:
print "wait on all announced participants"
participants = 0
while not masterq.empty():
participants += 1
worker = masterq.get()
workers.append(worker)
print "%d: %s is ready" % (participants, worker)
if participants == 0:
print "nobody found"
sys.exit(1)
else:
participants = options.participants
print "wait on %d participants" % participants
for i in xrange(participants): for i in xrange(participants):
worker = masterq.get() print "sending worker %d start message" % (i + 1,)
workers.append(worker) minionsq.put("start")
print "%d of %d: %s is ready" % (i+1, participants, worker)
if len(set(workers)) != len(workers): receive_start = time.time()
for i in workers: widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()]
minionsq.put("slay") pbar = ProgressBar(widgets=widgets, maxval=count)
raise Exception("duplicate workers reported") pbar.start()
for i in xrange(count):
cur_result = resultsq.get()
results.append(cur_result)
pbar.update(i + 1)
receive_end = time.time()
pbar.finish()
results = [] submit = submit_end - submit_start
receive = receive_end - receive_start
widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()] # shut down all participants
pbar = ProgressBar(widgets = widgets, maxval = count) for i in xrange(participants):
pbar.start() if options.last:
submit_start = time.time() minionsq.put("teardown")
for i in xrange(count):
X = np.random.random((1,3))[0]
tasksq.put((i, options.order, options.extra, X))
pbar.update(i+1)
submit_end = time.time()
pbar.finish()
for i in xrange(participants): # post processing
print "sending worker %d start message" % (i+1,) stats = {}
minionsq.put("start") stats['submit'] = float(submit)
stats['receive'] = float(receive)
stats['count'] = count
stats['participants'] = participants
stats['extra'] = options.extra
stats['order'] = options.order
receive_start = time.time() print "%s" % stats
widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()] log.error("stats: %s", stats)
pbar = ProgressBar(widgets = widgets, maxval = count)
pbar.start()
for i in xrange(count):
cur_result = resultsq.get()
results.append(cur_result)
pbar.update(i+1)
receive_end = time.time()
pbar.finish()
submit = submit_end - submit_start tasks_accomplished_by = defaultdict(int)
receive = receive_end - receive_start for i in results:
tasks_accomplished_by[i[1]] += 1
stats['tasks'] = tasks_accomplished_by
# shut down all participants npresults = np.array([(i[0], i[2], i[3], i[4], i[5]) for i in results])
for i in xrange(participants):
if options.last:
minionsq.put("teardown")
# post processing n = str(time.time())
stats = {} s = shelve.open(options.shelvename)
stats['submit' ] = float(submit) s[n] = {
stats['receive' ] = float(receive) 'stats': stats,
stats['count' ] = count 'results': npresults,
stats['participants'] = participants }
stats['extra' ] = options.extra s.close()
stats['order' ] = options.order
print "%s" % stats
log.error("stats: %s", stats)
tasks_accomplished_by = defaultdict(int)
for i in results:
tasks_accomplished_by[i[1]] += 1
stats['tasks'] = tasks_accomplished_by
# npresults = np.array([(i[0],i[2],i[3],i[4], i[5]) for i in results])
n = str(time.time())
s = shelve.open(options.shelvename)
s[n] = {
'stats' : stats,
# 'results' : npresults,
}
s.close()

9
bin/memtest.py
View File

@ -1,9 +0,0 @@
import sys
from interp.grid.gmsh import *
import time
s = time.time()
g = ggrid(sys.argv[1])
e = time.time()
print "time to parse mesh:", e-s

104
bin/minion.py
View File

@ -2,79 +2,71 @@
import sys import sys
import os import os
import time
from multiprocessing.managers import BaseManager
from optparse import OptionParser from optparse import OptionParser
import datetime
import numpy as np import numpy as np
from interp.grid.gmsh import ggrid from interp.grid.gmsh import ggrid
from interp.tools import baker_exact_3D as exact from interp.tools import baker_exact_3D as exact
from interp.cluster import QueueManager, get_qs from interp.cluster import QueueManager, get_qs
if __name__ == '__main__': if __name__ == '__main__':
parser = OptionParser(usage = "usage: %s [options] <server> <gmsh file>") parser = OptionParser(usage="usage: %s [options] <server> <gmsh file>")
parser.add_option("-v", "--verbose", parser.add_option("-v", "--verbose",
action="store_true", dest="verbose", default=False, action="store_true", dest="verbose", default=False,
help="verbose flag (default: %default)") help="verbose flag (default: %default)")
parser.add_option('-p', '--port', parser.add_option('-p', '--port',
type="int", dest="port", default=6666, type="int", dest="port", default=6666,
help="specify the port to use on the server (default: %default)") help="specify the port to use on the server (default: %default)")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
if len(args) != 2: if len(args) != 2:
parser.print_usage() parser.print_usage()
sys.exit(1) sys.exit(1)
server, input_file = args server, input_file = args
myname = "%s-%d" % (os.uname()[1], os.getpid()) myname = "%s-%d" % (os.uname()[1], os.getpid())
if options.verbose:
print "%s: started" % myname
m = QueueManager(address=(server, options.port), authkey='asdf')
m.connect()
tasksq, resultsq, masterq, minionsq = get_qs(m)
if options.verbose:
print "%s: starting parse input file" % myname
g = ggrid(input_file)
g.q = np.array([exact(x) for x in g.verts])
if options.verbose:
print "%s: done parsing input file" % myname
while True:
if options.verbose: if options.verbose:
print "%s: letting master know that I am ready" % myname print "%s: started" % myname
masterq.put(myname)
m = QueueManager(address=(server, options.port), authkey='asdf')
m.connect()
tasksq, resultsq, masterq, minionsq = get_qs(m)
if options.verbose: if options.verbose:
print "%s: waiting for master to tell me to start" % myname print "%s: starting parse input file" % myname
action = minionsq.get() g = ggrid(input_file)
g.q = np.array([exact(x) for x in g.verts])
if options.verbose: if options.verbose:
print "%s: master said go!!" % myname print "%s: done parsing input file" % myname
if action in ('teardown', 'slay'): while True:
break if options.verbose:
print "%s: letting master know that I am ready" % myname
masterq.put(myname)
while not tasksq.empty(): if options.verbose:
i, o, e, X = tasksq.get() print "%s: waiting for master to tell me to start" % myname
try: action = minionsq.get()
a = g.run_baker(X, order = o, extra_points = e) if options.verbose:
resultsq.put((i, myname, a['qlin'], a['error'], a['final'], exact(X))) print "%s: master said go!!" % myname
except Exception as e:
print X, e
resultsq.put((i, myname, 0.0, 0.0, 0.0, 0.0))
if options.verbose: if action in ('teardown', 'slay'):
print "%s: exiting" % myname break
while not tasksq.empty():
i, o, e, X = tasksq.get()
try:
a = g.run_baker(X, order=o, extra_points=e)
resultsq.put((i, myname, a.qlin, a.error, a.final, exact(X)))
except Exception as e:
print X, e
resultsq.put((i, myname, 0.0, 0.0, 0.0, 0.0))
if options.verbose:
print "%s: exiting" % myname

58
bin/resolution.2D.py
View File

@ -1,58 +0,0 @@
import sys
import pickle
import time
from progressbar import *
from interp.grid.DD import *
from interp.tools import *
from interp import bootstrap
EXTRA_POINTS = 64
RES_START = 10
RES_END = 100
RES_DELTA = 10
ORDER_START = 1
ORDER_END = 8
dest_verts, output_filename = sys.argv[1:]
dest_verts = pickle.load(open(dest_verts, 'r'))
output_file = open(output_filename, 'w')
count = len(dest_verts) * len(xrange(RES_START,RES_END + 1, RES_DELTA)) * len(xrange(ORDER_START, ORDER_END + 1))
widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()]
pbar = ProgressBar(widgets = widgets, maxval = count)
pbar.start()
i=0
for resolution in xrange(RES_START,RES_END + 1, RES_DELTA):
r = rect_grid(resolution, resolution)
for v in xrange(len(r.verts)):
r.q[v] = baker_exact_2D(r.verts[v])
output_file.write("%f" % (1.0 / resolution))
for order in xrange(ORDER_START, ORDER_END + 1):
errors = []
start = time.time()
for X in dest_verts:
a = r.run_baker(X, order, EXTRA_POINTS)
cur_exact = baker_exact_2D(X)
e = a['final'] - cur_exact
errors.append(e)
i+=1
pbar.update(i)
end = time.time()
rmserror = rms(errors)
output_file.write(" %e" % rmserror)
output_file.write("\n")
output_file.flush()
pbar.finish()

26
bin/server.py
View File

@ -1,21 +1,21 @@
#!/usr/bin/env python #!/usr/bin/env python
from interp.cluster import QueueManager from interp.cluster import QueueManager
from optparse import OptionParser from optparse import OptionParser
if __name__ == '__main__': if __name__ == '__main__':
parser = OptionParser(usage = "usage: %s [options] <server> <interp count>") parser = OptionParser(usage="usage: %s [options] <server> <interp count>")
parser.add_option('-p', '--port', parser.add_option('-p', '--port',
type="int", dest="port", default=6666, type="int", dest="port", default=6666,
help="specify the port to use on the server (default: %default)") help="specify the port to use on the server (default: %default)")
parser.add_option('-a', '--auth-key', parser.add_option('-a', '--auth-key',
type="str", dest="authkey", default='asdf', type="str", dest="authkey", default='asdf',
help="authkey (default: %default)") help="authkey (default: %default)")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
m = QueueManager(address=('', options.port), authkey=options.authkey) m = QueueManager(address=('', options.port), authkey=options.authkey)
s = m.get_server() s = m.get_server()
s.serve_forever() s.serve_forever()

160
bin/sheep.py
View File

@ -3,108 +3,102 @@
import sys import sys
import os import os
import time import time
from multiprocessing.managers import BaseManager
from optparse import OptionParser from optparse import OptionParser
import numpy as np import numpy as np
import interp.bootstrap from interp.grid.gmsh import ggrid
from interp.grid.gmsh import ggrid from interp.tools import (baker_exact_2D, baker_exact_3D,
from interp.tools import * friendly_exact_2D, friendly_exact_3D)
from interp.cluster import QueueManager, get_qs from interp.cluster import QueueManager, get_qs
if __name__ == '__main__': if __name__ == '__main__':
parser = OptionParser(usage = "usage: %s [options] <server> <gmsh file> <dimension> <baker|friendly>") parser = OptionParser(usage="usage: %s [options] <server>"
" <gmsh file> <dimension> <baker|friendly>")
parser.add_option("-v", "--verbose", parser.add_option("-v", "--verbose",
action="store_true", dest="verbose", default=False, action="store_true", dest="verbose", default=False,
help="verbose flag (default: %default)") help="verbose flag (default: %default)")
parser.add_option('-p', '--port', parser.add_option('-p', '--port',
type="int", dest="port", default=6666, type="int", dest="port", default=6666,
help="specify the port to use on the server (default: %default)") help="specify the port to use on the server (default: %default)")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
if len(args) != 4: if len(args) != 4:
parser.print_usage() parser.print_usage()
sys.exit(1) sys.exit(1)
server, input_file, dimension, exact_type = args server, input_file, dimension, exact_type = args
dimension = int(dimension) dimension = int(dimension)
if dimension == 2 and exact_type == 'baker': if dimension == 2 and exact_type == 'baker':
exact = baker_exact_2D exact = baker_exact_2D
elif dimension == 2 and exact_type == 'friendly': elif dimension == 2 and exact_type == 'friendly':
exact = friendly_exact_2D exact = friendly_exact_2D
elif dimension == 3 and exact_type == 'baker': elif dimension == 3 and exact_type == 'baker':
exact = baker_exact_3D exact = baker_exact_3D
elif dimension == 3 and exact_type == 'friendly': elif dimension == 3 and exact_type == 'friendly':
exact = friendly_exact_3D exact = friendly_exact_3D
myname = "%s-%d" % (os.uname()[1], os.getpid())
myname = "%s-%d" % (os.uname()[1], os.getpid())
if options.verbose:
print "%s: started" % myname
m = QueueManager(address=(server, options.port), authkey='asdf')
m.connect()
tasksq, resultsq, masterq, minionsq = get_qs(m)
if options.verbose:
print "%s: starting parse input file" % myname
g = ggrid(input_file, dimension)
g.q = np.array([exact(x) for x in g.verts])
if options.verbose:
print "%s: done parsing input file" % myname
while True:
if options.verbose: if options.verbose:
print "%s: letting master know that I am ready" % myname print "%s: started" % myname
masterq.put(myname)
m = QueueManager(address=(server, options.port), authkey='asdf')
m.connect()
tasksq, resultsq, masterq, minionsq = get_qs(m)
if options.verbose: if options.verbose:
print "%s: waiting for master to tell me to start" % myname print "%s: starting parse input file" % myname
action = minionsq.get() g = ggrid(input_file, dimension)
g.q = np.array([exact(x) for x in g.verts])
if options.verbose: if options.verbose:
print "%s: got message from master: %s" % (myname, action) print "%s: done parsing input file" % myname
if action in ('teardown', 'slay'): while True:
masterq.get() if options.verbose:
break print "%s: letting master know that I am ready" % myname
masterq.put(myname)
while not tasksq.empty(): if options.verbose:
i, order, extra, X = tasksq.get() print "%s: waiting for master to tell me to start" % myname
try: action = minionsq.get()
s = time.time() if options.verbose:
a = g.run_baker(X, order = order, extra_points = extra) print "%s: got message from master: %s" % (myname, action)
cur_qlin = a['qlin' ]
cur_error = a['error']
cur_final = a['final']
cur_exact = exact(X)
e = time.time()
duration = e-s
except Exception as e:
print >>sys.stderr, X, e
cur_qlin = 0.0
cur_error = 0.0
cur_final = 0.0
cur_exact = 0.0
duration = 0
resultsq.put((i, if action in ('teardown', 'slay'):
cur_qlin, masterq.get()
cur_error, break
cur_final,
cur_exact,
duration,
))
if options.verbose: while not tasksq.empty():
print "%s: exiting" % myname i, order, extra, X = tasksq.get()
try:
s = time.time()
a = g.run_baker(X, order=order, extra_points=extra)
cur_qlin = a.qlin
cur_error = a.error
cur_final = a.final
cur_exact = exact(X)
e = time.time()
duration = e - s
except Exception as e:
print >>sys.stderr, X, e
cur_qlin = 0.0
cur_error = 0.0
cur_final = 0.0
cur_exact = 0.0
duration = 0
resultsq.put((i,
cur_qlin,
cur_error,
cur_final,
cur_exact,
duration,
))
if options.verbose:
print "%s: exiting" % myname

206
bin/shepherd.py
View File

@ -1,131 +1,133 @@
#!/usr/bin/env python #!/usr/bin/env python
from optparse import OptionParser
import pickle
import sys import sys
import time import time
from progressbar import * from progressbar import ProgressBar, ETA, Percentage, Bar
from optparse import OptionParser
import pickle
import numpy as np
from interp.cluster import QueueManager, get_qs from interp.cluster import QueueManager, get_qs
if __name__ == '__main__': if __name__ == '__main__':
parser = OptionParser(usage = "%prog [options] <server> <destination vertexes file (pickle)> <results output file> <dimension>") parser = OptionParser(usage="%prog [options] <server>"
" <destination vertexes file (pickle)>"
" <results output file> <dimension>")
parser.add_option("-v", "--verbose", parser.add_option("-v", "--verbose",
action="store_true", dest="verbose", default=False, action="store_true", dest="verbose", default=False,
help="verbose flag") help="verbose flag")
parser.add_option('-p', '--port', parser.add_option('-p', '--port',
type="int", dest="port", default=6666, type="int", dest="port", default=6666,
help="specify the port to use on the server (default: %default)") help="specify the port to use on the server (default: %default)")
parser.add_option('-n', '--node-count', parser.add_option('-n', '--node-count',
type="int", dest="participants", default=None, type="int", dest="participants", default=None,
help="specify how many participants we should wait for (default: %default)") help="specify how many participants we should"
" wait for (default: %default)")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
if len(args) != 4: if len(args) != 4:
parser.print_usage() parser.print_usage()
sys.exit(1) sys.exit(1)
server, dest_verts, results_p_name, dimension = args server, dest_verts, results_p_name, dimension = args
dimension = int(dimension) dimension = int(dimension)
dest_verts = pickle.load(open(dest_verts, 'r')) dest_verts = pickle.load(open(dest_verts, 'r'))
orders = [2,3,4,5] orders = [2, 3, 4, 5]
extras = [4,6,8,12, 16,20,32,48, 64,96,128,192, 256] extras = [4, 6, 8, 12, 16, 20, 32, 48, 64, 96, 128, 192, 256]
count = len(orders) * len(extras) * len(dest_verts)
count = len(orders) * len(extras) * len(dest_verts) results = [None] * count
# results = [id, order, extra, X, qlin, err, final, exact, time]
results = [None] * count m = QueueManager(address=(server, options.port), authkey='asdf')
# results = [id, order, extra, X, qlin, err, final, exact, time] m.connect()
m = QueueManager(address=(server, options.port), authkey='asdf') tasksq, resultsq, masterq, minionsq = get_qs(m)
m.connect()
tasksq, resultsq, masterq, minionsq = get_qs(m) if not options.participants:
print "wait on all announced participants"
participants = 0
while not masterq.empty():
participants += 1
worker = masterq.get()
print "%s (%d)" % (worker, participants)
if participants == 0:
print "nobody found"
sys.exit(1)
else:
participants = options.participants
print "wait on %d participants" % participants
for i in xrange(participants):
worker = masterq.get()
print "%s (%d/%d)" % (worker, i + 1, participants)
print "Submitting %d pieces of workload" % count
widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()]
pbar = ProgressBar(widgets=widgets, maxval=count)
pbar.start()
submit_start = time.time()
i = 0
for order in orders:
for extra in extras:
for X in dest_verts:
cur_job = (i, order, extra, X)
results[i] = [order, extra, X]
tasksq.put(cur_job)
i += 1
pbar.update(i)
submit_end = time.time()
pbar.finish()
print("it took %0.2f seconds to submit"
" the workload" % (submit_end - submit_start,))
print "len(results)", len(results)
if not options.participants:
print "wait on all announced participants"
participants = 0
while not masterq.empty():
participants += 1
worker = masterq.get()
print "%s (%d)" % (worker,participants)
if participants == 0:
print "nobody found"
sys.exit(1)
else:
participants = options.participants
print "wait on %d participants" % participants
for i in xrange(participants): for i in xrange(participants):
worker = masterq.get() print "sending worker %d start message" % (i + 1,)
print "%s (%d/%d)" % (worker, i+1, participants) minionsq.put("start")
print "Submitting %d pieces of workload" % count receive_start = time.time()
widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()]
pbar = ProgressBar(widgets=widgets, maxval=count)
pbar.start()
for i in xrange(count):
rid, qlin, err, final, exact, duration = resultsq.get()
results[rid].extend((qlin, err, final, exact, duration))
pbar.update(i + 1)
receive_end = time.time()
pbar.finish()
widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()] # pickle.dump(results, open(results_p_name, 'w'))
pbar = ProgressBar(widgets = widgets, maxval = count) o = open(results_p_name, 'w')
pbar.start() for i in results:
if dimension == 2:
o.write("%d %d %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0],
i[2][1], i[3], i[4], i[5], i[6], i[7]))
elif dimension == 3:
o.write("%d %d %e %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0],
i[2][1], i[2][2], i[3], i[4], i[5], i[6], i[7]))
submit_start = time.time() submit = submit_end - submit_start
i = 0 receive = receive_end - receive_start
for order in orders:
for extra in extras:
for X in dest_verts:
cur_job = (i, order, extra, X)
results[i] = [ order, extra, X]
tasksq.put(cur_job)
i+=1
pbar.update(i)
submit_end = time.time()
pbar.finish()
print "it took %0.2f seconds to submit the workload" % (submit_end - submit_start,) # shut down all participants
for i in xrange(participants):
minionsq.put("teardown")
print "len(results)", len(results) # post processing
stats = {}
stats['submit'] = float(submit)
stats['receive'] = float(receive)
stats['count'] = count
stats['participants'] = participants
for i in xrange(participants): print "%s" % stats
print "sending worker %d start message" % (i+1,)
minionsq.put("start")
receive_start = time.time()
widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()]
pbar = ProgressBar(widgets = widgets, maxval = count)
pbar.start()
for i in xrange(count):
rid, qlin, err, final, exact, duration = resultsq.get()
results[rid].extend((qlin, err, final, exact, duration))
pbar.update(i+1)
receive_end = time.time()
pbar.finish()
# pickle.dump(results, open(results_p_name, 'w'))
o = open(results_p_name, 'w')
for i in results:
if dimension == 2:
o.write("%d %d %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0], i[2][1], i[3], i[4], i[5], i[6], i[7]))
elif dimension == 3:
o.write("%d %d %e %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0], i[2][1], i[2][2], i[3], i[4], i[5], i[6], i[7]))
submit = submit_end - submit_start
receive = receive_end - receive_start
# shut down all participants
for i in xrange(participants):
minionsq.put("teardown")
# post processing
stats = {}
stats['submit' ] = float(submit)
stats['receive' ] = float(receive)
stats['count' ] = count
stats['participants'] = participants
print "%s" % stats