From 09160230801fbc69ac0b6102d0450818e948194b Mon Sep 17 00:00:00 2001 From: "Stephen M. McQuay" Date: Tue, 27 Sep 2011 21:54:04 -0600 Subject: [PATCH] pep8/pyflakes on the bin folder --- bin/gbench.py | 153 ++++++++++++++--------------- bin/grid_regular.py | 58 ----------- bin/iqmgr.py | 142 ++++++++++++++------------- bin/master.py | 223 ++++++++++++++++++++++--------------------- bin/memtest.py | 9 -- bin/minion.py | 104 ++++++++++---------- bin/resolution.2D.py | 58 ----------- bin/server.py | 26 ++--- bin/sheep.py | 160 +++++++++++++++---------------- bin/shepherd.py | 206 +++++++++++++++++++-------------------- 10 files changed, 506 insertions(+), 633 deletions(-) delete mode 100644 bin/grid_regular.py delete mode 100644 bin/memtest.py delete mode 100644 bin/resolution.2D.py diff --git a/bin/gbench.py b/bin/gbench.py index 3c5c357..36cb16d 100644 --- a/bin/gbench.py +++ b/bin/gbench.py @@ -10,99 +10,100 @@ from optparse import OptionParser import progressbar import interp.bootstrap +interp.bootstrap -from interp.grid.gmsh import ggrid -from interp.grid.delaunay import dgrid +from interp.grid.gmsh import ggrid +from interp.grid.delaunay import dgrid +from interp.tools import improved_answer +from interp.tools import baker_exact_2D, baker_exact_3D +from interp.tools import friendly_exact_2D, friendly_exact_3D +from interp.tools import exact_me -from interp.tools import improved_answer -from interp.tools import baker_exact_2D, baker_exact_3D -from interp.tools import friendly_exact_2D, friendly_exact_3D def get_right_exact_func(options): - f = None - if options.dimension == 3: - if options.baker: - f = baker_exact_3D - else: - f =friendly_exact_3D - elif options.dimension == 2: - if options.baker: - f = baker_exact_2D - else: - f = friendly_exact_2D - return f + f = None + if options.dimension == 3: + if options.baker: + f = baker_exact_3D + else: + f = friendly_exact_3D + elif options.dimension == 2: + if options.baker: + f = baker_exact_2D + else: + f = friendly_exact_2D + return f if __name__ == '__main__': - parser = OptionParser(usage = "%prog [options] ") + parser = OptionParser(usage="%prog [options] ") - parser.add_option("-v", "--verbose", - action="store_true", dest="verbose", default=False, - help="verbose flag") + parser.add_option("-b", "--baker", + action="store_true", dest="baker", default=False, + help="use more impressive baker funcs") - parser.add_option("-b", "--baker", - action="store_true", dest="baker", default=False, - help="use more impressive baker funcs") + meshtype_options = ('gmsh', 'delaunay') + parser.add_option('-t', '--type', + type="str", dest="meshtype", default='delaunay', + help="specify mesh type (default: %default, options gmsh, delaunay)") - meshtype_options = ('gmsh', 'delaunay') - parser.add_option('-t', '--type', - type="str", dest="meshtype", default='delaunay', - help="specify mesh type (default: %default, options gmsh, delaunay)") + parser.add_option("-d", "--dimension", + type="int", dest="dimension", default=3, + help="use 3D or 2D gmsh lib (default: %default)") - parser.add_option("-d", "--dimension", - type="int", dest="dimension", default=3, - help="use 3D or 2D gmsh lib (default: %default)") + parser.add_option("-o", "--order", + type="int", dest="order", default=3, + help="order of interpolation (default: %default)") - parser.add_option("-o", "--order", - type="int", dest="order", default=3, - help="order of interpolation (default: %default)") + parser.add_option("-e", "--extra-points", + type="int", dest="extra", default=3, + help="number of extra points (default: %default)") - parser.add_option("-e", "--extra-points", - type="int", dest="extra", default=3, - help="number of extra points (default: %default)") + (options, args) = parser.parse_args() + if len(args) != 2: + parser.print_usage() + sys.exit(1) - (options, args) = parser.parse_args() - if len(args) != 2: - parser.print_usage() - sys.exit(1) + input_file, count = args + count = int(count) - input_file, count = args - count = int(count) + if options.meshtype == 'gmsh': + g = ggrid(input_file, options.dimension) + g.values = exact_me(g.points, get_right_exact_func(options)) + elif options.meshtype == 'delaunay': + point_count = int(input_file) + v = np.random.random((point_count, options.dimension)) + q = exact_me(v, get_right_exact_func(options)) + g = dgrid(v, q) + else: + raise Exception("improper mesh type specified. options: %s\n%s".format( + str(meshtype_options), options.meshtype)) - if options.meshtype == 'gmsh': - g = ggrid(input_file, options.dimension) - elif options.meshtype == 'delaunay': - point_count = int(input_file) - v = np.random.random((point_count, options.dimension)) - g = dgrid(v) - else: - raise Exception("improper mesh type specified. options: %s\n%s" % (str(meshtype_options), options.meshtype)) + results = {True: 0, False: 0} + i = 0 + outside = 0 + pbar = progressbar.ProgressBar().start() + start = time.time() + while i < count: + try: + X = np.random.rand(1, options.dimension)[0] - g.q = np.array([get_right_exact_func(options)(x) for x in g.verts]) + a = g.interpolate(X, order=options.order, + extra_points=options.extra) + e = get_right_exact_func(options)(X) - results = {True:0, False:0} - i = 0 - outside = 0 - pbar = progressbar.ProgressBar().start() - start = time.time() - while i < count: - try: - X = np.random.random((1,options.dimension))[0] + results[improved_answer(a, e)] += 1 + update = i / float(count) * 100 + pbar.update(update) + i += 1 + except Exception as e: + # print e + # print X, i, count + outside += 1 - a = g.run_baker(X, order = options.order, extra_points = options.extra) - e = get_right_exact_func(options)(X) - - results[improved_answer(a, e)] += 1 - update = i/float(count) * 100 - pbar.update(update) - i += 1 - except Exception as e: - # print e - # print X, i, count - outside += 1 - - pbar.finish() - end = time.time() - print "total skipped points: %d" % outside - print results, end - start + pbar.finish() + end = time.time() + print "total skipped points: %d" % outside + print results, end - start diff --git a/bin/grid_regular.py b/bin/grid_regular.py deleted file mode 100644 index 5b2fce4..0000000 --- a/bin/grid_regular.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python - -import sys -import pickle -import pdb - -from interp.grid.DD import rect_grid, random_grid -from interp.baker import run_baker -from interp.tools import baker_exact_2D as exact_func - -qfile = '/tmp/grid_regular.txt' - -if __name__ == '__main__': - try: - rx = int(sys.argv[1]) - ry = int(sys.argv[2]) - except (IndexError, ValueError) as e: - print "problem with argv: %s" % e - rx = 4 - ry = 4 * rx - - source_mesh = rect_grid(rx, ry) - - # print source_mesh - - X = [0.5, 0.5] - - try: - print "trying to get simplex and nearest points using nearest points (kdtree)" - (R, S) = source_mesh.get_simplex_and_nearest_points(X, extra_points=4) - print "R for nearest-neighbor:\n", R - print "S for nearest-neighbor:\n", S - print "trying to run baker" - print run_baker(X, R, S) - except Exception as e: - print "caught error: %s" % e - - print "trying to get simplex and nearest points using connectivity scheme" - (R, S) = source_mesh.get_points_conn(X) - print "R for connectivity:\n", R - print "S for connectivity:\n", S - print run_baker(X, R, S) - - - print "repeating the above just using the grid object:" - r = source_mesh.run_baker(X) - exact = exact_func(X) - print r - print 'exact', exact - print 'qlin' , r['qlin'] - print 'error', r['error'] - print 'final', r['final'] - - if abs(r['final'] - exact) <= abs(r['qlin'] - exact): - print "win" - else: - print "failure" - open(qfile, 'w').write(source_mesh.for_qhull()) diff --git a/bin/iqmgr.py b/bin/iqmgr.py index 3083983..b43d6a2 100755 --- a/bin/iqmgr.py +++ b/bin/iqmgr.py @@ -1,87 +1,93 @@ #!/usr/bin/env python -import sys import time -from interp.cluster import QueueManager, get_qs -from optparse import OptionParser +from interp.cluster import QueueManager, get_qs +from optparse import OptionParser if __name__ == '__main__': - parser = OptionParser(usage = "usage: %s [options] ") + parser = OptionParser(usage="usage: %s [options] ") - parser.add_option('-p', '--port', - type="int", dest="port", default=6666, - help="specify the port to use on the server (default: %default)") + parser.add_option('-p', '--port', + type="int", dest="port", default=6666, + help="specify the port to use on the server (default: %default)") - parser.add_option('-a', '--auth-key', - type="str", dest="authkey", default='asdf', - help="authkey (default: %default)") + parser.add_option('-a', '--auth-key', + type="str", dest="authkey", default='asdf', + help="authkey (default: %default)") - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() - if len(args) == 0: - cmd = 'status' - else: - cmd = args[0] - - m = QueueManager(address=('', options.port), authkey=options.authkey) - m.connect() - - tq,rq,mq,sq = get_qs(m) - - if cmd.startswith("st"): - print "interp queue status:" - print " tasksq : %d" % tq.qsize() - print " resultsq : %d" % rq.qsize() - print " masterq : %d" % mq.qsize() - print " minionsq : %d" % sq.qsize() - - if cmd.startswith("wa"): - if len(args) == 2: - sleeptime = float(args[1]) + if len(args) == 0: + cmd = 'status' else: - sleeptime = 1 + cmd = args[0] - i = 0 - while True: - time.sleep(sleeptime) - if i % 20 == 0: - print "tasksq resultsq masterq minionsq" - print "%d %d %d %d" % \ - (tq.qsize(), - rq.qsize(), - mq.qsize(), - sq.qsize(),) - i += 1 + m = QueueManager(address=('', options.port), authkey=options.authkey) + m.connect() - if cmd == 'add': - for i in xrange(int(args[1])): - mq.put('jane%d' % i) + tq, rq, mq, sq = get_qs(m) - if cmd == 'fresult': - for i in xrange(int(args[1])): - rq.put('fake.%d' % i) + if cmd.startswith("st"): + print "interp queue status:" + print " tasksq : %d" % tq.qsize() + print " resultsq : %d" % rq.qsize() + print " masterq : %d" % mq.qsize() + print " minionsq : %d" % sq.qsize() - if cmd == 'slay': - if len(args) == 1: - for i in xrange(mq.qsize()): - print i, "killing", mq.get() - sq.put("slay") - elif len(args) == 2: - for i in xrange(int(args[1])): - print i, "killing", mq.get() - sq.put("slay") + if cmd.startswith("wa"): + if len(args) == 2: + sleeptime = float(args[1]) + else: + sleeptime = 1 + i = 0 + while True: + time.sleep(sleeptime) + if i % 20 == 0: + print "tasksq resultsq masterq minionsq" + print "%d %d %d %d" % \ + (tq.qsize(), + rq.qsize(), + mq.qsize(), + sq.qsize(),) + i += 1 - if cmd == 'clear': - for i in xrange(tq.qsize()): print tq.get() - for i in xrange(rq.qsize()): print rq.get() + if cmd == 'add': + for i in xrange(int(args[1])): + mq.put('jane%d' % i) - if cmd == 'clearall': - for i in xrange(tq.qsize()): print tq.get() - for i in xrange(rq.qsize()): print rq.get() - for i in xrange(mq.qsize()): print mq.get() - for i in xrange(sq.qsize()): print sq.get() + if cmd == 'fresult': + for i in xrange(int(args[1])): + rq.put('fake.%d' % i) - if cmd == 'clearresults': - for i in xrange(rq.qsize()): print rq.get() + if cmd == 'slay': + if len(args) == 1: + for i in xrange(mq.qsize()): + print i, "killing", mq.get() + sq.put("slay") + elif len(args) == 2: + for i in xrange(int(args[1])): + print i, "killing", mq.get() + sq.put("slay") + + if cmd == 'clear': + for i in xrange(tq.qsize()): + print tq.get() + for i in xrange(rq.qsize()): + print rq.get() + + if cmd == 'clearall': + for i in xrange(tq.qsize()): + print tq.get() + for i in xrange(rq.qsize()): + print rq.get() + for i in xrange(mq.qsize()): + print mq.get() + for i in xrange(sq.qsize()): + print sq.get() + + if cmd == 'clearresults': + for i in xrange(rq.qsize()): + print rq.get() diff --git a/bin/master.py b/bin/master.py index c98aeb4..f4cb9cb 100644 --- a/bin/master.py +++ b/bin/master.py @@ -5,143 +5,146 @@ import os import time import shelve -from collections import defaultdict -from optparse import OptionParser +from collections import defaultdict +from optparse import OptionParser import logging log = logging.getLogger("interp") import numpy as np -from interp.cluster import QueueManager, get_qs +from interp.cluster import QueueManager, get_qs -from progressbar import * +from progressbar import ProgressBar, Percentage, Bar, ETA if __name__ == '__main__': - parser = OptionParser(usage = "usage: %s [options] ") + parser = OptionParser(usage="usage: %s [options] ") - parser.add_option("-l", "--last-time", - action="store_true", dest="last", default=False, - help="when finished, send shutdown signal to connected nodes (default: %default)") + parser.add_option("-l", "--last-time", + action="store_true", dest="last", default=False, + help="when finished, send shutdown signal to connected nodes" + "(default: %default)") - parser.add_option('-n', '--node-count', - type="int", dest="participants", default=None, - help="specify how many participants we should wait for (default: %default)") + parser.add_option('-n', '--node-count', + type="int", dest="participants", default=None, + help="specify how many participants we should wait for" + "(default: %default)") - parser.add_option('-p', '--port', - type="int", dest="port", default=6666, - help="specify the port to use on the server (default: %default)") + parser.add_option('-p', '--port', + type="int", dest="port", default=6666, + help="specify the port to use on the server (default: %default)") - parser.add_option("-o", "--order", - type="int", dest="order", default=2, - help="order of interpolation (default: %default)") + parser.add_option("-o", "--order", + type="int", dest="order", default=2, + help="order of interpolation (default: %default)") - parser.add_option("-e", "--extra-points", - type="int", dest="extra", default=3, - help="number of extra points (default: %default)") + parser.add_option("-e", "--extra-points", + type="int", dest="extra", default=3, + help="number of extra points (default: %default)") - parser.add_option('-s', '--shelve', - type="str", dest="shelvename", default=os.path.expanduser('~/interp.shelve'), - help="shelve output file (default: %default)") + parser.add_option('-s', '--shelve', + type="str", dest="shelvename", + default=os.path.expanduser('~/interp.shelve'), + help="shelve output file (default: %default)") - (options, args) = parser.parse_args() - if len(args) != 2: - parser.print_usage() - sys.exit(1) + (options, args) = parser.parse_args() + if len(args) != 2: + parser.print_usage() + sys.exit(1) - server = args[0] - count = int(float(args[1])) + server = args[0] + count = int(float(args[1])) - m = QueueManager(address=(server, options.port), authkey='asdf') - m.connect() + m = QueueManager(address=(server, options.port), authkey='asdf') + m.connect() - tasksq, resultsq, masterq, minionsq = get_qs(m) + tasksq, resultsq, masterq, minionsq = get_qs(m) - workers = [] + workers = [] + + if not options.participants: + print "wait on all announced participants" + participants = 0 + while not masterq.empty(): + participants += 1 + worker = masterq.get() + workers.append(worker) + print "%d: %s is ready" % (participants, worker) + if participants == 0: + print "nobody found" + sys.exit(1) + else: + participants = options.participants + print "wait on %d participants" % participants + for i in xrange(participants): + worker = masterq.get() + workers.append(worker) + print "%d of %d: %s is ready" % (i + 1, participants, worker) + + if len(set(workers)) != len(workers): + for i in workers: + minionsq.put("slay") + raise Exception("duplicate workers reported") + + results = [] + + widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()] + pbar = ProgressBar(widgets=widgets, maxval=count) + pbar.start() + submit_start = time.time() + for i in xrange(count): + X = np.random.random((1, 3))[0] + tasksq.put((i, options.order, options.extra, X)) + pbar.update(i + 1) + submit_end = time.time() + pbar.finish() - if not options.participants: - print "wait on all announced participants" - participants = 0 - while not masterq.empty(): - participants += 1 - worker = masterq.get() - workers.append(worker) - print "%d: %s is ready" % (participants, worker) - if participants == 0: - print "nobody found" - sys.exit(1) - else: - participants = options.participants - print "wait on %d participants" % participants for i in xrange(participants): - worker = masterq.get() - workers.append(worker) - print "%d of %d: %s is ready" % (i+1, participants, worker) + print "sending worker %d start message" % (i + 1,) + minionsq.put("start") - if len(set(workers)) != len(workers): - for i in workers: - minionsq.put("slay") - raise Exception("duplicate workers reported") + receive_start = time.time() + widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()] + pbar = ProgressBar(widgets=widgets, maxval=count) + pbar.start() + for i in xrange(count): + cur_result = resultsq.get() + results.append(cur_result) + pbar.update(i + 1) + receive_end = time.time() + pbar.finish() - results = [] + submit = submit_end - submit_start + receive = receive_end - receive_start - widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()] - pbar = ProgressBar(widgets = widgets, maxval = count) - pbar.start() - submit_start = time.time() - for i in xrange(count): - X = np.random.random((1,3))[0] - tasksq.put((i, options.order, options.extra, X)) - pbar.update(i+1) - submit_end = time.time() - pbar.finish() + # shut down all participants + for i in xrange(participants): + if options.last: + minionsq.put("teardown") - for i in xrange(participants): - print "sending worker %d start message" % (i+1,) - minionsq.put("start") + # post processing + stats = {} + stats['submit'] = float(submit) + stats['receive'] = float(receive) + stats['count'] = count + stats['participants'] = participants + stats['extra'] = options.extra + stats['order'] = options.order - receive_start = time.time() - widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()] - pbar = ProgressBar(widgets = widgets, maxval = count) - pbar.start() - for i in xrange(count): - cur_result = resultsq.get() - results.append(cur_result) - pbar.update(i+1) - receive_end = time.time() - pbar.finish() + print "%s" % stats + log.error("stats: %s", stats) - submit = submit_end - submit_start - receive = receive_end - receive_start + tasks_accomplished_by = defaultdict(int) + for i in results: + tasks_accomplished_by[i[1]] += 1 + stats['tasks'] = tasks_accomplished_by - # shut down all participants - for i in xrange(participants): - if options.last: - minionsq.put("teardown") + npresults = np.array([(i[0], i[2], i[3], i[4], i[5]) for i in results]) - # post processing - stats = {} - stats['submit' ] = float(submit) - stats['receive' ] = float(receive) - stats['count' ] = count - stats['participants'] = participants - stats['extra' ] = options.extra - stats['order' ] = options.order - - print "%s" % stats - log.error("stats: %s", stats) - - tasks_accomplished_by = defaultdict(int) - for i in results: - tasks_accomplished_by[i[1]] += 1 - stats['tasks'] = tasks_accomplished_by - - # npresults = np.array([(i[0],i[2],i[3],i[4], i[5]) for i in results]) - - n = str(time.time()) - s = shelve.open(options.shelvename) - s[n] = { - 'stats' : stats, - # 'results' : npresults, - } - s.close() + n = str(time.time()) + s = shelve.open(options.shelvename) + s[n] = { + 'stats': stats, + 'results': npresults, + } + s.close() diff --git a/bin/memtest.py b/bin/memtest.py deleted file mode 100644 index 5b0945f..0000000 --- a/bin/memtest.py +++ /dev/null @@ -1,9 +0,0 @@ -import sys -from interp.grid.gmsh import * -import time -s = time.time() -g = ggrid(sys.argv[1]) -e = time.time() - -print "time to parse mesh:", e-s - diff --git a/bin/minion.py b/bin/minion.py index f77ee54..cb14046 100644 --- a/bin/minion.py +++ b/bin/minion.py @@ -2,79 +2,71 @@ import sys import os -import time - -from multiprocessing.managers import BaseManager - from optparse import OptionParser -import datetime import numpy as np -from interp.grid.gmsh import ggrid -from interp.tools import baker_exact_3D as exact - +from interp.grid.gmsh import ggrid +from interp.tools import baker_exact_3D as exact from interp.cluster import QueueManager, get_qs if __name__ == '__main__': - parser = OptionParser(usage = "usage: %s [options] ") + parser = OptionParser(usage="usage: %s [options] ") - parser.add_option("-v", "--verbose", - action="store_true", dest="verbose", default=False, - help="verbose flag (default: %default)") + parser.add_option("-v", "--verbose", + action="store_true", dest="verbose", default=False, + help="verbose flag (default: %default)") - parser.add_option('-p', '--port', - type="int", dest="port", default=6666, - help="specify the port to use on the server (default: %default)") + parser.add_option('-p', '--port', + type="int", dest="port", default=6666, + help="specify the port to use on the server (default: %default)") - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() - if len(args) != 2: - parser.print_usage() - sys.exit(1) + if len(args) != 2: + parser.print_usage() + sys.exit(1) - server, input_file = args + server, input_file = args - myname = "%s-%d" % (os.uname()[1], os.getpid()) - if options.verbose: - print "%s: started" % myname - - - m = QueueManager(address=(server, options.port), authkey='asdf') - m.connect() - - tasksq, resultsq, masterq, minionsq = get_qs(m) - - if options.verbose: - print "%s: starting parse input file" % myname - g = ggrid(input_file) - g.q = np.array([exact(x) for x in g.verts]) - if options.verbose: - print "%s: done parsing input file" % myname - - - while True: + myname = "%s-%d" % (os.uname()[1], os.getpid()) if options.verbose: - print "%s: letting master know that I am ready" % myname - masterq.put(myname) + print "%s: started" % myname + + m = QueueManager(address=(server, options.port), authkey='asdf') + m.connect() + + tasksq, resultsq, masterq, minionsq = get_qs(m) if options.verbose: - print "%s: waiting for master to tell me to start" % myname - action = minionsq.get() + print "%s: starting parse input file" % myname + g = ggrid(input_file) + g.q = np.array([exact(x) for x in g.verts]) if options.verbose: - print "%s: master said go!!" % myname + print "%s: done parsing input file" % myname - if action in ('teardown', 'slay'): - break + while True: + if options.verbose: + print "%s: letting master know that I am ready" % myname + masterq.put(myname) - while not tasksq.empty(): - i, o, e, X = tasksq.get() - try: - a = g.run_baker(X, order = o, extra_points = e) - resultsq.put((i, myname, a['qlin'], a['error'], a['final'], exact(X))) - except Exception as e: - print X, e - resultsq.put((i, myname, 0.0, 0.0, 0.0, 0.0)) + if options.verbose: + print "%s: waiting for master to tell me to start" % myname + action = minionsq.get() + if options.verbose: + print "%s: master said go!!" % myname - if options.verbose: - print "%s: exiting" % myname + if action in ('teardown', 'slay'): + break + + while not tasksq.empty(): + i, o, e, X = tasksq.get() + try: + a = g.run_baker(X, order=o, extra_points=e) + resultsq.put((i, myname, a.qlin, a.error, a.final, exact(X))) + except Exception as e: + print X, e + resultsq.put((i, myname, 0.0, 0.0, 0.0, 0.0)) + + if options.verbose: + print "%s: exiting" % myname diff --git a/bin/resolution.2D.py b/bin/resolution.2D.py deleted file mode 100644 index 8597afe..0000000 --- a/bin/resolution.2D.py +++ /dev/null @@ -1,58 +0,0 @@ -import sys -import pickle - -import time - -from progressbar import * - -from interp.grid.DD import * -from interp.tools import * -from interp import bootstrap - -EXTRA_POINTS = 64 - -RES_START = 10 -RES_END = 100 -RES_DELTA = 10 - -ORDER_START = 1 -ORDER_END = 8 - -dest_verts, output_filename = sys.argv[1:] -dest_verts = pickle.load(open(dest_verts, 'r')) - -output_file = open(output_filename, 'w') - -count = len(dest_verts) * len(xrange(RES_START,RES_END + 1, RES_DELTA)) * len(xrange(ORDER_START, ORDER_END + 1)) - -widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()] -pbar = ProgressBar(widgets = widgets, maxval = count) -pbar.start() - -i=0 -for resolution in xrange(RES_START,RES_END + 1, RES_DELTA): - r = rect_grid(resolution, resolution) - - for v in xrange(len(r.verts)): - r.q[v] = baker_exact_2D(r.verts[v]) - - output_file.write("%f" % (1.0 / resolution)) - - for order in xrange(ORDER_START, ORDER_END + 1): - errors = [] - - start = time.time() - for X in dest_verts: - a = r.run_baker(X, order, EXTRA_POINTS) - cur_exact = baker_exact_2D(X) - e = a['final'] - cur_exact - errors.append(e) - i+=1 - pbar.update(i) - end = time.time() - - rmserror = rms(errors) - output_file.write(" %e" % rmserror) - output_file.write("\n") - output_file.flush() -pbar.finish() diff --git a/bin/server.py b/bin/server.py index 8011283..a1200d8 100644 --- a/bin/server.py +++ b/bin/server.py @@ -1,21 +1,21 @@ #!/usr/bin/env python -from interp.cluster import QueueManager -from optparse import OptionParser +from interp.cluster import QueueManager +from optparse import OptionParser if __name__ == '__main__': - parser = OptionParser(usage = "usage: %s [options] ") + parser = OptionParser(usage="usage: %s [options] ") - parser.add_option('-p', '--port', - type="int", dest="port", default=6666, - help="specify the port to use on the server (default: %default)") + parser.add_option('-p', '--port', + type="int", dest="port", default=6666, + help="specify the port to use on the server (default: %default)") - parser.add_option('-a', '--auth-key', - type="str", dest="authkey", default='asdf', - help="authkey (default: %default)") + parser.add_option('-a', '--auth-key', + type="str", dest="authkey", default='asdf', + help="authkey (default: %default)") - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() - m = QueueManager(address=('', options.port), authkey=options.authkey) - s = m.get_server() - s.serve_forever() + m = QueueManager(address=('', options.port), authkey=options.authkey) + s = m.get_server() + s.serve_forever() diff --git a/bin/sheep.py b/bin/sheep.py index a0ff354..ce40684 100644 --- a/bin/sheep.py +++ b/bin/sheep.py @@ -3,108 +3,102 @@ import sys import os import time - -from multiprocessing.managers import BaseManager - from optparse import OptionParser import numpy as np -import interp.bootstrap -from interp.grid.gmsh import ggrid -from interp.tools import * - +from interp.grid.gmsh import ggrid +from interp.tools import (baker_exact_2D, baker_exact_3D, + friendly_exact_2D, friendly_exact_3D) from interp.cluster import QueueManager, get_qs if __name__ == '__main__': - parser = OptionParser(usage = "usage: %s [options] ") + parser = OptionParser(usage="usage: %s [options] " + " ") - parser.add_option("-v", "--verbose", - action="store_true", dest="verbose", default=False, - help="verbose flag (default: %default)") + parser.add_option("-v", "--verbose", + action="store_true", dest="verbose", default=False, + help="verbose flag (default: %default)") - parser.add_option('-p', '--port', - type="int", dest="port", default=6666, - help="specify the port to use on the server (default: %default)") + parser.add_option('-p', '--port', + type="int", dest="port", default=6666, + help="specify the port to use on the server (default: %default)") - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() - if len(args) != 4: - parser.print_usage() - sys.exit(1) + if len(args) != 4: + parser.print_usage() + sys.exit(1) - server, input_file, dimension, exact_type = args - dimension = int(dimension) + server, input_file, dimension, exact_type = args + dimension = int(dimension) - if dimension == 2 and exact_type == 'baker': - exact = baker_exact_2D - elif dimension == 2 and exact_type == 'friendly': - exact = friendly_exact_2D - elif dimension == 3 and exact_type == 'baker': - exact = baker_exact_3D - elif dimension == 3 and exact_type == 'friendly': - exact = friendly_exact_3D + if dimension == 2 and exact_type == 'baker': + exact = baker_exact_2D + elif dimension == 2 and exact_type == 'friendly': + exact = friendly_exact_2D + elif dimension == 3 and exact_type == 'baker': + exact = baker_exact_3D + elif dimension == 3 and exact_type == 'friendly': + exact = friendly_exact_3D - - myname = "%s-%d" % (os.uname()[1], os.getpid()) - if options.verbose: - print "%s: started" % myname - - - m = QueueManager(address=(server, options.port), authkey='asdf') - m.connect() - - tasksq, resultsq, masterq, minionsq = get_qs(m) - - if options.verbose: - print "%s: starting parse input file" % myname - g = ggrid(input_file, dimension) - g.q = np.array([exact(x) for x in g.verts]) - if options.verbose: - print "%s: done parsing input file" % myname - - - while True: + myname = "%s-%d" % (os.uname()[1], os.getpid()) if options.verbose: - print "%s: letting master know that I am ready" % myname - masterq.put(myname) + print "%s: started" % myname + + m = QueueManager(address=(server, options.port), authkey='asdf') + m.connect() + + tasksq, resultsq, masterq, minionsq = get_qs(m) if options.verbose: - print "%s: waiting for master to tell me to start" % myname - action = minionsq.get() + print "%s: starting parse input file" % myname + g = ggrid(input_file, dimension) + g.q = np.array([exact(x) for x in g.verts]) if options.verbose: - print "%s: got message from master: %s" % (myname, action) + print "%s: done parsing input file" % myname - if action in ('teardown', 'slay'): - masterq.get() - break + while True: + if options.verbose: + print "%s: letting master know that I am ready" % myname + masterq.put(myname) - while not tasksq.empty(): - i, order, extra, X = tasksq.get() - try: - s = time.time() - a = g.run_baker(X, order = order, extra_points = extra) - cur_qlin = a['qlin' ] - cur_error = a['error'] - cur_final = a['final'] - cur_exact = exact(X) - e = time.time() - duration = e-s - except Exception as e: - print >>sys.stderr, X, e - cur_qlin = 0.0 - cur_error = 0.0 - cur_final = 0.0 - cur_exact = 0.0 - duration = 0 + if options.verbose: + print "%s: waiting for master to tell me to start" % myname + action = minionsq.get() + if options.verbose: + print "%s: got message from master: %s" % (myname, action) - resultsq.put((i, - cur_qlin, - cur_error, - cur_final, - cur_exact, - duration, - )) + if action in ('teardown', 'slay'): + masterq.get() + break - if options.verbose: - print "%s: exiting" % myname + while not tasksq.empty(): + i, order, extra, X = tasksq.get() + try: + s = time.time() + a = g.run_baker(X, order=order, extra_points=extra) + cur_qlin = a.qlin + cur_error = a.error + cur_final = a.final + cur_exact = exact(X) + e = time.time() + duration = e - s + except Exception as e: + print >>sys.stderr, X, e + cur_qlin = 0.0 + cur_error = 0.0 + cur_final = 0.0 + cur_exact = 0.0 + duration = 0 + + resultsq.put((i, + cur_qlin, + cur_error, + cur_final, + cur_exact, + duration, + )) + + if options.verbose: + print "%s: exiting" % myname diff --git a/bin/shepherd.py b/bin/shepherd.py index 3b18bb9..870ac16 100644 --- a/bin/shepherd.py +++ b/bin/shepherd.py @@ -1,131 +1,133 @@ #!/usr/bin/env python +from optparse import OptionParser +import pickle import sys - import time -from progressbar import * -from optparse import OptionParser -import pickle - -import numpy as np +from progressbar import ProgressBar, ETA, Percentage, Bar from interp.cluster import QueueManager, get_qs if __name__ == '__main__': - parser = OptionParser(usage = "%prog [options] ") + parser = OptionParser(usage="%prog [options] " + " " + " ") - parser.add_option("-v", "--verbose", - action="store_true", dest="verbose", default=False, - help="verbose flag") + parser.add_option("-v", "--verbose", + action="store_true", dest="verbose", default=False, + help="verbose flag") - parser.add_option('-p', '--port', - type="int", dest="port", default=6666, - help="specify the port to use on the server (default: %default)") + parser.add_option('-p', '--port', + type="int", dest="port", default=6666, + help="specify the port to use on the server (default: %default)") - parser.add_option('-n', '--node-count', - type="int", dest="participants", default=None, - help="specify how many participants we should wait for (default: %default)") + parser.add_option('-n', '--node-count', + type="int", dest="participants", default=None, + help="specify how many participants we should" + " wait for (default: %default)") - (options, args) = parser.parse_args() - if len(args) != 4: - parser.print_usage() - sys.exit(1) + (options, args) = parser.parse_args() + if len(args) != 4: + parser.print_usage() + sys.exit(1) - server, dest_verts, results_p_name, dimension = args - dimension = int(dimension) + server, dest_verts, results_p_name, dimension = args + dimension = int(dimension) - dest_verts = pickle.load(open(dest_verts, 'r')) + dest_verts = pickle.load(open(dest_verts, 'r')) - orders = [2,3,4,5] - extras = [4,6,8,12, 16,20,32,48, 64,96,128,192, 256] + orders = [2, 3, 4, 5] + extras = [4, 6, 8, 12, 16, 20, 32, 48, 64, 96, 128, 192, 256] + count = len(orders) * len(extras) * len(dest_verts) - count = len(orders) * len(extras) * len(dest_verts) + results = [None] * count + # results = [id, order, extra, X, qlin, err, final, exact, time] - results = [None] * count - # results = [id, order, extra, X, qlin, err, final, exact, time] + m = QueueManager(address=(server, options.port), authkey='asdf') + m.connect() - m = QueueManager(address=(server, options.port), authkey='asdf') - m.connect() + tasksq, resultsq, masterq, minionsq = get_qs(m) - tasksq, resultsq, masterq, minionsq = get_qs(m) + if not options.participants: + print "wait on all announced participants" + participants = 0 + while not masterq.empty(): + participants += 1 + worker = masterq.get() + print "%s (%d)" % (worker, participants) + if participants == 0: + print "nobody found" + sys.exit(1) + else: + participants = options.participants + print "wait on %d participants" % participants + for i in xrange(participants): + worker = masterq.get() + print "%s (%d/%d)" % (worker, i + 1, participants) + + print "Submitting %d pieces of workload" % count + + widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()] + pbar = ProgressBar(widgets=widgets, maxval=count) + pbar.start() + + submit_start = time.time() + i = 0 + for order in orders: + for extra in extras: + for X in dest_verts: + cur_job = (i, order, extra, X) + results[i] = [order, extra, X] + tasksq.put(cur_job) + i += 1 + pbar.update(i) + submit_end = time.time() + pbar.finish() + + print("it took %0.2f seconds to submit" + " the workload" % (submit_end - submit_start,)) + + print "len(results)", len(results) - if not options.participants: - print "wait on all announced participants" - participants = 0 - while not masterq.empty(): - participants += 1 - worker = masterq.get() - print "%s (%d)" % (worker,participants) - if participants == 0: - print "nobody found" - sys.exit(1) - else: - participants = options.participants - print "wait on %d participants" % participants for i in xrange(participants): - worker = masterq.get() - print "%s (%d/%d)" % (worker, i+1, participants) + print "sending worker %d start message" % (i + 1,) + minionsq.put("start") - print "Submitting %d pieces of workload" % count + receive_start = time.time() + widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()] + pbar = ProgressBar(widgets=widgets, maxval=count) + pbar.start() + for i in xrange(count): + rid, qlin, err, final, exact, duration = resultsq.get() + results[rid].extend((qlin, err, final, exact, duration)) + pbar.update(i + 1) + receive_end = time.time() + pbar.finish() - widgets = ['submit jobs: ', Percentage(), ' ', Bar(), ' ', ETA()] - pbar = ProgressBar(widgets = widgets, maxval = count) - pbar.start() + # pickle.dump(results, open(results_p_name, 'w')) + o = open(results_p_name, 'w') + for i in results: + if dimension == 2: + o.write("%d %d %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0], + i[2][1], i[3], i[4], i[5], i[6], i[7])) + elif dimension == 3: + o.write("%d %d %e %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0], + i[2][1], i[2][2], i[3], i[4], i[5], i[6], i[7])) - submit_start = time.time() - i = 0 - for order in orders: - for extra in extras: - for X in dest_verts: - cur_job = (i, order, extra, X) - results[i] = [ order, extra, X] - tasksq.put(cur_job) - i+=1 - pbar.update(i) - submit_end = time.time() - pbar.finish() + submit = submit_end - submit_start + receive = receive_end - receive_start - print "it took %0.2f seconds to submit the workload" % (submit_end - submit_start,) + # shut down all participants + for i in xrange(participants): + minionsq.put("teardown") - print "len(results)", len(results) + # post processing + stats = {} + stats['submit'] = float(submit) + stats['receive'] = float(receive) + stats['count'] = count + stats['participants'] = participants - for i in xrange(participants): - print "sending worker %d start message" % (i+1,) - minionsq.put("start") - - receive_start = time.time() - widgets = ['interpolate: ', Percentage(), ' ', Bar(), ' ', ETA()] - pbar = ProgressBar(widgets = widgets, maxval = count) - pbar.start() - for i in xrange(count): - rid, qlin, err, final, exact, duration = resultsq.get() - results[rid].extend((qlin, err, final, exact, duration)) - pbar.update(i+1) - receive_end = time.time() - pbar.finish() - - # pickle.dump(results, open(results_p_name, 'w')) - o = open(results_p_name, 'w') - for i in results: - if dimension == 2: - o.write("%d %d %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0], i[2][1], i[3], i[4], i[5], i[6], i[7])) - elif dimension == 3: - o.write("%d %d %e %e %e %e %e %e %e %e\n" % (i[0], i[1], i[2][0], i[2][1], i[2][2], i[3], i[4], i[5], i[6], i[7])) - - submit = submit_end - submit_start - receive = receive_end - receive_start - - # shut down all participants - for i in xrange(participants): - minionsq.put("teardown") - - # post processing - stats = {} - stats['submit' ] = float(submit) - stats['receive' ] = float(receive) - stats['count' ] = count - stats['participants'] = participants - - print "%s" % stats + print "%s" % stats