November 2009 - Triton-commits - mailman3.cels.anl.gov

[Gs-commits] Grayskull Repository branch, master, updated. git-migration-183-g8301a12
by noreply＠mcs.anl.gov 30 Nov '09

30 Nov '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 8301a12376500b6ae95e6d62b41f8741d197024b (commit) via 5a65b716ec7b0d33d097e424887fa72e80c65654 (commit) via 130512557b288c7c2c2fcdc5f8976c69ca9741f1 (commit) via 1816431b20550afe8fa723f985801bcbed2875ca (commit) via bbc8a410d34bc0f6a84f23f2c1808f141a1fcb68 (commit) from b968196f03039954f969f3699666391b36bfd909 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 8301a12376500b6ae95e6d62b41f8741d197024b Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Mon Nov 30 15:22:10 2009 -0600 New 3-phase rebuild. Now: after the fault is inflicted, build a list of copies to be performed, run them in the event-driven model, then actually copy objects that successfully copied. Allows us to interrupt the event-driven model with a successive fault. commit 5a65b716ec7b0d33d097e424887fa72e80c65654 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Mon Nov 30 12:22:27 2009 -0600 New avg(). commit 130512557b288c7c2c2fcdc5f8976c69ca9741f1 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Mon Nov 30 12:17:34 2009 -0600 Add $classpath for main build to enable colt. commit 1816431b20550afe8fa723f985801bcbed2875ca Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Mon Nov 30 12:17:05 2009 -0600 Display mean for exponential test. commit bbc8a410d34bc0f6a84f23f2c1808f141a1fcb68 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Mon Nov 30 12:00:47 2009 -0600 Link Exponential into Simulator and improve docs. ----------------------------------------------------------------------- Summary of changes: sim/gobs/build.xml | 3 +- sim/gobs/gobs/sim/Copy.java | 51 +++++++++++++ sim/gobs/gobs/sim/Event.java | 21 +++++ sim/gobs/gobs/sim/Metric.java | 24 ++++-- sim/gobs/gobs/sim/ParallelRUSH.java | 5 +- sim/gobs/gobs/sim/Placement.java | 59 +++++++-------- sim/gobs/gobs/sim/RUSHp.java | 5 +- sim/gobs/gobs/sim/Rebuild.java | 123 ++++++------------------------ sim/gobs/gobs/sim/Replicated.java | 13 +--- sim/gobs/gobs/sim/Simulator.java | 59 ++++++++++++--- sim/gobs/gobs/test/TestExponential.java | 6 +- sim/gobs/gobs/util/Stats.java | 8 ++ 12 files changed, 208 insertions(+), 169 deletions(-) create mode 100644 sim/gobs/gobs/sim/Copy.java create mode 100644 sim/gobs/gobs/sim/Event.java Diff of changes: diff --git a/sim/gobs/build.xml b/sim/gobs/build.xml index 5aa1299..f5c57cf 100644 --- a/sim/gobs/build.xml +++ b/sim/gobs/build.xml @@ -35,7 +35,7 @@ <target name="compile_sim" depends="compile_plot,compile_util"> <javac srcdir="gobs/sim" - classpath="." + classpath="${classpath}" listfiles="${lf}" debug="true" debuglevel="source,lines,vars"> @@ -98,6 +98,7 @@ includes="**/*.java"/> <link href="http://java.sun.com/javase/6/docs/api" /> <link href="http://www.jfree.org/jfreechart/api/javadoc" /> + <link href="http://acs.lbl.gov/~hoschek/colt/api" /> </javadoc> </target> diff --git a/sim/gobs/gobs/sim/Copy.java b/sim/gobs/gobs/sim/Copy.java new file mode 100644 index 0000000..6db2e98 --- /dev/null +++ b/sim/gobs/gobs/sim/Copy.java @@ -0,0 +1,51 @@ + +package gobs.sim; + +import java.text.DecimalFormat; +import java.util.*; + +import gobs.util.Bits; + +class Copy + extends Event +{ + gobs.sim.Object object; + Node source; + Node destination; + boolean complete = false; + double start = 0; + + static DecimalFormat df = new DecimalFormat("0.000000000"); + + Copy(gobs.sim.Object object, Node source, Node destination) + { + this.object = object; + this.source = source; + this.destination = destination; + } + + void startAt(double start) + { + this.start = start; + finish = start + object.size/Rebuild.rate; + } + + public String toString() + { + return log(); + } + + String log() + { + StringBuilder sb = new StringBuilder(128); + sb.append("copy: ") + .append(object).append(": ") + .append(source).append(" -> ").append(destination) + .append("(").append(df.format(start)); + if (finish == -1) + sb.append(")"); + else + sb.append(",").append(df.format(finish)).append(")"); + return sb.toString(); + } +} diff --git a/sim/gobs/gobs/sim/Event.java b/sim/gobs/gobs/sim/Event.java new file mode 100644 index 0000000..b1c5d3e --- /dev/null +++ b/sim/gobs/gobs/sim/Event.java @@ -0,0 +1,21 @@ + +package gobs.sim; + +class Event + implements Comparable<Event> +{ + double finish = -1; + + Event() + {} + + Event(double finish) + { + this.finish = finish; + } + + public int compareTo(Event event) + { + return new Double(finish).compareTo(new Double(event.finish)); + } +} diff --git a/sim/gobs/gobs/sim/Metric.java b/sim/gobs/gobs/sim/Metric.java index d3ad748..a541a98 100644 --- a/sim/gobs/gobs/sim/Metric.java +++ b/sim/gobs/gobs/sim/Metric.java @@ -35,8 +35,9 @@ public abstract class Metric return result; } - void rebuild(SingleObject object, List<Node> nodes) + Copy regimen(SingleObject object, List<Node> nodes) { + Copy result; List<Node> sites = sites(object, nodes); // System.out.println("sites.size(): " + sites.size()); Secondary secondary; @@ -44,6 +45,8 @@ public abstract class Metric Node primaryNode = sites.get(sites.size()-1); Node source = removeSource(object, sites); Node destination = null; + SingleObject payload; + if (object instanceof Primary) { Primary primary = (Primary) object; @@ -52,21 +55,22 @@ public abstract class Metric if (primarySite.contains(object.id)) { destination = locateDestination(object, sites); - secondary = (Secondary) primarySite.remove(object.id); - destination.add(secondary); + secondary = (Secondary) primarySite.remove(object.id); + primarySite.add(primary); + payload = secondary; } else { - destination = primarySite; + destination = primarySite; + payload = primary; } - primarySite.add(primary); } else { Primary primary = (Primary) primaryNode.get(object.id); secondary = primary.toSecondary(); - destination = locateDestination(object, sites); - destination.add(secondary); + destination = locateDestination(object, sites); + payload = secondary; } if (source == null) @@ -74,13 +78,15 @@ public abstract class Metric System.out.println("rebuild: lost: " + object.bitString(simulator.B)); simulator.lost(); - return; + return null; } source.pushes++; destination.pulls++; printRebuild(object, source, destination); - simulator.currentRebuild.add(object, source, destination); + // simulator.currentRebuild.add(object, source, destination); + result = new Copy(payload, source, destination); + return result; } } diff --git a/sim/gobs/gobs/sim/ParallelRUSH.java b/sim/gobs/gobs/sim/ParallelRUSH.java index 2fa345c..9fc315a 100644 --- a/sim/gobs/gobs/sim/ParallelRUSH.java +++ b/sim/gobs/gobs/sim/ParallelRUSH.java @@ -153,9 +153,10 @@ public class ParallelRUSH return nodes.get(index); } - void rebuild(SingleObject object, List<Node> nodes) + Copy regimen(SingleObject object, List<Node> nodes) { - place(object, nodes); + // place(object, nodes); + return null; } /** diff --git a/sim/gobs/gobs/sim/Placement.java b/sim/gobs/gobs/sim/Placement.java index 515da79..8612d5f 100644 --- a/sim/gobs/gobs/sim/Placement.java +++ b/sim/gobs/gobs/sim/Placement.java @@ -27,52 +27,45 @@ public abstract class Placement /** Generate an object set for a file. */ - // public Collection<SingleObject> allocateFile(File file, List<Node> nodes) - public abstract Collection<gobs.sim.Object> allocateFile(File file, List<Node> nodes); - /* - { - ArrayList<gobs.sim.Object> result = - new ArrayList<gobs.sim.Object>(file.width+2); - - BigInteger bigWidth = new BigInteger("" + file.width); - BigInteger s = M.divide(bigWidth); - BigInteger p = randomID(); - for (int i = 0; i < file.width; i++) - { - BigInteger id = p; - int bytes = file.objectSize(i); - gobs.sim.Object obj = - (gobs.sim.Object) new SingleObject(id, bytes, file.replicas); - result.add(obj); - p = p.add(s).mod(M); - } - - result.trimToSize(); - return result; - } -*/ - // public abstract Node lookup(BigInteger id, List<Node> nodes); - - void rebuild(gobs.sim.Object object, List<Node> nodes) + /** + Obtain a Copy that will repair the loss of this Object. + @return The Copy or null if the Object is lost. + */ + Copy regimen(gobs.sim.Object object, List<Node> nodes) { + Copy result; if (object instanceof SingleObject) - rebuild((SingleObject) object, nodes); + result = regimen((SingleObject) object, nodes); else if (object instanceof ObjectStrip) - rebuild((ObjectStrip) object, nodes); + result = regimen((ObjectStrip) object, nodes); + else + throw new IllegalArgumentException(); + return result; } /** - Simulate a rebuild. + Obtain a Copy that will repair the loss of this SingleObject. */ - abstract void rebuild(SingleObject object, List<Node> nodes); + abstract Copy regimen(SingleObject object, List<Node> nodes); - void rebuild(ObjectStrip object, List<Node> nodes) + /** + Obtain a Copy that will repair the loss of this ObjectStrip. + */ + Copy regimen(ObjectStrip object, List<Node> nodes) { throw new UnsupportedOperationException - ("rebuild(ObjectStrip) not supported!"); + ("regimen(ObjectStrip) not supported!"); + } + + /** + Actually perform this Copy. + */ + void copy(Copy copy) + { + copy.destination.add(copy.object); } /** diff --git a/sim/gobs/gobs/sim/RUSHp.java b/sim/gobs/gobs/sim/RUSHp.java index 3a0fbcc..e004738 100644 --- a/sim/gobs/gobs/sim/RUSHp.java +++ b/sim/gobs/gobs/sim/RUSHp.java @@ -103,9 +103,10 @@ public class RUSHp return nodes.get(index); } - void rebuild(SingleObject object, List<Node> nodes) + Copy regimen(SingleObject object, List<Node> nodes) { - place(object, nodes); + // place(object, nodes); + return null; } /** diff --git a/sim/gobs/gobs/sim/Rebuild.java b/sim/gobs/gobs/sim/Rebuild.java index a60314e..01c3382 100644 --- a/sim/gobs/gobs/sim/Rebuild.java +++ b/sim/gobs/gobs/sim/Rebuild.java @@ -4,14 +4,13 @@ package gobs.sim; import java.text.DecimalFormat; import java.util.*; -import gobs.util.Bits; +import gobs.util.*; /** Manages a rebuild after a fault. */ class Rebuild { - List<Copy> copies = new ArrayList<Copy>(); NavigableSet<Event> schedule = null; /** @@ -41,39 +40,30 @@ class Rebuild } /** - Add this copy to the Rebuild. + Schedule the next fault. */ - void add(Copy copy) + void nextFault(double t) { - copies.add(copy); - } - - /** - Construct a Copy and add it to this rebuild. - */ - void add(gobs.sim.Object object, Node source, Node destination) - { - Copy copy = new Copy(object, source, destination); - add(copy); + } /** - Actually perform the rebuild. + Actually perform the rebuild. + TODO: Handle cases where more than one copy is needed. @param time The time at which the copies start. @return The time at which all copies finished. */ - double rebuild(double time) + double rebuild(double time, Map<gobs.sim.Object,Copy> copies, + Set<gobs.sim.Object> todo, Set<gobs.sim.Object> done) { start = time; - Collections.shuffle(copies, Bits.rng); schedule = new TreeSet<Event>(); - Queue<Copy> todo = new LinkedList<Copy>(copies); - Set<Node> busy = new HashSet<Node>(copies.size()); + Queue<Copy> queue = order(copies); + Set<Node> busy = new HashSet<Node>(copies.size()); NavigableSet<Copy> inflight = new TreeSet<Copy>(); - int done = 0; - while (done < copies.size()) + while (done.size() < copies.size()) { - Copy copy = select(todo, busy); + Copy copy = select(queue, busy); if (copy != null) { copy.startAt(time); @@ -87,7 +77,6 @@ class Rebuild dumpInflight(inflight); time += selectTime; copy.startAt(time); - // System.exit(2); } if (simulator.bool("print.rebuilds")) System.out.println("start: " + copy.log()); @@ -96,20 +85,15 @@ class Rebuild { // dumpInflight(inflight); Copy finished = inflight.pollFirst(); - if (finished == null) - { - System.out.println("inflight.pollFirst(): null"); - System.out.println("inflight.size(): " + inflight.size()); - try { Thread.sleep(1000); } - catch (Exception e) {} - continue; - } + Tools.check(finished != null, + "inflight.pollFirst(): null"); busy.remove(finished.source); - busy.remove(finished.destination); + busy.remove(finished.destination); + todo.remove(finished.object); + done.add(finished.object); time = finished.finish; if (simulator.bool("print.rebuilds")) System.out.println("finish: " + finished.log()); - done++; } } return time; @@ -138,6 +122,13 @@ class Rebuild System.out.println("\t" + it.next()); } } + + Queue<Copy> order(Map<gobs.sim.Object,Copy> copies) + { + LinkedList<Copy> result = new LinkedList<Copy>(copies.values()); + Collections.shuffle(result, Bits.rng); + return result; + } Copy select(Queue<Copy> todo, Set<Node> busy) throws NoSuchElementException @@ -153,69 +144,5 @@ class Rebuild } } return null; - } - - class Event - implements Comparable<Event> - { - double finish = -1; - - Event() - {} - - Event(double finish) - { - this.finish = finish; - } - - public int compareTo(Event event) - { - return new Double(finish).compareTo(new Double(event.finish)); - } - } - - class Copy - extends Event - { - gobs.sim.Object object; - Node source; - Node destination; - boolean complete = false; - double start = 0; - - Copy(gobs.sim.Object object, Node source, Node destination) - { - - this.object = object; - this.source = source; - this.destination = destination; - } - - void startAt(double start) - { - this.start = start; - finish = start + object.size/rate; - } - - public String toString() - { - return log(); - } - - String log() - { - StringBuilder sb = new StringBuilder(128); - sb.append("copy: ") - .append(object).append(": ") - .append(source).append(" -> ").append(destination) - .append("(").append(df.format(start)); - if (finish == -1) - sb.append(")"); - else - sb.append(",").append(df.format(finish)).append(")"); - return sb.toString(); - } - } - - + } } diff --git a/sim/gobs/gobs/sim/Replicated.java b/sim/gobs/gobs/sim/Replicated.java index f1cdf21..2ac5e65 100644 --- a/sim/gobs/gobs/sim/Replicated.java +++ b/sim/gobs/gobs/sim/Replicated.java @@ -101,18 +101,7 @@ public abstract class Replicated ": does not support place(ObjectStrip)!"); } - /** - Obtain a list of nodes that may host the given object - with respect to the replica placement scheme used here. - - Closest node should be last. - */ - // abstract List<Node> sites(Object object, List<Node> nodes); - - /** - Place this object to rebuild. - */ - abstract void rebuild(SingleObject object, List<Node> nodes); + abstract Copy regimen(SingleObject object, List<Node> nodes); /** Locate a replica of the given object on one of the given nodes diff --git a/sim/gobs/gobs/sim/Simulator.java b/sim/gobs/gobs/sim/Simulator.java index cf8001d..79ef979 100644 --- a/sim/gobs/gobs/sim/Simulator.java +++ b/sim/gobs/gobs/sim/Simulator.java @@ -5,13 +5,16 @@ import java.math.BigInteger; import java.text.DecimalFormat; import java.util.*; +import cern.jet.random.engine.DRand; +import cern.jet.random.Exponential; + import gobs.util.*; import static gobs.util.Tools.bail; /** * <h3> GOBS placement simulator. </h3> * - * Command-line arguments: <br> + * Command-line arguments: {@link #main} * */ public class Simulator @@ -25,6 +28,8 @@ public class Simulator int faults; int reads; int writes; + + double mttf; public Placement placer; public NodeFactory hub; @@ -91,10 +96,16 @@ public class Simulator Cached value of printAccesses property */ boolean printAccesses = false; + + /** + Source of exponentially distributed random numbers + for MTTF samples. + */ + Exponential exponential; public Simulator() { - File.unique = 0; + File.unique = 0; } public Simulator(Properties properties) @@ -102,7 +113,13 @@ public class Simulator this(); initialize(properties); } - + + /** + Run a single Simulator instance. <br> + Command-line arguments: <br> + -p : Change properties file path <br> + key=value : Set property key to value + */ public static void main(String[] args) { if (args.length < 1) @@ -186,7 +203,12 @@ public class Simulator fileCount = integer("files"); faults = integer("faults"); reads = integer("reads"); - writes = integer("writes"); + writes = integer("writes"); + + mttf = decimal("mttf"); + DRand engine = new DRand(Bits.nextInt()); + double lambda = 1.0/mttf; + exponential = new Exponential(lambda, engine); } void setupFileFactory() @@ -400,28 +422,43 @@ public class Simulator void induceFaults() { System.out.println("faults: " + faults); - + Set<gobs.sim.Object> todo = + new LinkedHashSet<gobs.sim.Object>(); + Set<gobs.sim.Object> done = + new LinkedHashSet<gobs.sim.Object>(); for (int fault = 0; fault < faults; fault++) { - startRebuild(); + double nextFault = exponential.nextDouble(); + startRebuild(nextFault); int index = Bits.nextInt(nodes.size()); Node failed = nodes.remove(index); System.out.println("failed: " + failed.name(this) + " (" + failed.objects.size() + ")"); - Collection<gobs.sim.Object> replacements = failed.collection(); - for (gobs.sim.Object object : replacements) - placer.rebuild(object, nodes); + todo.addAll(failed.collection()); + Map<gobs.sim.Object,Copy> copies = + new HashMap<gobs.sim.Object,Copy>(); + for (gobs.sim.Object object : todo) + { + Copy copy = placer.regimen(object, nodes); + copies.put(object, copy); + } + currentRebuild.rebuild(time, copies, todo, done); + for (gobs.sim.Object object : done) + { + Copy copy = copies.get(object); + placer.copy(copy); + } if (string("print.nodes").equals("true")) printNodeObjects(nodes); - currentRebuild.rebuild(time); completeRebuild(); } } - Rebuild startRebuild() + Rebuild startRebuild(double nextFault) { Rebuild rebuild = new Rebuild(this); currentRebuild = rebuild; + rebuild.nextFault(nextFault); return rebuild; } diff --git a/sim/gobs/gobs/test/TestExponential.java b/sim/gobs/gobs/test/TestExponential.java index 312fdda..ab8e7e7 100644 --- a/sim/gobs/gobs/test/TestExponential.java +++ b/sim/gobs/gobs/test/TestExponential.java @@ -8,7 +8,9 @@ import cern.jet.random.Exponential; import org.jfree.data.xy.XYSeries; -import gobs.plot.SimplePlot; +import gobs.plot.SimplePlot; +import gobs.util.Stats; + import static gobs.util.Tools.check; /** @@ -43,6 +45,8 @@ class TestExponential { series.add(i, results.get(i)); } + + System.out.println("avg: " + Stats.avg(results)); SimplePlot.plot(series, "TestExponential", "step", "length", "testexponential.eps"); diff --git a/sim/gobs/gobs/util/Stats.java b/sim/gobs/gobs/util/Stats.java index 7c8792c..0de594f 100644 --- a/sim/gobs/gobs/util/Stats.java +++ b/sim/gobs/gobs/util/Stats.java @@ -19,6 +19,14 @@ public class Stats result = list.size(); return result; } + + public static double avg(List<Double> data) + { + double sum = 0.0; + for (Double d : data) + sum += d; + return sum/data.size(); + } /** Return the average of the corresponding hooks/post-receive -- Grayskull Repository

1 0

[Gs-commits] Grayskull Repository branch, master, updated. git-migration-178-gb968196
by noreply＠mcs.anl.gov 30 Nov '09

30 Nov '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via b968196f03039954f969f3699666391b36bfd909 (commit) from e056bfcc068a2d5429fec8fb34aa8fde5c6a523d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit b968196f03039954f969f3699666391b36bfd909 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Mon Nov 30 11:39:44 2009 -0600 Correction for Rebuild to handle Copies that finish at the same time. ----------------------------------------------------------------------- Summary of changes: sim/gobs/gobs/sim/Rebuild.java | 12 ++++++------ sim/gobs/gobs/sim/Simulator.java | 2 +- sim/gobs/sweeps/dc02kpv01.cfg | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) Diff of changes: diff --git a/sim/gobs/gobs/sim/Rebuild.java b/sim/gobs/gobs/sim/Rebuild.java index 1ce4fc7..a60314e 100644 --- a/sim/gobs/gobs/sim/Rebuild.java +++ b/sim/gobs/gobs/sim/Rebuild.java @@ -29,9 +29,9 @@ class Rebuild This must be large enough to differentiate all copies based on start time. */ - static double selectTime = 0.000001; + static double selectTime = 0.00001; - static DecimalFormat df = new DecimalFormat("0.0"); + static DecimalFormat df = new DecimalFormat("0.000000000"); Simulator simulator; @@ -80,15 +80,15 @@ class Rebuild schedule.add(copy); busy.add(copy.source); busy.add(copy.destination); - boolean b = inflight.add(copy); - if (!b) + while (!inflight.add(copy)) { System.out.println("add failed!"); System.out.println("" + copy); dumpInflight(inflight); - System.exit(2); + time += selectTime; + copy.startAt(time); + // System.exit(2); } - time += selectTime; if (simulator.bool("print.rebuilds")) System.out.println("start: " + copy.log()); } diff --git a/sim/gobs/gobs/sim/Simulator.java b/sim/gobs/gobs/sim/Simulator.java index fd1f896..cf8001d 100644 --- a/sim/gobs/gobs/sim/Simulator.java +++ b/sim/gobs/gobs/sim/Simulator.java @@ -289,7 +289,7 @@ public class Simulator Collection<gobs.sim.Object> set; do { - System.out.println("allocateFile(): iterate: " + tries); + // System.out.println("allocateFile(): iterate: " + tries); if (tries++ >= maxTries) bail("Could not allocate new object ids!"); diff --git a/sim/gobs/sweeps/dc02kpv01.cfg b/sim/gobs/sweeps/dc02kpv01.cfg index 77ab756..a8e3954 100644 --- a/sim/gobs/sweeps/dc02kpv01.cfg +++ b/sim/gobs/sweeps/dc02kpv01.cfg @@ -1,5 +1,5 @@ -include = kpv01.cfg system_size_1000.cfg +include = kpv01.cfg system_size_20-1000.cfg # This declustering test has no user accesses: hooks/post-receive -- Grayskull Repository

1 0

[Gs-commits] Grayskull Repository branch, master, updated. git-migration-177-ge056bfc
by noreply＠mcs.anl.gov 30 Nov '09

30 Nov '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via e056bfcc068a2d5429fec8fb34aa8fde5c6a523d (commit) via 0b36c45ef23c8246fbef038f0cc2dd53a8eaa0c9 (commit) via fbebe943b7959a802ce9cb5b7bd7faf3abb8f264 (commit) from a2f89b1e8aea34becc3956198d6b8ff8d07d2eb7 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit e056bfcc068a2d5429fec8fb34aa8fde5c6a523d Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Mon Nov 30 10:18:14 2009 -0600 New plot for wiki. commit 0b36c45ef23c8246fbef038f0cc2dd53a8eaa0c9 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Fri Nov 27 15:01:08 2009 -0600 Slightly bigger Rebuild01 case. commit fbebe943b7959a802ce9cb5b7bd7faf3abb8f264 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Fri Nov 27 15:00:26 2009 -0600 Better error checking and performance for placing Objects on Nodes. Should have done this a while ago. ----------------------------------------------------------------------- Summary of changes: sim/gobs/gobs/sim/Addressable.java | 25 +++++--- sim/gobs/gobs/sim/Node.java | 59 +++++++++---------- sim/gobs/gobs/sim/Object.java | 2 +- sim/gobs/gobs/sim/Simulator.java | 17 ++++-- sim/gobs/gobs/sim/SingleObject.java | 6 +- sim/gobs/gobs/sweep/ArraySweep.java | 7 +- sim/gobs/gobs/sweep/Rebuild01.java | 2 +- sim/gobs/gobs/test/TestELHS.java | 34 +++++++++++ sim/gobs/gobs/test/TestExponential.java | 2 +- sim/gobs/gobs/util/EnhancedLinkedHashSet.java | 4 +- .../{examples/params.cfg => sweeps/rb01-range.cfg} | 31 +++++++--- .../{examples/gobs.cfg => sweeps/rb_output.cfg} | 6 -- .../{examples/params.cfg => wiki/rb01-range.cfg} | 31 +++++++--- sim/gobs/{examples/gobs.cfg => wiki/rb_output.cfg} | 6 -- 14 files changed, 144 insertions(+), 88 deletions(-) create mode 100644 sim/gobs/gobs/test/TestELHS.java copy sim/gobs/{examples/params.cfg => sweeps/rb01-range.cfg} (69%) copy sim/gobs/{examples/gobs.cfg => sweeps/rb_output.cfg} (82%) copy sim/gobs/{examples/params.cfg => wiki/rb01-range.cfg} (69%) copy sim/gobs/{examples/gobs.cfg => wiki/rb_output.cfg} (82%) Diff of changes: diff --git a/sim/gobs/gobs/sim/Addressable.java b/sim/gobs/gobs/sim/Addressable.java index a089c30..83af737 100644 --- a/sim/gobs/gobs/sim/Addressable.java +++ b/sim/gobs/gobs/sim/Addressable.java @@ -8,6 +8,7 @@ import java.math.BigInteger; * */ class Addressable + implements Comparable<Addressable> { public BigInteger id; @@ -111,19 +112,25 @@ class Addressable return id.toString(); } + /** + Used by data structures. + */ public int hashCode() { - return id.hashCode(); + return id.intValue(); } - - public boolean equals(Object other) + + /** + Used by data structures. + */ + public boolean equals(java.lang.Object other) { - if (other instanceof Addressable) - { - if (((Addressable) other).id.equals(id)) - return true; - } - return false; + return (hashCode() == other.hashCode()); + } + + public int compareTo(Addressable other) + { + return id.compareTo(other.id); } /* diff --git a/sim/gobs/gobs/sim/Node.java b/sim/gobs/gobs/sim/Node.java index a0b1178..e229824 100644 --- a/sim/gobs/gobs/sim/Node.java +++ b/sim/gobs/gobs/sim/Node.java @@ -4,6 +4,8 @@ package gobs.sim; import java.math.BigInteger; import java.util.*; +import gobs.util.Tools; + /** * Simulates an object store. * */ @@ -14,7 +16,7 @@ public class Node /** Object store. */ - public List<gobs.sim.Object> objects; + public Map<BigInteger,gobs.sim.Object> objects; /** Counter for number of pushes during rebuilds. @@ -34,12 +36,17 @@ public class Node Node(BigInteger id) { super(id); - objects = new ArrayList<gobs.sim.Object>(); + objects = new TreeMap<BigInteger,gobs.sim.Object>(); } void add(gobs.sim.Object object) { - objects.add(object); + Tools.check(! objects.containsKey(object.id), + "Node.add(): " + id + " contains " + + object.toString(Simulator.current) + "\n" + + display()); + + objects.put(object.id, object); } /** @@ -47,16 +54,7 @@ public class Node */ gobs.sim.Object get(BigInteger id) { - for (Iterator<gobs.sim.Object> it = objects.iterator(); - it.hasNext(); ) - { - gobs.sim.Object object = it.next(); - if (object.id.equals(id)) - { - return object; - } - } - return null; + return objects.get(id); } /** @@ -64,20 +62,18 @@ public class Node */ gobs.sim.Object remove(BigInteger id) { - for (Iterator<gobs.sim.Object> it = objects.iterator(); - it.hasNext(); ) - { - gobs.sim.Object object = it.next(); - if (object.id.equals(id)) - { - it.remove(); - return object; - } - } - return null; + return objects.remove(id); } /** + Return a Collection of all of this Node's objects. + */ + public Collection<gobs.sim.Object> collection() + { + return objects.values(); + } + + /** Act as though this Node's disk failed and was replaced. Essentially just empties {@link #objects}. */ @@ -91,12 +87,7 @@ public class Node */ boolean contains(BigInteger id) { - for (gobs.sim.Object object : objects) - { - if (object.id.equals(id)) - return true; - } - return false; + return objects.containsKey(id); } String loadReport(int B) @@ -104,13 +95,19 @@ public class Node return bitString(B) + " " + objects.size(); } + public String display() + { + return display(Simulator.current); + } + public String display(Simulator simulator) { StringBuffer result = new StringBuffer(objects.size()*10); result.append(name(simulator)); result.append(":\t"); - for (gobs.sim.Object object : objects) + for (BigInteger i : objects.keySet()) { + gobs.sim.Object object = objects.get(i); result.append(object.toString(simulator)).append(" "); } result.append("\n"); diff --git a/sim/gobs/gobs/sim/Object.java b/sim/gobs/gobs/sim/Object.java index ab1190e..4100e9c 100644 --- a/sim/gobs/gobs/sim/Object.java +++ b/sim/gobs/gobs/sim/Object.java @@ -7,7 +7,7 @@ import java.math.BigInteger; * Tags objects. * */ -abstract class Object +public abstract class Object extends Addressable { Object(BigInteger id, int size, int replicas) diff --git a/sim/gobs/gobs/sim/Simulator.java b/sim/gobs/gobs/sim/Simulator.java index 04aa48e..fd1f896 100644 --- a/sim/gobs/gobs/sim/Simulator.java +++ b/sim/gobs/gobs/sim/Simulator.java @@ -80,7 +80,7 @@ public class Simulator */ Rebuild currentRebuild = null; - // static Simulator current; + static Simulator current; static final int maxTries = 10; DecimalFormat df = new DecimalFormat("0.00"); @@ -146,6 +146,8 @@ public class Simulator public void execute() { + Simulator.current = this; + setup(); System.out.println("\nexecute:"); @@ -157,7 +159,9 @@ public class Simulator induceFaults(); performAccesses(); - computeStatistics(); + computeStatistics(); + + Simulator.current = null; } /** @@ -285,6 +289,7 @@ public class Simulator Collection<gobs.sim.Object> set; do { + System.out.println("allocateFile(): iterate: " + tries); if (tries++ >= maxTries) bail("Could not allocate new object ids!"); @@ -293,7 +298,7 @@ public class Simulator while (objects.containsAny(set)); file.link(set); - files.add(file); + files.add(file); objects.addAll(set); } @@ -403,7 +408,7 @@ public class Simulator Node failed = nodes.remove(index); System.out.println("failed: " + failed.name(this) + " (" + failed.objects.size() + ")"); - List<gobs.sim.Object> replacements = failed.objects; + Collection<gobs.sim.Object> replacements = failed.collection(); for (gobs.sim.Object object : replacements) placer.rebuild(object, nodes); if (string("print.nodes").equals("true")) @@ -621,7 +626,7 @@ public class Simulator double interval = decimal("plot.traffic.interval"); List<Double> intervals = new ArrayList<Double>(); - List<List<Integer>> results = new ArrayList<List<Integer>>(); + List<List<Integer>> results = new ArrayList<List<Integer>>(); for (Rebuild rebuild : rebuilds) { List<Integer> sample = new ArrayList<Integer>(); @@ -644,7 +649,7 @@ public class Simulator intervals.add(d+=interval); List<Double> avg = Stats.avg(results); - System.out.println(avg); + // System.out.println(avg); stats.results.setProperty("traffic-x", intervals.toString()); stats.results.setProperty("traffic-y", avg.toString()); diff --git a/sim/gobs/gobs/sim/SingleObject.java b/sim/gobs/gobs/sim/SingleObject.java index a06e49c..2b9b6d2 100644 --- a/sim/gobs/gobs/sim/SingleObject.java +++ b/sim/gobs/gobs/sim/SingleObject.java @@ -7,10 +7,10 @@ import java.math.BigInteger; * Simulates a single object. * */ -class SingleObject - extends Object +public class SingleObject + extends gobs.sim.Object { - SingleObject(BigInteger id, int size, int replicas) + public SingleObject(BigInteger id, int size, int replicas) { super(id, size, replicas); } diff --git a/sim/gobs/gobs/sweep/ArraySweep.java b/sim/gobs/gobs/sweep/ArraySweep.java index c23087d..2155a86 100644 --- a/sim/gobs/gobs/sweep/ArraySweep.java +++ b/sim/gobs/gobs/sweep/ArraySweep.java @@ -46,7 +46,8 @@ class ArraySweep List<String> values, String yname) { repeats = Tools.integer(properties, "repeats"); - + System.out.println("repeats: " + repeats); + if (repeats <= 0) System.out.println("WARNING: repeats=" + repeats); @@ -61,8 +62,8 @@ class ArraySweep Simulator simulator = new Simulator(properties); simulator.execute(); stats.add(simulator.stats); - System.out.println("result: " + - Arrays.toString(result(simulator, yname))); + /* System.out.println("result: " + + Arrays.toString(result(simulator, yname))); */ results.add(result(simulator, yname)); } XYSeries series = new XYSeries(description + " " + diff --git a/sim/gobs/gobs/sweep/Rebuild01.java b/sim/gobs/gobs/sweep/Rebuild01.java index 14d33a1..073de6c 100644 --- a/sim/gobs/gobs/sweep/Rebuild01.java +++ b/sim/gobs/gobs/sweep/Rebuild01.java @@ -33,7 +33,7 @@ class Rebuild01 int minNodes = Tools.integer(properties, "nodes.min"); int maxNodes = Tools.integer(properties, "nodes.max"); - String step = Tools.string(properties, "nodes.step"); + String step = Tools.string(properties, "nodes.step"); List<String> values = Sweep.values(step, minNodes, maxNodes); System.out.println("Rebuild01: " + values); diff --git a/sim/gobs/gobs/test/TestELHS.java b/sim/gobs/gobs/test/TestELHS.java new file mode 100644 index 0000000..f7f7dd3 --- /dev/null +++ b/sim/gobs/gobs/test/TestELHS.java @@ -0,0 +1,34 @@ + +package gobs.test; + +import java.math.BigInteger; +import java.util.*; + +import gobs.sim.*; +import gobs.util.EnhancedLinkedHashSet; + +/** + * Test the EnhancedLinkedHashSet data structure. + * */ + +class TestELHS +{ + public static void main(String[] args) + { + int n = 10; + + EnhancedLinkedHashSet<gobs.sim.Object> set = + new EnhancedLinkedHashSet<gobs.sim.Object>(); + + for (int i = 0; i < n; i++) + { + List<gobs.sim.Object> list = new ArrayList<gobs.sim.Object>(); + for (int j = 0; j < n; j++) + list.add(new SingleObject(new BigInteger("" + j), 0, 0)); + if (set.containsAny(list)) + System.out.println("containsAny!"); + set.addAll(list); + } + System.out.println(set.size()); + } +} diff --git a/sim/gobs/gobs/test/TestExponential.java b/sim/gobs/gobs/test/TestExponential.java index ae438c8..312fdda 100644 --- a/sim/gobs/gobs/test/TestExponential.java +++ b/sim/gobs/gobs/test/TestExponential.java @@ -24,7 +24,7 @@ class TestExponential public static void main(String[] args) { check(args.length == 2, - "usage: TestExponential <lambda> <n>"); + "usage: TestExponential <lambda> <n>"); double lambda = Double.parseDouble(args[0]); int n = Integer.parseInt(args[1]); diff --git a/sim/gobs/gobs/util/EnhancedLinkedHashSet.java b/sim/gobs/gobs/util/EnhancedLinkedHashSet.java index 10d1e12..53980f2 100644 --- a/sim/gobs/gobs/util/EnhancedLinkedHashSet.java +++ b/sim/gobs/gobs/util/EnhancedLinkedHashSet.java @@ -17,10 +17,8 @@ public class EnhancedLinkedHashSet<T> public boolean containsAny(Collection<T> c) { for (T thing : c) - { if (contains(thing)) - return true; - } + return true; return false; } diff --git a/sim/gobs/examples/params.cfg b/sim/gobs/sweeps/rb01-range.cfg similarity index 69% copy from sim/gobs/examples/params.cfg copy to sim/gobs/sweeps/rb01-range.cfg index 0f23672..6bab4cb 100644 --- a/sim/gobs/examples/params.cfg +++ b/sim/gobs/sweeps/rb01-range.cfg @@ -1,4 +1,13 @@ +# Simulator configuration + +include = rb_output.cfg + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 1000 + ### Parameters for GOBS Simulator run ### General parameters: @@ -6,22 +15,26 @@ # Bit-length of addresses: B = 16 # Number of servers: -nodes = 128 +nodes.min = 20 +nodes.max = 200 +# Step through nodes: See AbstractSweep.values() +nodes.step = mids # Number of files: -files = 1000 +files = 100 # Number of faults: faults = 1 # Number of accesses: -reads = 10 +reads = 0 writes = 0 ### Choose a PlacementScheme implementation: #impl.placement = Kademlia -#impl.placement = Nearest -impl.placement = VertexGroups +impl.placement = Nearest +#impl.placement = VertexGroups -VertexGroupPrefix = 6 +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() replica.source = primary ### Choose a NodeScheme implementation: @@ -34,13 +47,13 @@ impl.nodefactory = SimpleNodes # FileFactory implementation PlainFiles impl.filefactory = PlainFiles # File size: -file.size = 140 +file.size = 14000 # Strip size: -file.strip = 4 +file.strip = 1000 # File/object replicas: file.replicas = 3 # Number of objects per file: -file.width = 3 +file.width = 4 # FileFactory implementation TraceFiles #impl.filefactory = TraceFiles diff --git a/sim/gobs/examples/gobs.cfg b/sim/gobs/sweeps/rb_output.cfg similarity index 82% copy from sim/gobs/examples/gobs.cfg copy to sim/gobs/sweeps/rb_output.cfg index ddb885c..99d53e4 100644 --- a/sim/gobs/examples/gobs.cfg +++ b/sim/gobs/sweeps/rb_output.cfg @@ -1,6 +1,4 @@ -# Simulator configuration - # Basics: # Print all properties at simulation start @@ -18,7 +16,3 @@ print.loaddiffs = false # Print user accesses print.accesses = false -# Statistics: - -# Rebuild traffic over time analysis: -plot.traffic.interval = 10 diff --git a/sim/gobs/examples/params.cfg b/sim/gobs/wiki/rb01-range.cfg similarity index 69% copy from sim/gobs/examples/params.cfg copy to sim/gobs/wiki/rb01-range.cfg index 0f23672..6bab4cb 100644 --- a/sim/gobs/examples/params.cfg +++ b/sim/gobs/wiki/rb01-range.cfg @@ -1,4 +1,13 @@ +# Simulator configuration + +include = rb_output.cfg + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 1000 + ### Parameters for GOBS Simulator run ### General parameters: @@ -6,22 +15,26 @@ # Bit-length of addresses: B = 16 # Number of servers: -nodes = 128 +nodes.min = 20 +nodes.max = 200 +# Step through nodes: See AbstractSweep.values() +nodes.step = mids # Number of files: -files = 1000 +files = 100 # Number of faults: faults = 1 # Number of accesses: -reads = 10 +reads = 0 writes = 0 ### Choose a PlacementScheme implementation: #impl.placement = Kademlia -#impl.placement = Nearest -impl.placement = VertexGroups +impl.placement = Nearest +#impl.placement = VertexGroups -VertexGroupPrefix = 6 +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() replica.source = primary ### Choose a NodeScheme implementation: @@ -34,13 +47,13 @@ impl.nodefactory = SimpleNodes # FileFactory implementation PlainFiles impl.filefactory = PlainFiles # File size: -file.size = 140 +file.size = 14000 # Strip size: -file.strip = 4 +file.strip = 1000 # File/object replicas: file.replicas = 3 # Number of objects per file: -file.width = 3 +file.width = 4 # FileFactory implementation TraceFiles #impl.filefactory = TraceFiles diff --git a/sim/gobs/examples/gobs.cfg b/sim/gobs/wiki/rb_output.cfg similarity index 82% copy from sim/gobs/examples/gobs.cfg copy to sim/gobs/wiki/rb_output.cfg index ddb885c..99d53e4 100644 --- a/sim/gobs/examples/gobs.cfg +++ b/sim/gobs/wiki/rb_output.cfg @@ -1,6 +1,4 @@ -# Simulator configuration - # Basics: # Print all properties at simulation start @@ -18,7 +16,3 @@ print.loaddiffs = false # Print user accesses print.accesses = false -# Statistics: - -# Rebuild traffic over time analysis: -plot.traffic.interval = 10 hooks/post-receive -- Grayskull Repository

1 0

[Gs-commits] Grayskull Repository branch, master, updated. git-migration-174-ga2f89b1
by noreply＠mcs.anl.gov 27 Nov '09

27 Nov '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via a2f89b1e8aea34becc3956198d6b8ff8d07d2eb7 (commit) via 77e566ec3b1e8ec621f538372796063742539818 (commit) via aa8d452d9cd0057f54eda33701ba3f3e88b76197 (commit) via 0ee0d931de87245e0ca535def4ad97a40486cca4 (commit) from f309bdd074af46cd897b3c5697594c958ad4675f (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit a2f89b1e8aea34becc3956198d6b8ff8d07d2eb7 Merge: 77e566ec3b1e8ec621f538372796063742539818 f309bdd074af46cd897b3c5697594c958ad4675f Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Fri Nov 27 13:08:11 2009 -0600 Merge branch 'master' of /homes/dkimpe/git/grayskull commit 77e566ec3b1e8ec621f538372796063742539818 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Fri Nov 27 13:05:11 2009 -0600 Add exponential distribution for MTBF experiments. Got Colt from http://acs.lbl.gov/~hoschek/colt commit aa8d452d9cd0057f54eda33701ba3f3e88b76197 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Fri Nov 27 13:03:47 2009 -0600 New check() function. commit 0ee0d931de87245e0ca535def4ad97a40486cca4 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Fri Nov 27 13:03:12 2009 -0600 Minor changes for wiki post. ----------------------------------------------------------------------- Summary of changes: sim/gobs/gobs/sweep/Rebuild01.java | 6 +++- sim/gobs/gobs/test/TestExponential.java | 50 ++++++++++++++++++++++++++++++ sim/gobs/gobs/util/Tools.java | 9 +++++ sim/gobs/prereqs/colt.jar | Bin 0 -> 581945 bytes sim/gobs/{sweeps => wiki}/rb01-small.cfg | 0 5 files changed, 64 insertions(+), 1 deletions(-) create mode 100644 sim/gobs/gobs/test/TestExponential.java create mode 100644 sim/gobs/prereqs/colt.jar copy sim/gobs/{sweeps => wiki}/rb01-small.cfg (100%) Diff of changes: diff --git a/sim/gobs/gobs/sweep/Rebuild01.java b/sim/gobs/gobs/sweep/Rebuild01.java index 0319f00..14d33a1 100644 --- a/sim/gobs/gobs/sweep/Rebuild01.java +++ b/sim/gobs/gobs/sweep/Rebuild01.java @@ -16,6 +16,9 @@ import gobs.plot.*; * Usage: Rebuild01 -n <repeats> *.cfg * */ +// good runs: +// seed=828 -n 1 + class Rebuild01 { static Properties properties; @@ -62,7 +65,8 @@ class Rebuild01 collection.addSeries(series); MultiplePlot.plot(collection, "Rebuild Traffic", - "time", "load fraction(%)", output); + "time from rebuild start", "copies in flight", + output); // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", // "nodes", "load", "tmp.txt"); diff --git a/sim/gobs/gobs/test/TestExponential.java b/sim/gobs/gobs/test/TestExponential.java new file mode 100644 index 0000000..ae438c8 --- /dev/null +++ b/sim/gobs/gobs/test/TestExponential.java @@ -0,0 +1,50 @@ + +package gobs.test; + +import java.util.*; + +import cern.jet.random.engine.DRand; +import cern.jet.random.Exponential; + +import org.jfree.data.xy.XYSeries; + +import gobs.plot.SimplePlot; +import static gobs.util.Tools.check; + +/** + * Test the Exponential distribution provided by Colt. + * + * Generates testexponential.eps + * + * Usage: TestExponential <lambda> <n> + * */ + +class TestExponential +{ + public static void main(String[] args) + { + check(args.length == 2, + "usage: TestExponential <lambda> <n>"); + + double lambda = Double.parseDouble(args[0]); + int n = Integer.parseInt(args[1]); + + DRand engine = new DRand(new Date()); + Exponential ed = new Exponential(lambda, engine); + + XYSeries series = new XYSeries("TestExponential"); + List<Double> results = new ArrayList<Double>(n); + for (int i = 0; i < n; i++) + { + results.add(ed.nextDouble()); + } + Collections.sort(results); + for (int i = 0; i < n; i++) + { + series.add(i, results.get(i)); + } + + SimplePlot.plot(series, "TestExponential", "step", "length", + "testexponential.eps"); + } +} diff --git a/sim/gobs/gobs/util/Tools.java b/sim/gobs/gobs/util/Tools.java index f65f60b..a3a25e3 100644 --- a/sim/gobs/gobs/util/Tools.java +++ b/sim/gobs/gobs/util/Tools.java @@ -83,6 +83,15 @@ public class Tools { return (properties.getProperty(name) != null); } + + /** + Call bail(msg) if condition is false. + */ + public static void check(boolean condition, String msg) + { + if (!condition) + bail(msg); + } public static void bail(String msg) { diff --git a/sim/gobs/prereqs/colt.jar b/sim/gobs/prereqs/colt.jar new file mode 100644 index 0000000..a7192f6 Binary files /dev/null and b/sim/gobs/prereqs/colt.jar differ diff --git a/sim/gobs/sweeps/rb01-small.cfg b/sim/gobs/wiki/rb01-small.cfg similarity index 100% copy from sim/gobs/sweeps/rb01-small.cfg copy to sim/gobs/wiki/rb01-small.cfg hooks/post-receive -- Grayskull Repository

1 0

[Gs-commits] Grayskull Repository branch, master, updated. git-migration-170-gf309bdd
by noreply＠mcs.anl.gov 24 Nov '09

24 Nov '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via f309bdd074af46cd897b3c5697594c958ad4675f (commit) via ba483aa2ecd20ea7e5ba529b86e0c3b3575e45ec (commit) via 0525078315616682c251e302f234ff5621b6be6a (commit) via 550c1e1758f054b61cf17f95d2610c1efdee54f8 (commit) via 3f10a11a24e81f4fd07efa040897677af271ab0d (commit) from 3fe810741014915db1802828dbdc0db69f22103c (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit f309bdd074af46cd897b3c5697594c958ad4675f Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Nov 24 17:13:22 2009 -0500 read log offset from disk when starting out commit ba483aa2ecd20ea7e5ba529b86e0c3b3575e45ec Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Nov 24 16:24:26 2009 -0500 stubbed in txn_close function commit 0525078315616682c251e302f234ff5621b6be6a Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Nov 24 15:14:19 2009 -0500 trivial testing of new write function commit 550c1e1758f054b61cf17f95d2610c1efdee54f8 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Nov 24 15:03:13 2009 -0500 implemented write function; untested commit 3f10a11a24e81f4fd07efa040897677af271ab0d Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Nov 24 12:10:36 2009 -0500 added txn_open function to cosd prototype ----------------------------------------------------------------------- Summary of changes: .../gsl/resources/cosd-prototype/cosd-prototype.c | 483 +++++++++++++++++++- .../resources/cosd-prototype/cosd-prototype.gsh | 34 ++ .../src/gsl/resources/cosd-prototype/test/cosd1.gs | 25 + 3 files changed, 539 insertions(+), 3 deletions(-) Diff of changes: diff --git a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c index 93d6765..38b40ec 100644 --- a/code/src/gsl/resources/cosd-prototype/cosd-prototype.c +++ b/code/src/gsl/resources/cosd-prototype/cosd-prototype.c @@ -1,4 +1,5 @@ #define _GNU_SOURCE +#define _XOPEN_SOURCE 600 #include <time.h> #include <sys/time.h> @@ -20,6 +21,7 @@ #include "include/gs-op.h" #include "include/gs-oplist.h" #include "include/gs-opcache.h" +#include "include/gs-hash.h" #include "resources/cosd-prototype/cosd-prototype.h" #define STORAGE_DEFAULT_OPCACHE_SIZE 1024 @@ -30,6 +32,8 @@ static gs_oplist_t cosd_oplist; static gs_mutex_t cosd_mutex = GS_MUTEX_INITIALIZER; static char* cosd_path = NULL; static char* cosd_log_path = NULL; +static struct gs_hash_table *txn_table = NULL; +static gs_mutex_t txn_mutex = GS_MUTEX_INITIALIZER; static DB *log_map_dbp = NULL; static DB *missing_ver_dbp = NULL; @@ -37,12 +41,42 @@ static DB *ver_dbp = NULL; static DB *log_offset_dbp = NULL; DB_ENV *envp = NULL; -/* temporary: a global fd used to access object */ -static global_fd = -1; +/* temporary: a global fd and log offset used to access object */ +static int global_fd = -1; +static int64_t global_log_offset = -1; +static gs_mutex_t global_log_mutex = GS_MUTEX_INITIALIZER; static int gs_cosd_resource_id; static enum progress_mode gs_cosd_progress_mode = GS_PROG_NONE; +/* TODO: find this programmatically? */ +#define DIRECT_ALIGN 512 + +/* uniquely identifies each active txn (in memory) */ +struct txn_accumulator +{ + /* txn identifier */ + uint64_t txn_number; + /* list of updates included in this txn */ + struct gs_list_link update_list; + + /* link into hash table of current txns */ + struct gs_hash_link hash_link; +}; + +/* describes each update within a txn (in memory) */ +struct txn_update +{ + /* size */ + int64_t size; + /* logical offset (as seen by caller) */ + int64_t logical_offset; + /* internal log offset */ + int64_t log_offset; + /* link into list of all updates for txn */ + struct gs_list_link list_link; +}; + struct cosd_op { union { @@ -50,6 +84,22 @@ struct cosd_op uint64_t requested_oid; uint64_t* out_oid; } create; + struct write_op{ + uint64_t oid; + uint64_t txn_number; + char** mem_offsets; + int64_t* mem_sizes; + int mem_count; + int64_t* obj_offsets; + int64_t* obj_sizes; + int obj_count; + } write; + struct txn_close_op{ + uint64_t oid; + uint64_t txn_number; + } txn_close; + +#if 0 struct remove_op{ uint64_t oid; } remove; @@ -69,7 +119,7 @@ struct cosd_op void** val_array; int* len_array; } getattr_list; - +#endif } u; gs_op_id_t op_id; struct gs_op op; @@ -96,6 +146,14 @@ static void* thread_fn(void* foo) return(NULL); } +static int txn_acc_compare(void *key, struct gs_hash_link *link) +{ + struct txn_accumulator *txn_acc; + + txn_acc = gs_hash_get_entry(link, struct txn_accumulator, hash_link); + return(*((uint64_t*)key) == txn_acc->txn_number); +} + static int gs_cosd_launch_op(struct gs_op *op, enum progress_mode mode) { int ret; @@ -369,6 +427,17 @@ int gs_cosd_init(enum progress_mode mode, const char* path) return(-1); } + gs_mutex_lock(&txn_mutex); + /* create hash table for open txns */ + txn_table = gs_hash_init(txn_acc_compare, gs_hash_64bit_hash, 1024); + if(!txn_table) + { + /* TODO: error handling */ + gs_mutex_unlock(&txn_mutex); + return(-1); + } + gs_mutex_unlock(&txn_mutex); + GS_OPLIST_INIT(&cosd_oplist); ret = GS_OPCACHE_INIT(struct cosd_op, op, STORAGE_DEFAULT_OPCACHE_SIZE, &cosd_opcache); @@ -416,9 +485,212 @@ int gs_cosd_finalize(void) ver_dbp->close(ver_dbp, 0); log_offset_dbp->close(log_offset_dbp, 0); + gs_mutex_lock(&txn_mutex); + gs_hash_finalize(txn_table); + gs_mutex_unlock(&txn_mutex); + return 0; } +/* write_op_worker() + * + * writes data to an object + */ +static int write_op_worker(struct gs_op* op) +{ + struct cosd_op *c_op; + int ret; + DB_TXN *txn; + DBT key, value; + char log_name[PATH_MAX]; + int done = 0; + int mem_index = 0; + int obj_index = 0; + struct txn_update *tmp_update; + char* mem_ptr; + void* scratch_buffer = NULL; + int scratch_size = 0; + int64_t aligned_size = 0; + struct txn_accumulator* txn_acc; + struct gs_hash_link* hash_link; + + c_op = gs_op_entry(op, struct cosd_op, op); + assert(c_op); + + /* only support one object for now */ + assert(c_op->u.write.oid == 1); + + assert(c_op->u.write.mem_count > 0); + assert(c_op->u.write.obj_count > 0); + + if(global_fd < 0) + { + /* open log file */ + sprintf(log_name, "%s/%llu.dat", cosd_log_path, + c_op->u.write.oid); + ret = open(log_name, O_RDWR|O_DIRECT|O_EXCL|O_NOATIME, + S_IRUSR|S_IWUSR); + if(ret < 0) + { + c_op->error_code = -errno; + return(1); + } + global_fd = ret; + } + + /* TODO: safety check that total mem size equals total obj size */ + /* TODO: safety check no zero buffers in either list */ + + while(!done) + { + /* structure to track this update in the txn */ + tmp_update = malloc(sizeof(*tmp_update)); + if(!tmp_update) + { + /* TODO: err handling */ + c_op->error_code = -errno; + return(1); + } + + /* find the next chunk that is contiguous in memory and disk */ + mem_ptr = c_op->u.write.mem_offsets[mem_index]; + tmp_update->logical_offset = c_op->u.write.obj_offsets[obj_index]; + tmp_update->size = c_op->u.write.mem_sizes[mem_index]; + if(tmp_update->size >= c_op->u.write.obj_sizes[obj_index]) + { + /* mem region is bigger than obj region */ + tmp_update->size = c_op->u.write.obj_sizes[obj_index]; + } + + /* advance pointers */ + c_op->u.write.mem_sizes[mem_index] -= tmp_update->size; + c_op->u.write.mem_offsets[mem_index] += tmp_update->size; + if(c_op->u.write.mem_sizes[mem_index] == 0) + { + if(mem_index == (c_op->u.write.mem_count -1)) + done = 1; + else + mem_index++; + } + c_op->u.write.obj_sizes[obj_index] -= tmp_update->size; + c_op->u.write.obj_offsets[obj_index] += tmp_update->size; + if(c_op->u.write.obj_sizes[obj_index] == 0) + { + if(obj_index == (c_op->u.write.obj_count -1)) + done = 1; + else + obj_index++; + } + + /* calculate aligned buffer size */ + if(tmp_update->size % DIRECT_ALIGN != 0) + aligned_size = + ((tmp_update->size / DIRECT_ALIGN)+1)*DIRECT_ALIGN; + else + aligned_size = tmp_update->size; + + /* TODO: think about how to track this per object; we need to + * atomically get a new log offset, but it doesn't have to be + * flushed to db until we close a transaction. For now using a + * global var and global mutex + */ + /* get next log offset and increment */ + gs_mutex_lock(&global_log_mutex); + if(global_log_offset < 0) + { + memset(&key, 0, sizeof(DBT)); + key.data = &c_op->u.write.oid; + key.size = sizeof(c_op->u.write.oid); + memset(&value, 0, sizeof(DBT)); + value.data = &global_log_offset; + value.ulen = sizeof(global_log_offset); + value.flags = DB_DBT_USERMEM; + + /* read log offset from db */ + ret = envp->txn_begin(envp, NULL, &txn, 0); + if(ret != 0) + { + /* TODO: error handling */ + c_op->error_code = -1; + return(1); + } + + ret = log_offset_dbp->get(log_offset_dbp, txn, &key, &value, 0); + if(ret != 0) + { + /* TODO: error handling */ + c_op->error_code = -1; + return(1); + } + + ret = txn->commit(txn, 0); + if(ret != 0) + { + /* TODO: error handling */ + assert(0); + } + } + + tmp_update->log_offset = global_log_offset; + global_log_offset += aligned_size; + gs_mutex_unlock(&global_log_mutex); + + if(((int)mem_ptr) % DIRECT_ALIGN || tmp_update->size != aligned_size) + { + /* mem buffer is not aligned */ + /* allocate a scratch buffer to copy to */ + if(scratch_size < aligned_size) + { + if(scratch_buffer) + free(scratch_buffer); + scratch_size = aligned_size; + ret = posix_memalign(&scratch_buffer, DIRECT_ALIGN, + aligned_size); + if(ret < 0) + { + /* TODO: err handling */ + c_op->error_code = -errno; + return(1); + } + memcpy(scratch_buffer, mem_ptr, tmp_update->size); + mem_ptr = scratch_buffer; + } + } + + /* actually write the data */ + ret = pwrite(global_fd, mem_ptr, aligned_size, + tmp_update->log_offset); + if(ret < 0) + { + /* TODO: err handling */ + c_op->error_code = -errno; + return(1); + } + + /* track this update in the txn accumulator */ + gs_mutex_lock(&txn_mutex); + hash_link = gs_hash_search(txn_table, &c_op->u.write.txn_number); + if(!hash_link) + { + /* TODO: txn is gone (which could be normal); error handling */ + c_op->error_code = -1; + gs_mutex_unlock(&txn_mutex); + return(1); + } + txn_acc = gs_hash_get_entry(hash_link, struct txn_accumulator, + hash_link); + gs_list_add(&tmp_update->list_link, &txn_acc->update_list); + gs_mutex_unlock(&txn_mutex); + } + + /* cleanup any leftover buffer */ + if(scratch_buffer) + free(scratch_buffer); + + c_op->error_code = 0; + return(1); +} + /* create_op_worker() * * creates an object @@ -468,6 +740,7 @@ static int create_op_worker(struct gs_op* op) memset(&value, 0, sizeof(DBT)); value.data = &val; value.size = sizeof(val); + value.flags = DB_DBT_USERMEM; ret = ver_dbp->put(ver_dbp, txn, &key, &value, 0); if(ret != 0) @@ -497,6 +770,63 @@ static int create_op_worker(struct gs_op* op) return(1); } +gs_ret_t gs_cosd_write_post( + uint64_t oid, + uint64_t txn_number, + char** mem_offsets, + int64_t* mem_sizes, + int mem_count, + int64_t* obj_offsets, + int64_t* obj_sizes, + int obj_count, + void (*callback)(void *ptr, int ret), + void *user_ptr, + gs_hints_t hints, + gs_context_t ctx, + gs_op_id_t *op_id) +{ + struct gs_op *op; + struct cosd_op *c_op; + + op = gs_opcache_get(cosd_opcache); + gs_op_fill(op, callback, user_ptr, hints, ctx); + + c_op = gs_op_entry(op, struct cosd_op, op); + c_op->op_id = gs_id_gen(gs_cosd_resource_id, (uint64_t)(op->cache_id)); + c_op->u.write.oid = oid; + c_op->u.write.txn_number = txn_number; + c_op->u.write.mem_offsets = mem_offsets; + c_op->u.write.mem_sizes = mem_sizes; + c_op->u.write.mem_count = mem_count; + c_op->u.write.obj_offsets = obj_offsets; + c_op->u.write.obj_sizes = obj_sizes; + c_op->u.write.obj_count = obj_count; + + *op_id = c_op->op_id; + + /* TODO: put this in the fill function if we keep it? */ + op->op_worker = write_op_worker; + + gs_cosd_launch_op(op, gs_cosd_progress_mode); + + return 0; +} +gs_ret_t (* gs_cosd_write)( + uint64_t oid, + uint64_t txn_number, + char** mem_offsets, + int64_t* mem_sizes, + int mem_count, + int64_t* obj_offsets, + int64_t* obj_sizes, + int obj_count, + void (*callback)(void *ptr, int ret), + void *user_ptr, + gs_hints_t hints, + gs_context_t ctx, + gs_op_id_t *op_id) = gs_cosd_write_post; + + gs_ret_t gs_cosd_create_post( uint64_t requested_oid, uint64_t* out_oid, @@ -535,6 +865,153 @@ gs_ret_t (* gs_cosd_create)( gs_context_t ctx, gs_op_id_t *op_id) = gs_cosd_create_post; +int gs_cosd_txn_open(uint64_t oid, uint64_t txn_number) +{ + struct txn_accumulator *txn_acc; + + txn_acc = malloc(sizeof(*txn_acc)); + if(!txn_acc) + { + return(-ENOMEM); + } + memset(txn_acc, 0, sizeof(*txn_acc)); + + txn_acc->txn_number = txn_number; + INIT_GS_LIST_HEAD(&txn_acc->update_list); + + gs_mutex_lock(&txn_mutex); + /* TODO: safety test for duplicate txn numbers */ + gs_hash_add(txn_table, &txn_acc->txn_number, &txn_acc->hash_link); + gs_mutex_unlock(&txn_mutex); + + return(0); +} + +/* txn_close_op_worker() + * + * closes a transaction + */ +static int txn_close_op_worker(struct gs_op* op) +{ + struct cosd_op *c_op; + int ret; + DB_TXN *txn; + int64_t val; + DBT key, value; + struct txn_accumulator* txn_acc; + struct gs_hash_link* hash_link; + + c_op = gs_op_entry(op, struct cosd_op, op); + assert(c_op); + + /* only support one object for now */ + assert(c_op->u.txn_close.oid == 1); + + /* pull txn accumulator out of hash so no one can touch it */ + gs_mutex_lock(&txn_mutex); + hash_link = gs_hash_search(txn_table, &c_op->u.txn_close.txn_number); + if(!hash_link) + { + /* TODO: txn is gone (which could be normal); error handling */ + c_op->error_code = -1; + gs_mutex_unlock(&txn_mutex); + return(1); + } + txn_acc = gs_hash_get_entry(hash_link, struct txn_accumulator, + hash_link); + gs_mutex_unlock(&txn_mutex); + + /* do db stuff */ + ret = envp->txn_begin(envp, NULL, &txn, 0); + if(ret != 0) + { + /* TODO: error handling */ + c_op->error_code = -1; + return(1); + } + + /* TODO: do something clever here */ + /* TODO: update all 4 dbs; logical map, missing vers, version, and + * log_offset + */ + /* TODO: be careful about deadlocks while holding the log offset mutex */ + + ret = txn->commit(txn, 0); + if(ret != 0) + { + /* TODO: error handling */ + assert(0); + } + + c_op->error_code = 0; + return(1); + +#if 0 + /* set initial version of 1 */ + val = 1; + memset(&key, 0, sizeof(DBT)); + key.data = &c_op->u.create.requested_oid; + key.size = sizeof(c_op->u.create.requested_oid); + memset(&value, 0, sizeof(DBT)); + value.data = &val; + value.size = sizeof(val); + + ret = ver_dbp->put(ver_dbp, txn, &key, &value, 0); + if(ret != 0) + { + /* TODO: error handling */ + assert(0); + } + + /* set initial log offset of 0 */ + val = 0; + ret = log_offset_dbp->put(log_offset_dbp, txn, &key, &value, 0); + if(ret != 0) + { + /* TODO: error handling */ + assert(0); + } + +#endif +} + +gs_ret_t gs_cosd_txn_close_post( + uint64_t oid, + uint64_t txn_number, + void (*callback)(void *ptr, int ret), + void *user_ptr, + gs_hints_t hints, + gs_context_t ctx, + gs_op_id_t *op_id) +{ + struct gs_op *op; + struct cosd_op *c_op; + + op = gs_opcache_get(cosd_opcache); + gs_op_fill(op, callback, user_ptr, hints, ctx); + + c_op = gs_op_entry(op, struct cosd_op, op); + c_op->op_id = gs_id_gen(gs_cosd_resource_id, (uint64_t)(op->cache_id)); + c_op->u.txn_close.oid = oid; + c_op->u.txn_close.txn_number = txn_number; + + *op_id = c_op->op_id; + + /* TODO: put this in the fill function if we keep it? */ + op->op_worker = txn_close_op_worker; + + gs_cosd_launch_op(op, gs_cosd_progress_mode); + + return 0; +} +gs_ret_t (* gs_cosd_txn_close)( + uint64_t oid, + uint64_t txn_number, + void (*callback)(void *ptr, int ret), + void *user_ptr, + gs_hints_t hints, + gs_context_t ctx, + gs_op_id_t *op_id) = gs_cosd_txn_close_post; /* diff --git a/code/src/gsl/resources/cosd-prototype/cosd-prototype.gsh b/code/src/gsl/resources/cosd-prototype/cosd-prototype.gsh index 1c5a499..e0ab173 100644 --- a/code/src/gsl/resources/cosd-prototype/cosd-prototype.gsh +++ b/code/src/gsl/resources/cosd-prototype/cosd-prototype.gsh @@ -53,6 +53,40 @@ __blocking int gs_cosd_create( uint64_t* out_oid /**< assigned identifier */ ); +/** + * Opens a txn + * \return 0 on success, -errno on failure + */ +int gs_cosd_txn_open( + uint64_t oid, /**< object to operate on */ + uint64_t txn_number /**< caller-assigned txn identifier */ +); + +/** + * Closes a txn + * \return 0 on success, -errno on failure + */ +__blocking int gs_cosd_txn_close( + uint64_t oid, /**< object to operate on */ + uint64_t txn_number /**< caller-assigned txn identifier */ +); + +/** + * Writes data into an object + * \return 0 on success, -errno on failure + */ +__blocking int gs_cosd_write( + uint64_t oid, /**< object to operate on */ + uint64_t txn_number, /**< txn identifier */ + char** mem_offsets, /**< memory region pointers */ + int64_t* mem_sizes, /**< memory region sizes */ + int mem_count, /**< memory region count */ + int64_t* obj_offsets, /**< object region offsets */ + int64_t* obj_sizes, /**< object region sizes */ + int obj_count /**< object region count */ +); + + #endif diff --git a/code/src/gsl/resources/cosd-prototype/test/cosd1.gs b/code/src/gsl/resources/cosd-prototype/test/cosd1.gs index dd62c03..b51376c 100644 --- a/code/src/gsl/resources/cosd-prototype/test/cosd1.gs +++ b/code/src/gsl/resources/cosd-prototype/test/cosd1.gs @@ -24,6 +24,12 @@ static __blocking int do_cosd(void) { int ret; uint64_t out_oid1; + int64_t buffer_sz = 1024; + char buffer[1024]; + char* buffer_offset = buffer; + int64_t obj_sz = 1024; + int64_t obj_offset = 0; + printf("Creating oid 1...\n"); ret = gs_cosd_create(1, &out_oid1); @@ -34,6 +40,25 @@ static __blocking int do_cosd(void) } printf("DONE\n"); + printf("opening txn 1...\n"); + ret = gs_cosd_txn_open(1, 1); + if(ret != 0) + { + printf("Error opening txn 1: %d\n", ret); + return 1; + } + printf("DONE\n"); + + printf("writing 0-1024...\n"); + ret = gs_cosd_write(1, 1, &buffer_offset, &buffer_sz, 1, &obj_offset, + &obj_sz, 1); + if(ret != 0) + { + printf("Error writing 0-1024: %d\n", ret); + return 1; + } + printf("DONE\n"); + return 0; } hooks/post-receive -- Grayskull Repository

1 0

[Gs-commits] Grayskull Repository branch, master, updated. git-migration-165-g3fe8107
by noreply＠mcs.anl.gov 24 Nov '09

24 Nov '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 3fe810741014915db1802828dbdc0db69f22103c (commit) from 6cef20ee9522a4754e925f050a55e4f8df75fc37 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 3fe810741014915db1802828dbdc0db69f22103c Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Nov 24 12:07:32 2009 -0500 fix bug in gs hash table locking The gs_hash_table includes a mutex lock, but it wasn't being initialized. Added a new gs_mutex_init() function for this purpose in addition to the existing static initializer. ----------------------------------------------------------------------- Summary of changes: code/src/gsl/include/gs-hash.h | 3 +++ code/src/gsl/include/gs.h | 4 ++++ 2 files changed, 7 insertions(+), 0 deletions(-) Diff of changes: diff --git a/code/src/gsl/include/gs-hash.h b/code/src/gsl/include/gs-hash.h index 4cf2fab..b06c486 100644 --- a/code/src/gsl/include/gs-hash.h +++ b/code/src/gsl/include/gs-hash.h @@ -72,6 +72,9 @@ static inline struct gs_hash_table *gs_hash_init( INIT_GS_LIST_HEAD(&new_table->array[i]); } + /* initialize mutex lock */ + gs_mutex_init(&new_table->lock, NULL); + return (new_table); } diff --git a/code/src/gsl/include/gs.h b/code/src/gsl/include/gs.h index affed73..44ef8de 100644 --- a/code/src/gsl/include/gs.h +++ b/code/src/gsl/include/gs.h @@ -38,8 +38,10 @@ uint64_t gs_id_lookup(gs_op_id_t id, int *resource_id); #ifdef __GS_POSIX_LOCKING__ typedef pthread_mutex_t gs_mutex_t; +typedef pthread_mutexattr_t gs_mutexattr_t; #define gs_mutex_lock(__mut) pthread_mutex_lock(__mut) #define gs_mutex_unlock(__mut) pthread_mutex_unlock(__mut) +#define gs_mutex_init(__mut, __attr) pthread_mutex_init(__mut, __attr) #define GS_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER typedef pthread_cond_t gs_cond_t; @@ -54,8 +56,10 @@ typedef pthread_condattr_t gs_condattr_t; #else typedef int gs_mutex_t; +typedef int gs_mutexattr_t; static inline int gs_mutex_lock(gs_mutex_t *__mut) { return 0; } static inline int gs_mutex_unlock(gs_mutex_t *__mut) { return 0; } +static inline int gs_mutex_init(gs_mutex_t *__mut, gs_mutexattr_t *__attr) {return 0; } #define GS_MUTEX_INITIALIZER 0 typedef int gs_cond_t; hooks/post-receive -- Grayskull Repository

1 0

[Gs-commits] Grayskull Repository branch, master, updated. git-migration-164-g6cef20e
by noreply＠mcs.anl.gov 24 Nov '09

24 Nov '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 6cef20ee9522a4754e925f050a55e4f8df75fc37 (commit) via a986b14df0aa23bbb7b2027144a21e308e8da712 (commit) from 3d119f79cd5a201a5d1c44af9008114a392ab91d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 6cef20ee9522a4754e925f050a55e4f8df75fc37 Merge: a986b14df0aa23bbb7b2027144a21e308e8da712 3d119f79cd5a201a5d1c44af9008114a392ab91d Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Nov 24 10:49:26 2009 -0500 Merge branch 'master' of [email protected]:~dkimpe/git/grayskull commit a986b14df0aa23bbb7b2027144a21e308e8da712 Author: Phil Carns <carns(a)mcs.anl.gov> Date: Tue Nov 24 10:48:43 2009 -0500 fixed parser dependency for generating headers ----------------------------------------------------------------------- Summary of changes: code/src/gsl/Makefile.in | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) Diff of changes: diff --git a/code/src/gsl/Makefile.in b/code/src/gsl/Makefile.in index 30eddeb..26aac5d 100644 --- a/code/src/gsl/Makefile.in +++ b/code/src/gsl/Makefile.in @@ -284,7 +284,7 @@ $(GSTRHDR_CPP): %.gsh.i: %.gsh $(E)$(call MCC_GS,$<) -I$(srcdir)/include/ -x c -E -o $*.gsh.i \ $(LIBCFLAGS) $(CFLAGS) $(CWARNS) $(call MODCFLAGS_GS,$<) $(srcdir)/$*.gsh -$(GSTRHDR): %.h: %.gsh.i +$(GSTRHDR): %.h: %.gsh.i parser $(E)$(builddir)/parser/gs-parser -j -r $(srcdir)/$*.gsh -o $@ $< $(LIBOBJS): %.o: %.d hooks/post-receive -- Grayskull Repository

1 0

[Gs-commits] Grayskull Repository branch, master, updated. git-migration-162-g3d119f7
by noreply＠mcs.anl.gov 23 Nov '09

23 Nov '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 3d119f79cd5a201a5d1c44af9008114a392ab91d (commit) via 32fc7e45af7fc8896a199b12240c9a9652349820 (commit) via f0c48f86c42364d03e20b0977d18b5c3f878aa83 (commit) from 3449f496d84d41d7a8c9cb3f0754b5e27fb0e987 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 3d119f79cd5a201a5d1c44af9008114a392ab91d Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Mon Nov 23 10:13:55 2009 -0600 Adding Rebuild01 test series. Improved properties (cfg) naming scheme. Renamed some poorly named classes. commit 32fc7e45af7fc8896a199b12240c9a9652349820 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Thu Nov 19 13:12:18 2009 -0600 Image from wiki. commit f0c48f86c42364d03e20b0977d18b5c3f878aa83 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Thu Nov 19 11:10:07 2009 -0800 Drafting Rebuild functionality. ----------------------------------------------------------------------- Summary of changes: sim/gobs/examples/gobs.cfg | 22 ++- sim/gobs/examples/params.cfg | 26 ++-- sim/gobs/gobs/sim/KClosest.java | 2 +- sim/gobs/gobs/sim/Metric.java | 20 +- .../gobs/sim/{NodeScheme.java => NodeFactory.java} | 4 +- .../sim/{PlacementScheme.java => Placement.java} | 6 +- sim/gobs/gobs/sim/PlainFiles.java | 8 +- sim/gobs/gobs/sim/Primary.java | 5 + sim/gobs/gobs/sim/RandomNodes.java | 2 +- sim/gobs/gobs/sim/Rebuild.java | 221 ++++++++++++++++++++ sim/gobs/gobs/sim/Replicated.java | 23 +-- sim/gobs/gobs/sim/SimpleNodes.java | 2 +- sim/gobs/gobs/sim/Simulator.java | 124 +++++++++-- sim/gobs/gobs/sweep/AbstractSweep.java | 34 ++- sim/gobs/gobs/sweep/ArraySweep.java | 9 +- sim/gobs/gobs/sweep/Balance01.java | 2 +- sim/gobs/gobs/sweep/Decluster02.java | 13 +- sim/gobs/gobs/sweep/Decluster03.java | 6 +- sim/gobs/gobs/sweep/Decluster04.java | 7 +- sim/gobs/gobs/sweep/Parallelism01.java | 8 +- .../sweep/{Decluster04.java => Rebuild01.java} | 36 ++-- sim/gobs/gobs/util/Bits.java | 20 +- sim/gobs/gobs/util/Stats.java | 81 +++++++ sim/gobs/gobs/util/Tools.java | 20 ++ sim/gobs/sweeps/bl01-small.cfg | 18 +- sim/gobs/sweeps/bl01.cfg | 18 +- sim/gobs/sweeps/bl01output.cfg | 12 +- sim/gobs/sweeps/dc02kpv01.cfg | 2 +- sim/gobs/sweeps/debug.cfg | 12 +- sim/gobs/sweeps/decluster02.zsh | 7 +- sim/gobs/sweeps/kpv01.cfg | 6 +- sim/gobs/sweeps/kpv02.cfg | 6 +- sim/gobs/sweeps/kpv03.cfg | 6 +- sim/gobs/sweeps/kpv04.cfg | 6 +- sim/gobs/sweeps/kpv05.cfg | 6 +- sim/gobs/sweeps/kpv06.cfg | 4 +- sim/gobs/sweeps/kpv_lit_big.cfg | 10 +- sim/gobs/sweeps/kpv_lit_small.cfg | 10 +- sim/gobs/sweeps/kpv_output.cfg | 12 +- sim/gobs/sweeps/kpv_scenario.cfg | 10 +- sim/gobs/sweeps/kpv_small.cfg | 10 +- sim/gobs/sweeps/output.cfg | 12 +- sim/gobs/sweeps/pll01.cfg | 6 +- sim/gobs/sweeps/pll_output.cfg | 12 +- sim/gobs/sweeps/quick04.cfg | 34 ++-- sim/gobs/sweeps/rb01-small.cfg | 89 ++++++++ ...ystem_size_20-1000.cfg => system_size_1000.cfg} | 12 +- sim/gobs/sweeps/system_size_100_1000.cfg | 10 +- sim/gobs/sweeps/system_size_20-1000.cfg | 10 +- sim/gobs/sweeps/system_size_pdsi_100-1000.cfg | 2 +- sim/gobs/sweeps/system_size_pdsi_20-40.cfg | 2 +- sim/gobs/wiki/PlacementHierarchy.odg | Bin 0 -> 15558 bytes sim/gobs/wiki/PlacementHierarchy.png | Bin 0 -> 17710 bytes sim/gobs/wiki/kpv01.cfg | 6 +- sim/gobs/wiki/kpv02.cfg | 6 +- sim/gobs/wiki/kpv03.cfg | 6 +- sim/gobs/wiki/kpv04.cfg | 6 +- sim/gobs/wiki/kpv05.cfg | 6 +- sim/gobs/wiki/kpv06.cfg | 4 +- sim/gobs/wiki/kpv_lit_big.cfg | 10 +- sim/gobs/wiki/kpv_lit_small.cfg | 10 +- sim/gobs/wiki/kpv_output.cfg | 12 +- sim/gobs/wiki/kpv_scenario.cfg | 10 +- sim/gobs/wiki/kpv_small.cfg | 10 +- sim/gobs/wiki/system_size_100_1000.cfg | 10 +- sim/gobs/wiki/system_size_20-1000.cfg | 10 +- 66 files changed, 830 insertions(+), 321 deletions(-) rename sim/gobs/gobs/sim/{NodeScheme.java => NodeFactory.java} (81%) rename sim/gobs/gobs/sim/{PlacementScheme.java => Placement.java} (96%) create mode 100644 sim/gobs/gobs/sim/Rebuild.java copy sim/gobs/gobs/sweep/{Decluster04.java => Rebuild01.java} (64%) create mode 100644 sim/gobs/gobs/util/Stats.java create mode 100644 sim/gobs/sweeps/rb01-small.cfg copy sim/gobs/sweeps/{system_size_20-1000.cfg => system_size_1000.cfg} (80%) create mode 100644 sim/gobs/wiki/PlacementHierarchy.odg create mode 100644 sim/gobs/wiki/PlacementHierarchy.png Diff of changes: diff --git a/sim/gobs/examples/gobs.cfg b/sim/gobs/examples/gobs.cfg index 4d9c679..ddb885c 100644 --- a/sim/gobs/examples/gobs.cfg +++ b/sim/gobs/examples/gobs.cfg @@ -1,14 +1,24 @@ # Simulator configuration -printProperties = true +# Basics: + +# Print all properties at simulation start +print.properties = true # Print nodes and objects at startup (true/false) -printInitialNodes = true +print.initialnodes = true # Print nodes and objects after each rebuild (true/false) -printNodes = false +print.nodes = false # Print object tranfers during rebuilds (true/false) -printRebuilds = true +print.rebuilds = true # Print file information (true/false) -printFiles = false +print.files = false # Print load change statistics (true/false) -printLoadDiffs = false +print.loaddiffs = false +# Print user accesses +print.accesses = false + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 10 diff --git a/sim/gobs/examples/params.cfg b/sim/gobs/examples/params.cfg index d3a7944..0f23672 100644 --- a/sim/gobs/examples/params.cfg +++ b/sim/gobs/examples/params.cfg @@ -17,38 +17,38 @@ writes = 0 ### Choose a PlacementScheme implementation: -#PlacementScheme = Kademlia -#PlacementScheme = Nearest -PlacementScheme = VertexGroups +#impl.placement = Kademlia +#impl.placement = Nearest +impl.placement = VertexGroups VertexGroupPrefix = 6 -ReplicaSource = primary +replica.source = primary ### Choose a NodeScheme implementation: -# NodeScheme = RandomNodes -NodeScheme = SimpleNodes +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes ### Choose one of the following FileFactory sections: # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 140 +file.size = 140 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 3 +file.width = 3 # FileFactory implementation TraceFiles -#FileFactory = TraceFiles +#impl.filefactory = TraceFiles # Trace filename: #filetrace = traces/test01.trace # FileFactory implementation RandomFiles -#FileFactory = RandomFiles +#impl.filefactory = RandomFiles # Max size of big files: #filebigmax = 100 # Min size of big files: diff --git a/sim/gobs/gobs/sim/KClosest.java b/sim/gobs/gobs/sim/KClosest.java index b3bc9a4..3c39645 100644 --- a/sim/gobs/gobs/sim/KClosest.java +++ b/sim/gobs/gobs/sim/KClosest.java @@ -7,7 +7,7 @@ import java.util.List; import gobs.util.BoundedSortedValues; /** - * Superclass for PlacementSchemes that place secondaries + * Superclass for Placements that place secondaries * in a k-closest manner. * */ diff --git a/sim/gobs/gobs/sim/Metric.java b/sim/gobs/gobs/sim/Metric.java index 48f2a8c..d3ad748 100644 --- a/sim/gobs/gobs/sim/Metric.java +++ b/sim/gobs/gobs/sim/Metric.java @@ -5,7 +5,7 @@ import java.math.BigInteger; import java.util.List; /** - * Superclass for PlacementSchemes with some concept of + * Superclass for Placements with some concept of * a distance metric. * */ @@ -37,37 +37,34 @@ public abstract class Metric void rebuild(SingleObject object, List<Node> nodes) { - // System.out.println("rebuild: " + object.toString(simulator)); List<Node> sites = sites(object, nodes); // System.out.println("sites.size(): " + sites.size()); Secondary secondary; - Node source = null, destination = null; + Node primarySite = lookup(object.id, nodes); + Node primaryNode = sites.get(sites.size()-1); + Node source = removeSource(object, sites); + Node destination = null; if (object instanceof Primary) { Primary primary = (Primary) object; - Node primarySite = lookup(object.id, nodes); + // System.out.println("primary: " + primarySite.name(simulator)); if (primarySite.contains(object.id)) { - source = locateSource(object, sites); destination = locateDestination(object, sites); secondary = (Secondary) primarySite.remove(object.id); destination.add(secondary); } else { - source = locateSource(object, sites); destination = primarySite; } primarySite.add(primary); } else { - Node primaryNode = sites.get(sites.size()-1); Primary primary = (Primary) primaryNode.get(object.id); - source = locateSource(object, sites); - gobs.sim.Object s = source.get(object.id); - secondary = ((SingleObject) s).toSecondary(primary); + secondary = primary.toSecondary(); destination = locateDestination(object, sites); destination.add(secondary); } @@ -83,6 +80,7 @@ public abstract class Metric source.pushes++; destination.pulls++; - printRebuild(object, source, destination); + printRebuild(object, source, destination); + simulator.currentRebuild.add(object, source, destination); } } diff --git a/sim/gobs/gobs/sim/NodeScheme.java b/sim/gobs/gobs/sim/NodeFactory.java similarity index 81% rename from sim/gobs/gobs/sim/NodeScheme.java rename to sim/gobs/gobs/sim/NodeFactory.java index 35a0974..25c8a00 100644 --- a/sim/gobs/gobs/sim/NodeScheme.java +++ b/sim/gobs/gobs/sim/NodeFactory.java @@ -7,11 +7,11 @@ import java.util.*; * Generates node addresses. * */ -public abstract class NodeScheme +public abstract class NodeFactory { Simulator simulator; - NodeScheme(Simulator simulator) + NodeFactory(Simulator simulator) { this.simulator = simulator; } diff --git a/sim/gobs/gobs/sim/PlacementScheme.java b/sim/gobs/gobs/sim/Placement.java similarity index 96% rename from sim/gobs/gobs/sim/PlacementScheme.java rename to sim/gobs/gobs/sim/Placement.java index f73f8bf..515da79 100644 --- a/sim/gobs/gobs/sim/PlacementScheme.java +++ b/sim/gobs/gobs/sim/Placement.java @@ -10,13 +10,13 @@ import gobs.util.*; * Interface to place a list of objects on a set of nodes. * */ -public abstract class PlacementScheme +public abstract class Placement { Simulator simulator; // int B; BigInteger M; - PlacementScheme(Simulator simulator) + Placement(Simulator simulator) { this.simulator = simulator; @@ -111,7 +111,7 @@ public abstract class PlacementScheme void printRebuild(SingleObject object, Node source, Node destination) { - if (simulator.bool("printRebuilds")) + if (simulator.bool("print.rebuilds")) System.out.println("rebuild: " + object.toString(simulator) + ": " + source.name(simulator) + " -> " + diff --git a/sim/gobs/gobs/sim/PlainFiles.java b/sim/gobs/gobs/sim/PlainFiles.java index ef78f72..6aa8a8c 100644 --- a/sim/gobs/gobs/sim/PlainFiles.java +++ b/sim/gobs/gobs/sim/PlainFiles.java @@ -18,10 +18,10 @@ class PlainFiles PlainFiles(Simulator simulator) { - size = simulator.integer("filesize"); - strip = simulator.integer("filestrip"); - width = simulator.integer("filewidth"); - replicas = simulator.integer("filereplicas"); + size = simulator.integer("file.size"); + strip = simulator.integer("file.strip"); + width = simulator.integer("file.width"); + replicas = simulator.integer("file.replicas"); } public File get() diff --git a/sim/gobs/gobs/sim/Primary.java b/sim/gobs/gobs/sim/Primary.java index 11760d6..d46db16 100644 --- a/sim/gobs/gobs/sim/Primary.java +++ b/sim/gobs/gobs/sim/Primary.java @@ -21,6 +21,11 @@ class Primary { super(id, size, replicas); } + + Secondary toSecondary() + { + return new Secondary(id, size, replicas, this); + } public String toString(Simulator simulator) { diff --git a/sim/gobs/gobs/sim/RandomNodes.java b/sim/gobs/gobs/sim/RandomNodes.java index c29f7b4..ff70893 100644 --- a/sim/gobs/gobs/sim/RandomNodes.java +++ b/sim/gobs/gobs/sim/RandomNodes.java @@ -11,7 +11,7 @@ import gobs.util.*; * */ class RandomNodes - extends NodeScheme + extends NodeFactory { RandomNodes(Simulator simulator) { diff --git a/sim/gobs/gobs/sim/Rebuild.java b/sim/gobs/gobs/sim/Rebuild.java new file mode 100644 index 0000000..1ce4fc7 --- /dev/null +++ b/sim/gobs/gobs/sim/Rebuild.java @@ -0,0 +1,221 @@ + +package gobs.sim; + +import java.text.DecimalFormat; +import java.util.*; + +import gobs.util.Bits; + +/** + Manages a rebuild after a fault. +*/ +class Rebuild +{ + List<Copy> copies = new ArrayList<Copy>(); + NavigableSet<Event> schedule = null; + + /** + Simulated time that rebuild started. + */ + double start = -1; + + /** + Rebuild rate in bytes/sec. + */ + static double rate = 1; + + /** + Time taken to select the next copy. + This must be large enough to differentiate + all copies based on start time. + */ + static double selectTime = 0.000001; + + static DecimalFormat df = new DecimalFormat("0.0"); + + Simulator simulator; + + Rebuild(Simulator simulator) + { + this.simulator = simulator; + } + + /** + Add this copy to the Rebuild. + */ + void add(Copy copy) + { + copies.add(copy); + } + + /** + Construct a Copy and add it to this rebuild. + */ + void add(gobs.sim.Object object, Node source, Node destination) + { + Copy copy = new Copy(object, source, destination); + add(copy); + } + + /** + Actually perform the rebuild. + @param time The time at which the copies start. + @return The time at which all copies finished. + */ + double rebuild(double time) + { + start = time; + Collections.shuffle(copies, Bits.rng); + schedule = new TreeSet<Event>(); + Queue<Copy> todo = new LinkedList<Copy>(copies); + Set<Node> busy = new HashSet<Node>(copies.size()); + NavigableSet<Copy> inflight = new TreeSet<Copy>(); + int done = 0; + while (done < copies.size()) + { + Copy copy = select(todo, busy); + if (copy != null) + { + copy.startAt(time); + schedule.add(copy); + busy.add(copy.source); + busy.add(copy.destination); + boolean b = inflight.add(copy); + if (!b) + { + System.out.println("add failed!"); + System.out.println("" + copy); + dumpInflight(inflight); + System.exit(2); + } + time += selectTime; + if (simulator.bool("print.rebuilds")) + System.out.println("start: " + copy.log()); + } + else + { + // dumpInflight(inflight); + Copy finished = inflight.pollFirst(); + if (finished == null) + { + System.out.println("inflight.pollFirst(): null"); + System.out.println("inflight.size(): " + inflight.size()); + try { Thread.sleep(1000); } + catch (Exception e) {} + continue; + } + busy.remove(finished.source); + busy.remove(finished.destination); + time = finished.finish; + if (simulator.bool("print.rebuilds")) + System.out.println("finish: " + finished.log()); + done++; + } + } + return time; + } + + int loadAt(double t) + { + int result = 0; + NavigableSet<Event> view = schedule.tailSet(new Event(t), false); + for (Iterator<Event> it = view.iterator(); it.hasNext(); ) + { + Copy copy = (Copy) it.next(); + if (copy.start <= t) + result++; + } + return result; + } + + /** + Debugging only. + */ + void dumpInflight(NavigableSet<Copy> inflight) + { + for (Iterator<Copy> it = inflight.iterator(); it.hasNext(); ) + { + System.out.println("\t" + it.next()); + } + } + + Copy select(Queue<Copy> todo, Set<Node> busy) + throws NoSuchElementException + { + for (Iterator<Copy> it = todo.iterator(); it.hasNext(); ) + { + Copy copy = it.next(); + if (! (busy.contains(copy.source) || + busy.contains(copy.destination))) + { + it.remove(); + return copy; + } + } + return null; + } + + class Event + implements Comparable<Event> + { + double finish = -1; + + Event() + {} + + Event(double finish) + { + this.finish = finish; + } + + public int compareTo(Event event) + { + return new Double(finish).compareTo(new Double(event.finish)); + } + } + + class Copy + extends Event + { + gobs.sim.Object object; + Node source; + Node destination; + boolean complete = false; + double start = 0; + + Copy(gobs.sim.Object object, Node source, Node destination) + { + + this.object = object; + this.source = source; + this.destination = destination; + } + + void startAt(double start) + { + this.start = start; + finish = start + object.size/rate; + } + + public String toString() + { + return log(); + } + + String log() + { + StringBuilder sb = new StringBuilder(128); + sb.append("copy: ") + .append(object).append(": ") + .append(source).append(" -> ").append(destination) + .append("(").append(df.format(start)); + if (finish == -1) + sb.append(")"); + else + sb.append(",").append(df.format(finish)).append(")"); + return sb.toString(); + } + } + + +} diff --git a/sim/gobs/gobs/sim/Replicated.java b/sim/gobs/gobs/sim/Replicated.java index fa0ebe8..f1cdf21 100644 --- a/sim/gobs/gobs/sim/Replicated.java +++ b/sim/gobs/gobs/sim/Replicated.java @@ -11,7 +11,7 @@ import gobs.util.*; * */ public abstract class Replicated - extends PlacementScheme + extends Placement { /** One of: primary, random, last. @@ -116,9 +116,10 @@ public abstract class Replicated /** Locate a replica of the given object on one of the given nodes - in accordance with the {@link #replicaSource} policy. + in accordance with the {@link #replicaSource} policy; + remove and return that replica from sites. */ - Node locateSource(SingleObject object, List<Node> sites) + Node removeSource(SingleObject object, List<Node> sites) { int maxTries = sites.size()+1; Node result = null; @@ -156,6 +157,8 @@ public abstract class Replicated { Node destination = null; + int origsize = sites.size(); + while (sites.size() > 0) { int r = Bits.nextInt(sites.size()); @@ -168,7 +171,10 @@ public abstract class Replicated } if (destination == null) - Tools.bail("No destination!"); + { + System.out.println(origsize); + Tools.bail("No destination: " + object); + } return destination; } @@ -184,15 +190,6 @@ public abstract class Replicated Tools.bail("Exit!"); } - - void printRebuild(SingleObject object, Node source, Node destination) - { - if (simulator.bool("printRebuilds")) - System.out.println("rebuild: " + object.toString(simulator) + - ": " + - source.name(simulator) + " -> " + - destination.name(simulator)); - } /** New id anywhere in the address space. diff --git a/sim/gobs/gobs/sim/SimpleNodes.java b/sim/gobs/gobs/sim/SimpleNodes.java index 07135b0..85ffabc 100644 --- a/sim/gobs/gobs/sim/SimpleNodes.java +++ b/sim/gobs/gobs/sim/SimpleNodes.java @@ -12,7 +12,7 @@ import gobs.util.*; * */ public class SimpleNodes - extends NodeScheme + extends NodeFactory { public SimpleNodes(Simulator simulator) { diff --git a/sim/gobs/gobs/sim/Simulator.java b/sim/gobs/gobs/sim/Simulator.java index 8214702..04aa48e 100644 --- a/sim/gobs/gobs/sim/Simulator.java +++ b/sim/gobs/gobs/sim/Simulator.java @@ -26,11 +26,16 @@ public class Simulator int reads; int writes; - public PlacementScheme placer; - public NodeScheme hub; + public Placement placer; + public NodeFactory hub; public FileFactory factory; /** + Simulated time. Starts at 0. + */ + double time = 0; + + /** Address id display mode. */ Addressable.Display display = Addressable.Display.BITS; @@ -64,6 +69,16 @@ public class Simulator Object load difference before and after faults. */ LinkedHashMap<Node,Integer> sortedDiffs; + + /** + History of rebuilds. + */ + List<Rebuild> rebuilds = new ArrayList<Rebuild>(); + + /** + Current rebuild in progress. + */ + Rebuild currentRebuild = null; // static Simulator current; static final int maxTries = 10; @@ -119,7 +134,7 @@ public class Simulator else Bits.init(true); - simulator.execute(); + simulator.execute(); } void path(String path) @@ -151,12 +166,12 @@ public class Simulator */ public void setup() { - if (bool("printProperties")) + if (bool("print.properties")) printProperties(); setupBasics(); setupFileFactory(); - setupPlacementScheme(); + setupPlacement(); setupNodes(); } @@ -172,9 +187,9 @@ public class Simulator void setupFileFactory() { - String factoryName = properties.getProperty("FileFactory"); + String factoryName = properties.getProperty("impl.filefactory"); if (factoryName == null) - bail("No FileFactory given!"); + bail("No impl.filefactory given!"); else if (factoryName.equals("PlainFiles")) factory = new PlainFiles(this); else if (factoryName.equals("TraceFiles")) @@ -187,11 +202,11 @@ public class Simulator bail("Bad filefactory name: " + factoryName); } - void setupPlacementScheme() + void setupPlacement() { - String placerName = string("PlacementScheme"); + String placerName = string("impl.placement"); ReplicaSource source = - ReplicaSource.parse(string("ReplicaSource")); + ReplicaSource.parse(string("replica.source")); if (placerName.equals("Nearest")) { display = Addressable.Display.NUMERIC; @@ -219,20 +234,20 @@ public class Simulator placer = new ParallelRUSH(this, source); } else - bail("Bad PlacementScheme name: " + placerName); + bail("Bad Placement name: " + placerName); M = placer.M; } void setupNodes() { - String nodesName = string("NodeScheme"); + String nodesName = string("impl.nodefactory"); if (nodesName.equals("RandomNodes")) hub = new RandomNodes(this); else if (nodesName.equals("SimpleNodes")) hub = new SimpleNodes(this); else - bail("Bad NodeScheme name: " + nodesName); + bail("Bad NodeFactory name: " + nodesName); nodes = new ArrayList<Node>(N); } @@ -243,7 +258,7 @@ public class Simulator stats.nodes = N; System.out.println("nodes: " + N); - if (bool("printInitialNodes")) + if (bool("print.initialnodes")) printNodes(nodes); } @@ -251,7 +266,7 @@ public class Simulator { allocateFiles(fileCount); - if (string("printFiles").equals("true")) + if (bool("print.files")) printFiles(); System.out.println("objects: " + objects.size()); } @@ -289,7 +304,7 @@ public class Simulator origCount = countObjects(nodes); // printNodes(nodes); - if (string("printInitialNodes").equals("true")) + if (bool("print.initialnodes")) printNodeObjects(nodes); } @@ -383,6 +398,7 @@ public class Simulator for (int fault = 0; fault < faults; fault++) { + startRebuild(); int index = Bits.nextInt(nodes.size()); Node failed = nodes.remove(index); System.out.println("failed: " + failed.name(this) + @@ -390,10 +406,25 @@ public class Simulator List<gobs.sim.Object> replacements = failed.objects; for (gobs.sim.Object object : replacements) placer.rebuild(object, nodes); - if (string("printNodes").equals("true")) - printNodeObjects(nodes); + if (string("print.nodes").equals("true")) + printNodeObjects(nodes); + currentRebuild.rebuild(time); + completeRebuild(); } } + + Rebuild startRebuild() + { + Rebuild rebuild = new Rebuild(this); + currentRebuild = rebuild; + return rebuild; + } + + void completeRebuild() + { + rebuilds.add(currentRebuild); + currentRebuild = null; + } void computeStatistics() { @@ -402,7 +433,8 @@ public class Simulator computePullDeclustering(total); computePushDeclustering(total); computeTotalDeclustering(total); - computeAccesses(); + computeAccesses(); + computeTraffic(); stats.results.setProperty("lostObjects", df.format(stats.lostObjects)); @@ -435,7 +467,7 @@ public class Simulator sortedDiffs = sorter.sort(diffs); // Display load differences... - if (string("printLoadDiffs").equals("true")) + if (string("print.loaddiffs").equals("true")) printLoadDifferences(sortedDiffs); } @@ -541,7 +573,7 @@ public class Simulator stats.results.setProperty("maxTotal", df.format(maxPercentTotal)); } - + void computeAccesses() { // Did the user perform any operations? @@ -576,6 +608,47 @@ public class Simulator System.out.println("accessbalance: " + df.format(ratio)); stats.results.setProperty("accessbalance", df.format(ratio)); } + + /** + Compute and print object movement traffic over time. + */ + void computeTraffic() + { + if (! has("plot.traffic.interval")) + return; + + System.out.println("TRAFFIC: "); + double interval = decimal("plot.traffic.interval"); + + List<Double> intervals = new ArrayList<Double>(); + List<List<Integer>> results = new ArrayList<List<Integer>>(); + for (Rebuild rebuild : rebuilds) + { + List<Integer> sample = new ArrayList<Integer>(); + double d = rebuild.start; + boolean done = false; + while (!done) + { + int traffic = rebuild.loadAt(d); + sample.add(traffic); + d += interval; + if (traffic == 0) + done = true; + } + results.add(sample); + } + + int length = Stats.length(results); + double d = 0; + for (int i = 0; i < length; i++) + intervals.add(d+=interval); + List<Double> avg = Stats.avg(results); + + System.out.println(avg); + + stats.results.setProperty("traffic-x", intervals.toString()); + stats.results.setProperty("traffic-y", avg.toString()); + } void lost() { @@ -649,6 +722,15 @@ public class Simulator /** Allow subordinate classes to obtain properties. + @return The value of property name as a double. + */ + public double decimal(String name) + { + return Tools.decimal(properties, name); + } + + /** + Allow subordinate classes to obtain properties. @return The value of property name. */ public String string(String name) diff --git a/sim/gobs/gobs/sweep/AbstractSweep.java b/sim/gobs/gobs/sweep/AbstractSweep.java index 8841f5f..441e132 100644 --- a/sim/gobs/gobs/sweep/AbstractSweep.java +++ b/sim/gobs/gobs/sweep/AbstractSweep.java @@ -96,20 +96,20 @@ abstract class AbstractSweep path = p; } - static List<String> values(String type, int min, int max) + static List<String> values(String step, int min, int max) { - if (type.equals("all")) + if (step.equals("all")) return all(min,max); - else if (type.equals("tens")) + else if (step.equals("tens")) return tens(min,max); - else if (type.equals("exps")) + else if (step.equals("exps")) return exps(min,max); - else if (type.equals("mids")) + else if (step.equals("mids")) return mids(min,max); - else if (type.startsWith("literal")) - return literal(type); + else if (step.startsWith("literal")) + return literal(step); else - Tools.bail("Sweep.values(): Bad values type: " + type); + Tools.bail("Sweep.values(): Bad step: " + step); return null; } @@ -215,15 +215,15 @@ abstract class AbstractSweep } /** - Set properties from memory. + Set properties from memory. */ void initialize(Properties properties) { - this.properties = properties; + this.properties = properties; } /** - Set properties from command-line. + Set properties from command-line. */ void initialize(String[] args) { @@ -237,6 +237,18 @@ abstract class AbstractSweep } /** + Seed the random number generator based on the "seed" property. + */ + static void seed(Properties properties) + { + if (Tools.has(properties, "seed")) + Bits.init(Tools.integer(properties, "seed")); + else + Bits.init(true); + Bits.lock(); + } + + /** Debugging only. */ public static void main(String[] args) diff --git a/sim/gobs/gobs/sweep/ArraySweep.java b/sim/gobs/gobs/sweep/ArraySweep.java index b966348..c23087d 100644 --- a/sim/gobs/gobs/sweep/ArraySweep.java +++ b/sim/gobs/gobs/sweep/ArraySweep.java @@ -65,7 +65,8 @@ class ArraySweep Arrays.toString(result(simulator, yname))); results.add(result(simulator, yname)); } - XYSeries series = new XYSeries(description + " N=" + v); + XYSeries series = new XYSeries(description + " " + + xname + "=" + v); arrayAvgToSeries(results, series); collection.add(series); } @@ -123,12 +124,6 @@ class ArraySweep return total/results.size(); } - static void reportRuntime() - { - long t = (new Date().getTime()-launch.getTime())/(60*1000); - System.out.println("took " + t + " minutes"); - } - /** Debugging only. */ diff --git a/sim/gobs/gobs/sweep/Balance01.java b/sim/gobs/gobs/sweep/Balance01.java index 91aec24..6ba7484 100644 --- a/sim/gobs/gobs/sweep/Balance01.java +++ b/sim/gobs/gobs/sweep/Balance01.java @@ -52,7 +52,7 @@ class Balance01 static Simulator execute(Properties properties, String scheme) { - properties.setProperty("PlacementScheme", scheme); + properties.setProperty("impl.placement", scheme); Simulator simulator = new Simulator(); simulator.initialize(properties); simulator.execute(); diff --git a/sim/gobs/gobs/sweep/Decluster02.java b/sim/gobs/gobs/sweep/Decluster02.java index 0df4748..5d03c88 100644 --- a/sim/gobs/gobs/sweep/Decluster02.java +++ b/sim/gobs/gobs/sweep/Decluster02.java @@ -29,9 +29,8 @@ class Decluster02 public static void main(String[] args) { - Bits.init(); - - properties = Sweep.getopts(args); + properties = Sweep.getopts(args); + AbstractSweep.seed(properties); String output = Tools.string(properties, "output"); int minNodes = Tools.integer(properties, "minNodes"); @@ -41,23 +40,23 @@ class Decluster02 final XYSeriesCollection collection = new XYSeriesCollection(); - properties.setProperty("PlacementScheme", "Nearest"); + properties.setProperty("impl.placement", "Nearest"); Sweep nearestSweep = new Sweep(properties); nearestSweep.repeats = Tools.integer(properties, "repeats"); nearestSweep.execute("PAST", "nodes", values, "maxPulls"); collection.addSeries(nearestSweep.series); - properties.setProperty("PlacementScheme", "Kademlia"); + properties.setProperty("impl.placement", "Kademlia"); Sweep kademliaSweep = new Sweep(properties); kademliaSweep.repeats = Tools.integer(properties, "repeats"); kademliaSweep.execute("Kademlia", "nodes", values, "maxPulls"); collection.addSeries(kademliaSweep.series); - properties.setProperty("PlacementScheme", "VertexGroups"); + properties.setProperty("impl.placement", "VertexGroups"); Sweep vgSweep = new Sweep(properties); vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "maxTotal"); + vgSweep.execute("Vertex Groups", "nodes", values, "maxPulls"); collection.addSeries(vgSweep.series); MultiLogXPlot.plot(collection, "Maximum Rebuild Load", diff --git a/sim/gobs/gobs/sweep/Decluster03.java b/sim/gobs/gobs/sweep/Decluster03.java index e16c688..3f94021 100644 --- a/sim/gobs/gobs/sweep/Decluster03.java +++ b/sim/gobs/gobs/sweep/Decluster03.java @@ -36,20 +36,20 @@ class Decluster03 final XYSeriesCollection collection = new XYSeriesCollection(); - properties.setProperty("PlacementScheme", "Nearest"); + properties.setProperty("impl.placement", "Nearest"); Sweep nearestSweep = new Sweep(properties); nearestSweep.repeats = Tools.integer(properties, "repeats"); nearestSweep.execute("PAST", "nodes", values, "maxTotal"); collection.addSeries(nearestSweep.series); - properties.setProperty("PlacementScheme", "Kademlia"); + properties.setProperty("impl.placement", "Kademlia"); Sweep kademliaSweep = new Sweep(properties); kademliaSweep.repeats = Tools.integer(properties, "repeats"); kademliaSweep.execute("Kademlia", "nodes", values, "maxTotal"); collection.addSeries(kademliaSweep.series); - properties.setProperty("PlacementScheme", "VertexGroups"); + properties.setProperty("impl.placement", "VertexGroups"); Sweep vgSweep = new Sweep(properties); vgSweep.repeats = Tools.integer(properties, "repeats"); vgSweep.execute("Vertex Groups", "nodes", values, "maxTotal"); diff --git a/sim/gobs/gobs/sweep/Decluster04.java b/sim/gobs/gobs/sweep/Decluster04.java index 029a98c..cc6d375 100644 --- a/sim/gobs/gobs/sweep/Decluster04.java +++ b/sim/gobs/gobs/sweep/Decluster04.java @@ -6,7 +6,6 @@ import java.util.*; import org.jfree.data.xy.XYSeries; import org.jfree.data.xy.XYSeriesCollection; -// import gobs.sim.*; import gobs.util.*; import gobs.plot.*; @@ -40,19 +39,19 @@ class Decluster04 List<XYSeries> allSeries = new ArrayList<XYSeries>(); final XYSeriesCollection collection = new XYSeriesCollection(); - properties.setProperty("PlacementScheme", "Nearest"); + properties.setProperty("impl.placement", "Nearest"); ArraySweep pSweep = new ArraySweep(properties); pSweep.repeats = Tools.integer(properties, "repeats"); pSweep.execute("PAST", "nodes", values, "distribution"); allSeries.addAll(pSweep.collection); - properties.setProperty("PlacementScheme", "Kademlia"); + properties.setProperty("impl.placement", "Kademlia"); ArraySweep kSweep = new ArraySweep(properties); kSweep.repeats = Tools.integer(properties, "repeats"); kSweep.execute("Kademlia", "nodes", values, "distribution"); allSeries.addAll(kSweep.collection); - properties.setProperty("PlacementScheme", "VertexGroups"); + properties.setProperty("impl.placement", "VertexGroups"); ArraySweep vgSweep = new ArraySweep(properties); vgSweep.repeats = Tools.integer(properties, "repeats"); vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); diff --git a/sim/gobs/gobs/sweep/Parallelism01.java b/sim/gobs/gobs/sweep/Parallelism01.java index 97e5dff..aa60684 100644 --- a/sim/gobs/gobs/sweep/Parallelism01.java +++ b/sim/gobs/gobs/sweep/Parallelism01.java @@ -10,7 +10,7 @@ import gobs.util.*; import gobs.plot.*; /** - * Modifies node count N and PlacementScheme. + * Modifies node count N and Placement. * Reports file access load balancing for files and replicas. * * Usage: Parallelism01 -n <repeats> *.cfg @@ -37,19 +37,19 @@ class Parallelism01 final XYSeriesCollection collection = new XYSeriesCollection(); - properties.setProperty("PlacementScheme", "Nearest"); + properties.setProperty("impl.placement", "Nearest"); Sweep nearestSweep = new Sweep(properties); nearestSweep.repeats = Tools.integer(properties, "repeats"); nearestSweep.execute("PAST", "nodes", values, "accessbalance"); collection.addSeries(nearestSweep.series); - properties.setProperty("PlacementScheme", "Kademlia"); + properties.setProperty("impl.placement", "Kademlia"); Sweep kademliaSweep = new Sweep(properties); kademliaSweep.repeats = Tools.integer(properties, "repeats"); kademliaSweep.execute("Kademlia", "nodes", values, "accessbalance"); collection.addSeries(kademliaSweep.series); - properties.setProperty("PlacementScheme", "VertexGroups"); + properties.setProperty("impl.placement", "VertexGroups"); Sweep vgSweep = new Sweep(properties); vgSweep.repeats = Tools.integer(properties, "repeats"); vgSweep.execute("Vertex Groups", "nodes", values, "accessbalance"); diff --git a/sim/gobs/gobs/sweep/Decluster04.java b/sim/gobs/gobs/sweep/Rebuild01.java similarity index 64% copy from sim/gobs/gobs/sweep/Decluster04.java copy to sim/gobs/gobs/sweep/Rebuild01.java index 029a98c..0319f00 100644 --- a/sim/gobs/gobs/sweep/Decluster04.java +++ b/sim/gobs/gobs/sweep/Rebuild01.java @@ -6,63 +6,63 @@ import java.util.*; import org.jfree.data.xy.XYSeries; import org.jfree.data.xy.XYSeriesCollection; -// import gobs.sim.*; import gobs.util.*; import gobs.plot.*; /** * Modifies node count. - * Reports object movement load (pushes+pulls) on all servers - * during rebuild. + * Reports rebuild activity over time during rebuild. * - * Usage: Decluster04 -n <repeats> *.cfg + * Usage: Rebuild01 -n <repeats> *.cfg * */ -class Decluster04 +class Rebuild01 { static Properties properties; public static void main(String[] args) { - Bits.init(); - properties = Sweep.getopts(args); + + AbstractSweep.seed(properties); String output = Tools.string(properties, "output"); - int minNodes = Tools.integer(properties, "minNodes"); - int maxNodes = Tools.integer(properties, "maxNodes"); - String step = Tools.string(properties, "step"); + int minNodes = Tools.integer(properties, "nodes.min"); + int maxNodes = Tools.integer(properties, "nodes.max"); + String step = Tools.string(properties, "nodes.step"); List<String> values = Sweep.values(step, minNodes, maxNodes); - System.out.println("Decluster04: " + values); + System.out.println("Rebuild01: " + values); List<XYSeries> allSeries = new ArrayList<XYSeries>(); final XYSeriesCollection collection = new XYSeriesCollection(); - properties.setProperty("PlacementScheme", "Nearest"); + properties.setProperty("impl.placement", "Nearest"); ArraySweep pSweep = new ArraySweep(properties); pSweep.repeats = Tools.integer(properties, "repeats"); - pSweep.execute("PAST", "nodes", values, "distribution"); + pSweep.execute("PAST", "nodes", values, "traffic-y"); allSeries.addAll(pSweep.collection); - properties.setProperty("PlacementScheme", "Kademlia"); + /* + properties.setProperty("impl.placement", "Kademlia"); ArraySweep kSweep = new ArraySweep(properties); kSweep.repeats = Tools.integer(properties, "repeats"); kSweep.execute("Kademlia", "nodes", values, "distribution"); allSeries.addAll(kSweep.collection); - properties.setProperty("PlacementScheme", "VertexGroups"); + properties.setProperty("impl.placement", "VertexGroups"); ArraySweep vgSweep = new ArraySweep(properties); vgSweep.repeats = Tools.integer(properties, "repeats"); vgSweep.execute("Vertex Groups", "nodes", values, "distribution"); allSeries.addAll(vgSweep.collection); - + */ + for (XYSeries series : allSeries) collection.addSeries(series); - MultiplePlot.plot(collection, "Rebuild Load Distribution", - "nodes involved", "load fraction(%)", output); + MultiplePlot.plot(collection, "Rebuild Traffic", + "time", "load fraction(%)", output); // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", // "nodes", "load", "tmp.txt"); diff --git a/sim/gobs/gobs/util/Bits.java b/sim/gobs/gobs/util/Bits.java index 4717afb..c50b6db 100644 --- a/sim/gobs/gobs/util/Bits.java +++ b/sim/gobs/gobs/util/Bits.java @@ -10,7 +10,7 @@ import java.util.*; public class Bits { - public static Random rng = null; + public static Random rng = null; static boolean locked = false; static boolean ready = false; @@ -23,16 +23,16 @@ public class Bits } /** - If not locked, seed the generator with the clock mod 100. + If not locked, seed the generator with the clock mod 1000. Useful for debugging. - @param dump If true, dump the seed. + @param print If true, print the seed. */ - public static long init(boolean dump) + public static long init(boolean print) { long seed = System.currentTimeMillis() % 1000; init(seed); - if (dump) + if (print) System.out.println("Bits.seed: " + seed); return seed; @@ -44,12 +44,10 @@ public class Bits */ public static void init(long seed) { - if (! locked) - { - // System.out.println("Seeding RNG..."); - rng = new Random(seed); - ready = true; - } + assert (!locked) : "Bits locked!"; + // System.out.println("Seeding RNG..."); + rng = new Random(seed); + ready = true; } /** diff --git a/sim/gobs/gobs/util/Stats.java b/sim/gobs/gobs/util/Stats.java new file mode 100644 index 0000000..7c8792c --- /dev/null +++ b/sim/gobs/gobs/util/Stats.java @@ -0,0 +1,81 @@ + +package gobs.util; + +import java.util.*; + +/** + Data analysis helpers +*/ +public class Stats +{ + /** + Return the length of the longest List. + */ + public static int length(List<List<Integer>> data) + { + int result = 0; + for (List<Integer> list : data) + if (list.size() > result) + result = list.size(); + return result; + } + + /** + Return the average of the corresponding + entries of each list. + */ + public static List<Double> avg(List<List<Integer>> data) + { + List<Double> result = new ArrayList<Double>(); + int n = data.size(); + + int i = 0; + while (true) + { + double total = 0; + boolean something = false; + for (List<Integer> list : data) + { + if (list.size() > i) + { + something = true; + total += list.get(i); + } + } + if (something) + result.add(total/n); + else + break; + i++; + } + return result; + } + + /** + Debugging only. + */ + public static void main(String[] args) + { + List<List<Integer>> all = new ArrayList<List<Integer>>(); + List<Integer> l1 = new ArrayList<Integer>(); + List<Integer> l2 = new ArrayList<Integer>(); + List<Integer> l3 = new ArrayList<Integer>(); + + l1.add(3); + l1.add(3); + l1.add(3); + l1.add(3); + l2.add(3); + l2.add(2); + l2.add(3); + l3.add(1); + l3.add(3); + + all.add(l1); + all.add(l2); + all.add(l3); + + List<Double> out = avg(all); + System.out.println(out); + } +} diff --git a/sim/gobs/gobs/util/Tools.java b/sim/gobs/gobs/util/Tools.java index d419dd4..f65f60b 100644 --- a/sim/gobs/gobs/util/Tools.java +++ b/sim/gobs/gobs/util/Tools.java @@ -30,6 +30,26 @@ public class Tools return result; } + public static double decimal(Properties properties, String variable) + { + double result = 0; + try + { + result = + Double.parseDouble(properties.getProperty(variable).trim()); + } + catch (NumberFormatException e) + { + bail("Could not read double from property: " + variable + + " = " + properties.getProperty(variable)); + } + catch (NullPointerException e) + { + bail("Required property not found: " + variable); + } + return result; + } + public static boolean bool(Properties properties, String variable) { diff --git a/sim/gobs/sweeps/bl01-small.cfg b/sim/gobs/sweeps/bl01-small.cfg index b64d96d..30df6e6 100644 --- a/sim/gobs/sweeps/bl01-small.cfg +++ b/sim/gobs/sweeps/bl01-small.cfg @@ -22,27 +22,27 @@ nodes = 10 VertexGroupPrefix = 5 # No rebuilds will actually be performed. This is filler: -ReplicaSource = primary +replica.source = primary ### Choose a NodeScheme implementation: -#NodeScheme = RandomNodes -NodeScheme = SimpleNodes +#impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes ### Choose one of the following FileFactory sections: # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 -FileFactory = TraceFiles +impl.filefactory = TraceFiles filetrace = traces/onebigfile.trace ### User read/writes: diff --git a/sim/gobs/sweeps/bl01.cfg b/sim/gobs/sweeps/bl01.cfg index cccaf81..76494bf 100644 --- a/sim/gobs/sweeps/bl01.cfg +++ b/sim/gobs/sweeps/bl01.cfg @@ -20,27 +20,27 @@ nodes = 10 # Chosen by sweep driver # No rebuilds will actually be performed. This is filler: -ReplicaSource = primary +replica.source = primary ### Choose a NodeScheme implementation: -#NodeScheme = RandomNodes -NodeScheme = SimpleNodes +#impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes ### Choose one of the following FileFactory sections: # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 -FileFactory = TraceFiles +impl.filefactory = TraceFiles filetrace = traces/onebigfile.trace ### User read/writes: diff --git a/sim/gobs/sweeps/bl01output.cfg b/sim/gobs/sweeps/bl01output.cfg index a4bed90..ba359e8 100644 --- a/sim/gobs/sweeps/bl01output.cfg +++ b/sim/gobs/sweeps/bl01output.cfg @@ -2,14 +2,14 @@ ### Simulator output configuration # Dump properties for each Simulator execution -printProperties = true +print.properties = true # Print nodes and objects at startup (true/false) -printInitialNodes = true +print.initialnodes = true # Print nodes and objects after each rebuild (true/false) -printNodes = false +print.nodes = false # Print object tranfers during rebuilds (true/false) -printRebuilds = false +print.rebuilds = false # Print file information (true/false) -printFiles = true +print.files = true # Print load change statistics (true/false) -printLoadDiffs = false +print.loaddiffs = false diff --git a/sim/gobs/sweeps/dc02kpv01.cfg b/sim/gobs/sweeps/dc02kpv01.cfg index a8e3954..77ab756 100644 --- a/sim/gobs/sweeps/dc02kpv01.cfg +++ b/sim/gobs/sweeps/dc02kpv01.cfg @@ -1,5 +1,5 @@ -include = kpv01.cfg system_size_20-1000.cfg +include = kpv01.cfg system_size_1000.cfg # This declustering test has no user accesses: diff --git a/sim/gobs/sweeps/debug.cfg b/sim/gobs/sweeps/debug.cfg index b15d3bb..0df7001 100644 --- a/sim/gobs/sweeps/debug.cfg +++ b/sim/gobs/sweeps/debug.cfg @@ -1,14 +1,14 @@ # Simulator configuration -printProperties = true +print.properties = true # Print nodes and objects at startup (true/false) -printInitialNodes = true +print.initialnodes = true # Print nodes and objects after each rebuild (true/false) -printNodes = true +print.nodes = true # Print object tranfers during rebuilds (true/false) -printRebuilds = true +print.rebuilds = true # Print file information (true/false) -printFiles = true +print.files = true # Print load change statistics (true/false) -printLoadDiffs = true +print.loaddiffs = true diff --git a/sim/gobs/sweeps/decluster02.zsh b/sim/gobs/sweeps/decluster02.zsh index d42e129..531f22e 100755 --- a/sim/gobs/sweeps/decluster02.zsh +++ b/sim/gobs/sweeps/decluster02.zsh @@ -2,9 +2,11 @@ NAME=$1 N=$2 +SEED=$3 [[ ${NAME} == "" ]] && print "No NAME!" && exit 1 -[[ ${N} == "" ]] && N=20 +[[ ${N} == "" ]] && N=20 +[[ ${SEED} != "" ]] && SEED="seed=${SEED}" NAME="dc02${NAME}" @@ -12,7 +14,8 @@ CFG="${NAME}.cfg" EPS="${NAME}.eps" OUT="${NAME}.out" -java gobs.sweep.Decluster02 -p sweeps -n ${N} -o ${EPS} ${CFG} >& ${OUT} +java -ea gobs.sweep.Decluster02 \ + -p sweeps -n ${N} -o ${EPS} ${SEED} ${CFG} >& ${OUT} ERR=$? [[ ${ERR} != 0 ]] && print "Crashed: ${NAME} ERR: ${ERR}" && exit 1 diff --git a/sim/gobs/sweeps/kpv01.cfg b/sim/gobs/sweeps/kpv01.cfg index e62d3d6..72fdcfd 100644 --- a/sim/gobs/sweeps/kpv01.cfg +++ b/sim/gobs/sweeps/kpv01.cfg @@ -7,11 +7,11 @@ include = kpv_output.cfg # Chosen by sweep driver. -ReplicaSource = primary +replica.source = primary VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -# NodeScheme = RandomNodes -NodeScheme = SimpleNodes +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/kpv02.cfg b/sim/gobs/sweeps/kpv02.cfg index 7453f0c..d877857 100644 --- a/sim/gobs/sweeps/kpv02.cfg +++ b/sim/gobs/sweeps/kpv02.cfg @@ -7,11 +7,11 @@ include = kpv_output.cfg # Chosen by sweep driver. -ReplicaSource = primary +replica.source = primary VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -NodeScheme = RandomNodes -# NodeScheme = SimpleNodes +impl.nodefactory = RandomNodes +# impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/kpv03.cfg b/sim/gobs/sweeps/kpv03.cfg index f178102..8207b16 100644 --- a/sim/gobs/sweeps/kpv03.cfg +++ b/sim/gobs/sweeps/kpv03.cfg @@ -7,11 +7,11 @@ include = kpv_output.cfg # Chosen by sweep driver. -ReplicaSource = random +replica.source = random VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -#NodeScheme = RandomNodes -NodeScheme = SimpleNodes +#impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/kpv04.cfg b/sim/gobs/sweeps/kpv04.cfg index 30c54ce..84c4332 100644 --- a/sim/gobs/sweeps/kpv04.cfg +++ b/sim/gobs/sweeps/kpv04.cfg @@ -7,10 +7,10 @@ include = kpv_output.cfg # Chosen by sweep driver. -ReplicaSource = random +replica.source = random VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -NodeScheme = RandomNodes -# NodeScheme = SimpleNodes +impl.nodefactory = RandomNodes +# impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/kpv05.cfg b/sim/gobs/sweeps/kpv05.cfg index e28afdc..0ccfc44 100644 --- a/sim/gobs/sweeps/kpv05.cfg +++ b/sim/gobs/sweeps/kpv05.cfg @@ -7,11 +7,11 @@ include = kpv_output.cfg # Chosen by sweep driver. -ReplicaSource = last +replica.source = last VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -#NodeScheme = RandomNodes -NodeScheme = SimpleNodes +#impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/kpv06.cfg b/sim/gobs/sweeps/kpv06.cfg index 1c45f36..ba02d69 100644 --- a/sim/gobs/sweeps/kpv06.cfg +++ b/sim/gobs/sweeps/kpv06.cfg @@ -7,10 +7,10 @@ include = kpv_output.cfg # Chosen by sweep driver. -ReplicaSource = last +replica.source = last VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -NodeScheme = RandomNodes +impl.nodefactory = RandomNodes diff --git a/sim/gobs/sweeps/kpv_lit_big.cfg b/sim/gobs/sweeps/kpv_lit_big.cfg index 90f6d6a..adef78f 100644 --- a/sim/gobs/sweeps/kpv_lit_big.cfg +++ b/sim/gobs/sweeps/kpv_lit_big.cfg @@ -14,12 +14,12 @@ minNodes = 0 maxNodes = 0 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/sweeps/kpv_lit_small.cfg b/sim/gobs/sweeps/kpv_lit_small.cfg index eb772cb..bdbbc08 100644 --- a/sim/gobs/sweeps/kpv_lit_small.cfg +++ b/sim/gobs/sweeps/kpv_lit_small.cfg @@ -14,12 +14,12 @@ minNodes = 0 maxNodes = 0 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/sweeps/kpv_output.cfg b/sim/gobs/sweeps/kpv_output.cfg index d285c26..204275f 100644 --- a/sim/gobs/sweeps/kpv_output.cfg +++ b/sim/gobs/sweeps/kpv_output.cfg @@ -2,14 +2,14 @@ ### Simulator output configuration # Dump properties for each Simulator execution -printProperties = true +print.properties = true # Print nodes and objects at startup (true/false) -printInitialNodes = false +print.initialnodes = true # Print nodes and objects after each rebuild (true/false) -printNodes = false +print.nodes = true # Print object tranfers during rebuilds (true/false) -printRebuilds = false +print.rebuilds = true # Print file information (true/false) -printFiles = false +print.files = false # Print load change statistics (true/false) -printLoadDiffs = false +print.loaddiffs = false diff --git a/sim/gobs/sweeps/kpv_scenario.cfg b/sim/gobs/sweeps/kpv_scenario.cfg index 22950b8..3c693ae 100644 --- a/sim/gobs/sweeps/kpv_scenario.cfg +++ b/sim/gobs/sweeps/kpv_scenario.cfg @@ -15,12 +15,12 @@ minNodes = 20 maxNodes = 1000 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/sweeps/kpv_small.cfg b/sim/gobs/sweeps/kpv_small.cfg index f2f51e3..22a8fee 100644 --- a/sim/gobs/sweeps/kpv_small.cfg +++ b/sim/gobs/sweeps/kpv_small.cfg @@ -15,12 +15,12 @@ minNodes = 20 maxNodes = 1000 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/sweeps/output.cfg b/sim/gobs/sweeps/output.cfg index d285c26..5f562c3 100644 --- a/sim/gobs/sweeps/output.cfg +++ b/sim/gobs/sweeps/output.cfg @@ -2,14 +2,14 @@ ### Simulator output configuration # Dump properties for each Simulator execution -printProperties = true +print.properties = true # Print nodes and objects at startup (true/false) -printInitialNodes = false +print.initialnodes = false # Print nodes and objects after each rebuild (true/false) -printNodes = false +print.nodes = false # Print object tranfers during rebuilds (true/false) -printRebuilds = false +print.rebuilds = false # Print file information (true/false) -printFiles = false +print.files = false # Print load change statistics (true/false) -printLoadDiffs = false +print.loaddiffs = false diff --git a/sim/gobs/sweeps/pll01.cfg b/sim/gobs/sweeps/pll01.cfg index 66e7f1d..276d6e9 100644 --- a/sim/gobs/sweeps/pll01.cfg +++ b/sim/gobs/sweeps/pll01.cfg @@ -9,11 +9,11 @@ faults = 0 # Chosen by sweep driver. -ReplicaSource = primary +replica.source = primary VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -# NodeScheme = RandomNodes -NodeScheme = SimpleNodes +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/sweeps/pll_output.cfg b/sim/gobs/sweeps/pll_output.cfg index 6011575..51ff2ae 100644 --- a/sim/gobs/sweeps/pll_output.cfg +++ b/sim/gobs/sweeps/pll_output.cfg @@ -2,16 +2,16 @@ ### Simulator output configuration # Dump properties for each Simulator execution -printProperties = true +print.properties = true # Print nodes and objects at startup (true/false) -printInitialNodes = false +print.initialnodes = false # Print nodes and objects after each rebuild (true/false) -printNodes = false +print.nodes = false # Print object tranfers during rebuilds (true/false) -printRebuilds = false +print.rebuilds = false # Print file information (true/false) -printFiles = false +print.files = false # Print file access information (true/false) printAccesses = false # Print load change statistics (true/false) -printLoadDiffs = false +print.loaddiffs = false diff --git a/sim/gobs/sweeps/quick04.cfg b/sim/gobs/sweeps/quick04.cfg index 6dda9b4..952307d 100644 --- a/sim/gobs/sweeps/quick04.cfg +++ b/sim/gobs/sweeps/quick04.cfg @@ -20,35 +20,35 @@ maxNodes = 1000 # Chosen by sweep driver -ReplicaSource = random +replica.source = random ### Choose a NodeScheme implementation: -NodeScheme = RandomNodes -# NodeScheme = SimpleNodes +impl.nodefactory = RandomNodes +# impl.nodefactory = SimpleNodes ### Choose one of the following FileFactory sections: # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 # FileFactory implementation TraceFiles -#FileFactory = TraceFiles +#impl.filefactory = TraceFiles # Trace filename: #filetrace = traces/test01.trace # FileFactory implementation RandomFiles -#FileFactory = RandomFiles +#impl.filefactory = RandomFiles # Strip size: -#filestrip = 4 +#file.strip = 4 # Max size of big files: #filebigmax = 100 # Min size of big files: @@ -61,14 +61,14 @@ filewidth = 10 ### Simulator output configuration # Dump properties for each Simulator execution -printProperties = false +print.properties = false # Print nodes and objects at startup (true/false) -printInitialNodes = false +print.initialnodes = false # Print nodes and objects after each rebuild (true/false) -printNodes = false +print.nodes = false # Print object tranfers during rebuilds (true/false) -printRebuilds = false +print.rebuilds = false # Print file information (true/false) -printFiles = false +print.files = false # Print load change statistics (true/false) -printLoadDiffs = false +print.loaddiffs = false diff --git a/sim/gobs/sweeps/rb01-small.cfg b/sim/gobs/sweeps/rb01-small.cfg new file mode 100644 index 0000000..540977d --- /dev/null +++ b/sim/gobs/sweeps/rb01-small.cfg @@ -0,0 +1,89 @@ + +# Simulator configuration + +# Basics: + +# Print all properties at simulation start +print.properties = true +# Print nodes and objects at startup (true/false) +print.initialnodes = true +# Print nodes and objects after each rebuild (true/false) +print.nodes = false +# Print object tranfers during rebuilds (true/false) +print.rebuilds = true +# Print file information (true/false) +print.files = false +# Print load change statistics (true/false) +print.loaddiffs = false +# Print user accesses +print.accesses = false + +# Statistics: + +# Rebuild traffic over time analysis: +plot.traffic.interval = 10 + +### Parameters for GOBS Simulator run + +repeats = 1 + +### General parameters: + +# Bit-length of addresses: +B = 16 +# Number of servers: +nodes.min = 8 +nodes.max = 16 +# Step through nodes: See AbstractSweep.values() +nodes.step = literal_8,16 +# Number of files: +files = 10 +# Number of faults: +faults = 1 +# Number of accesses: +reads = 0 +writes = 0 + +### Choose a PlacementScheme implementation: + +#impl.placement = Kademlia +impl.placement = Nearest +#impl.placement = VertexGroups + +vertexgroups.prefix = 6 +# Replica selection: See Replicated.locateSource() +replica.source = primary + +### Choose a NodeScheme implementation: + +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes + +### Choose one of the following FileFactory sections: + +# FileFactory implementation PlainFiles +impl.filefactory = PlainFiles +# File size: +file.size = 140 +# Strip size: +file.strip = 4 +# File/object replicas: +file.replicas = 3 +# Number of objects per file: +file.width = 3 + +# FileFactory implementation TraceFiles +#impl.filefactory = TraceFiles +# Trace filename: +#filetrace = traces/test01.trace + +# FileFactory implementation RandomFiles +#impl.filefactory = RandomFiles +# Max size of big files: +#filebigmax = 100 +# Min size of big files: +#filebigmin = 50 +# Max size of small files: +#filesmallmax = 20 +# Min size of small files: +#filesmallmin = 0 diff --git a/sim/gobs/sweeps/system_size_20-1000.cfg b/sim/gobs/sweeps/system_size_1000.cfg similarity index 80% copy from sim/gobs/sweeps/system_size_20-1000.cfg copy to sim/gobs/sweeps/system_size_1000.cfg index 2d988b4..5107cdc 100644 --- a/sim/gobs/sweeps/system_size_20-1000.cfg +++ b/sim/gobs/sweeps/system_size_1000.cfg @@ -4,7 +4,7 @@ ### 3000 files * 10 objects * 3 replicas = 90,000 objects # Number of servers: -minNodes = 20 +minNodes = 1000 maxNodes = 1000 step = exps @@ -18,12 +18,12 @@ files = 3000 faults = 1 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/sweeps/system_size_100_1000.cfg b/sim/gobs/sweeps/system_size_100_1000.cfg index 90f6d6a..adef78f 100644 --- a/sim/gobs/sweeps/system_size_100_1000.cfg +++ b/sim/gobs/sweeps/system_size_100_1000.cfg @@ -14,12 +14,12 @@ minNodes = 0 maxNodes = 0 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/sweeps/system_size_20-1000.cfg b/sim/gobs/sweeps/system_size_20-1000.cfg index 2d988b4..57e29f8 100644 --- a/sim/gobs/sweeps/system_size_20-1000.cfg +++ b/sim/gobs/sweeps/system_size_20-1000.cfg @@ -18,12 +18,12 @@ files = 3000 faults = 1 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/sweeps/system_size_pdsi_100-1000.cfg b/sim/gobs/sweeps/system_size_pdsi_100-1000.cfg index 2d6860d..aaf7a91 100644 --- a/sim/gobs/sweeps/system_size_pdsi_100-1000.cfg +++ b/sim/gobs/sweeps/system_size_pdsi_100-1000.cfg @@ -19,6 +19,6 @@ reads = 5000 writes = 10000 # FileFactory implementation PlainFiles -FileFactory = PDSIFiles +impl.filefactory = PDSIFiles # 20% size 0, 60% size 1K-100K, 20% size 100K-10M PDSIFilesPoints = 0.2,1K;0.8,100K;1.0,10M diff --git a/sim/gobs/sweeps/system_size_pdsi_20-40.cfg b/sim/gobs/sweeps/system_size_pdsi_20-40.cfg index bf57678..a8d9635 100644 --- a/sim/gobs/sweeps/system_size_pdsi_20-40.cfg +++ b/sim/gobs/sweeps/system_size_pdsi_20-40.cfg @@ -19,6 +19,6 @@ reads = 500 writes = 1000 # FileFactory implementation PlainFiles -FileFactory = PDSIFiles +impl.filefactory = PDSIFiles # 20% size 0, 60% size 1K-100K, 20% size 100K-10M PDSIFilesPoints = 0.2,1K;0.8,100K;1.0,10M diff --git a/sim/gobs/wiki/PlacementHierarchy.odg b/sim/gobs/wiki/PlacementHierarchy.odg new file mode 100644 index 0000000..9a136c4 Binary files /dev/null and b/sim/gobs/wiki/PlacementHierarchy.odg differ diff --git a/sim/gobs/wiki/PlacementHierarchy.png b/sim/gobs/wiki/PlacementHierarchy.png new file mode 100644 index 0000000..a055ffe Binary files /dev/null and b/sim/gobs/wiki/PlacementHierarchy.png differ diff --git a/sim/gobs/wiki/kpv01.cfg b/sim/gobs/wiki/kpv01.cfg index daf2808..8f5920d 100644 --- a/sim/gobs/wiki/kpv01.cfg +++ b/sim/gobs/wiki/kpv01.cfg @@ -7,10 +7,10 @@ include = kpv_output.cfg kpv_lit_big.cfg # Chosen by sweep driver. -ReplicaSource = primary +replica.source = primary VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -# NodeScheme = RandomNodes -NodeScheme = SimpleNodes +# impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/wiki/kpv02.cfg b/sim/gobs/wiki/kpv02.cfg index 8a360bc..ce9fc89 100644 --- a/sim/gobs/wiki/kpv02.cfg +++ b/sim/gobs/wiki/kpv02.cfg @@ -8,11 +8,11 @@ include = kpv_output.cfg kpv_lit_big.cfg # Chosen by sweep driver. -ReplicaSource = primary +replica.source = primary VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -NodeScheme = RandomNodes -# NodeScheme = SimpleNodes +impl.nodefactory = RandomNodes +# impl.nodefactory = SimpleNodes diff --git a/sim/gobs/wiki/kpv03.cfg b/sim/gobs/wiki/kpv03.cfg index 12aae3b..3530d1a 100644 --- a/sim/gobs/wiki/kpv03.cfg +++ b/sim/gobs/wiki/kpv03.cfg @@ -8,11 +8,11 @@ include = kpv_output.cfg kpv_lit_big.cfg # Chosen by sweep driver. -ReplicaSource = random +replica.source = random VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -#NodeScheme = RandomNodes -NodeScheme = SimpleNodes +#impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/wiki/kpv04.cfg b/sim/gobs/wiki/kpv04.cfg index 7620038..f5bfc14 100644 --- a/sim/gobs/wiki/kpv04.cfg +++ b/sim/gobs/wiki/kpv04.cfg @@ -8,10 +8,10 @@ include = kpv_output.cfg kpv_lit_big.cfg # Chosen by sweep driver. -ReplicaSource = random +replica.source = random VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -NodeScheme = RandomNodes -# NodeScheme = SimpleNodes +impl.nodefactory = RandomNodes +# impl.nodefactory = SimpleNodes diff --git a/sim/gobs/wiki/kpv05.cfg b/sim/gobs/wiki/kpv05.cfg index e328a75..a7a95cc 100644 --- a/sim/gobs/wiki/kpv05.cfg +++ b/sim/gobs/wiki/kpv05.cfg @@ -8,11 +8,11 @@ include = kpv_output.cfg kpv_lit_big.cfg # Chosen by sweep driver. -ReplicaSource = last +replica.source = last VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -#NodeScheme = RandomNodes -NodeScheme = SimpleNodes +#impl.nodefactory = RandomNodes +impl.nodefactory = SimpleNodes diff --git a/sim/gobs/wiki/kpv06.cfg b/sim/gobs/wiki/kpv06.cfg index 4cf100d..b6e7bd5 100644 --- a/sim/gobs/wiki/kpv06.cfg +++ b/sim/gobs/wiki/kpv06.cfg @@ -8,10 +8,10 @@ include = kpv_output.cfg kpv_lit_big.cfg # Chosen by sweep driver. -ReplicaSource = last +replica.source = last VertexGroupPrefix = 5 ### Choose a NodeScheme implementation: -NodeScheme = RandomNodes +impl.nodefactory = RandomNodes diff --git a/sim/gobs/wiki/kpv_lit_big.cfg b/sim/gobs/wiki/kpv_lit_big.cfg index 90f6d6a..adef78f 100644 --- a/sim/gobs/wiki/kpv_lit_big.cfg +++ b/sim/gobs/wiki/kpv_lit_big.cfg @@ -14,12 +14,12 @@ minNodes = 0 maxNodes = 0 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/wiki/kpv_lit_small.cfg b/sim/gobs/wiki/kpv_lit_small.cfg index eb772cb..bdbbc08 100644 --- a/sim/gobs/wiki/kpv_lit_small.cfg +++ b/sim/gobs/wiki/kpv_lit_small.cfg @@ -14,12 +14,12 @@ minNodes = 0 maxNodes = 0 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/wiki/kpv_output.cfg b/sim/gobs/wiki/kpv_output.cfg index d285c26..5f562c3 100644 --- a/sim/gobs/wiki/kpv_output.cfg +++ b/sim/gobs/wiki/kpv_output.cfg @@ -2,14 +2,14 @@ ### Simulator output configuration # Dump properties for each Simulator execution -printProperties = true +print.properties = true # Print nodes and objects at startup (true/false) -printInitialNodes = false +print.initialnodes = false # Print nodes and objects after each rebuild (true/false) -printNodes = false +print.nodes = false # Print object tranfers during rebuilds (true/false) -printRebuilds = false +print.rebuilds = false # Print file information (true/false) -printFiles = false +print.files = false # Print load change statistics (true/false) -printLoadDiffs = false +print.loaddiffs = false diff --git a/sim/gobs/wiki/kpv_scenario.cfg b/sim/gobs/wiki/kpv_scenario.cfg index 22950b8..3c693ae 100644 --- a/sim/gobs/wiki/kpv_scenario.cfg +++ b/sim/gobs/wiki/kpv_scenario.cfg @@ -15,12 +15,12 @@ minNodes = 20 maxNodes = 1000 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/wiki/kpv_small.cfg b/sim/gobs/wiki/kpv_small.cfg index f2f51e3..22a8fee 100644 --- a/sim/gobs/wiki/kpv_small.cfg +++ b/sim/gobs/wiki/kpv_small.cfg @@ -15,12 +15,12 @@ minNodes = 20 maxNodes = 1000 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/wiki/system_size_100_1000.cfg b/sim/gobs/wiki/system_size_100_1000.cfg index 90f6d6a..adef78f 100644 --- a/sim/gobs/wiki/system_size_100_1000.cfg +++ b/sim/gobs/wiki/system_size_100_1000.cfg @@ -14,12 +14,12 @@ minNodes = 0 maxNodes = 0 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 diff --git a/sim/gobs/wiki/system_size_20-1000.cfg b/sim/gobs/wiki/system_size_20-1000.cfg index 2d988b4..57e29f8 100644 --- a/sim/gobs/wiki/system_size_20-1000.cfg +++ b/sim/gobs/wiki/system_size_20-1000.cfg @@ -18,12 +18,12 @@ files = 3000 faults = 1 # FileFactory implementation PlainFiles -FileFactory = PlainFiles +impl.filefactory = PlainFiles # File size: -filesize = 14 +file.size = 14 # Strip size: -filestrip = 4 +file.strip = 4 # File/object replicas: -filereplicas = 3 +file.replicas = 3 # Number of objects per file: -filewidth = 10 +file.width = 10 hooks/post-receive -- Grayskull Repository

1 0

[Gs-commits] Grayskull Repository branch, master, updated. git-migration-159-g3449f49
by noreply＠mcs.anl.gov 18 Nov '09

18 Nov '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 3449f496d84d41d7a8c9cb3f0754b5e27fb0e987 (commit) from 5c66cfaf5e41007f85b7488ab4867f18cfee89cb (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 3449f496d84d41d7a8c9cb3f0754b5e27fb0e987 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Wed Nov 18 19:29:15 2009 -0800 Basic ParallelRUSH functionality. ----------------------------------------------------------------------- Summary of changes: sim/gobs/gobs/sim/Metric.java | 5 ++- sim/gobs/gobs/sim/Node.java | 28 ++++++++++--------- sim/gobs/gobs/sim/ParallelRUSH.java | 45 ++++++++++++++++++++++---------- sim/gobs/gobs/sim/PlacementScheme.java | 14 ++++++++++ sim/gobs/gobs/sim/Replicated.java | 8 +++++- sim/gobs/gobs/sim/Simulator.java | 9 +++++- sim/gobs/gobs/sweep/Balance01.java | 9 +++--- 7 files changed, 82 insertions(+), 36 deletions(-) Diff of changes: diff --git a/sim/gobs/gobs/sim/Metric.java b/sim/gobs/gobs/sim/Metric.java index 959cdc7..48f2a8c 100644 --- a/sim/gobs/gobs/sim/Metric.java +++ b/sim/gobs/gobs/sim/Metric.java @@ -65,8 +65,9 @@ public abstract class Metric { Node primaryNode = sites.get(sites.size()-1); Primary primary = (Primary) primaryNode.get(object.id); - source = locateSource(object, sites); - secondary = source.get(object.id).toSecondary(primary); + source = locateSource(object, sites); + gobs.sim.Object s = source.get(object.id); + secondary = ((SingleObject) s).toSecondary(primary); destination = locateDestination(object, sites); destination.add(secondary); } diff --git a/sim/gobs/gobs/sim/Node.java b/sim/gobs/gobs/sim/Node.java index 2a70873..a0b1178 100644 --- a/sim/gobs/gobs/sim/Node.java +++ b/sim/gobs/gobs/sim/Node.java @@ -14,7 +14,7 @@ public class Node /** Object store. */ - public List<SingleObject> objects; + public List<gobs.sim.Object> objects; /** Counter for number of pushes during rebuilds. @@ -34,22 +34,23 @@ public class Node Node(BigInteger id) { super(id); - objects = new ArrayList<SingleObject>(); + objects = new ArrayList<gobs.sim.Object>(); } - void add(SingleObject object) + void add(gobs.sim.Object object) { objects.add(object); } /** - @return An SingleObject with this id. + @return An gobs.sim.Object with this id. */ - SingleObject get(BigInteger id) + gobs.sim.Object get(BigInteger id) { - for (Iterator<SingleObject> it = objects.iterator(); it.hasNext(); ) + for (Iterator<gobs.sim.Object> it = objects.iterator(); + it.hasNext(); ) { - SingleObject object = it.next(); + gobs.sim.Object object = it.next(); if (object.id.equals(id)) { return object; @@ -59,13 +60,14 @@ public class Node } /** - Remove and return an SingleObject with this id. + Remove and return an gobs.sim.Object with this id. */ - SingleObject remove(BigInteger id) + gobs.sim.Object remove(BigInteger id) { - for (Iterator<SingleObject> it = objects.iterator(); it.hasNext(); ) + for (Iterator<gobs.sim.Object> it = objects.iterator(); + it.hasNext(); ) { - SingleObject object = it.next(); + gobs.sim.Object object = it.next(); if (object.id.equals(id)) { it.remove(); @@ -89,7 +91,7 @@ public class Node */ boolean contains(BigInteger id) { - for (SingleObject object : objects) + for (gobs.sim.Object object : objects) { if (object.id.equals(id)) return true; @@ -107,7 +109,7 @@ public class Node StringBuffer result = new StringBuffer(objects.size()*10); result.append(name(simulator)); result.append(":\t"); - for (SingleObject object : objects) + for (gobs.sim.Object object : objects) { result.append(object.toString(simulator)).append(" "); } diff --git a/sim/gobs/gobs/sim/ParallelRUSH.java b/sim/gobs/gobs/sim/ParallelRUSH.java index 28065b8..2fa345c 100644 --- a/sim/gobs/gobs/sim/ParallelRUSH.java +++ b/sim/gobs/gobs/sim/ParallelRUSH.java @@ -54,6 +54,26 @@ public class ParallelRUSH result.trimToSize(); return result; } + + void place(ObjectStrip object, List<Node> nodes) + { + // System.out.println("place: " + object); + List<Node> sites = sites(object, nodes); + simulator.stats.flexibilities.add(sites.size()); + PrimaryStrip primary = object.toPrimaryStrip(); + Node node = sites.remove(sites.size()-1); + node.add(primary); + for (int i = object.replicas-1; i >= 0; i--) + { + if (sites.size() == 0) + { + System.out.println("Could not place all replicas!"); + return; + } + node = sites.remove(i); + node.add(object.toSecondaryStrip(primary)); + } + } List<Node> sites(gobs.sim.Object object, List<Node> nodes) { @@ -61,29 +81,25 @@ public class ParallelRUSH return sites((ObjectStrip) object, nodes); else throw new IllegalArgumentException - ("Incorrect use ParallelRUSH: " + + ("Incorrect use of ParallelRUSH: " + "object must be an ObjectStrip."); } List<Node> sites(gobs.sim.ObjectStrip object, List<Node> nodes) { - List<Node> result = new ArrayList<Node>(); - Set<Node> sites = new HashSet<Node>(); - - Node primary = - lookup(object.id, object.number, 0, object.replicas, nodes); + List<Node> result = new ArrayList<Node>(object.replicas); - for (int r = 1; r <= object.replicas-1; r++) + for (int r = object.replicas-1; r >= 1; r--) { Node node = lookup(object.id, object.number, r, object.replicas, nodes); - - if (node != primary) - sites.add(node); + result.add(node); } - - result.addAll(sites); - result.add(primary); + + Node node = + lookup(object.id, object.number, 0, object.replicas, nodes); + + result.add(node); return result; } @@ -166,7 +182,8 @@ public class ParallelRUSH boolean isPrime(int z) { BigInteger i = new BigInteger(""+z); - return i.isProbablePrime(certainty); + boolean result = i.isProbablePrime(certainty); + return result; } /** diff --git a/sim/gobs/gobs/sim/PlacementScheme.java b/sim/gobs/gobs/sim/PlacementScheme.java index 3806c6b..f73f8bf 100644 --- a/sim/gobs/gobs/sim/PlacementScheme.java +++ b/sim/gobs/gobs/sim/PlacementScheme.java @@ -56,11 +56,25 @@ public abstract class PlacementScheme */ // public abstract Node lookup(BigInteger id, List<Node> nodes); + void rebuild(gobs.sim.Object object, List<Node> nodes) + { + if (object instanceof SingleObject) + rebuild((SingleObject) object, nodes); + else if (object instanceof ObjectStrip) + rebuild((ObjectStrip) object, nodes); + } + /** Simulate a rebuild. */ abstract void rebuild(SingleObject object, List<Node> nodes); + void rebuild(ObjectStrip object, List<Node> nodes) + { + throw new UnsupportedOperationException + ("rebuild(ObjectStrip) not supported!"); + } + /** Simply place each object on a node. */ diff --git a/sim/gobs/gobs/sim/Replicated.java b/sim/gobs/gobs/sim/Replicated.java index 1a29503..fa0ebe8 100644 --- a/sim/gobs/gobs/sim/Replicated.java +++ b/sim/gobs/gobs/sim/Replicated.java @@ -30,7 +30,6 @@ public abstract class Replicated /** Generate an object set for a file. */ -// public Collection<SingleObject> allocateFile(File file, List<Node> nodes) public Collection<gobs.sim.Object> allocateFile(File file, List<Node> nodes) { @@ -95,6 +94,13 @@ public abstract class Replicated } } + void place(ObjectStrip object, List<Node> nodes) + { + throw new UnsupportedOperationException + (this.getClass().toString() + + ": does not support place(ObjectStrip)!"); + } + /** Obtain a list of nodes that may host the given object with respect to the replica placement scheme used here. diff --git a/sim/gobs/gobs/sim/Simulator.java b/sim/gobs/gobs/sim/Simulator.java index ff7be10..8214702 100644 --- a/sim/gobs/gobs/sim/Simulator.java +++ b/sim/gobs/gobs/sim/Simulator.java @@ -213,6 +213,11 @@ public class Simulator display = Addressable.Display.NUMERIC; placer = new RUSHp(this, source); } + else if (placerName.equals("ParallelRUSH")) + { + display = Addressable.Display.NUMERIC; + placer = new ParallelRUSH(this, source); + } else bail("Bad PlacementScheme name: " + placerName); @@ -382,8 +387,8 @@ public class Simulator Node failed = nodes.remove(index); System.out.println("failed: " + failed.name(this) + " (" + failed.objects.size() + ")"); - List<SingleObject> replacements = failed.objects; - for (SingleObject object : replacements) + List<gobs.sim.Object> replacements = failed.objects; + for (gobs.sim.Object object : replacements) placer.rebuild(object, nodes); if (string("printNodes").equals("true")) printNodeObjects(nodes); diff --git a/sim/gobs/gobs/sweep/Balance01.java b/sim/gobs/gobs/sweep/Balance01.java index 4c5ebd2..91aec24 100644 --- a/sim/gobs/gobs/sweep/Balance01.java +++ b/sim/gobs/gobs/sweep/Balance01.java @@ -30,7 +30,10 @@ class Balance01 final XYSeriesCollection collection = new XYSeriesCollection(); String[] schemes = - new String[]{ "Nearest", "Kademlia", "VertexGroups" }; + /* new String[]{ "Nearest", "Kademlia", "VertexGroups", + "RUSHp" }; */ + new String[]{ "ParallelRUSH" }; + // new String[]{ "Nearest" }; for (String scheme : schemes) { Simulator simulator = execute(properties, scheme); @@ -66,11 +69,9 @@ class Balance01 Map<Node,Integer> sorted = sorter.sort(simulator.origCount); int i = 0; - for (Node node: sorted.keySet()) + for (Node node : sorted.keySet()) series.add(i++, sorted.get(node)); return series; } } - - hooks/post-receive -- Grayskull Repository

1 0

[Gs-commits] Grayskull Repository branch, master, updated. git-migration-158-g5c66cfa
by noreply＠mcs.anl.gov 18 Nov '09

18 Nov '09

A ref change was pushed to the repository containing the project "Grayskull Repository". The branch, master has been updated via 5c66cfaf5e41007f85b7488ab4867f18cfee89cb (commit) via 7b77556e0fea788e00f0b64236b869bb7dc9abd2 (commit) via 9254df5d7964055bc043974b20b102161fc3084b (commit) via 67075695fa2e8eae8ba982a550cc4b200c1167d7 (commit) via 75281d4e1c672bed66e498b9829595e89c7c7d84 (commit) via 0affc54384e05590552503de9e7e01bc40bbe123 (commit) via b62e734d71b21f9fe01f848a30ba4d4b9cb26ab2 (commit) via 9836b37814f6155dac3bf9f8b3ced5632aa08967 (commit) via bafe386783f881ecdfb870013e1aa90ee59f6a8b (commit) via 35777d6374eba26728bbd573376bee88cb4fc056 (commit) via 3c86df9c01a65ee31cdb8582701b491b72bf3bcf (commit) via d69e4c68aee2569f6aadd94ba4ac10f6ddc37dad (commit) via 86f1845b4cda2fe49d89dce9bb377069611fa98d (commit) via fe8a73bf6bbfe6bef23d3c0b53f1be43271bc230 (commit) from f1f49f85c8cc5713d9d87917d38504bf60170c37 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 5c66cfaf5e41007f85b7488ab4867f18cfee89cb Merge: 7b77556e0fea788e00f0b64236b869bb7dc9abd2 f1f49f85c8cc5713d9d87917d38504bf60170c37 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Wed Nov 18 21:24:00 2009 -0600 Merge branch 'master' of /homes/dkimpe/git/grayskull commit 7b77556e0fea788e00f0b64236b869bb7dc9abd2 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Nov 3 16:27:41 2009 -0600 Should be executable. commit 9254df5d7964055bc043974b20b102161fc3084b Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Nov 3 16:24:53 2009 -0600 Extremely simple test for balance during parallel writes. commit 67075695fa2e8eae8ba982a550cc4b200c1167d7 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Nov 3 16:17:27 2009 -0600 Better ASCII outputs. commit 75281d4e1c672bed66e498b9829595e89c7c7d84 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Nov 3 15:48:54 2009 -0600 New infrastructure for ObjectStrips. commit 0affc54384e05590552503de9e7e01bc40bbe123 Merge: 9836b37814f6155dac3bf9f8b3ced5632aa08967 b62e734d71b21f9fe01f848a30ba4d4b9cb26ab2 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sun Nov 1 21:37:36 2009 -0600 Merge branch 'master' of login.mcs.anl.gov:~/gs commit b62e734d71b21f9fe01f848a30ba4d4b9cb26ab2 Merge: bafe386783f881ecdfb870013e1aa90ee59f6a8b 4dabd068ec058c6fbf6f9503ad8bc75f4a6fbbe5 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Fri Oct 30 18:23:40 2009 -0500 Merge branch 'master' of /homes/dkimpe/git/grayskull commit 9836b37814f6155dac3bf9f8b3ced5632aa08967 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sun Oct 25 20:55:48 2009 -0500 Nearest successfully places one object (primary or secondary) per server. Primaries and secondaries do not overlap. commit bafe386783f881ecdfb870013e1aa90ee59f6a8b Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sun Oct 25 19:14:16 2009 -0500 New Parallelism tests. commit 35777d6374eba26728bbd573376bee88cb4fc056 Merge: 3c86df9c01a65ee31cdb8582701b491b72bf3bcf 31f700d6396ea91b7b002d328447ca97cbc78529 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Sun Oct 25 19:11:29 2009 -0500 Merge branch 'master' of /homes/dkimpe/git/grayskull commit 3c86df9c01a65ee31cdb8582701b491b72bf3bcf Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Oct 6 15:45:34 2009 -0500 Usage info now on wiki. commit d69e4c68aee2569f6aadd94ba4ac10f6ddc37dad Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Oct 6 15:43:14 2009 -0500 This is now in examples. commit 86f1845b4cda2fe49d89dce9bb377069611fa98d Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Oct 6 15:22:54 2009 -0500 Dropping PNGs from git. From now on, assuming everyone has ImageMagick. commit fe8a73bf6bbfe6bef23d3c0b53f1be43271bc230 Author: Justin Wozniak <wozniak(a)mcs.anl.gov> Date: Tue Oct 6 15:16:25 2009 -0500 Small Parallelism01 tests going. ----------------------------------------------------------------------- Summary of changes: sim/gobs/About.txt | 59 +-------- sim/gobs/debug.cfg | 14 -- sim/gobs/gobs/plot/AsciiPlot.java | 53 ++++++- sim/gobs/gobs/sim/File.java | 6 +- sim/gobs/gobs/sim/KClosest.java | 2 +- sim/gobs/gobs/sim/LayoutScheme.java | 2 +- sim/gobs/gobs/sim/Metric.java | 2 +- sim/gobs/gobs/sim/Node.java | 26 ++-- sim/gobs/gobs/sim/Obj.java | 60 -------- sim/gobs/gobs/sim/Object.java | 31 ++++ sim/gobs/gobs/sim/ObjectStrip.java | 47 ++++++ sim/gobs/gobs/sim/PDSIFiles.java | 46 +++++-- sim/gobs/gobs/sim/ParallelRUSH.java | 192 +++++++++++++++++++++++++ sim/gobs/gobs/sim/PlacementScheme.java | 34 +++-- sim/gobs/gobs/sim/Primary.java | 2 +- sim/gobs/gobs/sim/PrimaryStrip.java | 22 +++ sim/gobs/gobs/sim/RUSHp.java | 28 ++-- sim/gobs/gobs/sim/Replicated.java | 35 +++-- sim/gobs/gobs/sim/Secondary.java | 2 +- sim/gobs/gobs/sim/SecondaryStrip.java | 26 ++++ sim/gobs/gobs/sim/Simulator.java | 100 ++++++++++--- sim/gobs/gobs/sim/SingleObject.java | 39 +++++ sim/gobs/gobs/sim/TraceFiles.java | 33 ++++- sim/gobs/gobs/sim/VertexGroups.java | 12 ++- sim/gobs/gobs/sweep/AbstractSweep.java | 2 - sim/gobs/gobs/sweep/Balance01.java | 53 ++++++-- sim/gobs/gobs/sweep/Parallelism01.java | 11 +- sim/gobs/gobs/test/TestFileFactory.java | 16 ++- sim/gobs/gobs/util/EnhancedLinkedHashSet.java | 4 +- sim/gobs/gobs/util/LineReader.java | 1 + sim/gobs/sweeps/bl01-small.cfg | 52 +++++++ sim/gobs/sweeps/bl01-small.zsh | 3 + sim/gobs/sweeps/bl01.cfg | 51 ++----- sim/gobs/sweeps/bl01output.cfg | 15 ++ sim/gobs/sweeps/decluster02.zsh | 4 +- sim/gobs/sweeps/parallelism01.zsh | 20 +++ sim/gobs/sweeps/pll01.cfg | 19 +++ sim/gobs/sweeps/pll01case01.cfg | 8 + sim/gobs/sweeps/pll01case02.cfg | 9 ++ sim/gobs/sweeps/pll_output.cfg | 17 +++ sim/gobs/sweeps/system_size_pdsi_100-1000.cfg | 24 +++ sim/gobs/sweeps/system_size_pdsi_20-40.cfg | 24 +++ sim/gobs/traces/onebigfile.trace | 4 + sim/gobs/wiki/PDSI_distribution.png | Bin 17300 -> 0 bytes sim/gobs/wiki/dc02kpv01.png | Bin 21883 -> 0 bytes sim/gobs/wiki/dc02kpv02.png | Bin 23019 -> 0 bytes sim/gobs/wiki/dc02kpv03.png | Bin 21152 -> 0 bytes sim/gobs/wiki/dc02kpv04.png | Bin 21772 -> 0 bytes sim/gobs/wiki/dc02kpv05.png | Bin 21809 -> 0 bytes sim/gobs/wiki/dc02kpv06.png | Bin 20711 -> 0 bytes sim/gobs/wiki/dc03kpv01.png | Bin 21731 -> 0 bytes sim/gobs/wiki/dc03kpv02.png | Bin 20000 -> 0 bytes sim/gobs/wiki/dc03kpv03.png | Bin 20001 -> 0 bytes sim/gobs/wiki/dc03kpv04.png | Bin 22903 -> 0 bytes sim/gobs/wiki/dc03kpv05.png | Bin 20883 -> 0 bytes sim/gobs/wiki/dc03kpv06.png | Bin 24109 -> 0 bytes sim/gobs/wiki/dc04kpv01.png | Bin 31626 -> 0 bytes sim/gobs/wiki/dc04kpv02.png | Bin 31774 -> 0 bytes sim/gobs/wiki/dc04kpv03.png | Bin 30909 -> 0 bytes sim/gobs/wiki/dc04kpv04.png | Bin 33182 -> 0 bytes sim/gobs/wiki/dc04kpv05.png | Bin 31715 -> 0 bytes sim/gobs/wiki/dc04kpv06.png | Bin 31124 -> 0 bytes 62 files changed, 908 insertions(+), 302 deletions(-) delete mode 100644 sim/gobs/debug.cfg delete mode 100644 sim/gobs/gobs/sim/Obj.java create mode 100644 sim/gobs/gobs/sim/Object.java create mode 100644 sim/gobs/gobs/sim/ObjectStrip.java create mode 100644 sim/gobs/gobs/sim/ParallelRUSH.java create mode 100644 sim/gobs/gobs/sim/PrimaryStrip.java create mode 100644 sim/gobs/gobs/sim/SecondaryStrip.java create mode 100644 sim/gobs/gobs/sim/SingleObject.java create mode 100644 sim/gobs/sweeps/bl01-small.cfg create mode 100755 sim/gobs/sweeps/bl01-small.zsh create mode 100644 sim/gobs/sweeps/bl01output.cfg create mode 100755 sim/gobs/sweeps/parallelism01.zsh create mode 100644 sim/gobs/sweeps/pll01.cfg create mode 100644 sim/gobs/sweeps/pll01case01.cfg create mode 100644 sim/gobs/sweeps/pll01case02.cfg create mode 100644 sim/gobs/sweeps/pll_output.cfg create mode 100644 sim/gobs/sweeps/system_size_pdsi_100-1000.cfg create mode 100644 sim/gobs/sweeps/system_size_pdsi_20-40.cfg create mode 100644 sim/gobs/traces/onebigfile.trace delete mode 100644 sim/gobs/wiki/PDSI_distribution.png delete mode 100644 sim/gobs/wiki/dc02kpv01.png delete mode 100644 sim/gobs/wiki/dc02kpv02.png delete mode 100644 sim/gobs/wiki/dc02kpv03.png delete mode 100644 sim/gobs/wiki/dc02kpv04.png delete mode 100644 sim/gobs/wiki/dc02kpv05.png delete mode 100644 sim/gobs/wiki/dc02kpv06.png delete mode 100644 sim/gobs/wiki/dc03kpv01.png delete mode 100644 sim/gobs/wiki/dc03kpv02.png delete mode 100644 sim/gobs/wiki/dc03kpv03.png delete mode 100644 sim/gobs/wiki/dc03kpv04.png delete mode 100644 sim/gobs/wiki/dc03kpv05.png delete mode 100644 sim/gobs/wiki/dc03kpv06.png delete mode 100644 sim/gobs/wiki/dc04kpv01.png delete mode 100644 sim/gobs/wiki/dc04kpv02.png delete mode 100644 sim/gobs/wiki/dc04kpv03.png delete mode 100644 sim/gobs/wiki/dc04kpv04.png delete mode 100644 sim/gobs/wiki/dc04kpv05.png delete mode 100644 sim/gobs/wiki/dc04kpv06.png Diff of changes: diff --git a/sim/gobs/About.txt b/sim/gobs/About.txt index 80cc379..f76285a 100644 --- a/sim/gobs/About.txt +++ b/sim/gobs/About.txt @@ -1,59 +1,4 @@ -GOBS: Simulator +See the wiki for up-to-date usage information: -Compile and run with: - -javac gobs/Simulator.java -java gobs.Simulator - -or - -ant -java -jar GOBS.jar - -Run with arguments: - - [-p <path>] <file.cfg>* <name>=<value>* - -where: - -path resets the directory that is searched for each file.cfg -file.cfg is a Java properties file -assignments set - -The resulting properties must contain: - -1) Basic settings: - -B: the bit length of the addresses -nodes: the number of object stores -files: the number of files -faults: the number of induced faults - -2) Implementations: - -FileFactory: The name of an implementation of FileFactory - See the class docs for additional required properties -PlacementScheme: The name of an implementation of PlacementScheme - See the class docs for additional required properties - -3) Modifiers: - -ReplicaSource: From where will replacement replicas be copied? - primary: always pull the replica from the primary, if the - primary was lost, pull from a random secondary - last: pull from the secondary most distant from the primary - random: select a primary or secondary at random - -4) Output selectors: - -printProperties: Dump properties for each Simulator execution -printInitialNodes: Print nodes and objects at startup (true/false) -printNodes: Print nodes and objects after each rebuild (true/false) -printRebuilds: Print object tranfers during rebuilds (true/false) -printFiles: Print file information (true/false) -printLoadDiffs: Print load change statistics (true/false) - -Additional optional properties: -include: a space separated list of additional cfg files to include -printInclude: Report included files (true/false) +http://wiki.mcs.anl.gov/radix/grayskull/index.php/GOBS_Simulator diff --git a/sim/gobs/debug.cfg b/sim/gobs/debug.cfg deleted file mode 100644 index b15d3bb..0000000 --- a/sim/gobs/debug.cfg +++ /dev/null @@ -1,14 +0,0 @@ - -# Simulator configuration - -printProperties = true -# Print nodes and objects at startup (true/false) -printInitialNodes = true -# Print nodes and objects after each rebuild (true/false) -printNodes = true -# Print object tranfers during rebuilds (true/false) -printRebuilds = true -# Print file information (true/false) -printFiles = true -# Print load change statistics (true/false) -printLoadDiffs = true diff --git a/sim/gobs/gobs/plot/AsciiPlot.java b/sim/gobs/gobs/plot/AsciiPlot.java index d57b2ab..c6a192f 100644 --- a/sim/gobs/gobs/plot/AsciiPlot.java +++ b/sim/gobs/gobs/plot/AsciiPlot.java @@ -4,6 +4,9 @@ package gobs.plot; import java.io.*; import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.util.Tools; /** * Will be easy to tie in gnuplot. @@ -20,23 +23,57 @@ public class AsciiPlot { PrintWriter writer = new PrintWriter(output); - writer.println("# TITLE: " + title); - writer.println("# XLABEL: " + xlabel); - writer.println("# YLABEL: " + ylabel); - - for (int i = 0; i < series.getItemCount(); i++) + write(writer, series, title, xlabel, ylabel, output); + + writer.close(); + } + catch (IOException e) + { + Tools.bail("AsciiPlot: Output error to: " + output); + } + } + + /** + Uses series descriptions for titles in output. + */ + public static void plot(XYSeriesCollection collection, + String title, + String xlabel, String ylabel, + String output) + { + try + { + PrintWriter writer = new PrintWriter(output); + writer.println("# TITLE: " + title); + for (Object o : collection.getSeries()) { - writer.println("" + series.getX(i) + " " + series.getY(i)); + XYSeries series = (XYSeries) o; + String description = series.getDescription(); + write(writer, series, description, xlabel, ylabel, output); } - writer.close(); } catch (IOException e) { - System.out.println("AsciiPlot: Output error to: " + output); + Tools.bail("AsciiPlot: Output error to: " + output); } } + static void write(PrintWriter writer, + XYSeries series, + String title, String xlabel, String ylabel, + String output) + throws IOException + { + writer.println("# DESCRIPTION: " + title); + writer.println("# XLABEL: " + xlabel); + writer.println("# YLABEL: " + ylabel); + + for (int i = 0; i < series.getItemCount(); i++) + { + writer.println("" + series.getX(i) + " " + series.getY(i)); + } + } /** Debugging only. diff --git a/sim/gobs/gobs/sim/File.java b/sim/gobs/gobs/sim/File.java index a74c73b..a51fa7b 100644 --- a/sim/gobs/gobs/sim/File.java +++ b/sim/gobs/gobs/sim/File.java @@ -19,7 +19,7 @@ public class File static int unique = 0; - Collection<Obj> objects; + Collection<gobs.sim.Object> objects; File(long size, int strip, int width, int replicas) { @@ -31,7 +31,7 @@ public class File id = unique++; } - void link(Collection<Obj> objects) + void link(Collection<gobs.sim.Object> objects) { this.objects = objects; } @@ -54,7 +54,7 @@ public class File StringBuilder sb = new StringBuilder(100+objects.size()*10); sb.append("file[").append(id).append("]: "); - for (Obj object : objects) + for (gobs.sim.Object object : objects) sb.append(object.toString(simulator)).append(" "); return sb.toString(); } diff --git a/sim/gobs/gobs/sim/KClosest.java b/sim/gobs/gobs/sim/KClosest.java index a147c50..b3bc9a4 100644 --- a/sim/gobs/gobs/sim/KClosest.java +++ b/sim/gobs/gobs/sim/KClosest.java @@ -23,7 +23,7 @@ abstract class KClosest Obtain a list of nodes that may host the given object with respect to the replica placement scheme used here. */ - List<Node> sites(Obj object, List<Node> nodes) + List<Node> sites(gobs.sim.Object object, List<Node> nodes) { BoundedSortedValues<Node,BigInteger> sites = new BoundedSortedValues<Node,BigInteger>(object.replicas); diff --git a/sim/gobs/gobs/sim/LayoutScheme.java b/sim/gobs/gobs/sim/LayoutScheme.java index 1bf9d6e..c18a99d 100644 --- a/sim/gobs/gobs/sim/LayoutScheme.java +++ b/sim/gobs/gobs/sim/LayoutScheme.java @@ -20,5 +20,5 @@ abstract class LayoutScheme M = two.pow(B); } - abstract List<Obj> generate(int fileSize, int objSize); + abstract List<SingleObject> generate(int fileSize, int objSize); } diff --git a/sim/gobs/gobs/sim/Metric.java b/sim/gobs/gobs/sim/Metric.java index 6a12b4d..959cdc7 100644 --- a/sim/gobs/gobs/sim/Metric.java +++ b/sim/gobs/gobs/sim/Metric.java @@ -35,7 +35,7 @@ public abstract class Metric return result; } - void rebuild(Obj object, List<Node> nodes) + void rebuild(SingleObject object, List<Node> nodes) { // System.out.println("rebuild: " + object.toString(simulator)); List<Node> sites = sites(object, nodes); diff --git a/sim/gobs/gobs/sim/Node.java b/sim/gobs/gobs/sim/Node.java index 6f29647..2a70873 100644 --- a/sim/gobs/gobs/sim/Node.java +++ b/sim/gobs/gobs/sim/Node.java @@ -14,7 +14,7 @@ public class Node /** Object store. */ - public List<Obj> objects; + public List<SingleObject> objects; /** Counter for number of pushes during rebuilds. @@ -34,22 +34,22 @@ public class Node Node(BigInteger id) { super(id); - objects = new ArrayList<Obj>(); + objects = new ArrayList<SingleObject>(); } - void add(Obj object) + void add(SingleObject object) { objects.add(object); } /** - @return An Obj with this id. + @return An SingleObject with this id. */ - Obj get(BigInteger id) + SingleObject get(BigInteger id) { - for (Iterator<Obj> it = objects.iterator(); it.hasNext(); ) + for (Iterator<SingleObject> it = objects.iterator(); it.hasNext(); ) { - Obj object = it.next(); + SingleObject object = it.next(); if (object.id.equals(id)) { return object; @@ -59,13 +59,13 @@ public class Node } /** - Remove and return an Obj with this id. + Remove and return an SingleObject with this id. */ - Obj remove(BigInteger id) + SingleObject remove(BigInteger id) { - for (Iterator<Obj> it = objects.iterator(); it.hasNext(); ) + for (Iterator<SingleObject> it = objects.iterator(); it.hasNext(); ) { - Obj object = it.next(); + SingleObject object = it.next(); if (object.id.equals(id)) { it.remove(); @@ -89,7 +89,7 @@ public class Node */ boolean contains(BigInteger id) { - for (Obj object : objects) + for (SingleObject object : objects) { if (object.id.equals(id)) return true; @@ -107,7 +107,7 @@ public class Node StringBuffer result = new StringBuffer(objects.size()*10); result.append(name(simulator)); result.append(":\t"); - for (Obj object : objects) + for (SingleObject object : objects) { result.append(object.toString(simulator)).append(" "); } diff --git a/sim/gobs/gobs/sim/Obj.java b/sim/gobs/gobs/sim/Obj.java deleted file mode 100644 index d2a6750..0000000 --- a/sim/gobs/gobs/sim/Obj.java +++ /dev/null @@ -1,60 +0,0 @@ - -package gobs.sim; - -import java.math.BigInteger; - -/** - * Simulates a single object. - * Identifier "Object" is used by Java. - * */ - -class Obj - extends Addressable -{ - /** - Simulated size in bytes. - */ - int size; - /** - Number of replicas, including primary. - */ - int replicas; - - /* - Obj(int B, BigInteger id, int size, int replicas) - { - super(B, id); - this.size = size; - this.replicas = replicas; - } - */ - - Obj(BigInteger id, int size, int replicas) - { - super(id); - this.size = size; - this.replicas = replicas; - } - - Primary toPrimary() - { - // return new Primary(B, id, size, replicas); - return new Primary(id, size, replicas); - } - - Secondary toSecondary(Primary primary) - { - // return new Secondary(B, id, size, replicas, primary); - return new Secondary(id, size, replicas, primary); - } - - public String toString(Simulator simulator) - { - return name(simulator) + "(" + size + ")"; - } - - public String toString() - { - return id.toString(); - } -} diff --git a/sim/gobs/gobs/sim/Object.java b/sim/gobs/gobs/sim/Object.java new file mode 100644 index 0000000..ab1190e --- /dev/null +++ b/sim/gobs/gobs/sim/Object.java @@ -0,0 +1,31 @@ + +package gobs.sim; + +import java.math.BigInteger; + +/** + * Tags objects. + * */ + +abstract class Object + extends Addressable +{ + Object(BigInteger id, int size, int replicas) + { + super(id); + this.size = size; + this.replicas = replicas; + } + + /** + Simulated size in bytes. + */ + int size; + + /** + Number of replicas, including primary. + */ + int replicas; + + abstract String toString(Simulator simulator); +} diff --git a/sim/gobs/gobs/sim/ObjectStrip.java b/sim/gobs/gobs/sim/ObjectStrip.java new file mode 100644 index 0000000..034333f --- /dev/null +++ b/sim/gobs/gobs/sim/ObjectStrip.java @@ -0,0 +1,47 @@ + +package gobs.sim; + +import java.math.BigInteger; + +/** + * Simulates a part of striped object. + * Identifier "Object" is used by Java. + * */ + +class ObjectStrip + extends Object +{ + /** + The strip number. A striped object with strip width n + will have object strips with numbers 0...n-1. + */ + int number; + + ObjectStrip(BigInteger id, int number, int size, int replicas) + { + super(id, size, replicas); + this.number = number; + } + + PrimaryStrip toPrimaryStrip() + { + // return new Primary(B, id, size, replicas); + return new PrimaryStrip(id, number, size, replicas); + } + + SecondaryStrip toSecondaryStrip(PrimaryStrip primary) + { + // return new Secondary(B, id, size, replicas, primary); + return new SecondaryStrip(id, number, size, replicas, primary); + } + + public String toString(Simulator simulator) + { + return name(simulator) + "[" + number + "](" + size + ")"; + } + + public String toString() + { + return id.toString(); + } +} diff --git a/sim/gobs/gobs/sim/PDSIFiles.java b/sim/gobs/gobs/sim/PDSIFiles.java index acfe698..3f3ee51 100644 --- a/sim/gobs/gobs/sim/PDSIFiles.java +++ b/sim/gobs/gobs/sim/PDSIFiles.java @@ -15,6 +15,15 @@ import static java.lang.Math.*; * * If the file is smaller than 1MB, it is on a single strip, * else its width is the number of servers. + * + * If over 4MB, it has replica count 2, else replica count 3. + * + * Input is a 2-column array: e.g., [[0.2,1024][0.8,1M][1.0,1G]] + * In this case, 20% of the files would have size 0, + * 60% would have size in 1K-1M, and 20% would have size in 1M-1G. + * Thus the first segment is implied ([[0.0,0]]). + * The other ranges are interpolated in a piecewise exponential fashion; + * the coefficients are stored in k and m. * */ public class PDSIFiles @@ -63,8 +72,10 @@ public class PDSIFiles double[] m; /** + Obtain file distribution by interpolating points. + @param simulator Required only to get node count @param points points(i,j) indicates that fraction i - has size below j. + has size below j */ public PDSIFiles(Simulator simulator, double[][] points) { @@ -76,13 +87,24 @@ public class PDSIFiles } /** + Obtain file distribution by interpolation points parsed from + property PDSIFilesPoints. + @param simulator Required only to get node count and property + @see #parse + */ + public PDSIFiles(Simulator simulator) + { + this(simulator, parse(simulator.string("PDSIFilesPoints"))); + } + + /** Parse input data points for interpolation. Commas separate x,y numbers, semicolons separate pairs, suffixes K, M, G, T, are valid for kibi, mibi, gibi, and tebi-bytes. Example: "0.2,1024;1.0,1M" gives [[0.2][1024]][[1.0][1048576]]. */ - static double[][] parse(String data) + public static double[][] parse(String data) { Map<Double,Double> map = new LinkedHashMap<Double,Double>(); String[] pairs = data.split(";"); @@ -263,12 +285,8 @@ public class PDSIFiles System.out.println("needed " + i + " files to produce " + n + " bytes"); } - - /** - Debugging only. - */ - /* - public static void main(String[] args) + + public static void debugInterpolation(String[] args) { long n = Long.parseLong(args[0]); @@ -295,9 +313,8 @@ public class PDSIFiles // generateCount(n); generateBytes(factory, n); } - */ - public static void main(String[] args) + public static void debugParse(String[] args) { String v = args[0]; @@ -308,4 +325,13 @@ public class PDSIFiles dumpPoints(p); } + + /** + Debugging only. + */ + public static void main(String[] args) + { + // debugInterpolation(args); + debugParse(args); + } } diff --git a/sim/gobs/gobs/sim/ParallelRUSH.java b/sim/gobs/gobs/sim/ParallelRUSH.java new file mode 100644 index 0000000..28065b8 --- /dev/null +++ b/sim/gobs/gobs/sim/ParallelRUSH.java @@ -0,0 +1,192 @@ + +package gobs.sim; + +import java.math.BigInteger; +import java.util.*; + +import gobs.util.Tools; + +/** + * RUSH extension for parallel access to files + * */ + +public class ParallelRUSH + extends Replicated +{ + /** + Must take care to manage state of this RNG, + as it is used successively by several methods. + */ + Random rng = new Random(); + + /** + The certainty that our primes are really prime. + */ + int certainty = Integer.MAX_VALUE; + + /** + @param simulator Reference to the Simulator + @param source A ReplicaSource. + */ + ParallelRUSH(Simulator simulator, ReplicaSource source) + { + super(simulator, source); + } + + /** + Generate an object set for a file. + */ + public Collection<gobs.sim.Object> + allocateFile(File file, List<Node> nodes) + { + ArrayList<gobs.sim.Object> result = + new ArrayList<gobs.sim.Object>(file.width); + + BigInteger id = randomID(); + for (int i = 0; i < file.width; i++) + { + int bytes = file.objectSize(i); + ObjectStrip obj = + new ObjectStrip(id, i, bytes, file.replicas); + result.add((gobs.sim.Object) obj); + } + + result.trimToSize(); + return result; + } + + List<Node> sites(gobs.sim.Object object, List<Node> nodes) + { + if (object instanceof ObjectStrip) + return sites((ObjectStrip) object, nodes); + else + throw new IllegalArgumentException + ("Incorrect use ParallelRUSH: " + + "object must be an ObjectStrip."); + } + + List<Node> sites(gobs.sim.ObjectStrip object, List<Node> nodes) + { + List<Node> result = new ArrayList<Node>(); + Set<Node> sites = new HashSet<Node>(); + + Node primary = + lookup(object.id, object.number, 0, object.replicas, nodes); + + for (int r = 1; r <= object.replicas-1; r++) + { + Node node = + lookup(object.id, object.number, r, object.replicas, nodes); + + if (node != primary) + sites.add(node); + } + + result.addAll(sites); + result.add(primary); + return result; + } + + /* + List<Node> sites(SingleObject object, List<Node> nodes) + { + return null; + } + */ + + /** + <a href="http://wiki.mcs.anl.gov/radix/grayskull/index.php/Stripe_placement_and_rebu…">From the wiki.</a> + + @param x The object id (key). + @param i The strip number + @param r The replica number (in [0,x.replicas)). + @param R The number of replicas for x (x.replicas). + @param nodes The set of nodes. + */ + public Node lookup(BigInteger x, int i, int r, int R, + List<Node> nodes) + { + int j = 0; + rng.setSeed(x.longValue()); + advance(j); + int z = rng.nextInt(nodes.size()); + int p = randomPrime(nodes.size()); + int m = nodes.size(); + + BigInteger big_z = new BigInteger(""+z); + BigInteger big_i = new BigInteger(""+i); + BigInteger big_r = new BigInteger(""+r); + BigInteger big_p = new BigInteger(""+p); + BigInteger big_R = new BigInteger(""+R); + BigInteger big_m = new BigInteger(""+m); + + BigInteger irjp = big_i.multiply(big_R).add(big_r) + .multiply(big_p); + BigInteger v = x.add(big_z).add(irjp); + BigInteger z_ = big_z.add(irjp).mod(big_m); + + int index = -1; + if (m >= R && z_.compareTo(big_m) < 0) + index = v.mod(big_m).intValue(); + else if (m < R && z_.compareTo(big_R) < 0 && + v.mod(big_R).compareTo(big_m) < 0) + index = v.mod(big_R).intValue(); + else + Tools.bail("ParallelRUSH.lookup(): Error!"); + + return nodes.get(index); + } + + void rebuild(SingleObject object, List<Node> nodes) + { + place(object, nodes); + } + + /** + Advance the class member rng by j steps. + */ + void advance(int j) + { + for (int i = 0; i < j; j++) + rng.nextInt(); + } + + /** + Obtain a prime number in {@code [m,2m)}. + */ + int randomPrime(int m) + { + int z; + do + z = rng.nextInt(m)+m; + while (!isPrime(z)); + return z; + } + + boolean isPrime(int z) + { + BigInteger i = new BigInteger(""+z); + return i.isProbablePrime(certainty); + } + + /** + Prime checker. + */ + public static void main(String[] args) + { + if (args.length < 1) + Tools.bail("Need 1 number!"); + + int i = Integer.parseInt(args[0]); + + System.out.println("argument: " + i); + + Simulator simulator = new Simulator(); + RUSHp foop = new RUSHp(simulator, ReplicaSource.LAST); + + long start = System.currentTimeMillis(); + System.out.println("is prime? " + foop.isPrime(i)); + long stop = System.currentTimeMillis(); + System.out.println("took: " + (stop-start) + " millis"); + } +} diff --git a/sim/gobs/gobs/sim/PlacementScheme.java b/sim/gobs/gobs/sim/PlacementScheme.java index f3582bf..3806c6b 100644 --- a/sim/gobs/gobs/sim/PlacementScheme.java +++ b/sim/gobs/gobs/sim/PlacementScheme.java @@ -27,20 +27,25 @@ public abstract class PlacementScheme /** Generate an object set for a file. */ - public Collection<Obj> allocateFile(File file, List<Node> nodes) + // public Collection<SingleObject> allocateFile(File file, List<Node> nodes) + + public abstract Collection<gobs.sim.Object> + allocateFile(File file, List<Node> nodes); + /* { - ArrayList<Obj> result = new ArrayList<Obj>(file.width+2); + ArrayList<gobs.sim.Object> result = + new ArrayList<gobs.sim.Object>(file.width+2); BigInteger bigWidth = new BigInteger("" + file.width); BigInteger s = M.divide(bigWidth); BigInteger p = randomID(); - int i; - for (i = 0; i < file.width; i++) + for (int i = 0; i < file.width; i++) { BigInteger id = p; int bytes = file.objectSize(i); - Obj obj = new Obj(id, bytes, file.replicas); + gobs.sim.Object obj = + (gobs.sim.Object) new SingleObject(id, bytes, file.replicas); result.add(obj); p = p.add(s).mod(M); } @@ -48,17 +53,21 @@ public abstract class PlacementScheme result.trimToSize(); return result; } - +*/ // public abstract Node lookup(BigInteger id, List<Node> nodes); - abstract void rebuild(Obj object, List<Node> nodes); + /** + Simulate a rebuild. + */ + abstract void rebuild(SingleObject object, List<Node> nodes); /** Simply place each object on a node. */ - public void place(Collection<Obj> objects, List<Node> nodes) + public void place(Collection<gobs.sim.Object> objects, + List<Node> nodes) { - for (Obj object : objects) + for (gobs.sim.Object object : objects) place(object, nodes); } @@ -66,7 +75,8 @@ public abstract class PlacementScheme Initial placement: allocate replicas for object and place on nodes. */ - public abstract void place(Obj object, List<Node> nodes); + public abstract void place(gobs.sim.Object object, + List<Node> nodes); /** Obtain a list of nodes that may host the given object @@ -74,7 +84,7 @@ public abstract class PlacementScheme Closest node should be last. */ - abstract List<Node> sites(Obj object, List<Node> nodes); + abstract List<Node> sites(gobs.sim.Object object, List<Node> nodes); /** Pull a random Node from one of the given sites. @@ -85,7 +95,7 @@ public abstract class PlacementScheme return sites.get(index); } - void printRebuild(Obj object, Node source, Node destination) + void printRebuild(SingleObject object, Node source, Node destination) { if (simulator.bool("printRebuilds")) System.out.println("rebuild: " + object.toString(simulator) + diff --git a/sim/gobs/gobs/sim/Primary.java b/sim/gobs/gobs/sim/Primary.java index d9e6a46..11760d6 100644 --- a/sim/gobs/gobs/sim/Primary.java +++ b/sim/gobs/gobs/sim/Primary.java @@ -8,7 +8,7 @@ import java.math.BigInteger; * */ class Primary - extends Obj + extends SingleObject { /* Primary(int B, BigInteger id, int size, int replicas) diff --git a/sim/gobs/gobs/sim/PrimaryStrip.java b/sim/gobs/gobs/sim/PrimaryStrip.java new file mode 100644 index 0000000..32c77da --- /dev/null +++ b/sim/gobs/gobs/sim/PrimaryStrip.java @@ -0,0 +1,22 @@ + +package gobs.sim; + +import java.math.BigInteger; + +/** + * Simulates a primary object strip. + * */ + +class PrimaryStrip + extends ObjectStrip +{ + PrimaryStrip(BigInteger id, int number, int size, int replicas) + { + super(id, number, size, replicas); + } + + public String toString(Simulator simulator) + { + return name(simulator) + "[P][" + number + "](" + size + ")"; + } +} diff --git a/sim/gobs/gobs/sim/RUSHp.java b/sim/gobs/gobs/sim/RUSHp.java index 58e2963..3a0fbcc 100644 --- a/sim/gobs/gobs/sim/RUSHp.java +++ b/sim/gobs/gobs/sim/RUSHp.java @@ -34,7 +34,17 @@ public class RUSHp super(simulator, source); } - List<Node> sites(Obj object, List<Node> nodes) + List<Node> sites(gobs.sim.Object object, List<Node> nodes) + { + if (object instanceof SingleObject) + return sites((SingleObject) object, nodes); + else + throw new IllegalArgumentException + ("Incorrect use of RUSHp: " + + "object must be an SingleObject."); + } + + List<Node> sites(gobs.sim.SingleObject object, List<Node> nodes) { List<Node> result = new ArrayList<Node>(); Set<Node> sites = new HashSet<Node>(); @@ -93,7 +103,7 @@ public class RUSHp return nodes.get(index); } - void rebuild(Obj object, List<Node> nodes) + void rebuild(SingleObject object, List<Node> nodes) { place(object, nodes); } @@ -125,20 +135,6 @@ public class RUSHp return i.isProbablePrime(certainty); } - static BigInteger flip(int B, int i, BigInteger id) - { - int v = (int) Math.ceil(B/8.0); - byte[] val = new byte[v]; - int vi = (i-1)/8; - int vj = (B-i)%8; - byte b = new Integer(1 << vj).byteValue(); - - val[vi] = b; - - BigInteger op = new BigInteger(val); - return op.xor(id); - } - /** Prime checker. */ diff --git a/sim/gobs/gobs/sim/Replicated.java b/sim/gobs/gobs/sim/Replicated.java index 5356763..1a29503 100644 --- a/sim/gobs/gobs/sim/Replicated.java +++ b/sim/gobs/gobs/sim/Replicated.java @@ -30,9 +30,12 @@ public abstract class Replicated /** Generate an object set for a file. */ - public Collection<Obj> allocateFile(File file, List<Node> nodes) +// public Collection<SingleObject> allocateFile(File file, List<Node> nodes) + public Collection<gobs.sim.Object> + allocateFile(File file, List<Node> nodes) { - ArrayList<Obj> result = new ArrayList<Obj>(file.width+2); + ArrayList<Object> result = + new ArrayList<Object>(file.width); BigInteger bigWidth = new BigInteger("" + file.width); BigInteger s = M.divide(bigWidth); @@ -43,7 +46,7 @@ public abstract class Replicated { BigInteger id = p; int bytes = file.objectSize(i); - Obj obj = new Obj(id, bytes, file.replicas); + gobs.sim.Object obj = new SingleObject(id, bytes, file.replicas); result.add(obj); p = p.add(s).mod(M); } @@ -54,9 +57,9 @@ public abstract class Replicated // public abstract Node lookup(BigInteger id, List<Node> nodes); - public void place(Collection<Obj> objects, List<Node> nodes) + public void place(Collection<Object> objects, List<Node> nodes) { - for (Obj object : objects) + for (Object object : objects) place(object, nodes); } @@ -64,7 +67,15 @@ public abstract class Replicated Initial placement: allocate replicas for object and place on nodes. */ - public void place(Obj object, List<Node> nodes) + public void place(gobs.sim.Object object, List<Node> nodes) + { + if (object instanceof SingleObject) + place((SingleObject) object, nodes); + else if (object instanceof ObjectStrip) + place((ObjectStrip) object, nodes); + } + + public void place(SingleObject object, List<Node> nodes) { // System.out.println("place: " + object); List<Node> sites = sites(object, nodes); @@ -90,18 +101,18 @@ public abstract class Replicated Closest node should be last. */ - abstract List<Node> sites(Obj object, List<Node> nodes); + // abstract List<Node> sites(Object object, List<Node> nodes); /** Place this object to rebuild. */ - abstract void rebuild(Obj object, List<Node> nodes); + abstract void rebuild(SingleObject object, List<Node> nodes); /** Locate a replica of the given object on one of the given nodes in accordance with the {@link #replicaSource} policy. */ - Node locateSource(Obj object, List<Node> sites) + Node locateSource(SingleObject object, List<Node> sites) { int maxTries = sites.size()+1; Node result = null; @@ -135,7 +146,7 @@ public abstract class Replicated return result; } - Node locateDestination(Obj object, List<Node> sites) + Node locateDestination(SingleObject object, List<Node> sites) { Node destination = null; @@ -158,7 +169,7 @@ public abstract class Replicated /** Report failure to perform replication and exit. */ - void failLocateSource(Obj object, List<Node> sites) + void failLocateSource(SingleObject object, List<Node> sites) { System.out.println("locateSource(): Could not find source for: " + object); @@ -168,7 +179,7 @@ public abstract class Replicated Tools.bail("Exit!"); } - void printRebuild(Obj object, Node source, Node destination) + void printRebuild(SingleObject object, Node source, Node destination) { if (simulator.bool("printRebuilds")) System.out.println("rebuild: " + object.toString(simulator) + diff --git a/sim/gobs/gobs/sim/Secondary.java b/sim/gobs/gobs/sim/Secondary.java index c92f841..fa5315f 100644 --- a/sim/gobs/gobs/sim/Secondary.java +++ b/sim/gobs/gobs/sim/Secondary.java @@ -8,7 +8,7 @@ import java.math.BigInteger; * */ class Secondary - extends Obj + extends SingleObject { Primary primary; diff --git a/sim/gobs/gobs/sim/SecondaryStrip.java b/sim/gobs/gobs/sim/SecondaryStrip.java new file mode 100644 index 0000000..187cf24 --- /dev/null +++ b/sim/gobs/gobs/sim/SecondaryStrip.java @@ -0,0 +1,26 @@ + +package gobs.sim; + +import java.math.BigInteger; + +/** + * Simulates a secondary object strip. + * */ + +class SecondaryStrip + extends ObjectStrip +{ + PrimaryStrip primary; + + SecondaryStrip(BigInteger id, int number, int size, int replicas, + PrimaryStrip primary) + { + super(id, number, size, replicas); + this.primary = primary; + } + + public String toString(Simulator simulator) + { + return name(simulator) + "[S][" + number + "](" + size + ")"; + } +} diff --git a/sim/gobs/gobs/sim/Simulator.java b/sim/gobs/gobs/sim/Simulator.java index 1f4eff7..ff7be10 100644 --- a/sim/gobs/gobs/sim/Simulator.java +++ b/sim/gobs/gobs/sim/Simulator.java @@ -51,8 +51,8 @@ public class Simulator String path = null; public List<Node> nodes; - EnhancedLinkedHashSet<Obj> objects = - new EnhancedLinkedHashSet<Obj>(); + EnhancedLinkedHashSet<gobs.sim.Object> objects = + new EnhancedLinkedHashSet<gobs.sim.Object>(); List<File> files = new ArrayList<File>(); /** @@ -71,12 +71,20 @@ public class Simulator DecimalFormat df = new DecimalFormat("0.00"); public Statistics stats = new Statistics(); + + /** + Cached value of printAccesses property + */ + boolean printAccesses = false; public Simulator() - {} + { + File.unique = 0; + } public Simulator(Properties properties) { + this(); initialize(properties); } @@ -173,6 +181,8 @@ public class Simulator factory = new TraceFiles(this); else if (factoryName.equals("RandomFiles")) factory = new RandomFiles(this); + else if (factoryName.equals("PDSIFiles")) + factory = new PDSIFiles(this); else bail("Bad filefactory name: " + factoryName); } @@ -237,7 +247,8 @@ public class Simulator allocateFiles(fileCount); if (string("printFiles").equals("true")) - printFiles(); + printFiles(); + System.out.println("objects: " + objects.size()); } void allocateFiles(int count) @@ -251,17 +262,18 @@ public class Simulator File file = factory.get(); int tries = 0; - Collection<Obj> set; + Collection<gobs.sim.Object> set; do { if (tries++ >= maxTries) bail("Could not allocate new object ids!"); set = placer.allocateFile(file, nodes); - file.link(set); - files.add(file); } while (objects.containsAny(set)); + + file.link(set); + files.add(file); objects.addAll(set); } @@ -295,26 +307,65 @@ public class Simulator void performReads() { System.out.println("reads: " + reads); - + if (reads > 0) + printAccesses = bool("printAccesses"); + for (int read = 0; read < reads; read++) { - File file = randomFile(); - System.out.println("read: " + file.id); - - for (Obj object : file.objects) - { - List<Node> sites = placer.sites(object, nodes); - Node node = placer.random(sites); - System.out.println("access: node: " + node.id + - " object: " + object.id); - node.accesses++; - } + File file = randomFile(); + performRead(file); } } + void performRead(File file) + { + if (printAccesses) + System.out.println("read: " + file.id); + + for (gobs.sim.Object object : file.objects) + { + List<Node> sites = placer.sites(object, nodes); + Node node = placer.random(sites); + if (printAccesses) + System.out.println("access: object: " + object.id + + " node: " + node.id); + node.accesses++; + } + } + void performWrites() - {} + { + if (writes > 0) + printAccesses = bool("printAccesses"); + + for (int write = 0; write < writes; write++) + { + File file = randomFile(); + performWrite(file); + } + } + void performWrite(File file) + { + if (printAccesses) + System.out.println("write: " + file.id); + + for (gobs.sim.Object object : file.objects) + performWrite(object); + } + + void performWrite(gobs.sim.Object object) + { + List<Node> sites = placer.sites(object, nodes); + for (Node node : sites) + { + if (printAccesses) + System.out.println("access: object: " + object.id + + " node: " + node.id); + node.accesses++; + } + } + File randomFile() { int index = Bits.nextInt(files.size()); @@ -331,8 +382,8 @@ public class Simulator Node failed = nodes.remove(index); System.out.println("failed: " + failed.name(this) + " (" + failed.objects.size() + ")"); - List<Obj> replacements = failed.objects; - for (Obj object : replacements) + List<SingleObject> replacements = failed.objects; + for (SingleObject object : replacements) placer.rebuild(object, nodes); if (string("printNodes").equals("true")) printNodeObjects(nodes); @@ -488,6 +539,10 @@ public class Simulator void computeAccesses() { + // Did the user perform any operations? + if (reads+writes == 0) + return; + System.out.println("ACCESS_BALANCE: "); Node least = null; Node most = null; @@ -513,6 +568,7 @@ public class Simulator double m = (double) most.accesses; double l = (double) least.accesses; double ratio = m/l; + System.out.println("accessbalance: " + df.format(ratio)); stats.results.setProperty("accessbalance", df.format(ratio)); } diff --git a/sim/gobs/gobs/sim/SingleObject.java b/sim/gobs/gobs/sim/SingleObject.java new file mode 100644 index 0000000..a06e49c --- /dev/null +++ b/sim/gobs/gobs/sim/SingleObject.java @@ -0,0 +1,39 @@ + +package gobs.sim; + +import java.math.BigInteger; + +/** + * Simulates a single object. + * */ + +class SingleObject + extends Object +{ + SingleObject(BigInteger id, int size, int replicas) + { + super(id, size, replicas); + } + + Primary toPrimary() + { + // return new Primary(B, id, size, replicas); + return new Primary(id, size, replicas); + } + + Secondary toSecondary(Primary primary) + { + // return new Secondary(B, id, size, replicas, primary); + return new Secondary(id, size, replicas, primary); + } + + public String toString(Simulator simulator) + { + return name(simulator) + "(" + size + ")"; + } + + public String toString() + { + return id.toString(); + } +} diff --git a/sim/gobs/gobs/sim/TraceFiles.java b/sim/gobs/gobs/sim/TraceFiles.java index d70c968..4d7e957 100644 --- a/sim/gobs/gobs/sim/TraceFiles.java +++ b/sim/gobs/gobs/sim/TraceFiles.java @@ -9,6 +9,12 @@ import gobs.util.*; * Generates files from tracefile. * * Overwrites Simulator.fileCount! + * + * File format: "file <size> <strip> <width> <replicas>" + * + * If width is "OPT", width is the number of servers + * divided by replicas. If width is "MAX", width is + * the number of servers. * * Required properties settings: filetrace * */ @@ -19,9 +25,16 @@ class TraceFiles String filename; List<String> lines = null; int index = 0; - + + Simulator simulator; + + /** + Opens trace file based on Simulator property filetrace. + Maintains an internal list of lines generated by LineReader. + */ TraceFiles(Simulator simulator) { + this.simulator = simulator; filename = simulator.string("filetrace"); try { @@ -35,20 +48,23 @@ class TraceFiles Tools.bail("TraceFiles: Could not read: \"" + filename + "\""); } - simulator.fileCount = lines.size(); } + /** + Grab the next File from the trace file. + @return The {@link File} or null if exhausted. + */ public File get() { if (index > lines.size()) - Tools.bail("TraceFiles: Exceeded trace file length!"); + return null; String[] tokens = LineReader.tokenize(lines.get(index)); String file = tokens[0]; if (! file.equals("file")) - Tools.bail("Does not compute (should be file stats): " + + Tools.bail("Does not compute (see javadocs): " + lines.get(index)); int size = 0; @@ -57,10 +73,17 @@ class TraceFiles int replicas = 0; try { + String w; size = Integer.parseInt(tokens[1]); strip = Integer.parseInt(tokens[2]); - width = Integer.parseInt(tokens[3]); replicas = Integer.parseInt(tokens[4]); + w = tokens[3]; + if (w.equals("MAX")) + width = simulator.N; + else if (w.equals("OPT")) + width = simulator.N/replicas; + else + width = Integer.parseInt(w); } catch (NumberFormatException e) { diff --git a/sim/gobs/gobs/sim/VertexGroups.java b/sim/gobs/gobs/sim/VertexGroups.java index 5921e4f..d07c1b3 100644 --- a/sim/gobs/gobs/sim/VertexGroups.java +++ b/sim/gobs/gobs/sim/VertexGroups.java @@ -34,7 +34,17 @@ public class VertexGroups return id1.xor(id2); } - List<Node> sites(Obj object, List<Node> nodes) + List<Node> sites(gobs.sim.Object object, List<Node> nodes) + { + if (object instanceof SingleObject) + return sites((SingleObject) object, nodes); + else + throw new IllegalArgumentException + ("Incorrect use of VertexGroups: " + + "object must be an SingleObject."); + } + + List<Node> sites(SingleObject object, List<Node> nodes) { List<Node> result = new ArrayList<Node>(P+1); Set<Node> sites = new HashSet<Node>(2*(P+1)); diff --git a/sim/gobs/gobs/sweep/AbstractSweep.java b/sim/gobs/gobs/sweep/AbstractSweep.java index 851536e..8841f5f 100644 --- a/sim/gobs/gobs/sweep/AbstractSweep.java +++ b/sim/gobs/gobs/sweep/AbstractSweep.java @@ -209,9 +209,7 @@ abstract class AbstractSweep List<String> result = new ArrayList<String>(); for (String token : tokens) - { result.add(token); - } return result; } diff --git a/sim/gobs/gobs/sweep/Balance01.java b/sim/gobs/gobs/sweep/Balance01.java index 7067103..4c5ebd2 100644 --- a/sim/gobs/gobs/sweep/Balance01.java +++ b/sim/gobs/gobs/sweep/Balance01.java @@ -4,6 +4,7 @@ package gobs.sweep; import java.util.*; import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; import gobs.sim.*; import gobs.util.*; @@ -19,29 +20,57 @@ class Balance01 { public static void main(String[] args) { - Bits.init(); + Bits.init(true); Properties properties = Sweep.getopts(args); properties.setProperty("faults", "0"); - + + String output = Tools.string(properties, "output"); + + final XYSeriesCollection collection = new XYSeriesCollection(); + + String[] schemes = + new String[]{ "Nearest", "Kademlia", "VertexGroups" }; + for (String scheme : schemes) + { + Simulator simulator = execute(properties, scheme); + XYSeries series = extract(scheme, simulator); + collection.addSeries(series); + } + + MultiplePlot.plot(collection, "Load by Node", + "node", "load", output); + + /* + AsciiPlot.plot(collection, "Load by Node", + "nodes", "load", output); + */ + } + + static Simulator execute(Properties properties, String scheme) + { + properties.setProperty("PlacementScheme", scheme); Simulator simulator = new Simulator(); simulator.initialize(properties); simulator.execute(); - XYSeries series = new XYSeries("Balance"); + return simulator; + } + static XYSeries extract(String scheme, Simulator simulator) + { + XYSeries series = new XYSeries(scheme); + series.setDescription(scheme); + MapSorter<Node> sorter = new MapSorter<Node>(); Map<Node,Integer> sorted = sorter.sort(simulator.origCount); - + int i = 0; for (Node node: sorted.keySet()) - { - Integer load = sorted.get(node); - series.add(i++, load); - } + series.add(i++, sorted.get(node)); - String output = Tools.string(properties, "output"); - AsciiPlot.plot(series, "Load Distribution", "nodes", "load", - output); - } + return series; + } } + + diff --git a/sim/gobs/gobs/sweep/Parallelism01.java b/sim/gobs/gobs/sweep/Parallelism01.java index 7765f2e..97e5dff 100644 --- a/sim/gobs/gobs/sweep/Parallelism01.java +++ b/sim/gobs/gobs/sweep/Parallelism01.java @@ -43,23 +43,20 @@ class Parallelism01 nearestSweep.execute("PAST", "nodes", values, "accessbalance"); collection.addSeries(nearestSweep.series); - /* properties.setProperty("PlacementScheme", "Kademlia"); Sweep kademliaSweep = new Sweep(properties); kademliaSweep.repeats = Tools.integer(properties, "repeats"); - kademliaSweep.execute("Kademlia", "nodes", values, - "maxTotal"); + kademliaSweep.execute("Kademlia", "nodes", values, "accessbalance"); collection.addSeries(kademliaSweep.series); properties.setProperty("PlacementScheme", "VertexGroups"); Sweep vgSweep = new Sweep(properties); vgSweep.repeats = Tools.integer(properties, "repeats"); - vgSweep.execute("Vertex Groups", "nodes", values, "maxTotal"); + vgSweep.execute("Vertex Groups", "nodes", values, "accessbalance"); collection.addSeries(vgSweep.series); - */ - MultiLogXPlot.plot(collection, "Maximum Rebuild Load", - "nodes", "load fraction(%)", output); + MultiLogXPlot.plot(collection, "Load Imbalance", + "nodes", "imbalance", output); // AsciiPlot.plot(nearestSweep.series, "Maximum Rebuild Load", // "nodes", "load", "tmp.txt"); } diff --git a/sim/gobs/gobs/test/TestFileFactory.java b/sim/gobs/gobs/test/TestFileFactory.java index ebe6c0f..e22bc95 100644 --- a/sim/gobs/gobs/test/TestFileFactory.java +++ b/sim/gobs/gobs/test/TestFileFactory.java @@ -19,9 +19,10 @@ class TestFileFactory */ static int n; - static PriorityQueue<File> generate(int n) + static double[][] hardPoints() { double[][] points = new double[5][2]; + points[0][0] = 0.2; points[0][1] = 2*1024; points[1][0] = 0.6; @@ -33,6 +34,11 @@ class TestFileFactory points[4][0] = 1.0; points[4][1] = 1024.0*1024*1024*1024; + return points; + } + + static PriorityQueue<File> generate(double[][] points, int n) + { Simulator simulator = new Simulator(); simulator.nodes = new ArrayList<Node>(); @@ -99,8 +105,14 @@ class TestFileFactory Bits.init(); n = Integer.parseInt(args[0]); + + double[][] points; + if (args.length > 1) + points = PDSIFiles.parse(args[1]); + else + points = hardPoints(); - PriorityQueue<File> files = generate(n); + PriorityQueue<File> files = generate(points, n); dumpSizes(files); diff --git a/sim/gobs/gobs/util/EnhancedLinkedHashSet.java b/sim/gobs/gobs/util/EnhancedLinkedHashSet.java index ac78648..10d1e12 100644 --- a/sim/gobs/gobs/util/EnhancedLinkedHashSet.java +++ b/sim/gobs/gobs/util/EnhancedLinkedHashSet.java @@ -32,9 +32,9 @@ public class EnhancedLinkedHashSet<T> { EnhancedLinkedHashSet<Object> set = new EnhancedLinkedHashSet<Object>(); - Obj obj = new Obj(4, new BigInteger("3"), 0, 0); + SingleObject obj = new Obj(4, new BigInteger("3"), 0, 0); set.add(obj); - if (set.contains(new Obj(5, new BigInteger("3"), 1, 1))) + if (set.contains(new SingleObject(5, new BigInteger("3"), 1, 1))) System.out.println("true"); } */ diff --git a/sim/gobs/gobs/util/LineReader.java b/sim/gobs/gobs/util/LineReader.java index f090ac0..c3d3da3 100644 --- a/sim/gobs/gobs/util/LineReader.java +++ b/sim/gobs/gobs/util/LineReader.java @@ -51,6 +51,7 @@ public class LineReader result.add(line); } } + reader.close(); } catch (java.io.IOException e) { diff --git a/sim/gobs/sweeps/bl01-small.cfg b/sim/gobs/sweeps/bl01-small.cfg new file mode 100644 index 0000000..b64d96d --- /dev/null +++ b/sim/gobs/sweeps/bl01-small.cfg @@ -0,0 +1,52 @@ + +### BALANCE 01 + +include = bl01output.cfg + +### General parameters: + +# Bit-length of addresses: +B = 8 +# Number of files: +files = 2 +# Number of faults: +faults = 0 + +# Number of servers: +nodes = 10 + +### Choose a PlacementScheme implementation: + +# Chosen by sweep driver + +VertexGroupPrefix = 5 + +# No rebuilds will actually be performed. This is filler: +ReplicaSource = primary + +### Choose a NodeScheme implementation: + +#NodeScheme = RandomNodes +NodeScheme = SimpleNodes + +### Choose one of the following FileFactory sections: + +# FileFactory implementation PlainFiles +FileFactory = PlainFiles +# File size: +filesize = 14 +# Strip size: +filestrip = 4 +# File/object replicas: +filereplicas = 3 +# Number of objects per file: +filewidth = 10 + +FileFactory = TraceFiles +filetrace = traces/onebigfile.trace + +### User read/writes: + +# No user operations... +reads = 0 +writes = 0 diff --git a/sim/gobs/sweeps/bl01-small.zsh b/sim/gobs/sweeps/bl01-small.zsh new file mode 100755 index 0000000..ba3fe74 --- /dev/null +++ b/sim/gobs/sweeps/bl01-small.zsh @@ -0,0 +1,3 @@ +#!/bin/zsh + +java gobs.sweep.Balance01 -p sweeps bl01-small.cfg -o bl01-small.eps diff --git a/sim/gobs/sweeps/bl01.cfg b/sim/gobs/sweeps/bl01.cfg index aee4eea..cccaf81 100644 --- a/sim/gobs/sweeps/bl01.cfg +++ b/sim/gobs/sweeps/bl01.cfg @@ -1,27 +1,26 @@ -### Kademlia vs. PAST vs. Vertex Groups 03 +### BALANCE 01 -step = mids +include = bl01output.cfg ### General parameters: # Bit-length of addresses: B = 32 # Number of files: -files = 3000 +files = 1 # Number of faults: -faults = 1 +faults = 0 # Number of servers: -minNodes = 10 -maxNodes = 1000 +nodes = 10 ### Choose a PlacementScheme implementation: # Chosen by sweep driver -PlacementScheme = Nearest -ReplicaSource = random +# No rebuilds will actually be performed. This is filler: +ReplicaSource = primary ### Choose a NodeScheme implementation: @@ -41,35 +40,11 @@ filereplicas = 3 # Number of objects per file: filewidth = 10 -# FileFactory implementation TraceFiles -#FileFactory = TraceFiles -# Trace filename: -#filetrace = traces/test01.trace +FileFactory = TraceFiles +filetrace = traces/onebigfile.trace -# FileFactory implementation RandomFiles -#FileFactory = RandomFiles -# Strip size: -#filestrip = 4 -# Max size of big files: -#filebigmax = 100 -# Min size of big files: -#filebigmin = 50 -# Max size of small files: -#filesmallmax = 20 -# Min size of small files: -#filesmallmin = 0 - -### Simulator output configuration +### User read/writes: -# Dump properties for each Simulator execution -printProperties = true -# Print nodes and objects at startup (true/false) -printInitialNodes = false -# Print nodes and objects after each rebuild (true/false) -printNodes = false -# Print object tranfers during rebuilds (true/false) -printRebuilds = false -# Print file information (true/false) -printFiles = false -# Print load change statistics (true/false) -printLoadDiffs = false +# No user operations... +reads = 0 +writes = 0 diff --git a/sim/gobs/sweeps/bl01output.cfg b/sim/gobs/sweeps/bl01output.cfg new file mode 100644 index 0000000..a4bed90 --- /dev/null +++ b/sim/gobs/sweeps/bl01output.cfg @@ -0,0 +1,15 @@ + +### Simulator output configuration + +# Dump properties for each Simulator execution +printProperties = true +# Print nodes and objects at startup (true/false) +printInitialNodes = true +# Print nodes and objects after each rebuild (true/false) +printNodes = false +# Print object tranfers during rebuilds (true/false) +printRebuilds = false +# Print file information (true/false) +printFiles = true +# Print load change statistics (true/false) +printLoadDiffs = false diff --git a/sim/gobs/sweeps/decluster02.zsh b/sim/gobs/sweeps/decluster02.zsh index 386483a..d42e129 100755 --- a/sim/gobs/sweeps/decluster02.zsh +++ b/sim/gobs/sweeps/decluster02.zsh @@ -1,8 +1,10 @@ #!/bin/zsh NAME=$1 +N=$2 [[ ${NAME} == "" ]] && print "No NAME!" && exit 1 +[[ ${N} == "" ]] && N=20 NAME="dc02${NAME}" @@ -10,7 +12,7 @@ CFG="${NAME}.cfg" EPS="${NAME}.eps" OUT="${NAME}.out" -java gobs.sweep.Decluster02 -p sweeps -n 20 -o ${EPS} ${CFG} >& ${OUT} +java gobs.sweep.Decluster02 -p sweeps -n ${N} -o ${EPS} ${CFG} >& ${OUT} ERR=$? [[ ${ERR} != 0 ]] && print "Crashed: ${NAME} ERR: ${ERR}" && exit 1 diff --git a/sim/gobs/sweeps/parallelism01.zsh b/sim/gobs/sweeps/parallelism01.zsh new file mode 100755 index 0000000..fc715d7 --- /dev/null +++ b/sim/gobs/sweeps/parallelism01.zsh @@ -0,0 +1,20 @@ +#!/bin/zsh + +# PARALLELISM 01 + +NAME=$1 + +[[ ${NAME} == "" ]] && print "No NAME!" && exit 1 + +NAME="pll01${NAME}" + +CFG="${NAME}.cfg" +EPS="${NAME}.eps" +OUT="${NAME}.out" + +java gobs.sweep.Parallelism01 -p sweeps -n 2 -o ${EPS} ${CFG} >& ${OUT} +ERR=$? + +[[ ${ERR} != 0 ]] && print "Crashed: ${NAME}" && exit 1 + +exit 0 diff --git a/sim/gobs/sweeps/pll01.cfg b/sim/gobs/sweeps/pll01.cfg new file mode 100644 index 0000000..66e7f1d --- /dev/null +++ b/sim/gobs/sweeps/pll01.cfg @@ -0,0 +1,19 @@ + +### Kademlia vs. PAST vs. Vertex Groups 01 + +include = pll_output.cfg + +faults = 0 + +### Choose a PlacementScheme implementation: + +# Chosen by sweep driver. + +ReplicaSource = primary +VertexGroupPrefix = 5 + +### Choose a NodeScheme implementation: + +# NodeScheme = RandomNodes +NodeScheme = SimpleNodes + diff --git a/sim/gobs/sweeps/pll01case01.cfg b/sim/gobs/sweeps/pll01case01.cfg new file mode 100644 index 0000000..b5dd12e --- /dev/null +++ b/sim/gobs/sweeps/pll01case01.cfg @@ -0,0 +1,8 @@ + +# PARALLELISM 01 CASE 01 + +include = pll01.cfg system_size_pdsi_100-1000.cfg +# system_size_pdsi_20-40.cfg +# # + +faults = 0 diff --git a/sim/gobs/sweeps/pll01case02.cfg b/sim/gobs/sweeps/pll01case02.cfg new file mode 100644 index 0000000..c19f3c9 --- /dev/null +++ b/sim/gobs/sweeps/pll01case02.cfg @@ -0,0 +1,9 @@ + +# PARALLELISM 01 CASE 01 + +include = pll01.cfg system_size_pdsi_100-1000.cfg +# system_size_pdsi_20-40.cfg +# +# + +faults = 1 diff --git a/sim/gobs/sweeps/pll_output.cfg b/sim/gobs/sweeps/pll_output.cfg new file mode 100644 index 0000000..6011575 --- /dev/null +++ b/sim/gobs/sweeps/pll_output.cfg @@ -0,0 +1,17 @@ + +### Simulator output configuration + +# Dump properties for each Simulator execution +printProperties = true +# Print nodes and objects at startup (true/false) +printInitialNodes = false +# Print nodes and objects after each rebuild (true/false) +printNodes = false +# Print object tranfers during rebuilds (true/false) +printRebuilds = false +# Print file information (true/false) +printFiles = false +# Print file access information (true/false) +printAccesses = false +# Print load change statistics (true/false) +printLoadDiffs = false diff --git a/sim/gobs/sweeps/system_size_pdsi_100-1000.cfg b/sim/gobs/sweeps/system_size_pdsi_100-1000.cfg new file mode 100644 index 0000000..2d6860d --- /dev/null +++ b/sim/gobs/sweeps/system_size_pdsi_100-1000.cfg @@ -0,0 +1,24 @@ + +### System size: servers and files/objects +### 100-1000 servers, selected exponentially with midpoints +### TODO: 0 files * 10 objects * 3 replicas = 90,000 objects + +# Number of servers: +minNodes = 100 +maxNodes = 1000 +step = literal_100,400,1000 + +### General parameters: + +# Bit-length of addresses: +B = 32 +# Number of files: +files = 1000 +# User accesses: +reads = 5000 +writes = 10000 + +# FileFactory implementation PlainFiles +FileFactory = PDSIFiles +# 20% size 0, 60% size 1K-100K, 20% size 100K-10M +PDSIFilesPoints = 0.2,1K;0.8,100K;1.0,10M diff --git a/sim/gobs/sweeps/system_size_pdsi_20-40.cfg b/sim/gobs/sweeps/system_size_pdsi_20-40.cfg new file mode 100644 index 0000000..bf57678 --- /dev/null +++ b/sim/gobs/sweeps/system_size_pdsi_20-40.cfg @@ -0,0 +1,24 @@ + +### System size: servers and files/objects +### 20-40 servers, selected exponentially with midpoints +### TODO: 0 files * 10 objects * 3 replicas = 90,000 objects + +# Number of servers: +minNodes = 20 +maxNodes = 40 +step = mids + +### General parameters: + +# Bit-length of addresses: +B = 32 +# Number of files: +files = 100 +# User accesses: +reads = 500 +writes = 1000 + +# FileFactory implementation PlainFiles +FileFactory = PDSIFiles +# 20% size 0, 60% size 1K-100K, 20% size 100K-10M +PDSIFilesPoints = 0.2,1K;0.8,100K;1.0,10M diff --git a/sim/gobs/traces/onebigfile.trace b/sim/gobs/traces/onebigfile.trace new file mode 100644 index 0000000..6ad73b3 --- /dev/null +++ b/sim/gobs/traces/onebigfile.trace @@ -0,0 +1,4 @@ + +# Format: file <size> <strip> <width> <replicas> + +file 1000000 100 OPT 3 diff --git a/sim/gobs/wiki/PDSI_distribution.png b/sim/gobs/wiki/PDSI_distribution.png deleted file mode 100644 index fd1f252..0000000 Binary files a/sim/gobs/wiki/PDSI_distribution.png and /dev/null differ diff --git a/sim/gobs/wiki/dc02kpv01.png b/sim/gobs/wiki/dc02kpv01.png deleted file mode 100644 index 58cb631..0000000 Binary files a/sim/gobs/wiki/dc02kpv01.png and /dev/null differ diff --git a/sim/gobs/wiki/dc02kpv02.png b/sim/gobs/wiki/dc02kpv02.png deleted file mode 100644 index 42c9971..0000000 Binary files a/sim/gobs/wiki/dc02kpv02.png and /dev/null differ diff --git a/sim/gobs/wiki/dc02kpv03.png b/sim/gobs/wiki/dc02kpv03.png deleted file mode 100644 index 778d579..0000000 Binary files a/sim/gobs/wiki/dc02kpv03.png and /dev/null differ diff --git a/sim/gobs/wiki/dc02kpv04.png b/sim/gobs/wiki/dc02kpv04.png deleted file mode 100644 index b90d15d..0000000 Binary files a/sim/gobs/wiki/dc02kpv04.png and /dev/null differ diff --git a/sim/gobs/wiki/dc02kpv05.png b/sim/gobs/wiki/dc02kpv05.png deleted file mode 100644 index ed0591b..0000000 Binary files a/sim/gobs/wiki/dc02kpv05.png and /dev/null differ diff --git a/sim/gobs/wiki/dc02kpv06.png b/sim/gobs/wiki/dc02kpv06.png deleted file mode 100644 index ca64a50..0000000 Binary files a/sim/gobs/wiki/dc02kpv06.png and /dev/null differ diff --git a/sim/gobs/wiki/dc03kpv01.png b/sim/gobs/wiki/dc03kpv01.png deleted file mode 100644 index 0ffd888..0000000 Binary files a/sim/gobs/wiki/dc03kpv01.png and /dev/null differ diff --git a/sim/gobs/wiki/dc03kpv02.png b/sim/gobs/wiki/dc03kpv02.png deleted file mode 100644 index 3fde50a..0000000 Binary files a/sim/gobs/wiki/dc03kpv02.png and /dev/null differ diff --git a/sim/gobs/wiki/dc03kpv03.png b/sim/gobs/wiki/dc03kpv03.png deleted file mode 100644 index ba6dd5c..0000000 Binary files a/sim/gobs/wiki/dc03kpv03.png and /dev/null differ diff --git a/sim/gobs/wiki/dc03kpv04.png b/sim/gobs/wiki/dc03kpv04.png deleted file mode 100644 index 0894cc5..0000000 Binary files a/sim/gobs/wiki/dc03kpv04.png and /dev/null differ diff --git a/sim/gobs/wiki/dc03kpv05.png b/sim/gobs/wiki/dc03kpv05.png deleted file mode 100644 index b472293..0000000 Binary files a/sim/gobs/wiki/dc03kpv05.png and /dev/null differ diff --git a/sim/gobs/wiki/dc03kpv06.png b/sim/gobs/wiki/dc03kpv06.png deleted file mode 100644 index 28a7bf6..0000000 Binary files a/sim/gobs/wiki/dc03kpv06.png and /dev/null differ diff --git a/sim/gobs/wiki/dc04kpv01.png b/sim/gobs/wiki/dc04kpv01.png deleted file mode 100644 index 09c3973..0000000 Binary files a/sim/gobs/wiki/dc04kpv01.png and /dev/null differ diff --git a/sim/gobs/wiki/dc04kpv02.png b/sim/gobs/wiki/dc04kpv02.png deleted file mode 100644 index a223b30..0000000 Binary files a/sim/gobs/wiki/dc04kpv02.png and /dev/null differ diff --git a/sim/gobs/wiki/dc04kpv03.png b/sim/gobs/wiki/dc04kpv03.png deleted file mode 100644 index 8366e5c..0000000 Binary files a/sim/gobs/wiki/dc04kpv03.png and /dev/null differ diff --git a/sim/gobs/wiki/dc04kpv04.png b/sim/gobs/wiki/dc04kpv04.png deleted file mode 100644 index d1ac581..0000000 Binary files a/sim/gobs/wiki/dc04kpv04.png and /dev/null differ diff --git a/sim/gobs/wiki/dc04kpv05.png b/sim/gobs/wiki/dc04kpv05.png deleted file mode 100644 index 42b36fc..0000000 Binary files a/sim/gobs/wiki/dc04kpv05.png and /dev/null differ diff --git a/sim/gobs/wiki/dc04kpv06.png b/sim/gobs/wiki/dc04kpv06.png deleted file mode 100644 index cf52717..0000000 Binary files a/sim/gobs/wiki/dc04kpv06.png and /dev/null differ hooks/post-receive -- Grayskull Repository

1 0