This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via 2bc720e77101e9607286a6929c970bfc7ccda624 (commit) via bbf831093c352a0f082437d7346acd2560d2b7cd (commit) via 7adac0875c07e759725479ae4d5b9be64fb227bf (commit) via 82d106fa55ee2ed434e3beb1d615b877a36fe0b9 (commit) via d401a351336d2d36084f2259abbb3cf2269c8129 (commit) via 1d58968df2a502ec57c4ec444ad88d2b8820f511 (commit) via 057455a078ea259ef2d0d7acc1bdd12c6d8d0304 (commit) from 157bbdfc15eb02caff6b61e990e51232fbe25463 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 2bc720e77101e9607286a6929c970bfc7ccda624 Author: U-JUSTINWOZNIAK\wozniak <wozniak@justinwozniak.(none)> Date: Mon Mar 8 10:31:20 2010 -0600 Large range of minor fixes. commit bbf831093c352a0f082437d7346acd2560d2b7cd Author: U-JUSTINWOZNIAK\wozniak <wozniak@justinwozniak.(none)> Date: Mon Mar 8 10:27:00 2010 -0600 Handle rare inflight Copy collision. commit 7adac0875c07e759725479ae4d5b9be64fb227bf Author: U-JUSTINWOZNIAK\wozniak <wozniak@justinwozniak.(none)> Date: Thu Mar 4 15:16:01 2010 -0600 New output for decluster tests. commit 82d106fa55ee2ed434e3beb1d615b877a36fe0b9 Author: U-JUSTINWOZNIAK\wozniak <wozniak@justinwozniak.(none)> Date: Thu Mar 4 15:13:09 2010 -0600 Minor fixes to improve plots. commit d401a351336d2d36084f2259abbb3cf2269c8129 Author: U-JUSTINWOZNIAK\wozniak <wozniak@justinwozniak.(none)> Date: Thu Mar 4 10:58:17 2010 -0600 Fixing up old Declustering tests. commit 1d58968df2a502ec57c4ec444ad88d2b8820f511 Author: U-JUSTINWOZNIAK\wozniak <wozniak@justinwozniak.(none)> Date: Thu Mar 4 10:57:40 2010 -0600 Minor improvements. commit 057455a078ea259ef2d0d7acc1bdd12c6d8d0304 Author: U-JUSTINWOZNIAK\wozniak <wozniak@justinwozniak.(none)> Date: Wed Mar 3 13:14:12 2010 -0600 Make executable. ----------------------------------------------------------------------- Summary of changes: sim/gobs/src/gobs/plot/MultiLogXPlot.java | 6 + sim/gobs/src/gobs/plot/MultiplePlot.java | 10 +- sim/gobs/src/gobs/sim/Fault.java | 10 +- sim/gobs/src/gobs/sim/File.java | 2 +- sim/gobs/src/gobs/sim/KClosest.java | 23 +- sim/gobs/src/gobs/sim/Metric.java | 2 +- sim/gobs/src/gobs/sim/Node.java | 48 ++-- sim/gobs/src/gobs/sim/OSD.java | 52 ++-- sim/gobs/src/gobs/sim/PDSIFiles.java | 8 +- sim/gobs/src/gobs/sim/RAID.java | 266 ++++++++++---------- sim/gobs/src/gobs/sim/Rebuild.java | 42 +++- sim/gobs/src/gobs/sim/Replicated.java | 1 + sim/gobs/src/gobs/sim/Simulator.java | 81 ++++--- sim/gobs/src/gobs/sim/VertexGroups.java | 4 +- sim/gobs/src/gobs/sweep/AbstractSweep.java | 5 +- sim/gobs/src/gobs/sweep/ArraySweep.java | 14 +- sim/gobs/src/gobs/sweep/Helpers.java | 84 +++--- sim/gobs/src/gobs/sweep/Sweep.java | 11 + sim/gobs/src/gobs/sweep/x/DataLoss02.java | 10 +- sim/gobs/src/gobs/sweep/x/Decluster03.java | 52 +++-- sim/gobs/src/gobs/sweep/x/Decluster04.java | 68 +++--- sim/gobs/src/gobs/sweep/x/RAID01.java | 116 +++++----- sim/gobs/src/gobs/sweep/x/Rebuild01.java | 25 +- sim/gobs/src/gobs/sweep/x/Rebuild02.java | 28 +- sim/gobs/src/gobs/util/BoundedSortedValues.java | 2 +- sim/gobs/src/gobs/util/Entry.java | 102 ++++---- sim/gobs/src/gobs/util/Tools.java | 11 +- sim/gobs/sweeps/dc03kpv01.cfg | 6 +- sim/gobs/sweeps/dc03kpv03.cfg | 6 +- sim/gobs/sweeps/dc03kpv05.cfg | 7 +- sim/gobs/sweeps/dc04kpv01.cfg | 7 +- sim/gobs/sweeps/dc04kpv03.cfg | 7 +- sim/gobs/sweeps/dc04kpv05.cfg | 7 +- sim/gobs/sweeps/{dl_output.cfg => dc_output.cfg} | 3 +- sim/gobs/sweeps/decluster03.zsh | 12 +- sim/gobs/sweeps/decluster04.zsh | 12 +- sim/gobs/sweeps/disks-full.cfg | 2 +- sim/gobs/sweeps/disks-medium.cfg | 2 + sim/gobs/sweeps/kpv01.cfg | 2 - sim/gobs/sweeps/kpv03.cfg | 2 - sim/gobs/sweeps/rb01-medium.cfg | 1 + sim/gobs/sweeps/rb01-small.cfg | 3 +- .../{system_20-1000.cfg => system_100,600.cfg} | 20 ++- .../{system_20-1000.cfg => system_100-300-600.cfg} | 20 ++- .../{system_20-1000.cfg => system_100-600.cfg} | 20 ++- sim/gobs/sweeps/system_20-1000.cfg | 20 ++- .../{system_20-1000.cfg => system_20-600.cfg} | 20 ++- .../sweeps/{system_20-1000.cfg => system_600.cfg} | 20 ++- 48 files changed, 702 insertions(+), 580 deletions(-) copy sim/gobs/sweeps/{dl_output.cfg => dc_output.cfg} (90%) mode change 100644 => 100755 sim/gobs/sweeps/rb01-medium.zsh copy sim/gobs/sweeps/{system_20-1000.cfg => system_100,600.cfg} (66%) copy sim/gobs/sweeps/{system_20-1000.cfg => system_100-300-600.cfg} (66%) copy sim/gobs/sweeps/{system_20-1000.cfg => system_100-600.cfg} (67%) copy sim/gobs/sweeps/{system_20-1000.cfg => system_20-600.cfg} (67%) copy sim/gobs/sweeps/{system_20-1000.cfg => system_600.cfg} (66%) Diff of changes: diff --git a/sim/gobs/src/gobs/plot/MultiLogXPlot.java b/sim/gobs/src/gobs/plot/MultiLogXPlot.java index 2f5149f..d275a0a 100644 --- a/sim/gobs/src/gobs/plot/MultiLogXPlot.java +++ b/sim/gobs/src/gobs/plot/MultiLogXPlot.java @@ -30,6 +30,8 @@ import gobs.util.*; public class MultiLogXPlot { + public static boolean bw = false; + /** Generate simple plot. @param collection The x,y data @@ -83,8 +85,12 @@ public class MultiLogXPlot // Turn on shapes... XYLineAndShapeRenderer renderer = new XYLineAndShapeRenderer(); + if (bw) + for (int i = 0; i < plot.getSeriesCount(); i++) + renderer.setSeriesPaint(i, Color.BLACK); plot.setRenderer(renderer); + // Cache default fonts... ValueAxis rangeAxis = plot.getRangeAxis(); Font labelFont = rangeAxis.getLabelFont(); diff --git a/sim/gobs/src/gobs/plot/MultiplePlot.java b/sim/gobs/src/gobs/plot/MultiplePlot.java index 8231307..1c7521d 100644 --- a/sim/gobs/src/gobs/plot/MultiplePlot.java +++ b/sim/gobs/src/gobs/plot/MultiplePlot.java @@ -28,6 +28,8 @@ import gobs.util.Bits; public class MultiplePlot { + public static boolean bw = false; + /** Generate simple plot. @param collection The x,y data. @@ -75,11 +77,13 @@ public class MultiplePlot (title, xlabel, ylabel, collection, PlotOrientation.VERTICAL, withLegend, false, false); - XYLineAndShapeRenderer renderer = - new XYLineAndShapeRenderer(); XYPlot plot = chart.getXYPlot(); - plot.setBackgroundPaint(Color.WHITE); + XYLineAndShapeRenderer renderer = new XYLineAndShapeRenderer(); + if (bw) + for (int i = 0; i < plot.getSeriesCount(); i++) + renderer.setSeriesPaint(i, Color.BLACK); plot.setRenderer(renderer); + plot.setBackgroundPaint(Color.WHITE); chart.draw(g2d, rectangle); try diff --git a/sim/gobs/src/gobs/sim/Fault.java b/sim/gobs/src/gobs/sim/Fault.java index 8c73e6e..95d7d7d 100644 --- a/sim/gobs/src/gobs/sim/Fault.java +++ b/sim/gobs/src/gobs/sim/Fault.java @@ -12,11 +12,11 @@ class Fault */ boolean dataLost = false; - /** - The Rebuild that responded to this Fault. + /** + The Rebuild that responded to this Fault. */ - Rebuild rebuild; - + Rebuild rebuild; + Fault(Simulator simulator) { super(simulator); @@ -30,7 +30,7 @@ class Fault { this.rebuild = rebuild; } - + String log() { return "Fault(" + Time.toString(finish) + ")"; diff --git a/sim/gobs/src/gobs/sim/File.java b/sim/gobs/src/gobs/sim/File.java index 8e45665..d4621b7 100644 --- a/sim/gobs/src/gobs/sim/File.java +++ b/sim/gobs/src/gobs/sim/File.java @@ -18,7 +18,7 @@ public class File int width; int replicas; int id; - + Collection<Gob> objects = null; static int unique = 0; diff --git a/sim/gobs/src/gobs/sim/KClosest.java b/sim/gobs/src/gobs/sim/KClosest.java index 6c83c56..2194113 100644 --- a/sim/gobs/src/gobs/sim/KClosest.java +++ b/sim/gobs/src/gobs/sim/KClosest.java @@ -30,14 +30,14 @@ abstract class KClosest { Sites result = new Sites(object); - /* + BoundedSortedValues<Node,BigInteger> sorter = sites(object.id, nodes, object.replicas); List<Node> list = sorter.get(); - */ + /* List<Node> list = sites(object.id, nodes, object.replicas); - + */ boolean first = true; for (Iterator<Node> it = list.iterator(); it.hasNext(); ) { @@ -59,7 +59,7 @@ abstract class KClosest return result; } - /* + BoundedSortedValues<Node,BigInteger> sites(BigInteger id, List<Node> nodes, int count) { @@ -74,9 +74,9 @@ abstract class KClosest return sorter; } - */ - List<Node> + /* + List<Node> sites(BigInteger id, List<Node> nodes, int count) { Entry<Node,BigInteger>[] best = new Entry[count]; @@ -97,16 +97,16 @@ abstract class KClosest } if (n == null) { - Entry<Node,BigInteger> entry = best[0]; + Entry<Node,BigInteger> entry = best[0]; BigInteger c = entry.getValue(); if (d.compareTo(c) < 0) { n = new Entry<Node,BigInteger>(node, d); - best[0] = n; + best[0] = n; } } if (n == null) - continue; + continue; // Bubble: for ( ; j < count-1; j++) { @@ -115,7 +115,7 @@ abstract class KClosest if (d.compareTo(c) < 0) { best[j] = best[j+1]; - best[j+1] = n; + best[j+1] = n; } } } @@ -125,7 +125,8 @@ abstract class KClosest result.add(entry.thing); return result; } - + */ + public void register(Node node) { return; diff --git a/sim/gobs/src/gobs/sim/Metric.java b/sim/gobs/src/gobs/sim/Metric.java index b9cca58..c834137 100644 --- a/sim/gobs/src/gobs/sim/Metric.java +++ b/sim/gobs/src/gobs/sim/Metric.java @@ -190,7 +190,7 @@ public abstract class Metric source == null) Tools.bail("regimen(): could not handle: " + object); - return new Copy(simulator, Cause.FAULT, payload, + return new Copy(simulator, Cause.FAULT, payload, source, destination); } diff --git a/sim/gobs/src/gobs/sim/Node.java b/sim/gobs/src/gobs/sim/Node.java index 9bed4df..8a9b0ee 100644 --- a/sim/gobs/src/gobs/sim/Node.java +++ b/sim/gobs/src/gobs/sim/Node.java @@ -9,10 +9,10 @@ import gobs.util.Tools; /** * Simulates an object store. - * - * Current Fault model: A Node that is down may have lost - * no OSDs or one OSD. A Node that is down will not lose - * other OSDs while down. + * + * Current Fault model: A Node that is down may have lost + * no OSDs or one OSD. A Node that is down will not lose + * other OSDs while down. * */ public class Node @@ -25,12 +25,12 @@ public class Node */ public OSD[] osd; - /** - If a data loss fault has occurred, this will note the index - of the failed OSD. Otherwise -1. + /** + If a data loss fault has occurred, this will note the index + of the failed OSD. Otherwise -1. */ - int failed = -1; - + int failed = -1; + /** Counter for number of pushes during rebuilds. */ @@ -45,7 +45,7 @@ public class Node Counter for number of user read/write accesses. */ int accesses = 0; - + Status status = Status.UP; Set<Node> neighbors = new LinkedHashSet<Node>(); @@ -78,7 +78,7 @@ public class Node ZAPPED } - /** + /** Add this object to a random OSD on this Node. */ void add(Gob object) @@ -87,8 +87,8 @@ public class Node add(r, object); } - /** - Add this object to the given OSD on this Node. + /** + Add this object to the given OSD on this Node. @param d The OSD index. */ void add(int d, Gob object) @@ -100,7 +100,7 @@ public class Node displayObjects()); osd[d].put(object.id, object); } - + /** Return the index of the OSD that serves id. If not found, return -1. @@ -160,7 +160,7 @@ public class Node result.addAll(d.keySet()); return result; } - + /** Return a Collection of all of this Node's objects. */ @@ -189,8 +189,8 @@ public class Node /** After a Repair, remove all objects. - @return A Set of Copies to be performed to restore this Node, - or null if there is nothing to do. + @return A Set of Copies to be performed to restore this Node, + or null if there is nothing to do. */ Set<Copy> reboot(Repair repair) { @@ -203,17 +203,17 @@ public class Node simulator.print("rebuilds", "reboot(zapped): " + this); else simulator.print("rebuilds", "reboot(clean): " + this); - + for (Gob object : collection()) repair.rebuild.validate(object, this); - + if (status == Status.ZAPPED) { osd[failed].status = OSD.Status.UP; copies = rejoin(repair); failed = -1; } - + status = Status.UP; return copies; } @@ -237,8 +237,8 @@ public class Node for (Gob object : objects) { Sites sites = simulator.placer.sites(object); - if (sites.contains(this) && - (!contains(object.id)) && + if (sites.contains(this) && + (!contains(object.id)) && ids.add(object.id)) { Node source = simulator.placer.removeSource(object, sites); @@ -327,7 +327,7 @@ public class Node /** Simulate a disk failure. - @return The index of the lost OSD. + @return The index of the lost OSD. */ int damage() { @@ -351,7 +351,7 @@ public class Node boolean contains(BigInteger id) { int i = lookup(id); - boolean b = (i >= 0); + boolean b = (i >= 0); return b; } diff --git a/sim/gobs/src/gobs/sim/OSD.java b/sim/gobs/src/gobs/sim/OSD.java index 0442a4c..616a19f 100755 --- a/sim/gobs/src/gobs/sim/OSD.java +++ b/sim/gobs/src/gobs/sim/OSD.java @@ -1,26 +1,26 @@ -package gobs.sim; - -import java.math.BigInteger; -import java.util.LinkedHashMap; - -/** - Simulate a disk or RAID array. - */ -public class OSD extends LinkedHashMap<BigInteger,Gob> -{ - Status status = Status.UP; - - Load load = Load.IDLE; - - enum Load - { - IDLE, BUSY - } - - enum Status - { - UP, DOWN, LOCALREBUILD - } - - static final long serialVersionUID = 1; -} +package gobs.sim; + +import java.math.BigInteger; +import java.util.LinkedHashMap; + +/** + Simulate a disk or RAID array. + */ +public class OSD extends LinkedHashMap<BigInteger,Gob> +{ + Status status = Status.UP; + + Load load = Load.IDLE; + + enum Load + { + IDLE, BUSY + } + + enum Status + { + UP, DOWN, LOCALREBUILD + } + + static final long serialVersionUID = 1; +} diff --git a/sim/gobs/src/gobs/sim/PDSIFiles.java b/sim/gobs/src/gobs/sim/PDSIFiles.java index 99f3277..4c9a065 100644 --- a/sim/gobs/src/gobs/sim/PDSIFiles.java +++ b/sim/gobs/src/gobs/sim/PDSIFiles.java @@ -56,7 +56,7 @@ public class PDSIFiles int disksPerNode = 0; int presetReplicas = 0; - + /** Obtain file distribution by interpolating points. @param points points(i,j) indicates that fraction i @@ -72,7 +72,7 @@ public class PDSIFiles disksPerNode = Tools.integer(properties, "node.osds"); if (Tools.has(properties, "replicas")) presetReplicas = Tools.integer(properties, "replicas"); - + interpolate(); } @@ -231,8 +231,8 @@ public class PDSIFiles } if (presetReplicas > 0) - replicas = presetReplicas; - + replicas = presetReplicas; + return new File(size, strip, width, replicas); } diff --git a/sim/gobs/src/gobs/sim/RAID.java b/sim/gobs/src/gobs/sim/RAID.java index 6f6c44e..6f38770 100755 --- a/sim/gobs/src/gobs/sim/RAID.java +++ b/sim/gobs/src/gobs/sim/RAID.java @@ -1,133 +1,133 @@ - -package gobs.sim; - -import java.math.BigInteger; -import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import gobs.util.Tools; - -public class RAID -{ - int data, parity; - String spec; - - double size = -1; - double speed; - - static DecimalFormat df = new DecimalFormat("0.0"); - - public RAID(String spec) - { - this.spec = spec; - scan(spec); - } - - public RAID(String spec, double size, double speed) - { - this(spec); - this.size = size; - this.speed = speed; - } - - public double computeMTTF(double mttf) - { - if (size < 0) - Tools.bail("RAID not set up properly!"); - return computeMTTF(size, speed, mttf); - } - - public double computeMTTF(double diskSize, double diskSpeed, - double mttf) - { - double mttr = diskSize/diskSpeed; - return computeMTTF(data, parity, mttf, mttr); - } - - public static double computeMTTF(int data, int parity, - double mttf, double mttr) - { - double result = 0; - - if (parity == 0) - { - result = mttf/data; - } - else - { - double mttfSmaller = computeMTTF(data, parity-1, mttf, mttr); - result = mttfSmaller * mttf/((data+parity)*mttr); - } - - return result; - } - - /** - Compute RAID MTTFs in String form given a list of - disk MTTFs in String form. - Useful for parameter sweeps. - */ - public List<String> mttfValues(List<String> input) - { - List<String> result = new ArrayList<String>(input.size()); - - for (String s : input) - { - double mttf = Double.parseDouble(s); - double raidMTTF = computeMTTF(mttf); - result.add(df.format(raidMTTF)); - } - - return result; - } - - /** - Given a workload and node properties, - determine how many nodes should be used. - Useful for parameter sweeps. - */ - public int estimateNodes(Properties properties, - int nodeDevices) - { - int nodes = -1; - int files = Tools.integer(properties, "files"); - FileFactory factory = Simulator.setupFileFactory(properties); - - BigInteger total = BigInteger.ZERO; - for (int i = 0; i < files; i++) - { - BigInteger size = factory.getSize(); - total = total.add(size); - } - System.out.println("ESTIMATE_FS: " + File.toString(total)); - - int nodeDisks = nodeDevices * data; - BigInteger nd = BigInteger.valueOf(nodeDisks); - BigInteger ds = BigInteger.valueOf((long) size); - BigInteger nodeSpace = nd.multiply(ds); - nodes = total.divide(nodeSpace).add(BigInteger.ONE).intValue(); - - return nodes; - } - - public void scan(String input) - { - int i = input.indexOf("+"); - String s1 = input.substring(0, i); - data = Integer.parseInt(s1); - String s2 = input.substring(i+1); - parity = Integer.parseInt(s2); - } - - public int data() - { - return data; - } - - public String toString() - { - return spec; - } -} + +package gobs.sim; + +import java.math.BigInteger; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import gobs.util.Tools; + +public class RAID +{ + int data, parity; + String spec; + + double size = -1; + double speed; + + static DecimalFormat df = new DecimalFormat("0.0"); + + public RAID(String spec) + { + this.spec = spec; + scan(spec); + } + + public RAID(String spec, double size, double speed) + { + this(spec); + this.size = size; + this.speed = speed; + } + + public double computeMTTF(double mttf) + { + if (size < 0) + Tools.bail("RAID not set up properly!"); + return computeMTTF(size, speed, mttf); + } + + public double computeMTTF(double diskSize, double diskSpeed, + double mttf) + { + double mttr = diskSize/diskSpeed; + return computeMTTF(data, parity, mttf, mttr); + } + + public static double computeMTTF(int data, int parity, + double mttf, double mttr) + { + double result = 0; + + if (parity == 0) + { + result = mttf/data; + } + else + { + double mttfSmaller = computeMTTF(data, parity-1, mttf, mttr); + result = mttfSmaller * mttf/((data+parity)*mttr); + } + + return result; + } + + /** + Compute RAID MTTFs in String form given a list of + disk MTTFs in String form. + Useful for parameter sweeps. + */ + public List<String> mttfValues(List<String> input) + { + List<String> result = new ArrayList<String>(input.size()); + + for (String s : input) + { + double mttf = Double.parseDouble(s); + double raidMTTF = computeMTTF(mttf); + result.add(df.format(raidMTTF)); + } + + return result; + } + + /** + Given a workload and node properties, + determine how many nodes should be used. + Useful for parameter sweeps. + */ + public int estimateNodes(Properties properties, + int nodeDevices) + { + int nodes = -1; + int files = Tools.integer(properties, "files"); + FileFactory factory = Simulator.setupFileFactory(properties); + + BigInteger total = BigInteger.ZERO; + for (int i = 0; i < files; i++) + { + BigInteger size = factory.getSize(); + total = total.add(size); + } + System.out.println("ESTIMATE_FS: " + File.toString(total)); + + int nodeDisks = nodeDevices * data; + BigInteger nd = BigInteger.valueOf(nodeDisks); + BigInteger ds = BigInteger.valueOf((long) size); + BigInteger nodeSpace = nd.multiply(ds); + nodes = total.divide(nodeSpace).add(BigInteger.ONE).intValue(); + + return nodes; + } + + public void scan(String input) + { + int i = input.indexOf("+"); + String s1 = input.substring(0, i); + data = Integer.parseInt(s1); + String s2 = input.substring(i+1); + parity = Integer.parseInt(s2); + } + + public int data() + { + return data; + } + + public String toString() + { + return spec; + } +} diff --git a/sim/gobs/src/gobs/sim/Rebuild.java b/sim/gobs/src/gobs/sim/Rebuild.java index 197d2f3..0cda51c 100644 --- a/sim/gobs/src/gobs/sim/Rebuild.java +++ b/sim/gobs/src/gobs/sim/Rebuild.java @@ -47,11 +47,11 @@ public class Rebuild */ Repair repair; - /** + /** The fault that caused this Rebuild. */ Fault fault; - + /** The fault that will occur after this Rebuild has started. May or may not interrupt this Rebuild. @@ -75,7 +75,7 @@ public class Rebuild Rebuild(Simulator simulator, Fault fault, Fault next, Repair repair) { this.simulator = simulator; - this.fault = fault; + this.fault = fault; this.next = next; this.repair = repair; fault.rebuild(this); @@ -171,11 +171,11 @@ public class Rebuild copy.destination.stopStreamTo(copy); List<Copy> extras = performCopy(copy); add(extras); - } + } - /** - Perform the Copy and validate it. - Return any new work to do. + /** + Perform the Copy and validate it. + Return any new work to do. */ List<Copy> performCopy(Copy copy) { @@ -183,12 +183,12 @@ public class Rebuild List<Copy> extras = validate(copy.object, copy.destination); return extras; } - + /** Ensure that the replica count is restored. Remove extraneous replicas or return a new Copy if needed. @parameter object The object in question. - @parameter node The node which holds the new or restored replica. + @parameter node The node which holds the new or restored replica. @return The Copies needed. */ List<Copy> validate(Gob object, Node node) @@ -503,6 +503,13 @@ public class Rebuild } if (copy.source.busy(copy.object.id)) continue; + + if (isInFlight(copy)) + { + it.remove(); + continue; + } + int b = copy.destination.busy(); if (b >= 0) { @@ -513,4 +520,21 @@ public class Rebuild } return null; } + + boolean isInFlight(Copy copy) + { + for (Event event : simulator.inflight) + { + if (! (event instanceof Copy)) + continue; + Copy c = (Copy) event; + if (copy.destination.equals(c.destination) && + copy.object.equals(c.object)) + { + System.out.println("already inflight: " + copy); + return true; + } + } + return false; + } } diff --git a/sim/gobs/src/gobs/sim/Replicated.java b/sim/gobs/src/gobs/sim/Replicated.java index eb3189c..9de21ce 100644 --- a/sim/gobs/src/gobs/sim/Replicated.java +++ b/sim/gobs/src/gobs/sim/Replicated.java @@ -48,6 +48,7 @@ public abstract class Replicated } result.trimToSize(); + return result; } diff --git a/sim/gobs/src/gobs/sim/Simulator.java b/sim/gobs/src/gobs/sim/Simulator.java index 6a60b3e..703d1e9 100644 --- a/sim/gobs/src/gobs/sim/Simulator.java +++ b/sim/gobs/src/gobs/sim/Simulator.java @@ -72,7 +72,7 @@ public class Simulator /** Simulated objects. */ - EnhancedLinkedHashSet<Gob> objects = + EnhancedLinkedHashSet<Gob> objects = new EnhancedLinkedHashSet<Gob>(); /** @@ -81,7 +81,8 @@ public class Simulator List<File> files = new ArrayList<File>(); /** - Original allocation of objects. Read by gobs.sweep.Balance. + Original allocation of objects. + Read by {@link gobs.sweep.x.Balance01}. */ public Map<Node,Integer> origCount; @@ -126,11 +127,11 @@ public class Simulator */ Set<BigInteger> lost = new HashSet<BigInteger>(); - /** - Nodes that are temporarily down. + /** + Nodes that are temporarily down. */ Set<Node> down = new LinkedHashSet<Node>(); - + /** Current rebuild in progress. */ @@ -159,22 +160,22 @@ public class Simulator */ boolean printAccesses = false; - /** - The type of RAID we are using. + /** + The type of RAID we are using. */ - RAID raid; - + RAID raid; + /** - MTTF for all fault types. Per RAID device. + MTTF for all fault types. Per RAID device. @see dataLossFraction */ double mttf; - /** + /** MTT crash (non-data-loss Fault). */ double mttc; - + /** Fraction of Faults that cause data loss. */ @@ -326,7 +327,7 @@ public class Simulator double diskSpeed = decimal("disk.speed"); String spec = string("raid"); raid = new RAID(spec, diskSize, diskSpeed); - + rebuildRate = diskSpeed*raid.data(); bigRebuildRate = BigInteger.valueOf((long) rebuildRate); @@ -355,27 +356,27 @@ public class Simulator void setupExponentials() { devicesPerNode = integer("node.osds"); - + mttrReboot = decimal("mttr.reboot"); mttrDisk = decimal("mttr.disk"); - - double mttfDisk = decimal("disk.mttf"); + + double mttfDisk = decimal("disk.mttf"); double mttfLoss = raid.computeMTTF(mttfDisk); mttc = decimal("mttc")/N; mttfLoss = mttfLoss/(N*devicesPerNode); - - // Imaginary long run: + + // Imaginary long run: double span = mttc*mttfLoss; - // Number of events in span: + // Number of events in span: double events = mttc+mttfLoss; - mttf = span/events; - // MTTC = span/mttfLoss = # loss events, so: + mttf = span/events; + // MTTC = span/mttfLoss = # loss events, so: dataLossFraction = mttc/events; - + print("mttf: " + df.format(mttf)); print("dataLossFraction: " + df.format(dataLossFraction)); - + DRand engine = new DRand(Bits.nextInt()); mttfExponential = new Exponential(1.0/mttf, engine); mttrRebootExponential = new Exponential(1.0/mttrReboot, engine); @@ -479,7 +480,7 @@ public class Simulator if (bool("print.files")) printFiles(); - else if (bool("print.filesobjects")) + if (bool("print.filesobjects")) printFilesObjects(); System.out.println("FS_SIZE: " + File.toString(fsSize())); @@ -528,11 +529,11 @@ public class Simulator void place() { - print("OBJECT_COUNT: " + countObjects()); - print("allocating: " + objects.size()); + print("allocating: " + objects.size()); placer.place(objects, nodes); origCount = countObjects(nodes); + // print("OBJECT_COUNT: " + objects.size()); // printNodes(nodes); if (bool("print.initialnodes")) @@ -549,6 +550,17 @@ public class Simulator return result; } + int totalObjects(Map<Node,Integer> input) + { + int result = 0; + for (Node node : input.keySet()) + { + int t = input.get(node); + result += t; + } + return result; + } + void performAccesses() { System.out.println("ACCESSES: " + timef()); @@ -852,7 +864,7 @@ public class Simulator } List<Copy> extras = fault.rebuild.performCopy(copy); for (Copy extra : extras) - print("rebuilds", "extra: " + extra); + print("rebuilds", "extra: " + extra); it.remove(); } } @@ -868,12 +880,12 @@ public class Simulator printNodeObjects(nodes); } - /** - Wrap up after the induceFaults() loop. - Perform any outstanding copies + /** + Wrap up after the induceFaults() loop. + Perform any outstanding copies in {@link inflight} or {@link todo}. - @param last The final fault. This and its corresponding - Rebuild will be charged with any remaining work. + @param last The final fault. This and its corresponding + Rebuild will be charged with any remaining work. */ void finish(Fault last) { @@ -885,7 +897,7 @@ public class Simulator if (bool("print.nodes")) printNodeObjects(nodes); } - + /** Set the rebuild rate. */ @@ -1206,7 +1218,7 @@ public class Simulator for (Node node : down) if (node.contains(object.id)) { - found = true; + found = true; break; } if (found) @@ -1353,7 +1365,6 @@ public class Simulator busy.clear(); todo.clear(); inflight.clear(); - System.gc(); } /** diff --git a/sim/gobs/src/gobs/sim/VertexGroups.java b/sim/gobs/src/gobs/sim/VertexGroups.java index 34f2d04..6c491e8 100644 --- a/sim/gobs/src/gobs/sim/VertexGroups.java +++ b/sim/gobs/src/gobs/sim/VertexGroups.java @@ -46,13 +46,13 @@ public class VertexGroups Sites sites(SingleObject object, List<Node> nodes) { - Sites result = new Sites(object); + Sites result = new Sites(object); Node primary = lookup(object.id, nodes); result.primary(primary); int i = 1; - for (; i < object.replicas-2; i++) + for (; i < object.replicas-1; i++) { BigInteger t = flip(simulator.B, i, object.id); Node node = lookup(t, nodes); diff --git a/sim/gobs/src/gobs/sweep/AbstractSweep.java b/sim/gobs/src/gobs/sweep/AbstractSweep.java index 708b1a3..bb4c1c5 100644 --- a/sim/gobs/src/gobs/sweep/AbstractSweep.java +++ b/sim/gobs/src/gobs/sweep/AbstractSweep.java @@ -288,8 +288,11 @@ public abstract class AbstractSweep static void free(Simulator simulator) { - if (Tools.bool(simulator.properties, "sweep.free")) + if (! Tools.disabled(simulator.properties, "sweep.free")) + { simulator.free(); + System.gc(); + } } public static void reportRuntime() diff --git a/sim/gobs/src/gobs/sweep/ArraySweep.java b/sim/gobs/src/gobs/sweep/ArraySweep.java index 87b1b47..ffe07ec 100644 --- a/sim/gobs/src/gobs/sweep/ArraySweep.java +++ b/sim/gobs/src/gobs/sweep/ArraySweep.java @@ -91,8 +91,7 @@ public class ArraySweep xresult.add(result(simulator, dname)); yresult.add(result(simulator, yname)); } - XYSeries series = new XYSeries(description + " " + - xname + "=" + v); + XYSeries series = new XYSeries(description + v); if (dname == null) arrayAvgToSeries(yresult, series); else @@ -115,7 +114,16 @@ public class ArraySweep double[] result = new double[tokens.length]; int i = 0; for (String token : tokens) - result[i++] = Double.parseDouble(token.trim()); + { + try + { + result[i++] = Double.parseDouble(token.trim()); + } + catch (Exception e) + { + Tools.bail("ArraySweep.result(): Bad token: " + token); + } + } return result; } diff --git a/sim/gobs/src/gobs/sweep/Helpers.java b/sim/gobs/src/gobs/sweep/Helpers.java index 7f1d9e9..4ec29e2 100755 --- a/sim/gobs/src/gobs/sweep/Helpers.java +++ b/sim/gobs/src/gobs/sweep/Helpers.java @@ -1,42 +1,42 @@ - -package gobs.sweep; - -import gobs.sim.RAID; -import gobs.util.Tools; - -import java.util.*; - -/** - * GOBS-specific sweep helpers. - * */ - -public class Helpers -{ - public static List<String> valuesMTTF(Properties properties) - { - int minMTTF = Tools.integer(properties, "disk.mttf.min"); - int maxMTTF = Tools.integer(properties, "disk.mttf.max"); - String stepMTTF = Tools.string(properties, "disk.mttf.step"); - List<String> valuesMTTF = - Sweep.values(stepMTTF, minMTTF, maxMTTF); - return valuesMTTF; - } - - public static RAID setupRAID(Properties properties, String system) - { - double diskSize = Tools.decimal(properties, "disk.size"); - double diskSpeed = Tools.decimal(properties, "disk.speed"); - int nodeDevices = Tools.integer(properties, - system+".node.osds"); - String spec = Tools.string(properties, system+".raid"); - RAID raid = new RAID(spec, diskSize, diskSpeed); - int nodes = raid.estimateNodes(properties, nodeDevices); - properties.setProperty("raid", spec); - properties.setProperty("node.count", ""+nodes); - properties.setProperty("node.osds", ""+nodeDevices); - int replicas = Tools.integer(properties, system+".replicas"); - properties.setProperty("replicas", ""+replicas); - return raid; - } - -} + +package gobs.sweep; + +import gobs.sim.RAID; +import gobs.util.Tools; + +import java.util.*; + +/** + * GOBS-specific sweep helpers. + * */ + +public class Helpers +{ + public static List<String> valuesMTTF(Properties properties) + { + int minMTTF = Tools.integer(properties, "disk.mttf.min"); + int maxMTTF = Tools.integer(properties, "disk.mttf.max"); + String stepMTTF = Tools.string(properties, "disk.mttf.step"); + List<String> valuesMTTF = + Sweep.values(stepMTTF, minMTTF, maxMTTF); + return valuesMTTF; + } + + public static RAID setupRAID(Properties properties, String system) + { + double diskSize = Tools.decimal(properties, "disk.size"); + double diskSpeed = Tools.decimal(properties, "disk.speed"); + int nodeDevices = Tools.integer(properties, + system+".node.osds"); + String spec = Tools.string(properties, system+".raid"); + RAID raid = new RAID(spec, diskSize, diskSpeed); + int nodes = raid.estimateNodes(properties, nodeDevices); + properties.setProperty("raid", spec); + properties.setProperty("node.count", ""+nodes); + properties.setProperty("node.osds", ""+nodeDevices); + int replicas = Tools.integer(properties, system+".replicas"); + properties.setProperty("replicas", ""+replicas); + return raid; + } + +} diff --git a/sim/gobs/src/gobs/sweep/Sweep.java b/sim/gobs/src/gobs/sweep/Sweep.java index 2be1cee..ce2263e 100644 --- a/sim/gobs/src/gobs/sweep/Sweep.java +++ b/sim/gobs/src/gobs/sweep/Sweep.java @@ -170,6 +170,7 @@ public class Sweep simulator.execute(); System.out.println("result: " + result(simulator, yname)); total += result(simulator, yname); + free(simulator); } System.out.println("avg: " + total/repeats); series.add(Double.parseDouble(v), total/repeats); @@ -194,6 +195,16 @@ public class Sweep System.out.println("TOOK: " + t + " minutes"); } + public void free() + { + if (! Tools.disabled(properties, "sweep.free")) + { + for (Simulator sim : sims.values()) + sim.free(); + System.gc(); + } + } + /** Debugging only. */ diff --git a/sim/gobs/src/gobs/sweep/x/DataLoss02.java b/sim/gobs/src/gobs/sweep/x/DataLoss02.java index 3b75f38..61973e0 100755 --- a/sim/gobs/src/gobs/sweep/x/DataLoss02.java +++ b/sim/gobs/src/gobs/sweep/x/DataLoss02.java @@ -61,7 +61,7 @@ public class DataLoss02 properties.setProperty("node.osds", ""+nodeDevices); Sweep pSweep = new Sweep(properties); pSweep.repeats = Tools.integer(properties, "repeats"); - series = pSweep.perform(system, "disk.mttf", + series = pSweep.perform(system, "disk.mttf", valuesMTTF, "lost"); allSeries.add(series); } @@ -91,8 +91,8 @@ public class DataLoss02 // "nodes", "load", "tmp.txt"); } - /** - Replace the X coordinates in series with those in values. + /** + Replace the X coordinates in series with those in values. */ static XYSeries setX(List<String> values, XYSeries series) { @@ -104,9 +104,9 @@ public class DataLoss02 Number y = series.getY(i); result.add(d, y); } - return result; + return result; } - + /** Plot average number of lost objects for each MTTF. @param label E.g., "PAST" diff --git a/sim/gobs/src/gobs/sweep/x/Decluster03.java b/sim/gobs/src/gobs/sweep/x/Decluster03.java index 559bc29..ace67ab 100644 --- a/sim/gobs/src/gobs/sweep/x/Decluster03.java +++ b/sim/gobs/src/gobs/sweep/x/Decluster03.java @@ -3,6 +3,7 @@ package gobs.sweep.x; import java.util.*; +import org.jfree.data.xy.XYSeries; import org.jfree.data.xy.XYSeriesCollection; import gobs.plot.*; @@ -27,37 +28,48 @@ class Decluster03 properties = Sweep.getopts(args); - int minNodes = Tools.integer(properties, "minNodes"); - int maxNodes = Tools.integer(properties, "maxNodes"); - String step = Tools.string(properties, "step"); + String output = Tools.string(properties, "output"); + + int minNodes = Tools.integer(properties, "nodes.min"); + int maxNodes = Tools.integer(properties, "nodes.max"); + String step = Tools.string(properties, "nodes.step"); List<String> values = Sweep.values(step, minNodes, maxNodes); System.out.println("Decluster03: " + values); + XYSeries series; final XYSeriesCollection collection = new XYSeriesCollection(); - properties.setProperty("impl.placement", "Nearest"); - Sweep nearestSweep = new Sweep(properties); - nearestSweep.repeats = Tools.integer(properties, "repeats"); - collection.addSeries - (nearestSweep.perform("PAST", "nodes", values, "maxTotal")); + for (String r : new String[] {"3", "4"}) + { + properties.setProperty("file.replicas", r); - properties.setProperty("impl.placement", "Kademlia"); - Sweep kademliaSweep = new Sweep(properties); - kademliaSweep.repeats = Tools.integer(properties, "repeats"); - collection.addSeries - (kademliaSweep.perform("Kademlia", "nodes", values, "maxTotal")); + properties.setProperty("impl.placement", "Nearest"); + Sweep nearestSweep = new Sweep(properties); + nearestSweep.repeats = Tools.integer(properties, "repeats"); + series = nearestSweep.perform("NEAREST R="+r, "node.count", + values, "maxTotal"); + collection.addSeries(series); + properties.setProperty("impl.placement", "Kademlia"); + Sweep kademliaSweep = new Sweep(properties); + kademliaSweep.repeats = Tools.integer(properties, "repeats"); + series = kademliaSweep.perform("XOR R="+r, "node.count", + values, "maxTotal"); + collection.addSeries(series); + } + /* properties.setProperty("impl.placement", "VertexGroups"); Sweep vgSweep = new Sweep(properties); vgSweep.repeats = Tools.integer(properties, "repeats"); - collection.addSeries - (vgSweep.perform("Vertex Groups", "nodes", values, "maxTotal")); + series = vgSweep.perform("Vertex Groups", "node.count", + values, "maxTotal"); + collection.addSeries(series); + */ - String output = Tools.string(properties, "output"); - MultiLogXPlot.plot(collection, "Maximum Rebuild Load", - "nodes", "load fraction(%)", output); - // AsciiPlot.plot(nearestSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); + if (Tools.enabled(properties, "plot.blackandwhite")) + MultiLogXPlot.bw = true; + MultiLogXPlot.plot(collection, "", + "node.count", "load fraction(%)", output); } } diff --git a/sim/gobs/src/gobs/sweep/x/Decluster04.java b/sim/gobs/src/gobs/sweep/x/Decluster04.java index fb03d73..fe6be91 100644 --- a/sim/gobs/src/gobs/sweep/x/Decluster04.java +++ b/sim/gobs/src/gobs/sweep/x/Decluster04.java @@ -30,42 +30,52 @@ class Decluster04 String output = Tools.string(properties, "output"); - int minNodes = Tools.integer(properties, "minNodes"); - int maxNodes = Tools.integer(properties, "maxNodes"); - String step = Tools.string(properties, "step"); + int minNodes = Tools.integer(properties, "nodes.min"); + int maxNodes = Tools.integer(properties, "nodes.max"); + String step = Tools.string(properties, "nodes.step"); List<String> values = Sweep.values(step, minNodes, maxNodes); System.out.println("Decluster04: " + values); + List<XYSeries> series; List<XYSeries> allSeries = new ArrayList<XYSeries>(); final XYSeriesCollection collection = new XYSeriesCollection(); - properties.setProperty("impl.placement", "Nearest"); - ArraySweep pSweep = new ArraySweep(properties); - pSweep.repeats = Tools.integer(properties, "repeats"); - allSeries.addAll - (pSweep.execute("PAST", "nodes", values, "distribution")); - - properties.setProperty("impl.placement", "Kademlia"); - ArraySweep kSweep = new ArraySweep(properties); - kSweep.repeats = Tools.integer(properties, "repeats"); - allSeries.addAll - (kSweep.execute("Kademlia", "nodes", values, "distribution")); - - properties.setProperty("impl.placement", "VertexGroups"); - ArraySweep vgSweep = new ArraySweep(properties); - vgSweep.repeats = Tools.integer(properties, "repeats"); - allSeries.addAll - (vgSweep.execute - ("Vertex Groups", "nodes", values, "distribution")); - - for (XYSeries series : allSeries) - collection.addSeries(series); - - MultiplePlot.plot(collection, "Rebuild Load Distribution", + for (String r : new String[] {"3", "4"}) + { + properties.setProperty("file.replicas", r); + + properties.setProperty("impl.placement", "Nearest"); + ArraySweep pSweep = new ArraySweep(properties); + pSweep.repeats = Tools.integer(properties, "repeats"); + series = pSweep.execute("NEAREST R="+r+" N=", "node.count", + values, "distribution"); + allSeries.addAll(series); + + properties.setProperty("impl.placement", "Kademlia"); + ArraySweep kSweep = new ArraySweep(properties); + kSweep.repeats = Tools.integer(properties, "repeats"); + series = kSweep.execute("XOR R="+r+" N=", "node.count", + values, "distribution"); + allSeries.addAll(series); + } + + int spaces = 4; + String w = " "; + String t = ""; + for (int i = 0; i < spaces; i++) + t = t+w; + + for (XYSeries s : allSeries) + { + String d = (String) s.getKey(); + s.setKey(d+t); + collection.addSeries(s); + } + + if (Tools.enabled(properties, "plot.blackandwhite")) + MultiplePlot.bw = true; + MultiplePlot.plot(collection, "", "nodes involved", "load fraction(%)", output); - - // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); } } diff --git a/sim/gobs/src/gobs/sweep/x/RAID01.java b/sim/gobs/src/gobs/sweep/x/RAID01.java index 08d48b1..128bb18 100755 --- a/sim/gobs/src/gobs/sweep/x/RAID01.java +++ b/sim/gobs/src/gobs/sweep/x/RAID01.java @@ -1,58 +1,58 @@ - -package gobs.sweep.x; - -/** - * Plot RAID MTTF given a range of individual disk MTTFs. - */ - -import java.util.List; -import java.util.Properties; - -import org.jfree.data.xy.XYSeries; -import org.jfree.data.xy.XYSeriesCollection; - -import gobs.plot.MultiplePlot; -import gobs.sim.RAID; -import gobs.sweep.AbstractSweep; -import gobs.sweep.Sweep; -import gobs.util.Tools; - -public class RAID01 -{ - static Properties properties; - - public static void main(String[] args) - { - properties = Sweep.getopts(args); - String output = Tools.string(properties, "output"); - - int min = Tools.integer(properties, "mttf.min"); - int max = Tools.integer(properties, "mttf.max"); - String step = Tools.string(properties, "mttf.step"); - List<String> values = AbstractSweep.values(step, min, max); - - double diskSpeed = Tools.decimal(properties, "disk.speed"); - double diskSize = Tools.decimal(properties, "disk.size"); - - String rs = Tools.string(properties, "raids"); - List<String> raids = AbstractSweep.values(rs); - - XYSeriesCollection collection = new XYSeriesCollection(); - for (String r : raids) - { - RAID raid = new RAID(r); - XYSeries series = new XYSeries("RAID: " + raid); - for (String v : values) - { - double mttf = Double.parseDouble(v); - double result = raid.computeMTTF(diskSpeed, diskSize, mttf); - series.add(mttf, result); - } - collection.addSeries(series); - } - - MultiplePlot.plot(collection, "", - "Mean Time to Disk Failure", - "Mean Time to RAID Failure", output); - } -} + +package gobs.sweep.x; + +/** + * Plot RAID MTTF given a range of individual disk MTTFs. + */ + +import java.util.List; +import java.util.Properties; + +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; + +import gobs.plot.MultiplePlot; +import gobs.sim.RAID; +import gobs.sweep.AbstractSweep; +import gobs.sweep.Sweep; +import gobs.util.Tools; + +public class RAID01 +{ + static Properties properties; + + public static void main(String[] args) + { + properties = Sweep.getopts(args); + String output = Tools.string(properties, "output"); + + int min = Tools.integer(properties, "mttf.min"); + int max = Tools.integer(properties, "mttf.max"); + String step = Tools.string(properties, "mttf.step"); + List<String> values = AbstractSweep.values(step, min, max); + + double diskSpeed = Tools.decimal(properties, "disk.speed"); + double diskSize = Tools.decimal(properties, "disk.size"); + + String rs = Tools.string(properties, "raids"); + List<String> raids = AbstractSweep.values(rs); + + XYSeriesCollection collection = new XYSeriesCollection(); + for (String r : raids) + { + RAID raid = new RAID(r); + XYSeries series = new XYSeries("RAID: " + raid); + for (String v : values) + { + double mttf = Double.parseDouble(v); + double result = raid.computeMTTF(diskSpeed, diskSize, mttf); + series.add(mttf, result); + } + collection.addSeries(series); + } + + MultiplePlot.plot(collection, "", + "Mean Time to Disk Failure", + "Mean Time to RAID Failure", output); + } +} diff --git a/sim/gobs/src/gobs/sweep/x/Rebuild01.java b/sim/gobs/src/gobs/sweep/x/Rebuild01.java index 639b955..7152eb6 100644 --- a/sim/gobs/src/gobs/sweep/x/Rebuild01.java +++ b/sim/gobs/src/gobs/sweep/x/Rebuild01.java @@ -18,9 +18,6 @@ import gobs.util.*; * Usage: Rebuild01 -n <repeats> *.cfg * */ -// good runs: -// seed=828 -n 1 - class Rebuild01 { static Properties properties; @@ -32,13 +29,14 @@ class Rebuild01 AbstractSweep.seed(properties); String output = Tools.string(properties, "output"); - + List<String> valuesMTTF = Helpers.valuesMTTF(properties); - + System.out.println("Rebuild01: " + valuesMTTF); List<XYSeries> allSeries = new ArrayList<XYSeries>(); - final XYSeriesCollection collection = new XYSeriesCollection(); + final XYSeriesCollection collection = + new XYSeriesCollection(); properties.setProperty("impl.placement", "Nearest"); List<String> systems = Tools.list(properties, "systems"); @@ -49,12 +47,12 @@ class Rebuild01 properties.setProperty("mttc", mttc); ArraySweep pSweep = new ArraySweep(properties); pSweep.repeats = Tools.integer(properties, "repeats"); - List<XYSeries> series = - pSweep.execute("PAST-"+system, "disk.mttf", valuesMTTF, - "traffic-x", "traffic-y"); + List<XYSeries> series = + pSweep.execute("NEAREST-"+system+" MTTF=", "disk.mttf", + valuesMTTF, "traffic-x", "traffic-y"); allSeries.addAll(series); } - + /* properties.setProperty("impl.placement", "Kademlia"); ArraySweep kSweep = new ArraySweep(properties); @@ -72,11 +70,10 @@ class Rebuild01 for (XYSeries series : allSeries) collection.addSeries(series); - MultiplePlot.plot(collection, "Rebuild Traffic", + if (Tools.enabled(properties, "plot.blackandwhite")) + MultiplePlot.bw = true; + MultiplePlot.plot(collection, "", "time", "copies in flight", output); - - // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", - // "nodes", "load", "tmp.txt"); } } diff --git a/sim/gobs/src/gobs/sweep/x/Rebuild02.java b/sim/gobs/src/gobs/sweep/x/Rebuild02.java index 638affc..fc4b7fd 100644 --- a/sim/gobs/src/gobs/sweep/x/Rebuild02.java +++ b/sim/gobs/src/gobs/sweep/x/Rebuild02.java @@ -15,8 +15,8 @@ import gobs.util.*; /** * Modifies node count. * Reports rebuild activity over time during rebuild. - * - * Allows for use of regimen.skip . + * + * Allows for use of regimen.skip . * * Usage: Rebuild02 -n <repeats> -o <output.eps> *.cfg * */ @@ -37,13 +37,13 @@ class Rebuild02 double diskSize = Tools.decimal(properties, "disk.size"); double diskSpeed = Tools.decimal(properties, "disk.speed"); - + int minMTTF = Tools.integer(properties, "disk.mttf.min"); int maxMTTF = Tools.integer(properties, "disk.mttf.max"); String stepMTTF = Tools.string(properties, "disk.mttf.step"); List<String> valuesMTTF = Sweep.values(stepMTTF, minMTTF, maxMTTF); - + System.out.println("Rebuild02: " + valuesMTTF); final XYSeriesCollection collection = new XYSeriesCollection(); @@ -58,27 +58,27 @@ class Rebuild02 int nodeDevices = Tools.integer(properties, system+".node.osds"); String spec = Tools.string(properties, system+".raid"); - RAID raid = new RAID(spec, diskSize, diskSpeed); + RAID raid = new RAID(spec, diskSize, diskSpeed); int nodes = raid.estimateNodes(properties, nodeDevices); properties.setProperty("raid", spec); properties.setProperty("node.count", ""+nodes); properties.setProperty("node.osds", ""+nodeDevices); properties.setProperty("regimen.skip", "false"); properties.setProperty("system", system); - + Sweep pySweep = new Sweep(properties); pySweep.repeats = Tools.integer(properties, "repeats"); pySweep.execute("disk.mttf", valuesMTTF); sims.addAll(pySweep.sims.values()); } - - List<XYSeries> pySeries = plots(sims, "PAST-Y", - "system", systems, - "disk.mttf", valuesMTTF, + + List<XYSeries> pySeries = plots(sims, "PAST-Y", + "system", systems, + "disk.mttf", valuesMTTF, interval); allSeries.addAll(pySeries); - + /* properties.setProperty("impl.placement", "Nearest"); Sweep pnSweep = new Sweep(properties); @@ -112,7 +112,7 @@ class Rebuild02 MultiplePlot.plot(collection, "Rebuild Traffic", "time", "copies in flight", - output); + output); // AsciiPlot.plot(pSweep.series, "Maximum Rebuild Load", // "nodes", "load", "tmp.txt"); @@ -131,11 +131,11 @@ class Rebuild02 { for (String v2 : values2) { - XYSeries series = + XYSeries series = new XYSeries(label + " SYSTEM=" + v1 + ":" + v2); List<Rebuild> rebuilds = new ArrayList<Rebuild>(); for (Simulator simulator : sims) - if (Tools.string(simulator.properties, x1).equals(v1) && + if (Tools.string(simulator.properties, x1).equals(v1) && Tools.string(simulator.properties, x2).equals(v2)) rebuilds.addAll(simulator.rebuilds); loadsToSeries(rebuilds, interval, series); diff --git a/sim/gobs/src/gobs/util/BoundedSortedValues.java b/sim/gobs/src/gobs/util/BoundedSortedValues.java index 5e8f4ec..14934fc 100644 --- a/sim/gobs/src/gobs/util/BoundedSortedValues.java +++ b/sim/gobs/src/gobs/util/BoundedSortedValues.java @@ -43,7 +43,7 @@ public class BoundedSortedValues<T,V extends Comparable<V>> list.size() < bound) { int i = 0; - for (Iterator<Entry<T,V>> it = list.iterator(); + for (Iterator<Entry<T,V>> it = list.iterator(); it.hasNext(); ) { Entry<T,V> pair = it.next(); diff --git a/sim/gobs/src/gobs/util/Entry.java b/sim/gobs/src/gobs/util/Entry.java index 36a0fba..48f7e2e 100755 --- a/sim/gobs/src/gobs/util/Entry.java +++ b/sim/gobs/src/gobs/util/Entry.java @@ -1,51 +1,51 @@ - -package gobs.util; - -import java.util.*; - -public class Entry<T,V> - implements Map.Entry<T,V> -{ - public T thing; - public V value; - - public Entry() - {} - - public Entry(T thing, V value) - { - this.thing = thing; - this.value = value; - } - - public boolean equals(Object o) - { - return thing.equals(o); - } - - public T getKey() - { - return thing; - } - - public V getValue() - { - return value; - } - - public int hashCode() - { - return thing.hashCode() + value.hashCode(); - } - - public V setValue(V value) - { - this.value = value; // .intValue(); - return value; - } - - public String toString() - { - return "<" + thing + "," + value +">"; - } -} + +package gobs.util; + +import java.util.*; + +public class Entry<T,V> + implements Map.Entry<T,V> +{ + public T thing; + public V value; + + public Entry() + {} + + public Entry(T thing, V value) + { + this.thing = thing; + this.value = value; + } + + public boolean equals(Object o) + { + return thing.equals(o); + } + + public T getKey() + { + return thing; + } + + public V getValue() + { + return value; + } + + public int hashCode() + { + return thing.hashCode() + value.hashCode(); + } + + public V setValue(V value) + { + this.value = value; // .intValue(); + return value; + } + + public String toString() + { + return "<" + thing + "," + value +">"; + } +} diff --git a/sim/gobs/src/gobs/util/Tools.java b/sim/gobs/src/gobs/util/Tools.java index bde9927..fa0221f 100644 --- a/sim/gobs/src/gobs/util/Tools.java +++ b/sim/gobs/src/gobs/util/Tools.java @@ -129,6 +129,15 @@ public class Tools return (properties.getProperty(name) != null); } + public static boolean enabled(Properties properties, + String string) + { + if (has(properties, string) && + bool(properties, string)) + return true; + return false; + } + public static boolean disabled(Properties properties, String string) { @@ -138,7 +147,7 @@ public class Tools return false; } - public static List<String> list(Properties properties, + public static List<String> list(Properties properties, String string) { String text = string(properties, string); diff --git a/sim/gobs/sweeps/dc03kpv01.cfg b/sim/gobs/sweeps/dc03kpv01.cfg index 50636bd..a51bc26 100644 --- a/sim/gobs/sweeps/dc03kpv01.cfg +++ b/sim/gobs/sweeps/dc03kpv01.cfg @@ -1,7 +1,3 @@ -include = kpv01.cfg system_size_20-1000.cfg +include = kpv01.cfg system_20-1000.cfg dc_output.cfg user00.cfg -# This declustering test has no user accesses: - -reads = 0 -writes = 0 diff --git a/sim/gobs/sweeps/dc03kpv03.cfg b/sim/gobs/sweeps/dc03kpv03.cfg index 016f47b..24e03f5 100644 --- a/sim/gobs/sweeps/dc03kpv03.cfg +++ b/sim/gobs/sweeps/dc03kpv03.cfg @@ -1,7 +1,3 @@ -include = kpv03.cfg system_size_20-1000.cfg +include = kpv03.cfg system_20-1000.cfg dc_output.cfg user00.cfg -# This declustering test has no user accesses: - -reads = 0 -writes = 0 diff --git a/sim/gobs/sweeps/dc03kpv05.cfg b/sim/gobs/sweeps/dc03kpv05.cfg index ecbdca2..aab6a3f 100644 --- a/sim/gobs/sweeps/dc03kpv05.cfg +++ b/sim/gobs/sweeps/dc03kpv05.cfg @@ -1,7 +1,2 @@ -include = kpv05.cfg system_size_20-1000.cfg - -# This declustering test has no user accesses: - -reads = 0 -writes = 0 +include = kpv05.cfg system_20-1000.cfg dc_output.cfg user00.cfg diff --git a/sim/gobs/sweeps/dc04kpv01.cfg b/sim/gobs/sweeps/dc04kpv01.cfg index 876cab9..9db537c 100644 --- a/sim/gobs/sweeps/dc04kpv01.cfg +++ b/sim/gobs/sweeps/dc04kpv01.cfg @@ -1,7 +1,2 @@ -include = kpv01.cfg system_size_100_1000.cfg - -# This declustering test has no user accesses: - -reads = 0 -writes = 0 +include = kpv01.cfg system_600.cfg dc_output.cfg user00.cfg diff --git a/sim/gobs/sweeps/dc04kpv03.cfg b/sim/gobs/sweeps/dc04kpv03.cfg index 05d4b8b..9db537c 100644 --- a/sim/gobs/sweeps/dc04kpv03.cfg +++ b/sim/gobs/sweeps/dc04kpv03.cfg @@ -1,7 +1,2 @@ -include = kpv03.cfg system_size_100_1000.cfg - -# This declustering test has no user accesses: - -reads = 0 -writes = 0 +include = kpv01.cfg system_600.cfg dc_output.cfg user00.cfg diff --git a/sim/gobs/sweeps/dc04kpv05.cfg b/sim/gobs/sweeps/dc04kpv05.cfg index f7143ce..9db537c 100644 --- a/sim/gobs/sweeps/dc04kpv05.cfg +++ b/sim/gobs/sweeps/dc04kpv05.cfg @@ -1,7 +1,2 @@ -include = kpv05.cfg system_size_100_1000.cfg - -# This declustering test has no user accesses: - -reads = 0 -writes = 0 +include = kpv01.cfg system_600.cfg dc_output.cfg user00.cfg diff --git a/sim/gobs/sweeps/dl_output.cfg b/sim/gobs/sweeps/dc_output.cfg similarity index 90% copy from sim/gobs/sweeps/dl_output.cfg copy to sim/gobs/sweeps/dc_output.cfg index c9372d3..b3b7d43 100644 --- a/sim/gobs/sweeps/dl_output.cfg +++ b/sim/gobs/sweeps/dc_output.cfg @@ -11,9 +11,10 @@ print.nodes = true print.rebuilds = true # Print file information (true/false) print.files = true -print.filesobjects = false +print.filesobjects = true # Print load change statistics (true/false) print.loaddiffs = true # Print user accesses print.accesses = false +plot.blackandwhite = true diff --git a/sim/gobs/sweeps/decluster03.zsh b/sim/gobs/sweeps/decluster03.zsh index bf93ae6..4b9cc9a 100755 --- a/sim/gobs/sweeps/decluster03.zsh +++ b/sim/gobs/sweeps/decluster03.zsh @@ -6,13 +6,9 @@ NAME=$1 NAME="dc03${NAME}" -CFG="${NAME}.cfg" -EPS="${NAME}.eps" -OUT="${NAME}.out" +# SEED="seed=821" +CLASS=gobs.sweep.x.Decluster03 -java gobs.sweep.Decluster03 -p sweeps -n 20 -o ${EPS} ${CFG} >& ${OUT} -ERR=$? +SWEEPDIR=$( dirname $0 ) -[[ ${ERR} != 0 ]] && print "Crashed: ${NAME}" && exit 1 - -exit 0 +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/decluster04.zsh b/sim/gobs/sweeps/decluster04.zsh index 862cfdf..aad88dc 100755 --- a/sim/gobs/sweeps/decluster04.zsh +++ b/sim/gobs/sweeps/decluster04.zsh @@ -6,13 +6,9 @@ NAME=$1 NAME="dc04${NAME}" -CFG="${NAME}.cfg" -EPS="${NAME}.eps" -OUT="${NAME}.out" +# SEED="seed=821" +CLASS=gobs.sweep.x.Decluster04 -java gobs.sweep.Decluster04 -p sweeps -n 20 -o ${EPS} ${CFG} >& ${OUT} -ERR=$? +SWEEPDIR=$( dirname $0 ) -[[ ${ERR} != 0 ]] && print "Crashed: ${NAME} ERR: ${ERR}" && exit 1 - -exit 0 +source ${SWEEPDIR}/sweep.zsh diff --git a/sim/gobs/sweeps/disks-full.cfg b/sim/gobs/sweeps/disks-full.cfg index fbc5106..93f56f4 100644 --- a/sim/gobs/sweeps/disks-full.cfg +++ b/sim/gobs/sweeps/disks-full.cfg @@ -2,7 +2,7 @@ mttr.reboot = 100 mttr.disk = 500 -# 30 TB T G M K +# 30 TB T G M K disk.size = 30000000000000 disk.speed = 400000000 diff --git a/sim/gobs/sweeps/disks-medium.cfg b/sim/gobs/sweeps/disks-medium.cfg index 7f40893..0be9e02 100644 --- a/sim/gobs/sweeps/disks-medium.cfg +++ b/sim/gobs/sweeps/disks-medium.cfg @@ -2,7 +2,9 @@ mttr.reboot = 100 mttr.disk = 500 +# 5GB disk.size = 5000000000 +# 100MB/s disk.speed = 100000000 disk.mttf.min = 20000 diff --git a/sim/gobs/sweeps/kpv01.cfg b/sim/gobs/sweeps/kpv01.cfg index 67c3d63..bb890be 100644 --- a/sim/gobs/sweeps/kpv01.cfg +++ b/sim/gobs/sweeps/kpv01.cfg @@ -1,8 +1,6 @@ ### Kademlia vs. PAST vs. Vertex Groups 01 -include = kpv_output.cfg - ### Choose a PlacementScheme implementation: # Chosen by sweep driver. diff --git a/sim/gobs/sweeps/kpv03.cfg b/sim/gobs/sweeps/kpv03.cfg index 5c9a51b..905fbfe 100644 --- a/sim/gobs/sweeps/kpv03.cfg +++ b/sim/gobs/sweeps/kpv03.cfg @@ -1,8 +1,6 @@ ### Kademlia vs. PAST vs. Vertex Groups 03 -include = kpv_output.cfg - ### Choose a PlacementScheme implementation: # Chosen by sweep driver. diff --git a/sim/gobs/sweeps/rb01-medium.cfg b/sim/gobs/sweeps/rb01-medium.cfg index ad85230..1f4c671 100644 --- a/sim/gobs/sweeps/rb01-medium.cfg +++ b/sim/gobs/sweeps/rb01-medium.cfg @@ -10,6 +10,7 @@ include = disks-medium.cfg user00.cfg \ # Rebuild traffic over time analysis: plot.traffic.interval = 10 +plot.blackandwhite = true ### Parameters for GOBS Simulator run diff --git a/sim/gobs/sweeps/rb01-medium.zsh b/sim/gobs/sweeps/rb01-medium.zsh old mode 100644 new mode 100755 diff --git a/sim/gobs/sweeps/rb01-small.cfg b/sim/gobs/sweeps/rb01-small.cfg index 819891d..4e52ab7 100644 --- a/sim/gobs/sweeps/rb01-small.cfg +++ b/sim/gobs/sweeps/rb01-small.cfg @@ -9,7 +9,8 @@ include = disks-small.cfg user00.cfg \ # Statistics: # Rebuild traffic over time analysis: -plot.traffic.interval = 100 +plot.traffic.interval = 300 +plot.blackandwhite = true ### Parameters for GOBS Simulator run diff --git a/sim/gobs/sweeps/system_20-1000.cfg b/sim/gobs/sweeps/system_100,600.cfg similarity index 66% copy from sim/gobs/sweeps/system_20-1000.cfg copy to sim/gobs/sweeps/system_100,600.cfg index 6199fc7..e3af48b 100644 --- a/sim/gobs/sweeps/system_20-1000.cfg +++ b/sim/gobs/sweeps/system_100,600.cfg @@ -4,16 +4,16 @@ ### 3000 files * 10 objects * 3 replicas = 90,000 objects # Number of servers: -minNodes = 20 -maxNodes = 1000 -step = exps +nodes.min = 100 +nodes.max = 600 +nodes.step = literal:100,600 ### General parameters: # Bit-length of addresses: B = 32 # Number of files: -files = 3000 +files = 100 # FileFactory implementation PlainFiles impl.filefactory = PlainFiles @@ -22,9 +22,19 @@ file.size = 14 # Strip size: file.strip = 4 # File/object replicas: -file.replicas = 3 +# file.replicas = 3 # Number of objects per file: file.width = 10 mttf = 1 mttr = 1 + +node.osds = 1 +disk.size = 500000000 +disk.speed = 1000000 +disk.mttf = 1 +mttc = 1000 +mttr.reboot = 1 +mttr.disk = 1 +raid = 1+0 +fault.count = 1 diff --git a/sim/gobs/sweeps/system_20-1000.cfg b/sim/gobs/sweeps/system_100-300-600.cfg similarity index 66% copy from sim/gobs/sweeps/system_20-1000.cfg copy to sim/gobs/sweeps/system_100-300-600.cfg index 6199fc7..e3af48b 100644 --- a/sim/gobs/sweeps/system_20-1000.cfg +++ b/sim/gobs/sweeps/system_100-300-600.cfg @@ -4,16 +4,16 @@ ### 3000 files * 10 objects * 3 replicas = 90,000 objects # Number of servers: -minNodes = 20 -maxNodes = 1000 -step = exps +nodes.min = 100 +nodes.max = 600 +nodes.step = literal:100,600 ### General parameters: # Bit-length of addresses: B = 32 # Number of files: -files = 3000 +files = 100 # FileFactory implementation PlainFiles impl.filefactory = PlainFiles @@ -22,9 +22,19 @@ file.size = 14 # Strip size: file.strip = 4 # File/object replicas: -file.replicas = 3 +# file.replicas = 3 # Number of objects per file: file.width = 10 mttf = 1 mttr = 1 + +node.osds = 1 +disk.size = 500000000 +disk.speed = 1000000 +disk.mttf = 1 +mttc = 1000 +mttr.reboot = 1 +mttr.disk = 1 +raid = 1+0 +fault.count = 1 diff --git a/sim/gobs/sweeps/system_20-1000.cfg b/sim/gobs/sweeps/system_100-600.cfg similarity index 67% copy from sim/gobs/sweeps/system_20-1000.cfg copy to sim/gobs/sweeps/system_100-600.cfg index 6199fc7..e6c2192 100644 --- a/sim/gobs/sweeps/system_20-1000.cfg +++ b/sim/gobs/sweeps/system_100-600.cfg @@ -4,16 +4,16 @@ ### 3000 files * 10 objects * 3 replicas = 90,000 objects # Number of servers: -minNodes = 20 -maxNodes = 1000 -step = exps +nodes.min = 100 +nodes.max = 600 +nodes.step = exps ### General parameters: # Bit-length of addresses: B = 32 # Number of files: -files = 3000 +files = 100 # FileFactory implementation PlainFiles impl.filefactory = PlainFiles @@ -22,9 +22,19 @@ file.size = 14 # Strip size: file.strip = 4 # File/object replicas: -file.replicas = 3 +# file.replicas = 3 # Number of objects per file: file.width = 10 mttf = 1 mttr = 1 + +node.osds = 1 +disk.size = 500000000 +disk.speed = 1000000 +disk.mttf = 1 +mttc = 1000 +mttr.reboot = 1 +mttr.disk = 1 +raid = 1+0 +fault.count = 1 diff --git a/sim/gobs/sweeps/system_20-1000.cfg b/sim/gobs/sweeps/system_20-1000.cfg index 6199fc7..e3881f0 100644 --- a/sim/gobs/sweeps/system_20-1000.cfg +++ b/sim/gobs/sweeps/system_20-1000.cfg @@ -4,16 +4,16 @@ ### 3000 files * 10 objects * 3 replicas = 90,000 objects # Number of servers: -minNodes = 20 -maxNodes = 1000 -step = exps +nodes.min = 20 +nodes.max = 600 +nodes.step = exps ### General parameters: # Bit-length of addresses: B = 32 # Number of files: -files = 3000 +files = 100 # FileFactory implementation PlainFiles impl.filefactory = PlainFiles @@ -22,9 +22,19 @@ file.size = 14 # Strip size: file.strip = 4 # File/object replicas: -file.replicas = 3 +# file.replicas = 3 # Number of objects per file: file.width = 10 mttf = 1 mttr = 1 + +node.osds = 1 +disk.size = 500000000 +disk.speed = 1000000 +disk.mttf = 1 +mttc = 1000 +mttr.reboot = 1 +mttr.disk = 1 +raid = 1+0 +fault.count = 1 diff --git a/sim/gobs/sweeps/system_20-1000.cfg b/sim/gobs/sweeps/system_20-600.cfg similarity index 67% copy from sim/gobs/sweeps/system_20-1000.cfg copy to sim/gobs/sweeps/system_20-600.cfg index 6199fc7..e6c2192 100644 --- a/sim/gobs/sweeps/system_20-1000.cfg +++ b/sim/gobs/sweeps/system_20-600.cfg @@ -4,16 +4,16 @@ ### 3000 files * 10 objects * 3 replicas = 90,000 objects # Number of servers: -minNodes = 20 -maxNodes = 1000 -step = exps +nodes.min = 100 +nodes.max = 600 +nodes.step = exps ### General parameters: # Bit-length of addresses: B = 32 # Number of files: -files = 3000 +files = 100 # FileFactory implementation PlainFiles impl.filefactory = PlainFiles @@ -22,9 +22,19 @@ file.size = 14 # Strip size: file.strip = 4 # File/object replicas: -file.replicas = 3 +# file.replicas = 3 # Number of objects per file: file.width = 10 mttf = 1 mttr = 1 + +node.osds = 1 +disk.size = 500000000 +disk.speed = 1000000 +disk.mttf = 1 +mttc = 1000 +mttr.reboot = 1 +mttr.disk = 1 +raid = 1+0 +fault.count = 1 diff --git a/sim/gobs/sweeps/system_20-1000.cfg b/sim/gobs/sweeps/system_600.cfg similarity index 66% copy from sim/gobs/sweeps/system_20-1000.cfg copy to sim/gobs/sweeps/system_600.cfg index 6199fc7..c476879 100644 --- a/sim/gobs/sweeps/system_20-1000.cfg +++ b/sim/gobs/sweeps/system_600.cfg @@ -4,16 +4,16 @@ ### 3000 files * 10 objects * 3 replicas = 90,000 objects # Number of servers: -minNodes = 20 -maxNodes = 1000 -step = exps +nodes.min = 100 +nodes.max = 600 +nodes.step = literal:600 ### General parameters: # Bit-length of addresses: B = 32 # Number of files: -files = 3000 +files = 200 # FileFactory implementation PlainFiles impl.filefactory = PlainFiles @@ -22,9 +22,19 @@ file.size = 14 # Strip size: file.strip = 4 # File/object replicas: -file.replicas = 3 +# file.replicas = 3 # Number of objects per file: file.width = 10 mttf = 1 mttr = 1 + +node.osds = 1 +disk.size = 500000000 +disk.speed = 1000000 +disk.mttf = 1 +mttc = 1000 +mttr.reboot = 1 +mttr.disk = 1 +raid = 1+0 +fault.count = 1 hooks/post-receive -- Triton Repository