This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "".
The branch, master has been updated
via 0660e0935a55374a1adcf1e750878285ebc252c1 (commit)
via 3b5fec29f561d02052e65e003569c97d98590e73 (commit)
from 0d7a6545b063644a4c849af97ec0044bc7942b9e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 0660e0935a55374a1adcf1e750878285ebc252c1
Author: Phil Carns <carns(a)mcs.anl.gov>
Date: Fri Jun 29 14:14:50 2012 -0400
fill in recover test program
- also add test script, but disable for now. It fails when deleting
objects because there are no stubs created yet, leading to an
inconsistent replication factor.
commit 3b5fec29f561d02052e65e003569c97d98590e73
Author: Phil Carns <carns(a)mcs.anl.gov>
Date: Thu Jun 28 16:52:44 2012 -0400
stub for recovery test program
-----------------------------------------------------------------------
Summary of changes:
code/configure.ac | 1 +
code/src/rebuild/tests/module.mk.in | 15 ++
.../tests/recover-objects.ae} | 213 ++++++++++----------
.../tests/recover-objects.sh} | 10 +-
4 files changed, 122 insertions(+), 117 deletions(-)
create mode 100644 code/src/rebuild/tests/module.mk.in
copy code/src/{replicated-osd/tests/rosd-traffic-cop-stall.ae => rebuild/tests/recover-objects.ae} (58%)
copy code/src/{state/tests/rosd-injection.sh => rebuild/tests/recover-objects.sh} (65%)
Diff of changes:
diff --git a/code/configure.ac b/code/configure.ac
index a6ac366..19f0432 100644
--- a/code/configure.ac
+++ b/code/configure.ac
@@ -416,6 +416,7 @@ src/fakess/tests/module.mk
src/placement/module.mk
src/placement/tests/module.mk
src/rebuild/module.mk
+src/rebuild/tests/module.mk
src/zeroconf/module.mk
src/net/mock/module.mk
src/kv/module.mk
diff --git a/code/src/rebuild/tests/module.mk.in b/code/src/rebuild/tests/module.mk.in
new file mode 100644
index 0000000..1695d46
--- /dev/null
+++ b/code/src/rebuild/tests/module.mk.in
@@ -0,0 +1,15 @@
+DIR := src/rebuild/tests
+
+ifneq (,$(BUILD_MPI))
+
+AETESTSRC += $(DIR)/recover-objects.ae
+MODCFLAGS_$(DIR)/recover-objects = $(MPICFLAGS)
+MODLDFLAGS_$(DIR)/recover-objects = $(MPILDFLAGS)
+MODLIBS_$(DIR)/recover-objects = $(MPILIBS)
+MODCC_$(DIR)/recover-objects = $(MPICC)
+
+TEST_RESULTS += $(DIR)/recover-objects.result
+
+endif # BUILD_MPI
+
+MODLIBS_$(DIR) = -lpthread -ldb
diff --git a/code/src/replicated-osd/tests/rosd-traffic-cop-stall.ae b/code/src/rebuild/tests/recover-objects.ae
similarity index 58%
copy from code/src/replicated-osd/tests/rosd-traffic-cop-stall.ae
copy to code/src/rebuild/tests/recover-objects.ae
index 927e33d..e3599f2 100644
--- a/code/src/replicated-osd/tests/rosd-traffic-cop-stall.ae
+++ b/code/src/rebuild/tests/recover-objects.ae
@@ -30,14 +30,13 @@
/* NOTES:
*
- * Run this test with exactly two processes. One will be a client and one
- * will be a server. The client will create and write an object, and then
- * spend 15 seconds issuing concurrent read operations. The server will
- * start the servicing engine, wait 5 seconds, introduce a stall in the
- * traffic cop, sleep 1 second, and then resume.
- *
- * The client should detect that some number of rosd operations were stalled
- * based on the amount of time they took to respond.
+ * Run this test with 3 or more processes. One process (the last one) will
+ * be a client, and the remainder will be servers. N*2 objects will be
+ * created, where N is the number of servers. Each server will be the
+ * master server for two objects immediately adjacent (+/-1 from its server
+ * id). The replication factor of each object will be N-1. A single fault
+ * will then be injected on the first server. The surviving servers will
+ * recover all objects.
*/
#include "mpi.h"
@@ -82,9 +81,9 @@ int main(int argc, char *argv[])
return(-1);
}
- if(options.nservers !=1 || (nprocs - options.nservers) < 1)
+ if(options.nservers < 3 || (nprocs - options.nservers) < 1)
{
- fprintf(stderr, "Error: wrong number of procs.\n");
+ fprintf(stderr, "Error: not enough procs.\n");
return(-1);
}
@@ -109,75 +108,65 @@ int main(int argc, char *argv[])
return 0;
}
-static __blocking triton_ret_t test_reads(void)
+static __blocking void read_and_remove_oid(uint128_t oid)
{
- uint128_t target_oid;
- struct rosd_read_req req;
- double test_start;
- int flagged = 0;
- triton_ret_t tret;
-
- /* set long timeout on rpc operations to make sure that it waits through
- * the operation stalls on the server side.
- */
- tret = triton_zeroconf_set("triton.traffic_cop.static_timeout", "3000");
- triton_error_assert(tret);
+ triton_ret_t ret;
+ char oidbuf[TRITON_UINT128_STRLEN];
+ struct rosd_read_resp resp;
+ uint128_t* validate_oid;
- printf("# reading object.\n");
+ triton_uint128_to_string(oidbuf, TRITON_UINT128_STRLEN, oid);
+ printf("recover-objects: reading oid %s\n", oidbuf);
- target_oid = triton_uint128_from_uint64(1);
+ ret = client_rosd_read(oid, 0, sizeof(oid), 0, 0, &resp);
+ triton_error_assert(ret);
- test_start = MPI_Wtime();
- pwait
+ validate_oid = (uint128_t*)resp.buffer.buffer;
+ if(triton_cmp_uint128(oid, *validate_oid) != 0)
{
- pprivate int i;
- pprivate struct rosd_read_resp resp;
- pprivate triton_ret_t pret;
- pprivate double service_start;
- pprivate double service_end;
-
- for(i=0; i<16; i++)
- {
- pbranch
- {
- do
- {
- service_start = MPI_Wtime();
- pret = client_rosd_read(target_oid, 0, 128, 0, 0, &resp);
- if(pret != TRITON_SUCCESS)
- {
- triton_error_print(pret, "Error: rosd_read() failure.");
- triton_error_assert(pret);
- }
- else
- {
- aer_destroy_struct_rosd_read_resp(&resp);
- }
- service_end = MPI_Wtime();
- if(!flagged && service_end-service_start > 1.0)
- {
- flagged = 1;
- printf("Stall detected.\n");
- }
-
- /* keep going for 15 seconds */
- } while(MPI_Wtime() - test_start < 15);
- }
- }
+ fprintf(stderr, "ERROR: buffer validation failed.\n");
+ assert(0);
}
+ aer_destroy_struct_rosd_read_resp(&resp);
- return(TRITON_SUCCESS);
+ ret = client_rosd_remove(oid, 0);
+ triton_error_assert(ret);
+
+ return;
+}
+
+
+static __blocking void create_and_write_oid(uint128_t oid, int replication_factor)
+{
+ triton_ret_t ret;
+ triton_buffer_t tbuf;
+ char oidbuf[TRITON_UINT128_STRLEN];
+
+ triton_uint128_to_string(oidbuf, TRITON_UINT128_STRLEN, oid);
+ printf("recover-objects: creating oid %s\n", oidbuf);
+
+ ret = client_rosd_create(oid, replication_factor, 0);
+ triton_error_assert(ret);
+
+ /* write the oid value itself (in binary form) into the object */
+ ret = triton_buffer_init(&tbuf, &oid, sizeof(oid));
+ triton_error_assert(ret);
+
+ ret = client_rosd_write(oid, 0, tbuf, 0, 0);
+ triton_error_assert(ret);
+
+ return;
}
static __blocking triton_ret_t rosd_client_fn(int nservers, int nclients, void* data, MPI_Comm client_comm)
{
triton_ret_t ret;
- char* hostfile;
- double create_elapsed_time, create_min_elapsed_time, create_rate;
- double write_elapsed_time, write_min_elapsed_time, write_rate;
- double remove_elapsed_time, remove_min_elapsed_time, remove_rate;
int mpi_ret;
- struct rosd_test_harness_options* options = (struct rosd_test_harness_options*)data;
+ triton_node_t* node_array;
+ int node_array_count;
+ int i;
+ uint128_t* oid_array;
+ int oid_array_count;
/* start up triton */
ret = triton_init("triton.client");
@@ -186,23 +175,58 @@ static __blocking triton_ret_t rosd_client_fn(int nservers, int nclients, void*
/* attach to storage system */
ret = triton_attach("mpi://0");
triton_error_assert(ret);
-
- printf("# creating object.\n");
- ret = rosd_test_harness_create(options, client_comm, &create_min_elapsed_time, &create_elapsed_time, &create_rate);
+
+ /* list servers in the system */
+ ret = triton_state_list_nodes(&node_array, &node_array_count);
triton_error_assert(ret);
- printf("# writing object.\n");
- ret = rosd_test_harness_write(options, client_comm, &write_min_elapsed_time, &write_elapsed_time, &write_rate);
- triton_error_assert(ret);
+ /* build an array of oids to access. We want two per server, (one to the
+ * "left" and one to the "right" of the server id in terms of placement)
+ */
+ oid_array_count = node_array_count*2;
+ oid_array = malloc(oid_array_count*sizeof(*oid_array));
+ assert(oid_array);
- ret = test_reads();
- triton_error_assert(ret);
+ for(i=0; i<node_array_count; i++)
+ {
+ oid_array[i*2] = node_array[i];
+ oid_array[i*2+1] = node_array[i];
+ oid_array[i*2].l++;
+ oid_array[i*2+1].l--;
+
+ if(triton_cmp_uint128(oid_array[i*2+1], triton_node_null) == 0)
+ {
+ /* wrap around case; if oid is zero, set it to max values
+ * instead
+ */
+ oid_array[i*2+1].u = UINT64_MAX;
+ oid_array[i*2+1].l = UINT64_MAX;
+ }
+ }
- printf("# deleting object.\n");
- ret = rosd_test_harness_remove(options, client_comm,
- &remove_min_elapsed_time, &remove_elapsed_time, &remove_rate);
+ /* create objects and write a known value into each */
+ for(i=0; i<oid_array_count; i++)
+ {
+ create_and_write_oid(oid_array[i], nservers-1);
+ }
+
+ /* inject a failure */
+ ret = triton_status_set(node_array[0], TRITON_STATUS_INJECTED);
triton_error_assert(ret);
+ /* wait a bit */
+ triton_timer(1000);
+
+ /* read objects, confirm value, and delete */
+ for(i=0; i<oid_array_count; i++)
+ {
+ read_and_remove_oid(oid_array[i]);
+ }
+
+ /* TODO: something to dump objects and confirm if they are in the right
+ * places?
+ */
+
return(TRITON_SUCCESS);
}
@@ -248,7 +272,7 @@ static triton_ret_t parseargs(int argc, char** argv, int rank,
options->nobjects = 1;
options->nconcurrent = 1;
options->nservers = nprocs - 1;
- options->replication_factor = nprocs-1;
+ options->replication_factor = nprocs-2;
return(TRITON_SUCCESS);
}
@@ -266,7 +290,6 @@ static __blocking triton_ret_t rosd_server_fn(int nservers, int nclients, void*
{
triton_ret_t ret;
triton_ret_t signal_ret;
- char* hostfile;
struct rosd_test_harness_options* options = (struct rosd_test_harness_options*)data;
triton_node_t self;
triton_string_t key;
@@ -306,42 +329,10 @@ static __blocking triton_ret_t rosd_server_fn(int nservers, int nclients, void*
pbranch
{
triton_ret_t tc_ret;
- /* NOTE: we don't want recovery to kick off for now, so don't
- * monitor peers.
- */
- triton_zeroconf_set("triton.traffic_cop.monitor_peers", "no");
tc_ret = traffic_cop_engine_start();
/* TODO: error handling */
triton_error_assert(tc_ret);
}
- pbranch
- {
- /* TODO: this would be the mechanism to kick off failure
- * scenarios based on signals sent to a server. Currently
- * unused.
- */
- signal_ret = triton_signal_wait(SIGUSR1);
- if(signal_ret == TRITON_SUCCESS)
- {
- fprintf(stderr, "FOO: server recieved signal.\n");
- }
- }
- pbranch
- {
- int num_flushed;
-
- /* NOTE: this branch is for testing purposes. */
- printf("Server waiting 5 seconds...\n");
- triton_timer(5000);
- printf("Server traffic cop stalling...\n");
- traffic_cop_rpc_svc_stall(&num_flushed);
- printf("Server traffic cop flushed %d operations.\n", num_flushed);
- printf("Server waiting 1 second...\n");
- triton_timer(1000);
- printf("Server traffic cop resuming...\n");
- traffic_cop_rpc_svc_resume();
- printf("Server traffic cop normal.\n");
- }
}
/* will exit when it gets a shutdown request from a client */
diff --git a/code/src/state/tests/rosd-injection.sh b/code/src/rebuild/tests/recover-objects.sh
similarity index 65%
copy from code/src/state/tests/rosd-injection.sh
copy to code/src/rebuild/tests/recover-objects.sh
index 7e5c37b..9347895 100755
--- a/code/src/state/tests/rosd-injection.sh
+++ b/code/src/rebuild/tests/recover-objects.sh
@@ -7,15 +7,13 @@ SCRIPT=$0
BIN=${SCRIPT//+(${SRCDIR}\/|.sh)}
OUT=${BIN}.out
-# export triton_fakess_method=fault
-# export aesop_remote_default_net=fault
-export triton_traffic_cop_static_timeout=10
-
rm -rf /tmp/rosd.db /tmp/rosd-log
+echo SKIPPING TEST CASE 2012-06-29
+exit 0
+
# Test with sorted input...
-mpiexec -l -n 6 ${VALGRIND} ${BIN} /tmp/rosd.db /tmp/rosd-log \
- 1 1 3 1 > ${OUT} 2>&1
+mpiexec -l -n 6 ${VALGRIND} ${BIN} /tmp/rosd.db /tmp/rosd-log > ${OUT} 2>&1
#
[[ $? == 0 ]] || exit 1
hooks/post-receive
--
