This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via c552472cb9f0ed25add75de7a1ffa33279633aec (commit) from 750297eae624a973404b80fc85d9ea7cdbae3a8d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit c552472cb9f0ed25add75de7a1ffa33279633aec Author: Phil Carns <[email protected]> Date: Mon Jun 4 20:49:49 2012 -0400 rpc wrapper and error handling for s2s remove ----------------------------------------------------------------------- Summary of changes: code/src/replicated-osd/rosd.aer | 141 +++++++++++++++++++++++++++++-------- 1 files changed, 110 insertions(+), 31 deletions(-) Diff of changes: diff --git a/code/src/replicated-osd/rosd.aer b/code/src/replicated-osd/rosd.aer index 3d88b00..9d8a045 100644 --- a/code/src/replicated-osd/rosd.aer +++ b/code/src/replicated-osd/rosd.aer @@ -36,6 +36,28 @@ #include "src/replicated-osd/rosd.haer" #include "src/replicated-osd/rosd-s2s.haer" +/* TODO: should we be using ERR_NOENT or ERR_NOTFOUND when an object does + * not exist? + */ + +/* TODO: think about what happens in this scenario: + * - client issues a remove() to a 3-way replicated object + * - first 2 replicas delete + * - 3rd server fails + * - placement function on 2nd server sees failure and forwards to 4th + * server + * - 4th server has not started recovery yet + * + * I'm not sure that there is a way to guarantee that the 4th server will be + * in recovery when we hit it. Probably what needs to happen is the 1st + * server needs to go ahead and map out which servers it expects to do the + * remove, and pass that information along in the s2s requests? If the map + * changes before the operation completes, then a retryable error code can + * be sent back to the client. + * + * Think about if this needs to happen on any other request types + */ + /* TODO: doxygen format */ #include "src/transactional-osd/transactional-osd.hae" @@ -64,6 +86,7 @@ static __blocking triton_ret_t get_placement_info( triton_addr_t* next_peer); static __blocking void trigger_server_fault(triton_ret_t tret); static int is_usage_error_create(triton_ret_t tret); +static int is_usage_error_remove(triton_ret_t tret); static int is_usage_error_write(triton_ret_t tret); static __blocking triton_ret_t interpret_errors( triton_ret_t local_error_code, @@ -93,6 +116,12 @@ static __blocking triton_ret_t client_rosd_s2s_write( uint64_t txn_number, int current_position, uint64_t niid); +static __blocking triton_ret_t client_rosd_s2s_remove( + uint128_t oid, + uint32_t flags, + uint32_t replication_factor, + int current_position, + uint64_t niid); static triton_ret_t rosd_init(void) { @@ -329,13 +358,12 @@ static __blocking triton_ret_t rosd_remove_do_work( uint32_t flags, uint32_t replication_factor, int my_position, - triton_addr_t peer_addr, - uint64_t niid) + uint64_t niid, + int from_client_flag) { triton_ret_t local_tret; - triton_ret_t remote_tret; - aer_remote_ctx_t rctx; - struct rosd_s2s_remove_req req; + triton_ret_t remote_tret = TRITON_SUCCESS; + triton_ret_t tret; /* right now only chained replication is supported */ assert(!(flags & ROSD_FLAG_FANOUT)); @@ -347,25 +375,13 @@ static __blocking triton_ret_t rosd_remove_do_work( pbranch { /* forward on to peers if necessary */ - if(my_position == replication_factor-1) - { - /* end of the chain; no one further to forward to */ - remote_tret = TRITON_SUCCESS; - } - else + if(my_position < replication_factor-1) { triton_uint128_to_string(oid_str, TRITON_UINT128_STRLEN, oid); triton_debug(rosd_dbg_mask, "ROSD forwarding remove of oid %s to %d'th server.\n", oid_str, my_position+1); - aesop_hints_get("triton.remote.context", sizeof(rctx), &rctx); - /* TODO: error check above; we are in trouble if hint not - * found - */ - remote_tret = aer_init_struct_rosd_s2s_remove_req(&req, &oid, &flags, &replication_factor, &niid); - if(triton_error_equal(remote_tret, TRITON_SUCCESS)) - { - remote_tret = remote_rosd_s2s_remove(rctx, peer_addr, &req, NULL); - aer_destroy_struct_rosd_s2s_remove_req(&req); - } + + remote_tret = client_rosd_s2s_remove(oid, flags, + replication_factor, my_position, niid); } } pbranch @@ -377,12 +393,14 @@ static __blocking triton_ret_t rosd_remove_do_work( } } - /* TODO: handle cases where one of local or remote succeeded, but not - * both. What to clean up, and what to leave to the resync process? - */ - triton_error_assert(remote_tret); - triton_error_assert(local_tret); - return(local_tret); + if(my_position < replication_factor -1) + tret = interpret_errors(local_tret, remote_tret, my_position, + is_usage_error_remove, from_client_flag); + else + tret = interpret_error(local_tret, my_position, is_usage_error_remove, + from_client_flag); + + return(tret); } @@ -567,8 +585,8 @@ __remote __blocking triton_ret_t rosd_s2s_remove( req->flags, req->replication_factor, my_position, - peer_addr, - req->niid); + req->niid, + 0); return(tret); } @@ -662,8 +680,8 @@ __remote __blocking triton_ret_t rosd_remove( req->flags, replication_factor, my_position, - peer_addr, - niid); + niid, + 1); return(tret); } @@ -966,6 +984,31 @@ static int is_usage_error_write(triton_ret_t tret) * usage error rather than an a storage fault. Returns 1 if usage * error code, 0 if storage fault. */ +static int is_usage_error_remove(triton_ret_t tret) +{ + /* these are the scenarios that a remove could fail because of a + * usage (ie, semantic) error rather than because of a system fault. + */ + if(triton_error_equal(tret, TRITON_ERR_NO_SERVERS) || + triton_error_equal(tret, TRITON_ERR_NOENT) || + triton_error_equal(tret, TRITON_ERR_ACCES)) + { + /* if there aren't enough servers to satisfy the replication + * request, or if the object does not exist, or if + * permission was denied. + */ + return(1); + } + + return(0); +} + + +/** + * Checks the triton_ret_t to determine if the error code represents a + * usage error rather than an a storage fault. Returns 1 if usage + * error code, 0 if storage fault. + */ static int is_usage_error_create(triton_ret_t tret) { /* these are the scenarios that a create could fail because of a @@ -1219,7 +1262,43 @@ static __blocking triton_ret_t client_rosd_s2s_write( } +static __blocking triton_ret_t client_rosd_s2s_remove( + uint128_t oid, + uint32_t flags, + uint32_t replication_factor, + int current_position, + uint64_t niid) +{ + struct rosd_s2s_remove_req req; + triton_ret_t tret; + int retry; + uint64_t internal_niid = 0; + triton_addr_t addr; + int next_position = 0; + int critical = 0; + + tret = aer_init_struct_rosd_s2s_remove_req(&req, &oid, &flags, &replication_factor, &niid); + if(tret != TRITON_SUCCESS) + return(tret); + + retry = traffic_cop_rpc_control_s2s(&internal_niid, oid, replication_factor, + current_position, &next_position, &addr, &tret, &critical); + while(retry) + { + tret = remote_rosd_s2s_remove(AER_DEFAULT_CTX, addr, &req, NULL); + retry = traffic_cop_rpc_control_s2s(&internal_niid, oid, replication_factor, + current_position, &next_position, &addr, &tret, &critical); + } + aer_destroy_struct_rosd_s2s_remove_req(&req); + + if(critical) + { + trigger_server_fault(tret); + return(0); + } + return(tret); +} /* hooks/post-receive --