This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "Triton Repository". The branch, master has been updated via 442f5ddc7e78a1d8188bff82c0464c8dfb4eaff7 (commit) via 43882f7d0372d89c24e52021feb521bda125a025 (commit) via 22857f67b106c8d999a3ff05e914fd4fa57b61db (commit) via 9b2b5a289195613a58de49fe348b2e4284ed34dc (commit) via ecba12136743898983d15721746396285e366ecb (commit) from b74518d1dc5755ac305f19c6d13da3dfdae5c881 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 442f5ddc7e78a1d8188bff82c0464c8dfb4eaff7 Author: Phil Carns <[email protected]> Date: Wed Feb 9 15:44:07 2011 -0500 updated comments on plan for vosd_autotune commit 43882f7d0372d89c24e52021feb521bda125a025 Author: Phil Carns <[email protected]> Date: Wed Feb 9 14:55:31 2011 -0500 reorganize threading modes for bdb again commit 22857f67b106c8d999a3ff05e914fd4fa57b61db Author: Phil Carns <[email protected]> Date: Wed Feb 9 11:56:46 2011 -0500 bug fixes commit 9b2b5a289195613a58de49fe348b2e4284ed34dc Author: Phil Carns <[email protected]> Date: Wed Feb 9 11:29:13 2011 -0500 make bdb progress mode more flexible commit ecba12136743898983d15721746396285e366ecb Author: Phil Carns <[email protected]> Date: Wed Feb 9 09:55:14 2011 -0500 stubs for vosd autotuner fn ----------------------------------------------------------------------- Summary of changes: .../prototype/bdb-resource/bdb-resource.c | 170 ++++++++------------ .../prototype/bdb-resource/bdb-resource.hae | 4 +- .../src/versioned-osd/prototype/tests/module.mk.in | 1 + .../prototype/tests/vosd-bench-create.ae | 7 +- code/src/versioned-osd/prototype/versioned-osd.ae | 58 +++++++ code/src/versioned-osd/prototype/versioned-osd.hae | 6 + 6 files changed, 136 insertions(+), 110 deletions(-) Diff of changes: diff --git a/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c b/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c index 32ebbcc..09b23ca 100644 --- a/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c +++ b/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.c @@ -28,24 +28,27 @@ #define BDB_DEFAULT_OPCACHE_SIZE 1024 static ae_opcache_t bdb_opcache; -static ae_ops_t bdb_oplist; +static ae_ops_t bdb_poll_oplist; +static ae_ops_t bdb_thread_oplist; static triton_mutex_t bdb_mutex = TRITON_MUTEX_INITIALIZER; static int triton_bdb_resource_id; -static enum bdb_progress_mode triton_bdb_progress_mode = TRITON_BDB_PROG_NONE; static triton_debug_mask_t bdb_r_mask; static unsigned long deadlock_counter = 0; #define BDB_CONFKEY_PREFIX "triton.bdb" -static const char *progress_mode_confkey = BDB_CONFKEY_PREFIX ".progress_mode"; +static const char *txn_progress_mode_confkey = BDB_CONFKEY_PREFIX ".txn_progress_mode"; +static const char *flush_progress_mode_confkey = BDB_CONFKEY_PREFIX ".flush_progress_mode"; + +static int txn_progress_mode = TRITON_BDB_PROG_THREAD_PER_OP; +static int flush_progress_mode = TRITON_BDB_PROG_THREAD_PER_OP; struct bdb_op { ae_op_id_t op_id; struct ae_op op; triton_ret_t return_code; - int in_place_flag; int (*work_fn)(struct ae_op* op); void (*cleanup_fn)(struct ae_op* op); union @@ -199,24 +202,22 @@ static triton_ret_t bdb_launch_op(struct ae_op *op, enum bdb_progress_mode mode) struct bdb_op *b_op; b_op = ae_op_entry(op, struct bdb_op, op); - if(b_op->in_place_flag) - { - bdb_launch_op_here(op); - return(TRITON_SUCCESS); - } switch(mode) { + case TRITON_BDB_PROG_IN_PLACE: + bdb_launch_op_here(op); + break; case TRITON_BDB_PROG_POLL: triton_mutex_lock(&bdb_mutex); - ae_ops_enqueue(op, &bdb_oplist); + ae_ops_enqueue(op, &bdb_poll_oplist); ae_resource_request_poll(NULL, triton_bdb_resource_id); triton_mutex_unlock(&bdb_mutex); break; case TRITON_BDB_PROG_THREAD_PER_OP: /* put on the queue */ triton_mutex_lock(&bdb_mutex); - ae_ops_enqueue(op, &bdb_oplist); + ae_ops_enqueue(op, &bdb_thread_oplist); triton_mutex_unlock(&bdb_mutex); /* launch a thread to do the work */ @@ -248,15 +249,6 @@ static triton_ret_t bdb_launch_op(struct ae_op *op, enum bdb_progress_mode mode) break; } - return(0); -} - -/* bdb_poll_thread_per_op() - * - * poll function in the thread-per-op mode that doesn't do anything. - */ -static triton_ret_t bdb_poll_thread_per_op(ae_context_t context) -{ return(TRITON_SUCCESS); } @@ -266,16 +258,16 @@ static triton_ret_t bdb_poll(ae_context_t context) struct bdb_op *b_op; /* just servicing one op per call in this mode */ - /* NOTE: poll doesn't need locking, because it is not used in the - * threaded modes - */ - a_op = ae_ops_dequeue(&bdb_oplist); + triton_mutex_lock(&bdb_mutex); + a_op = ae_ops_dequeue(&bdb_poll_oplist); if(!a_op) { + triton_mutex_unlock(&bdb_mutex); /* nothing to work on */ return(TRITON_SUCCESS); } + triton_mutex_unlock(&bdb_mutex); b_op = ae_op_entry(a_op, struct bdb_op, op); @@ -288,7 +280,7 @@ static triton_ret_t bdb_poll(ae_context_t context) /* trigger completion of the operation */ ae_opcache_complete_op(bdb_opcache, a_op, triton_ret_t, b_op->return_code); - if(!ae_ops_empty(&bdb_oplist)) + if(!ae_ops_empty(&bdb_poll_oplist)) { /* need to poll more */ ae_resource_request_poll(NULL, triton_bdb_resource_id); @@ -297,28 +289,11 @@ static triton_ret_t bdb_poll(ae_context_t context) return TRITON_SUCCESS; } -/* TODO: does this have to handle the case of the op already having - * disappeared, or does aesop protect against that? - */ static triton_ret_t bdb_cancel(ae_context_t triton_ctx, ae_op_id_t op_id) { - int cache_id, resource_id; - triton_mutex_lock(&bdb_mutex); - struct ae_op *op; - struct bdb_op *b_op; - - cache_id = ae_id_lookup(op_id, &resource_id); - assert(resource_id == triton_bdb_resource_id); - - op = ae_opcache_lookup(bdb_opcache, (int)cache_id); - b_op = ae_op_entry(op, struct bdb_op, op); -#if 0 - ae_ops_del(op); - ae_opcache_put(bdb_opcache, op); -#endif - - triton_mutex_unlock(&bdb_mutex); - + /* TODO: maybe come back and at least figure out if the operation is on + * the queue? If so we can cancel it, otherwise we do nothing. + */ return TRITON_SUCCESS; } @@ -342,15 +317,24 @@ triton_ret_t bdb_init(void) return(ret); } - ret = triton_zeroconf_register(progress_mode_confkey, - "2", progress_mode_updater, - "Specify the progress mode for the Berkeley DB resource (1 for polling, 2 for threaded)"); + ret = triton_zeroconf_register(txn_progress_mode_confkey, + "THREAD_PER_OP", progress_mode_updater, + "Progress mode for BDB transactions (THREAD_PER_OP"); + if(ret != TRITON_SUCCESS && ret != TRITON_ERR_EXIST) + { + return(ret); + } + + ret = triton_zeroconf_register(flush_progress_mode_confkey, + "THREAD_PER_OP", progress_mode_updater, + "Progress mode for flushing BDB data (IN_PLACE, POLL, or THREAD_PER_OP"); if(ret != TRITON_SUCCESS && ret != TRITON_ERR_EXIST) { return(ret); } - ae_ops_init(&bdb_oplist); + ae_ops_init(&bdb_poll_oplist); + ae_ops_init(&bdb_thread_oplist); ret = AE_OPCACHE_INIT(struct bdb_op, op, BDB_DEFAULT_OPCACHE_SIZE, &bdb_opcache); @@ -359,19 +343,6 @@ triton_ret_t bdb_init(void) return(ret); } - /* modify poll function depending on requested mode */ - switch(triton_bdb_progress_mode) - { - case TRITON_BDB_PROG_POLL: - triton_bdb_resource.poll_context = bdb_poll; - break; - case TRITON_BDB_PROG_THREAD_PER_OP: - triton_bdb_resource.poll_context = bdb_poll_thread_per_op; - break; - default: - return(TRITON_ERR_INVAL); - } - ret = ae_resource_register(&triton_bdb_resource, &triton_bdb_resource_id); if(ret != TRITON_SUCCESS) { @@ -379,8 +350,6 @@ triton_ret_t bdb_init(void) return(ret); } - triton_debug(bdb_r_mask, - "bdb resource initialized with progress mode %d.\n", triton_bdb_progress_mode); return TRITON_SUCCESS; } @@ -422,11 +391,10 @@ ae_define_post(triton_ret_t, bdb_txn_begin, b_op->u.txn_begin.flags = flags; b_op->work_fn = txn_begin_work_fn; b_op->cleanup_fn = NULL; - /* this shouldn't block, so lets skip threading and queueing */ - b_op->in_place_flag = 1; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + /* hard code this one; it shouldn't ever block on locks or I/O */ + bdb_launch_op(op, TRITON_BDB_PROG_IN_PLACE); return(TRITON_SUCCESS); } @@ -449,10 +417,9 @@ ae_define_post(triton_ret_t, bdb_log_flush, b_op->u.log_flush.lsn = lsn; b_op->work_fn = log_flush_work_fn; b_op->cleanup_fn = NULL; - b_op->in_place_flag = 0; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, flush_progress_mode); return(TRITON_SUCCESS); } @@ -497,10 +464,9 @@ ae_define_post(triton_ret_t, bdb_txn_checkpoint, b_op->u.txn_checkpoint.flags = flags; b_op->work_fn = txn_checkpoint_work_fn; b_op->cleanup_fn = NULL; - b_op->in_place_flag = 0; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, flush_progress_mode); return(TRITON_SUCCESS); } @@ -565,10 +531,9 @@ ae_define_post(triton_ret_t, bdb_txn_commit, b_op->u.txn_commit.flags = flags; b_op->work_fn = txn_commit_work_fn; b_op->cleanup_fn = NULL; - b_op->in_place_flag = 0; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, txn_progress_mode); return(TRITON_SUCCESS); } @@ -608,10 +573,9 @@ ae_define_post(triton_ret_t, bdb_txn_abort, b_op->u.txn_abort.tid = tid; b_op->work_fn = txn_abort_work_fn; b_op->cleanup_fn = NULL; - b_op->in_place_flag = 0; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, txn_progress_mode); return(TRITON_SUCCESS); } @@ -664,10 +628,9 @@ ae_define_post(triton_ret_t, bdb_get, b_op->u.get.flags = flags; b_op->work_fn = get_work_fn; b_op->cleanup_fn = NULL; - b_op->in_place_flag = 0; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, txn_progress_mode); return(TRITON_SUCCESS); } @@ -715,10 +678,9 @@ ae_define_post(triton_ret_t, bdb_cursor_get, b_op->u.cursor_get.flags = flags; b_op->work_fn = cursor_get_work_fn; b_op->cleanup_fn = NULL; - b_op->in_place_flag = 0; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, txn_progress_mode); return(TRITON_SUCCESS); } @@ -761,10 +723,9 @@ ae_define_post(triton_ret_t, bdb_cursor_del, b_op->u.cursor_del.flags = flags; b_op->work_fn = cursor_del_work_fn; b_op->cleanup_fn = NULL; - b_op->in_place_flag = 0; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, txn_progress_mode); return(TRITON_SUCCESS); } @@ -803,10 +764,9 @@ ae_define_post(triton_ret_t, bdb_cursor_close, b_op->u.cursor_close.DBcursor = DBcursor; b_op->work_fn = cursor_close_work_fn; b_op->cleanup_fn = NULL; - b_op->in_place_flag = 0; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, txn_progress_mode); return(TRITON_SUCCESS); } @@ -852,11 +812,9 @@ ae_define_post(triton_ret_t, bdb_put, b_op->u.put.flags = flags; b_op->work_fn = put_work_fn; b_op->cleanup_fn = NULL; - /* this shouldn't block, so lets skip threading and queueing */ - b_op->in_place_flag = 1; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, txn_progress_mode); return(TRITON_SUCCESS); } @@ -904,10 +862,9 @@ ae_define_post(triton_ret_t, bdb_del, b_op->u.del.flags = flags; b_op->work_fn = del_work_fn; b_op->cleanup_fn = NULL; - b_op->in_place_flag = 0; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, txn_progress_mode); return(TRITON_SUCCESS); } @@ -954,10 +911,9 @@ ae_define_post(triton_ret_t, bdb_cursor, b_op->u.cursor.flags = flags; b_op->work_fn = cursor_work_fn; b_op->cleanup_fn = NULL; - b_op->in_place_flag = 0; *__ae_op_id = b_op->op_id; - bdb_launch_op(op, triton_bdb_progress_mode); + bdb_launch_op(op, txn_progress_mode); return(TRITON_SUCCESS); } @@ -1006,28 +962,32 @@ static triton_ret_t progress_mode_updater(const char *key, const char *value) { int s; int tmp_value; - assert(!strcmp(key, progress_mode_confkey)); - /* TODO: change this to be run-time configurable once we switch to the - * pipe/poll model - */ - if(triton_init_check("triton.vosd.bdb") == TRITON_SUCCESS) + if(!strcmp(key, txn_progress_mode_confkey)) { - triton_err(triton_log_default, "Error: %s parameter cannot be modified after bdb resource has been initialized.\n", progress_mode_confkey); - return TRITON_ERR_INVAL; + if(!strcmp(value, "THREAD_PER_OP")) + txn_progress_mode = TRITON_BDB_PROG_THREAD_PER_OP; + else + return(TRITON_ERR_INVAL); } - - s = sscanf(value, "%d", &tmp_value); - if(s < 1 || - tmp_value <= TRITON_BDB_PROG_NONE || - tmp_value >= TRITON_BDB_PROG_MAX ) + else if(!strcmp(key, flush_progress_mode_confkey)) + { + if(!strcmp(value, "IN_PLACE")) + flush_progress_mode = TRITON_BDB_PROG_IN_PLACE; + else if(!strcmp(value, "POLL")) + flush_progress_mode = TRITON_BDB_PROG_POLL; + else if(!strcmp(value, "THREAD_PER_OP")) + flush_progress_mode = TRITON_BDB_PROG_THREAD_PER_OP; + else + return(TRITON_ERR_INVAL); + } + else { - return TRITON_ERR_INVAL; + return(TRITON_ERR_INVAL); } - triton_bdb_progress_mode = tmp_value; triton_debug(bdb_r_mask, - "bdb progress mode set to: %d\n", tmp_value); + "%s set to: %s\n", key, value); return TRITON_SUCCESS; } diff --git a/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.hae b/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.hae index 6a657fc..4a6c857 100644 --- a/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.hae +++ b/code/src/versioned-osd/prototype/bdb-resource/bdb-resource.hae @@ -19,12 +19,10 @@ /** modes of making progress on posted storage operations */ enum bdb_progress_mode { - /* TODO: warning! Do not change the order of these enumerations. The - * default zeroconf parameters are hard coded to numerical values. - */ TRITON_BDB_PROG_NONE = 0, /**< invalid mode */ TRITON_BDB_PROG_POLL, /**< requires active polling by client */ TRITON_BDB_PROG_THREAD_PER_OP, /**< spawns new thread for each op */ + TRITON_BDB_PROG_IN_PLACE, /**< immediate execution */ TRITON_BDB_PROG_MAX, /**< invalid mode */ }; diff --git a/code/src/versioned-osd/prototype/tests/module.mk.in b/code/src/versioned-osd/prototype/tests/module.mk.in index 10fc6bb..6f4ee0d 100644 --- a/code/src/versioned-osd/prototype/tests/module.mk.in +++ b/code/src/versioned-osd/prototype/tests/module.mk.in @@ -1,6 +1,7 @@ DIR := src/versioned-osd/prototype/tests AETESTSRC += $(DIR)/vosd1.ae +AETESTSRC += $(DIR)/vosd-autotune.ae AETESTSRC += $(DIR)/vosd-niid.ae AETESTSRC += $(DIR)/fake1.ae AETESTSRC += $(DIR)/vosd-multi-object.ae diff --git a/code/src/versioned-osd/prototype/tests/vosd-bench-create.ae b/code/src/versioned-osd/prototype/tests/vosd-bench-create.ae index 4998675..36cadc3 100644 --- a/code/src/versioned-osd/prototype/tests/vosd-bench-create.ae +++ b/code/src/versioned-osd/prototype/tests/vosd-bench-create.ae @@ -187,10 +187,13 @@ int main(int argc, char *argv[]) tret = triton_zeroconf_set("triton.vosd.data_coalescing_highwater", argv[5]); triton_error_assert(tret); - tret = triton_init("triton.vosd"); + /* NOTE: we have to initialize more than just the vosd so that the + * retry_count hint gets registered + */ + tret = triton_init("triton.server"); triton_error_assert(tret); - ae_context_create(&ctx, "bdb", "sched"); + ae_context_create(&ctx, "bdb", "sched", "file"); done = 0; ae_post_blocking(do_vosd_test, done_callback, NULL, NULL, ctx, &op_id); diff --git a/code/src/versioned-osd/prototype/versioned-osd.ae b/code/src/versioned-osd/prototype/versioned-osd.ae index da4fc82..d81d453 100644 --- a/code/src/versioned-osd/prototype/versioned-osd.ae +++ b/code/src/versioned-osd/prototype/versioned-osd.ae @@ -3381,6 +3381,64 @@ __blocking triton_ret_t vosd_check_niid(uint128_t oid, uint64_t niid, int op_typ return(tret); } +__blocking triton_ret_t vosd_autotune(void) +{ + /* TODO: + * - go ahead and switch some of the more obvious operations to either + * execute immediately or else run in non threaded mode + * - examples: open, close, and fstat in file resource should probably + * run in non threaded mode + * - examples: bdb txn open, etc. should just execute immediately + */ + + /* TODO: + * - need a way to toggle O_DIRECT mode on the fly rather than forcing + * it at init time: + * - restrictions: all file descriptors must be idle (so that we can + * close them out) before allowing the switch + */ + + /* things to test: + * + * - pick a starting point for coalescing + * + * - vosd_create() with niids enabled and vosd_discard_niids(), + * stresses bdb write txn performance + * - toggle the txn_progress_mode + * - sweep across coalescing values? + * + * - using results from above, do: + * + * - vosd_write small (4k read) + * - with and without o_direct + * - with and without threads + * - sweep coalescing values again? + * - vosd_read small (4k read) + * - with and without o_direct + * - with and without threads + * - vosd_write large (4M read) + * - with and without o_direct + * - with and without threads + * - sweep coalescing values again? + * - vosd_read large (4M read) + * - with and without o_direct + * - with and without threads + * + * - if run as root, then try to drop buffer caches before each read + * test + * + * - run everything in parallel; we don't really care so much about + * sequential performance + * - maybe do test duration based on elapsed time rather than number of ops? + * + * - other things we could do (probably just hold these at default level + * for the moment?): + * - vary the alignment (for non-odirect use) + */ + + return(TRITON_ERR_NOSYS); +} + /* * Local variables: * c-indent-level: 4 diff --git a/code/src/versioned-osd/prototype/versioned-osd.hae b/code/src/versioned-osd/prototype/versioned-osd.hae index 82ab551..76ab13f 100644 --- a/code/src/versioned-osd/prototype/versioned-osd.hae +++ b/code/src/versioned-osd/prototype/versioned-osd.hae @@ -159,6 +159,12 @@ __blocking triton_ret_t vosd_discard_niids( */ __blocking triton_ret_t vosd_check_niid(uint128_t oid, uint64_t niid, int op_type); +/** + * Performs a series of tests to determine which tuning parameters are + * fastest + */ +__blocking triton_ret_t vosd_autotune(void); + #endif /* __VERSIONED_OSD_HAE__ */ /* @} */ hooks/post-receive -- Triton Repository