aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid Disseldorp <ddiss@samba.org>2018-07-19 11:55:23 +0200
committerKarolin Seeger <kseeger@samba.org>2018-08-23 10:38:27 +0200
commit7cebb4897550e2dd8109c4dcc5c9f885dcbefb83 (patch)
treeaf2758ead3ed119e74b33f42eec4082aa29c5094
parentc245fdce393c9c11b52576335802a5fdceeeaf96 (diff)
downloadsamba-7cebb4897550e2dd8109c4dcc5c9f885dcbefb83.tar.gz
samba-7cebb4897550e2dd8109c4dcc5c9f885dcbefb83.tar.xz
samba-7cebb4897550e2dd8109c4dcc5c9f885dcbefb83.zip
ctdb_mutex_ceph_rados_helper: fix deadlock via lock renewals
RADOS locks without expiry persist indefinitely. This results in CTDB deadlock during failover if the recovery master dies unexpectedly, as subsequently elected recovery master nodes can't obtain the recovery lock. Avoid deadlock by using a lock expiration time (10s by default), and renewing it periodically. Bug: https://bugzilla.samba.org/show_bug.cgi?id=13540 Signed-off-by: David Disseldorp <ddiss@samba.org> Reviewed-by: Samuel Cabrero <scabrero@suse.de> Reviewed-by: Amitay Isaacs <amitay@gmail.com> (cherry picked from commit ce289e89e5c469cf2c5626dc7f2666b945dba3bd)
-rw-r--r--ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c107
1 files changed, 98 insertions, 9 deletions
diff --git a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c
index 9f020bf57d5..7ef76c26e02 100644
--- a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c
+++ b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c
@@ -1,7 +1,7 @@
/*
CTDB mutex helper using Ceph librados locks
- Copyright (C) David Disseldorp 2016
+ Copyright (C) David Disseldorp 2016-2018
Based on ctdb_mutex_fcntl_helper.c, which is:
Copyright (C) Martin Schwenke 2015
@@ -29,6 +29,11 @@
#define CTDB_MUTEX_CEPH_LOCK_NAME "ctdb_reclock_mutex"
#define CTDB_MUTEX_CEPH_LOCK_COOKIE CTDB_MUTEX_CEPH_LOCK_NAME
#define CTDB_MUTEX_CEPH_LOCK_DESC "CTDB recovery lock"
+/*
+ * During failover it may take up to <lock duration> seconds before the
+ * newly elected recovery master can obtain the lock.
+ */
+#define CTDB_MUTEX_CEPH_LOCK_DURATION_SECS_DEFAULT 10
#define CTDB_MUTEX_STATUS_HOLDING "0"
#define CTDB_MUTEX_STATUS_CONTENDED "1"
@@ -89,16 +94,19 @@ static int ctdb_mutex_rados_ctx_create(const char *ceph_cluster_name,
}
static int ctdb_mutex_rados_lock(rados_ioctx_t *ioctx,
- const char *oid)
+ const char *oid,
+ uint64_t lock_duration_s,
+ uint8_t flags)
{
int ret;
+ struct timeval tv = { lock_duration_s, 0 };
ret = rados_lock_exclusive(ioctx, oid,
- CTDB_MUTEX_CEPH_LOCK_NAME,
+ CTDB_MUTEX_CEPH_LOCK_NAME,
CTDB_MUTEX_CEPH_LOCK_COOKIE,
CTDB_MUTEX_CEPH_LOCK_DESC,
- NULL, /* infinite duration */
- 0);
+ lock_duration_s == 0 ? NULL : &tv,
+ flags);
if ((ret == -EEXIST) || (ret == -EBUSY)) {
/* lock contention */
return ret;
@@ -138,11 +146,13 @@ struct ctdb_mutex_rados_state {
const char *ceph_auth_name;
const char *pool_name;
const char *object;
+ uint64_t lock_duration_s;
int ppid;
struct tevent_context *ev;
struct tevent_signal *sigterm_ev;
struct tevent_signal *sigint_ev;
struct tevent_timer *ppid_timer_ev;
+ struct tevent_timer *renew_timer_ev;
rados_t ceph_cluster;
rados_ioctx_t ioctx;
};
@@ -200,6 +210,49 @@ err_ctx_cleanup:
exit(ret ? 1 : 0);
}
+#define USECS_IN_SEC 1000000
+
+static void ctdb_mutex_rados_lock_renew_timer_cb(struct tevent_context *ev,
+ struct tevent_timer *te,
+ struct timeval current_time,
+ void *private_data)
+{
+ struct ctdb_mutex_rados_state *cmr_state = private_data;
+ struct timeval tv;
+ int ret;
+
+ ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object,
+ cmr_state->lock_duration_s,
+ LIBRADOS_LOCK_FLAG_RENEW);
+ if (ret == -EBUSY) {
+ /* should never get -EEXIST on renewal */
+ fprintf(stderr, "Lock contention during renew: %d\n", ret);
+ goto err_ctx_cleanup;
+ } else if (ret < 0) {
+ fprintf(stderr, "Lock renew failed\n");
+ goto err_ctx_cleanup;
+ }
+
+ tv = tevent_timeval_current_ofs(0,
+ cmr_state->lock_duration_s * (USECS_IN_SEC / 2));
+ cmr_state->renew_timer_ev = tevent_add_timer(cmr_state->ev,
+ cmr_state,
+ tv,
+ ctdb_mutex_rados_lock_renew_timer_cb,
+ cmr_state);
+ if (cmr_state->renew_timer_ev == NULL) {
+ fprintf(stderr, "Failed to create timer event\n");
+ goto err_ctx_cleanup;
+ }
+
+ return;
+
+err_ctx_cleanup:
+ /* drop lock (via destructor) and exit */
+ talloc_free(cmr_state);
+ exit(1);
+}
+
static int ctdb_mutex_rados_state_destroy(struct ctdb_mutex_rados_state *cmr_state)
{
if (cmr_state->holding_mutex) {
@@ -221,9 +274,10 @@ int main(int argc, char *argv[])
progname = argv[0];
- if (argc != 5) {
+ if ((argc != 5) && (argc != 6)) {
fprintf(stderr, "Usage: %s <Ceph Cluster> <Ceph user> "
- "<RADOS pool> <RADOS object>\n",
+ "<RADOS pool> <RADOS object> "
+ "[lock duration secs]\n",
progname);
ret = -EINVAL;
goto err_out;
@@ -246,6 +300,19 @@ int main(int argc, char *argv[])
cmr_state->ceph_auth_name = argv[2];
cmr_state->pool_name = argv[3];
cmr_state->object = argv[4];
+ if (argc == 6) {
+ /* optional lock duration provided */
+ char *endptr = NULL;
+ cmr_state->lock_duration_s = strtoull(argv[5], &endptr, 0);
+ if ((endptr == argv[5]) || (*endptr != '\0')) {
+ fprintf(stdout, CTDB_MUTEX_STATUS_ERROR);
+ ret = -EINVAL;
+ goto err_ctx_cleanup;
+ }
+ } else {
+ cmr_state->lock_duration_s
+ = CTDB_MUTEX_CEPH_LOCK_DURATION_SECS_DEFAULT;
+ }
cmr_state->ppid = getppid();
if (cmr_state->ppid == 1) {
@@ -313,7 +380,9 @@ int main(int argc, char *argv[])
goto err_ctx_cleanup;
}
- ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object);
+ ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object,
+ cmr_state->lock_duration_s,
+ 0);
if ((ret == -EEXIST) || (ret == -EBUSY)) {
fprintf(stdout, CTDB_MUTEX_STATUS_CONTENDED);
goto err_ctx_cleanup;
@@ -321,8 +390,28 @@ int main(int argc, char *argv[])
fprintf(stdout, CTDB_MUTEX_STATUS_ERROR);
goto err_ctx_cleanup;
}
-
cmr_state->holding_mutex = true;
+
+ if (cmr_state->lock_duration_s != 0) {
+ /*
+ * renew (reobtain) the lock, using a period of half the lock
+ * duration. Convert to usecs to avoid rounding.
+ */
+ struct timeval tv = tevent_timeval_current_ofs(0,
+ cmr_state->lock_duration_s * (USECS_IN_SEC / 2));
+ cmr_state->renew_timer_ev = tevent_add_timer(cmr_state->ev,
+ cmr_state,
+ tv,
+ ctdb_mutex_rados_lock_renew_timer_cb,
+ cmr_state);
+ if (cmr_state->renew_timer_ev == NULL) {
+ fprintf(stderr, "Failed to create timer event\n");
+ fprintf(stdout, CTDB_MUTEX_STATUS_ERROR);
+ ret = -ENOMEM;
+ goto err_ctx_cleanup;
+ }
+ }
+
fprintf(stdout, CTDB_MUTEX_STATUS_HOLDING);
/* wait for the signal / timer events to do their work */