From: Jeff Layton <jlayton(a)redhat.com&gt; Change-Id: I28dc92a615a4759b3b3b296f5c83ab378e1678d6 Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/MainNFSD/nfs_reaper_thread.c | 8 ++++++++ src/include/nfs_core.h | 1 + 2 files changed, 9 insertions(+) diff --git a/src/MainNFSD/nfs_reaper_thread.c b/src/MainNFSD/nfs_reaper_thread.c index 10ca197baae7..918a3b7f4256 100644 --- a/src/MainNFSD/nfs_reaper_thread.c +++ b/src/MainNFSD/nfs_reaper_thread.c @@ -264,6 +264,14 @@ int reaper_init(void) return 0; } +void reaper_wake(void) +{ + struct fridgethr *frt = reaper_fridge; + + if (frt) + fridgethr_wake(frt); +} + int reaper_shutdown(void) { int rc = fridgethr_sync_command(reaper_fridge, diff --git a/src/include/nfs_core.h b/src/include/nfs_core.h index fa1b78ce56f3..521db7b002b8 100644 --- a/src/include/nfs_core.h +++ b/src/include/nfs_core.h @@ -197,6 +197,7 @@ int nfs_Init_ip_name(void); void nfs_rpc_destroy_chan(rpc_call_channel_t *chan); int reaper_init(void); +void reaper_wake(void); int reaper_shutdown(void); #endif /* !NFS_CORE_H */ -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Change-Id: I3b3bcf6bf1c1100bc6a261030a0e93094a93b27e Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/SAL/recovery/recovery_rados.h | 6 ++++++ src/SAL/recovery/recovery_rados_kv.c | 7 +++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/SAL/recovery/recovery_rados.h b/src/SAL/recovery/recovery_rados.h index 6c068b7fb893..f15c88463845 100644 --- a/src/SAL/recovery/recovery_rados.h +++ b/src/SAL/recovery/recovery_rados.h @@ -28,6 +28,7 @@ extern rados_t rados_recov_cluster; extern rados_ioctx_t rados_recov_io_ctx; extern char rados_recov_oid[NI_MAXHOST + 6]; +extern char rados_recov_old_oid[NI_MAXHOST + 4]; struct rados_kv_parameter { /** Connection to ceph cluster */ @@ -51,10 +52,15 @@ typedef void (*pop_clid_entry_t)(char *, char *, struct pop_args *); int rados_kv_connect(rados_ioctx_t *io_ctx, const char *userid, const char *conf, const char *pool); void rados_kv_shutdown(void); +int rados_kv_put(char *key, char *val, char *object); int rados_kv_get(char *key, char *val, char *object); +void rados_kv_add_clid(nfs_client_id_t *clientid); +void rados_kv_rm_clid(nfs_client_id_t *clientid); +void rados_kv_add_revoke_fh(nfs_client_id_t *delr_clid, nfs_fh4 *delr_handle); void rados_kv_create_key(nfs_client_id_t *clientid, char *key); void rados_kv_create_val(nfs_client_id_t *clientid, char *val); int rados_kv_traverse(pop_clid_entry_t callback, struct pop_args *args, const char *object); void rados_kv_add_revoke_fh(nfs_client_id_t *delr_clid, nfs_fh4 *delr_handle); +void rados_kv_pop_clid_entry(char *key, char *val, struct pop_args *pop_args); #endif /* _RECOVERY_RADOS_H */ diff --git a/src/SAL/recovery/recovery_rados_kv.c b/src/SAL/recovery/recovery_rados_kv.c index e3ca628f80f7..daa6a9c8f6c7 100644 --- a/src/SAL/recovery/recovery_rados_kv.c +++ b/src/SAL/recovery/recovery_rados_kv.c @@ -21,7 +21,7 @@ static rados_t clnt; rados_ioctx_t rados_recov_io_ctx; char rados_recov_oid[NI_MAXHOST + 6]; -static char rados_recov_old_oid[NI_MAXHOST + 4]; +char rados_recov_old_oid[NI_MAXHOST + 4]; struct rados_kv_parameter rados_kv_param; @@ -131,7 +131,7 @@ void rados_kv_create_val(nfs_client_id_t *clientid, char *val) LogDebug(COMPONENT_CLIENTID, "Created client name [%s]", val); } -static int rados_kv_put(char *key, char *val, char *object) +int rados_kv_put(char *key, char *val, char *object) { int ret; char *keys[1]; @@ -448,8 +448,7 @@ void rados_kv_rm_clid(nfs_client_id_t *clientid) clientid->cid_recov_tag = NULL; } -static void rados_kv_pop_clid_entry(char *key, char *val, - struct pop_args *pop_args) +void rados_kv_pop_clid_entry(char *key, char *val, struct pop_args *pop_args) { int ret; char *dupval; -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Add a new maybe_start_grace operation that will check to see if a sibling has requested a grace period and start one if so. Call this function from the reaper thread. Note that we only enforce the grace period and do not reload the recovery db. All of the current node's state is still intact, so we don't need to initiate any reclaim; only enforce the grace period until the other node's reclaim is complete. Change-Id: I85f54907efb220510af0b87baa1798d3f1d7a041 Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/MainNFSD/nfs_reaper_thread.c | 3 +++ src/SAL/nfs4_recovery.c | 15 +++++++++++++++ src/include/sal_functions.h | 2 ++ 3 files changed, 20 insertions(+) diff --git a/src/MainNFSD/nfs_reaper_thread.c b/src/MainNFSD/nfs_reaper_thread.c index 918a3b7f4256..07ef670810fd 100644 --- a/src/MainNFSD/nfs_reaper_thread.c +++ b/src/MainNFSD/nfs_reaper_thread.c @@ -203,6 +203,9 @@ static void reaper_run(struct fridgethr_context *ctx) SetNameFunction("reaper"); + /* see if we need to start a grace period */ + nfs_maybe_start_grace(); + /* * Try to lift the grace period, unless we're shutting down. * Ordinarily, we'd take the mutex to check this, but this is just a diff --git a/src/SAL/nfs4_recovery.c b/src/SAL/nfs4_recovery.c index 120819e621c9..cc4f7cf9facf 100644 --- a/src/SAL/nfs4_recovery.c +++ b/src/SAL/nfs4_recovery.c @@ -198,6 +198,21 @@ bool nfs_in_grace(void) return atomic_fetch_time_t(&current_grace); } +/** + * @brief Enter the grace period if another node in the cluster needs it + * + * Singleton servers generally won't use this operation. Clustered servers + * call this function to check whether another node might need a grace period. + */ +void nfs_maybe_start_grace(void) +{ + if (recovery_backend->maybe_start_grace) { + if (nfs_in_grace()) + return; + recovery_backend->maybe_start_grace(); + } +} + void nfs_try_lift_grace(void) { bool in_grace = true; diff --git a/src/include/sal_functions.h b/src/include/sal_functions.h index 7e30e51eeabf..2829f574c8e7 100644 --- a/src/include/sal_functions.h +++ b/src/include/sal_functions.h @@ -976,6 +976,7 @@ void blocked_lock_polling(struct fridgethr_context *ctx); void nfs_start_grace(nfs_grace_start_t *gsp); bool nfs_in_grace(void); bool simple_try_lift_grace(void); +void nfs_maybe_start_grace(void); void nfs_try_lift_grace(void); void nfs4_add_clid(nfs_client_id_t *); void nfs4_rm_clid(nfs_client_id_t *); @@ -1023,6 +1024,7 @@ struct nfs4_recovery_backend { void (*add_clid)(nfs_client_id_t *); void (*rm_clid)(nfs_client_id_t *); void (*add_revoke_fh)(nfs_client_id_t *, nfs_fh4 *); + void (*maybe_start_grace)(void); bool (*try_lift_grace)(void); }; -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Add a new recovery backend operation to tell whether the grace period is being fully enforced. Singleton backends that don't define this operation will just have nfs_grace_enforcing() return true. Change-Id: I8020723238396efe0949f29e1906c58b0a5e26c9 Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/SAL/nfs4_recovery.c | 14 ++++++++++++++ src/include/sal_functions.h | 1 + 2 files changed, 15 insertions(+) diff --git a/src/SAL/nfs4_recovery.c b/src/SAL/nfs4_recovery.c index debc6cab1bdb..400d8b528d26 100644 --- a/src/SAL/nfs4_recovery.c +++ b/src/SAL/nfs4_recovery.c @@ -227,6 +227,20 @@ void nfs_maybe_start_grace(void) } } +/** + * @brief Are all hosts in cluster enforcing the grace period? + * + * Singleton servers always return true here since the only grace period that + * matters is the local one. Clustered backends should check to make sure that + * the whole cluster is in grace. + */ +bool nfs_grace_enforcing(void) +{ + if (recovery_backend->grace_enforcing) + return recovery_backend->grace_enforcing(); + return true; +} + void nfs_try_lift_grace(void) { bool in_grace = true; diff --git a/src/include/sal_functions.h b/src/include/sal_functions.h index ac01a6eba497..96a4eb40a50b 100644 --- a/src/include/sal_functions.h +++ b/src/include/sal_functions.h @@ -1027,6 +1027,7 @@ struct nfs4_recovery_backend { void (*maybe_start_grace)(void); bool (*try_lift_grace)(void); void (*set_enforcing)(void); + bool (*grace_enforcing)(void); }; void fs_backend_init(struct nfs4_recovery_backend **); -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Add a new support library for coordinating a grace period between multiple nodes. The basic idea is to use a shared RADOS object to mediate requests for a grace period. Typically, each node will perform a read operation on the database and (then possibly) a write operation. In order to ensure atomicity, we assert that the object version has not changed between the read and write ops. If it has, then we start over again from scratch. Change-Id: Idfff61f8d0092fd9095dc14390dfb434f0ad6696 Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/SAL/recovery/recovery_rados_kv.c | 5 +- src/include/rados_grace.h | 85 ++++ src/support/CMakeLists.txt | 4 + src/support/rados_grace.c | 675 +++++++++++++++++++++++++++ 4 files changed, 766 insertions(+), 3 deletions(-) create mode 100644 src/include/rados_grace.h create mode 100644 src/support/rados_grace.c diff --git a/src/SAL/recovery/recovery_rados_kv.c b/src/SAL/recovery/recovery_rados_kv.c index daa6a9c8f6c7..34fba4bd34ac 100644 --- a/src/SAL/recovery/recovery_rados_kv.c +++ b/src/SAL/recovery/recovery_rados_kv.c @@ -12,10 +12,9 @@ #include "fsal.h" #include "netdb.h" #include <rados/librados.h> +#include <rados_grace.h> #include "recovery_rados.h" -#define DEFAULT_POOL "nfs-ganesha" - #define MAX_ITEMS 1024 /* relaxed */ static rados_t clnt; @@ -30,7 +29,7 @@ static struct config_item rados_kv_params[] = { rados_kv_parameter, ceph_conf), CONF_ITEM_STR("userid", 1, MAXPATHLEN, NULL, rados_kv_parameter, userid), - CONF_ITEM_STR("pool", 1, MAXPATHLEN, DEFAULT_POOL, + CONF_ITEM_STR("pool", 1, MAXPATHLEN, DEFAULT_RADOS_GRACE_POOL, rados_kv_parameter, pool), CONFIG_EOL }; diff --git a/src/include/rados_grace.h b/src/include/rados_grace.h new file mode 100644 index 000000000000..97bd8a14107b --- /dev/null +++ b/src/include/rados_grace.h @@ -0,0 +1,85 @@ +#ifndef _RADOS_GRACE_H +#define _RADOS_GRACE_H + +#include <stdbool.h> + +#define DEFAULT_RADOS_GRACE_POOL "nfs-ganesha" +#define DEFAULT_RADOS_GRACE_OID "grace" + +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA + * + * --------------------------------------- + */ +int rados_grace_create(rados_ioctx_t io_ctx, const char *oid); +int rados_grace_dump(rados_ioctx_t io_ctx, const char *oid); +int rados_grace_epochs(rados_ioctx_t io_ctx, const char *oid, + uint64_t *cur, uint64_t *rec); +int rados_grace_enforcing_toggle(rados_ioctx_t io_ctx, const char *oid, + int nodes, const char * const *nodeids, uint64_t *pcur, + uint64_t *prec, bool start); +int rados_grace_enforcing_check(rados_ioctx_t io_ctx, const char *oid); +int rados_grace_join_bulk(rados_ioctx_t io_ctx, const char *oid, int nodes, + const char * const *nodeids, uint64_t *pcur, uint64_t *prec, + bool start); +int rados_grace_lift_bulk(rados_ioctx_t io_ctx, const char *oid, int nodes, + const char * const *nodeids, uint64_t *pcur, uint64_t *prec, + bool remove); + +static inline int +rados_grace_enforcing_on(rados_ioctx_t io_ctx, const char *oid, + const char *nodeid, uint64_t *pcur, uint64_t *prec) +{ + const char *nodeids[1]; + + nodeids[0] = nodeid; + return rados_grace_enforcing_toggle(io_ctx, oid, 1, nodeids, pcur, + prec, true); +} + +static inline int +rados_grace_enforcing_off(rados_ioctx_t io_ctx, const char *oid, + const char *nodeid, uint64_t *pcur, uint64_t *prec) +{ + const char *nodeids[1]; + + nodeids[0] = nodeid; + return rados_grace_enforcing_toggle(io_ctx, oid, 1, nodeids, pcur, + prec, false); +} + +static inline int +rados_grace_join(rados_ioctx_t io_ctx, const char *oid, const char *nodeid, + uint64_t *pcur, uint64_t *prec, bool start) +{ + const char *nodeids[1]; + + nodeids[0] = nodeid; + return rados_grace_join_bulk(io_ctx, oid, 1, nodeids, pcur, prec, + start); +} + +static inline int +rados_grace_lift(rados_ioctx_t io_ctx, const char *oid, const char *nodeid, + uint64_t *pcur, uint64_t *prec) +{ + const char *nodeids[1]; + + nodeids[0] = nodeid; + return rados_grace_lift_bulk(io_ctx, oid, 1, nodeids, pcur, prec, + false); +} +#endif /* _RADOS_GRACE_H */ diff --git a/src/support/CMakeLists.txt b/src/support/CMakeLists.txt index 42dc7ab7b589..380f01d3d83b 100644 --- a/src/support/CMakeLists.txt +++ b/src/support/CMakeLists.txt @@ -41,6 +41,10 @@ set(netgroup_cache_SRCS add_library(netgroup_cache STATIC ${netgroup_cache_SRCS}) add_sanitizers(netgroup_cache) +if (USE_RADOS_RECOV) + add_library(rados_grace STATIC rados_grace.c) +endif(USE_RADOS_RECOV) + ########### next target ############### SET(support_STAT_SRCS diff --git a/src/support/rados_grace.c b/src/support/rados_grace.c new file mode 100644 index 000000000000..e715f872e3ab --- /dev/null +++ b/src/support/rados_grace.c @@ -0,0 +1,675 @@ +/* + * vim:noexpandtab:shiftwidth=8:tabstop=8: + * + * Copyright 2018 Red Hat, Inc. and/or its affiliates. + * Author: Jeff Layton <jlayton(a)redhat.com&gt; + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * The rados_grace database is a rados object with a well-known name that + * with which all cluster nodes can interact to coordinate grace-period + * enforcement. + * + * It consists of two parts: + * + * 1) 2 uint64_t epoch values (stored LE) that indicate the serial number of + * the current grace period (C) and the serial number of the grace period that + * from which recovery is currently allowed (R). These are stored as object + * data. + * + * 2) An omap containing a key value pair for each cluster node. The key is + * the hostname of the node running ganesha, and the value is a byte with a + * set of flags. + * + * Consider a single server epoch (E) of an individual NFS server to be the + * period between reboots. That consists of an initial grace period and + * a regular operation period. An epoch value of 0 is never valid. + * + * The first value (C) indicates the current server epoch. The client recovery + * db should be tagged with this value on creation, or when updating the db + * after the grace period has been fully lifted. + * + * The second uint64_t value in the data tells the NFS server from what + * recovery db it is allowed to reclaim. A value of 0 in this field means that + * we are out of the cluster-wide grace period and that no recovery is allowed. + * + * The omap contains a key for each host in the cluster. Typically, nodes join + * the cluster by setting their omap key. The value of the omap is a single + * byte that contains a set of flags that indicates their current need for a + * grace period and whether they are locally enforcing one. + * + * The grace period handling engine will update and store the flags, and it + * can be queried to determine whether other nodes may need a grace period or + * are enforcing. + */ +#include "config.h" +#include <stdio.h> +#include <stdint.h> +#include <endian.h> +#include <rados/librados.h> +#include <errno.h> +#include <getopt.h> +#include <stdlib.h> +#include <limits.h> +#include <stdbool.h> + +/* Each cluster node needs a slot here */ +#define MAX_ITEMS 1024 + +/* Flags for the omap value flags field */ + +/* Does this node currently require a grace period? */ +#define RADOS_GRACE_NEED_GRACE 0x1 + +/* Is this node currently enforcing its grace period locally? */ +#define RADOS_GRACE_ENFORCING 0x2 + +static void rados_grace_notify(rados_ioctx_t io_ctx, const char *oid) +{ + static char *buf; + static size_t len; + + /* FIXME: we don't really want or need this to be synchronous */ + rados_notify2(io_ctx, oid, "", 0, 3000, &buf, &len); + rados_buffer_free(buf); +} + +int +rados_grace_create(rados_ioctx_t io_ctx, const char *oid) +{ + int ret; + rados_write_op_t op = NULL; + uint64_t cur = htole64(1); // starting epoch + uint64_t rec = htole64(0); // no recovery yet + char buf[sizeof(uint64_t) * 2]; + + /* + * 2 uint64_t's + * + * The first denotes the current epoch serial number, the epoch serial + * number under which new recovery records should be created. + * + * The second number denotes the epoch from which clients are allowed + * to reclaim. + * + * An epoch of zero is never allowed, so if rec=0, then the grace + * period is no longer in effect, and can't be joined. + */ + memcpy(buf, (char *)&cur, sizeof(cur)); + memcpy(buf + sizeof(cur), (char *)&rec, sizeof(rec)); + + op = rados_create_write_op(); + /* Create the object */ + rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL); + + /* Set serial numbers if we created the object */ + rados_write_op_write_full(op, buf, sizeof(buf)); + + ret = rados_write_op_operate(op, io_ctx, oid, NULL, 0); + rados_release_write_op(op); + return ret; +} + +int +rados_grace_dump(rados_ioctx_t io_ctx, const char *oid) +{ + int ret; + rados_omap_iter_t iter; + rados_read_op_t op; + char *key_out = NULL; + char *val_out = NULL; + unsigned char more = '\0'; + size_t len_out = 0; + char buf[sizeof(uint64_t) * 2]; + uint64_t cur, rec; + + op = rados_create_read_op(); + rados_read_op_read(op, 0, sizeof(buf), buf, &len_out, NULL); + rados_read_op_omap_get_vals2(op, "", "", MAX_ITEMS, &iter, &more, NULL); + ret = rados_read_op_operate(op, io_ctx, oid, 0); + if (ret < 0) { + printf("%s: ret=%d", __func__, ret); + goto out; + } + + if (len_out != sizeof(buf)) { + ret = -ENOTRECOVERABLE; + goto out; + } + + if (more) { + ret = -ENOTRECOVERABLE; + goto out; + } + + cur = le64toh(*(uint64_t *)buf); + rec = le64toh(*(uint64_t *)(buf + sizeof(uint64_t))); + printf("cur=%lu rec=%lu\n", cur, rec); + printf("======================================================\n"); + for (;;) { + char need = ' ', enforcing = ' '; + + rados_omap_get_next(iter, &key_out, &val_out, &len_out); + if (key_out == NULL || val_out == NULL) + break; + if (*val_out & RADOS_GRACE_NEED_GRACE) + need = 'N'; + if (*val_out & RADOS_GRACE_ENFORCING) + enforcing = 'E'; + printf("%s\t%c%c\n", key_out, need, enforcing); + } + rados_omap_get_end(iter); +out: + rados_release_read_op(op); + return ret; +} + +int +rados_grace_epochs(rados_ioctx_t io_ctx, const char *oid, + uint64_t *cur, uint64_t *rec) +{ + int ret; + rados_read_op_t op; + size_t len_out = 0; + char buf[sizeof(uint64_t) * 2]; + + op = rados_create_read_op(); + rados_read_op_read(op, 0, sizeof(buf), buf, &len_out, NULL); + ret = rados_read_op_operate(op, io_ctx, oid, 0); + if (ret < 0) + goto out; + + ret = -ENOTRECOVERABLE; + if (len_out != sizeof(buf)) + goto out; + + *cur = le64toh(*(uint64_t *)buf); + *rec = le64toh(*(uint64_t *)(buf + sizeof(uint64_t))); + ret = 0; +out: + rados_release_read_op(op); + return ret; +} + +int +rados_grace_enforcing_toggle(rados_ioctx_t io_ctx, const char *oid, int nodes, + const char * const *nodeids, uint64_t *pcur, + uint64_t *prec, bool enable) +{ + int i, ret; + char *flags = NULL; + char **vals = NULL; + size_t *lens = NULL; + uint64_t cur, rec, ver; + + /* allocate an array of flag bytes and set ENFORCING on all of them */ + flags = calloc(nodes, 1); + if (!flags) { + ret = -ENOMEM; + goto out; + } + + /* pointers to each val byte */ + vals = calloc(nodes, sizeof(char *)); + if (!vals) { + ret = -ENOMEM; + goto out; + } + + /* lengths */ + lens = calloc(nodes, sizeof(size_t)); + if (!lens) { + ret = -ENOMEM; + goto out; + } + + /* each val is one flag byte */ + for (i = 0; i < nodes; ++i) { + vals[i] = &flags[i]; + lens[i] = 1; + } + + do { + rados_write_op_t wop; + rados_read_op_t rop; + rados_omap_iter_t iter; + size_t len = 0; + unsigned char more = 0; + char buf[sizeof(uint64_t) * 2]; + + /* read epoch blob */ + rop = rados_create_read_op(); + rados_read_op_read(rop, 0, sizeof(buf), buf, &len, NULL); + rados_read_op_omap_get_vals2(rop, "", "", MAX_ITEMS, &iter, + &more, NULL); + ret = rados_read_op_operate(rop, io_ctx, oid, 0); + if (ret < 0) { + rados_release_read_op(rop); + break; + } + + if (more || (len != sizeof(buf))) { + ret = -ENOTRECOVERABLE; + rados_release_read_op(rop); + break; + } + + ver = rados_get_last_version(io_ctx); + + /* + * Walk the returned kv pairs and flip on any existing flags + * in the matching nodeid (if there is one) + */ + for (;;) { + char *key, *val; + + rados_omap_get_next(iter, &key, &val, &len); + if (!key) + break; + for (i = 0; i < nodes; ++i) { + if (strcmp(key, nodeids[i])) + continue; + flags[i] = *val; + if (enable) + flags[i] |= RADOS_GRACE_ENFORCING; + else + flags[i] &= ~RADOS_GRACE_ENFORCING; + break; + } + } + rados_omap_get_end(iter); + rados_release_read_op(rop); + + /* Get old epoch numbers and version */ + cur = le64toh(*(uint64_t *)buf); + rec = le64toh(*(uint64_t *)(buf + sizeof(uint64_t))); + + /* Attempt to update object */ + wop = rados_create_write_op(); + + /* Ensure that nothing has changed */ + rados_write_op_assert_version(wop, ver); + + /* Set omap values to given ones */ + rados_write_op_omap_set(wop, nodeids, + (const char * const*)vals, lens, nodes); + + ret = rados_write_op_operate(wop, io_ctx, oid, NULL, 0); + rados_release_write_op(wop); + if (ret >= 0) + rados_grace_notify(io_ctx, oid); + } while (ret == -ERANGE); + + if (!ret) { + *pcur = cur; + *prec = rec; + } +out: + free(lens); + free(flags); + free(vals); + return ret; +} + +int +rados_grace_enforcing_check(rados_ioctx_t io_ctx, const char *oid) +{ + int ret; + rados_read_op_t rop; + rados_omap_iter_t iter; + unsigned char more = 0; + + rop = rados_create_read_op(); + rados_read_op_omap_get_vals2(rop, "", "", MAX_ITEMS, &iter, + &more, NULL); + ret = rados_read_op_operate(rop, io_ctx, oid, 0); + if (ret < 0) { + rados_release_read_op(rop); + goto out; + } + + if (more) { + ret = -ENOTRECOVERABLE; + rados_release_read_op(rop); + goto out; + } + + ret = 0; + for (;;) { + char *key, *val; + size_t len = 0; + + rados_omap_get_next(iter, &key, &val, &len); + if (!key) + break; + /* If anyone isn't enforcing, then return an err */ + if (!(*val & RADOS_GRACE_ENFORCING)) { + ret = -EL2NSYNC; + break; + } + } + rados_omap_get_end(iter); + rados_release_read_op(rop); +out: + return ret; +} + +int +rados_grace_join_bulk(rados_ioctx_t io_ctx, const char *oid, int nodes, + const char * const *nodeids, uint64_t *pcur, + uint64_t *prec, bool start) +{ + int i, ret; + char *flags = NULL; + char **vals = NULL; + size_t *lens = NULL; + uint64_t cur, rec, ver; + + /* flag bytes */ + flags = malloc(nodes); + if (!flags) { + ret = -ENOMEM; + goto out; + } + + /* pointers to each val byte */ + vals = calloc(nodes, sizeof(char *)); + if (!vals) { + ret = -ENOMEM; + goto out; + } + + /* lengths */ + lens = calloc(nodes, sizeof(size_t)); + if (!lens) { + ret = -ENOMEM; + goto out; + } + + /* each val is one flag byte */ + for (i = 0; i < nodes; ++i) { + vals[i] = &flags[i]; + lens[i] = 1; + } + + do { + rados_write_op_t wop; + rados_read_op_t rop; + rados_omap_iter_t iter; + size_t len = 0; + unsigned char more = 0; + char buf[sizeof(uint64_t) * 2]; + + /* read epoch blob */ + rop = rados_create_read_op(); + rados_read_op_read(rop, 0, sizeof(buf), buf, &len, NULL); + rados_read_op_omap_get_vals2(rop, "", "", MAX_ITEMS, &iter, + &more, NULL); + ret = rados_read_op_operate(rop, io_ctx, oid, 0); + if (ret < 0) { + rados_release_read_op(rop); + break; + } + + if (more || (len != sizeof(buf))) { + ret = -ENOTRECOVERABLE; + rados_release_read_op(rop); + break; + } + + ver = rados_get_last_version(io_ctx); + + /* + * Walk the returned kv pairs and flip on any existing flags + * in the matching nodeid (if there is one) + */ + memset(flags, RADOS_GRACE_NEED_GRACE|RADOS_GRACE_ENFORCING, + nodes); + for (;;) { + char *key, *val; + + rados_omap_get_next(iter, &key, &val, &len); + if (!key) + break; + for (i = 0; i < nodes; ++i) { + if (!strcmp(key, nodeids[i])) { + flags[i] |= *val; + break; + } + } + } + rados_omap_get_end(iter); + rados_release_read_op(rop); + + /* Get old epoch numbers and version */ + cur = le64toh(*(uint64_t *)buf); + rec = le64toh(*(uint64_t *)(buf + sizeof(uint64_t))); + + /* Only start a new grace period if start bool is set */ + /* FIXME: do we need this with real membership? */ + if (rec == 0 && !start) + break; + + /* Attempt to update object */ + wop = rados_create_write_op(); + + /* Ensure that nothing has changed */ + rados_write_op_assert_version(wop, ver); + + /* Update the object data iff rec == 0 */ + if (rec == 0) { + uint64_t tc, tr; + + rec = cur; + ++cur; + tc = htole64(cur); + tr = htole64(rec); + memcpy(buf, (char *)&tc, sizeof(tc)); + memcpy(buf + sizeof(tc), (char *)&tr, sizeof(tr)); + rados_write_op_write_full(wop, buf, sizeof(buf)); + } + + /* Set omap values to given ones */ + rados_write_op_omap_set(wop, nodeids, + (const char * const*)vals, lens, nodes); + + ret = rados_write_op_operate(wop, io_ctx, oid, NULL, 0); + rados_release_write_op(wop); + if (ret >= 0) + rados_grace_notify(io_ctx, oid); + } while (ret == -ERANGE); + + if (!ret) { + *pcur = cur; + *prec = rec; + } +out: + free(lens); + free(flags); + free(vals); + return ret; +} + +int +rados_grace_lift_bulk(rados_ioctx_t io_ctx, const char *oid, int nodes, + const char * const *nodeids, uint64_t *pcur, + uint64_t *prec, bool remove) +{ + int ret; + char *flags = NULL; + char **vals = NULL; + size_t *lens = NULL; + const char **keys = NULL; + uint64_t cur, rec; + + keys = calloc(nodes, sizeof(char *)); + if (!keys) { + ret = -ENOMEM; + goto out; + } + + /* We don't need these arrays if we're just removing the keys */ + if (!remove) { + flags = calloc(nodes, 1); + if (!flags) { + ret = -ENOMEM; + goto out; + } + + /* pointers to each val byte */ + vals = calloc(nodes, sizeof(char *)); + if (!vals) { + ret = -ENOMEM; + goto out; + } + + lens = calloc(nodes, sizeof(size_t)); + if (!lens) { + ret = -ENOMEM; + goto out; + } + } + + do { + int i, k, need; + rados_write_op_t wop; + rados_read_op_t rop; + rados_omap_iter_t iter; + char *key, *val; + size_t len; + char buf[sizeof(uint64_t) * 2]; + unsigned char more = '\0'; + uint64_t ver; + + /* read epoch blob and omap keys */ + rop = rados_create_read_op(); + rados_read_op_read(rop, 0, sizeof(buf), buf, &len, NULL); + rados_read_op_omap_get_vals2(rop, "", "", MAX_ITEMS, &iter, + &more, NULL); + ret = rados_read_op_operate(rop, io_ctx, oid, 0); + if (ret < 0) { + rados_release_read_op(rop); + break; + } + + if (more) { + ret = -ENOTRECOVERABLE; + rados_release_read_op(rop); + break; + } + + if (len != sizeof(buf)) { + ret = -ENOTRECOVERABLE; + rados_release_read_op(rop); + break; + } + + /* Get old epoch numbers and version */ + ver = rados_get_last_version(io_ctx); + cur = le64toh(*(uint64_t *)buf); + rec = le64toh(*(uint64_t *)(buf + sizeof(uint64_t))); + + /* + * Walk omap keys, see if it's in nodeids array. Add any that + * are and have NEED_GRACE set to the "keys" array along with + * the the flags field with the NEED_GRACE flag cleared. + * + * If the remove flag is set, then just remove them from the + * omap and don't bother with changing the flags. + */ + need = 0; + k = 0; + for (;;) { + ret = rados_omap_get_next(iter, &key, &val, &len); + if (!key) + break; + + if (*val & RADOS_GRACE_NEED_GRACE) + ++need; + else if (!remove) + continue; + + for (i = 0; i < nodes; ++i) { + if (strcmp(key, nodeids[i])) + continue; + keys[k] = nodeids[i]; + /* + * For the removal case, we just need the + * keys. + */ + if (!remove) { + flags[k] = + *val & ~RADOS_GRACE_NEED_GRACE; + vals[k] = &flags[k]; + lens[k] = 1; + } + ++k; + break; + } + }; + rados_omap_get_end(iter); + rados_release_read_op(rop); + + /* No matching keys? Nothing to do. */ + if (k == 0) + break; + + /* Attempt to update object */ + wop = rados_create_write_op(); + + /* Ensure that nothing has changed */ + rados_write_op_assert_version(wop, ver); + + /* Set or remove any keys we matched earlier */ + if (remove) + rados_write_op_omap_rm_keys(wop, keys, k); + else + rados_write_op_omap_set(wop, keys, + (const char * const *)vals, + lens, k); + + /* + * If number of omap records we're setting or removing is the + * same as the number of hosts that have NEED_GRACE set, then + * fully lift the grace period. + */ + if (need == k) { + uint64_t tc, tr; + + rec = 0; + tr = htole64(rec); + tc = htole64(cur); + memcpy(buf, (char *)&tc, sizeof(tc)); + memcpy(buf + sizeof(tc), (char *)&tr, sizeof(tr)); + rados_write_op_write_full(wop, buf, sizeof(buf)); + } + + ret = rados_write_op_operate(wop, io_ctx, oid, NULL, 0); + rados_release_write_op(wop); + if (ret >= 0) + rados_grace_notify(io_ctx, oid); + } while (ret == -ERANGE); + if (!ret) { + *pcur = cur; + *prec = rec; + } +out: + free(vals); + free(lens); + free(flags); + free(keys); + return ret; +} -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Add a new clustered RADOS recovery backend driver. This uses a common RADOS object to coordinate a cluster-wide grace period. We use the hostname as a persistent nodeid. Change-Id: Ic1ec91f5df7c5cbfa5254c646757b2b29e434dfb Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/SAL/CMakeLists.txt | 3 +- src/SAL/nfs4_recovery.c | 7 +- src/SAL/recovery/recovery_rados_cluster.c | 406 ++++++++++++++++++++++ src/doc/man/ganesha-core-config.rst | 1 + src/include/sal_functions.h | 1 + 5 files changed, 413 insertions(+), 5 deletions(-) create mode 100644 src/SAL/recovery/recovery_rados_cluster.c diff --git a/src/SAL/CMakeLists.txt b/src/SAL/CMakeLists.txt index 115ff04c97ad..8af718949b98 100644 --- a/src/SAL/CMakeLists.txt +++ b/src/SAL/CMakeLists.txt @@ -38,6 +38,7 @@ if(USE_RADOS_RECOV) ${sal_STAT_SRCS} recovery/recovery_rados_kv.c recovery/recovery_rados_ng.c + recovery/recovery_rados_cluster.c ) endif(USE_RADOS_RECOV) @@ -46,7 +47,7 @@ add_sanitizers(sal) if(USE_RADOS_RECOV) include_directories(${RADOS_INCLUDE_DIR}) - target_link_libraries(sal ${RADOS_LIBRARIES}) + target_link_libraries(sal rados_grace ${RADOS_LIBRARIES}) endif(USE_RADOS_RECOV) ########### install files ############### diff --git a/src/SAL/nfs4_recovery.c b/src/SAL/nfs4_recovery.c index a0bd986ef247..22524903e17c 100644 --- a/src/SAL/nfs4_recovery.c +++ b/src/SAL/nfs4_recovery.c @@ -220,11 +220,8 @@ bool nfs_in_grace(void) */ void nfs_maybe_start_grace(void) { - if (recovery_backend->maybe_start_grace) { - if (nfs_in_grace()) - return; + if (recovery_backend->maybe_start_grace) recovery_backend->maybe_start_grace(); - } } /** @@ -485,6 +482,8 @@ static int load_backend(const char *name) rados_kv_backend_init(&recovery_backend); else if (!strcmp(name, "rados_ng")) rados_ng_backend_init(&recovery_backend); + else if (!strcmp(name, "rados_cluster")) + rados_cluster_backend_init(&recovery_backend); #endif else if (!strcmp(name, "fs_ng")) fs_ng_backend_init(&recovery_backend); diff --git a/src/SAL/recovery/recovery_rados_cluster.c b/src/SAL/recovery/recovery_rados_cluster.c new file mode 100644 index 000000000000..f582f110acf9 --- /dev/null +++ b/src/SAL/recovery/recovery_rados_cluster.c @@ -0,0 +1,406 @@ +/* + * vim:noexpandtab:shiftwidth=8:tabstop=8: + * + * Copyright 2017 Red Hat, Inc. and/or its affiliates. + * Author: Jeff Layton <jlayton(a)redhat.com&gt; + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * recovery_rados_cluster: a clustered recovery backing store + * + * We assume that each node has a unique nodeid, with a corresponding slot in + * the grace omap, and a rados_kv store for each server epoch. + * + * When the grace period is started, call into the rados_grace infrastructure + * to determine whether we're in a grace period and from what epoch we're + * allowed to recover state. Set the proper oid strings, and load the recovery + * db if applicable, and start a new one (or clear the old). + * + * Similarly, when we want to lift the grace period we'll call down into the + * rados_grace engine to clear its NEED_GRACE flag and see whether anyone else + * still needs one. If they do, we return appropriately so that the local + * grace period is not lifted, and we just re-poll for it again later. + * + * When one node needs a grace period, all nodes are obligated to begin grace + * enforcement as soon as possible. Note that they needn't allow any recovery, + * and any existing state can still be used. The important bit is that they do + * not allow the establishment of new state. This is done on each node by + * enabling the local grace period. + * + * Finally, set and clear the enforcing flag appropriately as we change the + * local grace period. When the driver notes that all nodes are in an enforcing + * state, it'll notify all exported FSALs. This allows them to safely clear + * old state held by a previous instance from the backend filesystem in order + * to handle recovery. + */ + +#include "config.h" +#include <netdb.h> +#include <rados/librados.h> +#include <rados_grace.h> +#include "log.h" +#include "nfs_core.h" +#include "sal_functions.h" +#include "recovery_rados.h" + +/* Use hostname as nodeid in cluster */ +char *nodeid; +static uint64_t rados_watch_cookie; + +static void rados_grace_watchcb(void *arg, uint64_t notify_id, uint64_t handle, + uint64_t notifier_id, void *data, + size_t data_len) +{ + int ret; + + /* ACK it first, so we keep things moving along */ + ret = rados_notify_ack(rados_recov_io_ctx, DEFAULT_RADOS_GRACE_OID, + notify_id, rados_watch_cookie, NULL, 0); + if (ret < 0) + LogEvent(COMPONENT_CLIENTID, + "rados_notify_ack failed: %d", ret); + + /* Now kick the reaper to check things out */ + nfs_notify_grace_waiters(); + reaper_wake(); +} + +static void rados_cluster_init(void) +{ + int ret; + long maxlen; + + maxlen = sysconf(_SC_HOST_NAME_MAX); + nodeid = gsh_malloc(maxlen); + ret = gethostname(nodeid, maxlen); + if (ret) { + LogEvent(COMPONENT_CLIENTID, "gethostname failed: %d", errno); + goto out_free_nodeid; + } + + ret = rados_kv_connect(&rados_recov_io_ctx, rados_kv_param.userid, + rados_kv_param.ceph_conf, rados_kv_param.pool); + if (ret < 0) { + LogEvent(COMPONENT_CLIENTID, + "Failed to connect to cluster: %d", ret); + goto out_shutdown; + } + + ret = rados_grace_create(rados_recov_io_ctx, DEFAULT_RADOS_GRACE_OID); + if (ret < 0 && ret != -EEXIST) { + LogEvent(COMPONENT_CLIENTID, + "Failed to create grace db: %d", ret); + goto out_shutdown; + } + + /* FIXME: not sure about the 30s timeout value here */ + ret = rados_watch3(rados_recov_io_ctx, DEFAULT_RADOS_GRACE_OID, + &rados_watch_cookie, rados_grace_watchcb, NULL, + 30, NULL); + if (ret < 0) { + LogEvent(COMPONENT_CLIENTID, + "Failed to set watch on grace db: %d", ret); + goto out_shutdown; + } + return; + +out_shutdown: + rados_kv_shutdown(); +out_free_nodeid: + gsh_free(nodeid); + nodeid = NULL; +} + +/* Try to delete old recovery db */ +static void rados_cluster_end_grace(void) +{ + int ret; + rados_write_op_t wop; + uint64_t cur, rec; + + if (rados_recov_old_oid[0] == '\0') + return; + + ret = rados_grace_enforcing_off(rados_recov_io_ctx, + DEFAULT_RADOS_GRACE_OID, nodeid, + &cur, &rec); + if (ret) + LogEvent(COMPONENT_CLIENTID, + "Failed to set grace off for %s: %d", nodeid, ret); + + wop = rados_create_write_op(); + rados_write_op_remove(wop); + ret = rados_write_op_operate(wop, rados_recov_io_ctx, + rados_recov_old_oid, NULL, 0); + if (ret) + LogEvent(COMPONENT_CLIENTID, "Failed to remove %s: %d", + rados_recov_old_oid, ret); + + memset(rados_recov_old_oid, '\0', sizeof(rados_recov_old_oid)); +} + +static void rados_cluster_read_clids(nfs_grace_start_t *gsp, + add_clid_entry_hook add_clid_entry, + add_rfh_entry_hook add_rfh_entry) +{ + int ret; + uint64_t cur, rec; + rados_write_op_t wop; + struct pop_args args = { + .add_clid_entry = add_clid_entry, + .add_rfh_entry = add_rfh_entry, + }; + + if (gsp) { + LogEvent(COMPONENT_CLIENTID, + "Clustered rados backend does not support takeover!"); + return; + } + + /* Start or join a grace period */ + ret = rados_grace_join(rados_recov_io_ctx, DEFAULT_RADOS_GRACE_OID, + nodeid, &cur, &rec, true); + if (ret) { + LogEvent(COMPONENT_CLIENTID, + "Failed to join grace period: %d", ret); + return; + } + + /* + * Recovery db names are "rec-cccccccccccccccc:hostname" + * + * "rec-" followed by epoch in 16 hex digits + nodeid. + */ + snprintf(rados_recov_oid, sizeof(rados_recov_oid), + "rec-%16.16lx:%s", cur, nodeid); + wop = rados_create_write_op(); + rados_write_op_create(wop, LIBRADOS_CREATE_IDEMPOTENT, NULL); + rados_write_op_omap_clear(wop); + ret = rados_write_op_operate(wop, rados_recov_io_ctx, + rados_recov_oid, NULL, 0); + rados_release_write_op(wop); + if (ret < 0) { + LogEvent(COMPONENT_CLIENTID, "Failed to create recovery db"); + return; + }; + + snprintf(rados_recov_old_oid, sizeof(rados_recov_old_oid), + "rec-%16.16lx:%s", rec, nodeid); + ret = rados_kv_traverse(rados_kv_pop_clid_entry, &args, + rados_recov_old_oid); + if (ret < 0) + LogEvent(COMPONENT_CLIENTID, + "Failed to traverse recovery db: %d", ret); +} + +static bool rados_cluster_try_lift_grace(void) +{ + int ret; + uint64_t cur, rec; + + ret = rados_grace_lift(rados_recov_io_ctx, DEFAULT_RADOS_GRACE_OID, + nodeid, &cur, &rec); + if (ret) { + LogEvent(COMPONENT_CLIENTID, + "Attempt to lift grace failed: %d", ret); + return false; + } + + /* Non-zero rec means grace is still in force */ + return (rec == 0); +} + +struct rados_cluster_kv_pairs { + size_t slots; /* Current array size */ + size_t num; /* Count of populated elements */ + char **keys; /* Array of key strings */ + char **vals; /* Array of value blobs */ + size_t *lens; /* Array of value lengths */ +}; + +/* + * FIXME: Since each hash tree is protected by its own mutex, we can't ensure + * that we'll get an accurate count before allocating. For now, we just + * have a fixed-size cap of 1024 entries in the db, but we should allow + * there to be an arbitrary number of entries. + */ +#define RADOS_KV_STARTING_SLOTS 1024 + +static void rados_set_client_cb(struct rbt_node *pn, void *arg) +{ + struct hash_data *addr = RBT_OPAQ(pn); + nfs_client_id_t *clientid = addr->val.addr; + struct rados_cluster_kv_pairs *kvp = arg; + char ckey[RADOS_KEY_MAX_LEN]; + char cval[RADOS_VAL_MAX_LEN]; + + /* FIXME: resize arrays in this case? */ + if (kvp->num >= kvp->slots) { + LogEvent(COMPONENT_CLIENTID, "too many clients to copy!"); + return; + } + + rados_kv_create_key(clientid, ckey); + rados_kv_create_val(clientid, cval); + + kvp->keys[kvp->num] = strdup(ckey); + kvp->vals[kvp->num] = strdup(cval); + kvp->lens[kvp->num] = strlen(cval); + ++kvp->num; +} + +/** + * @brief Start local grace period if we're in a global one + * + * In clustered setups, other machines in the cluster can start a new + * grace period. Check for that and enter the grace period if so. + */ +static void rados_cluster_maybe_start_grace(void) +{ + int ret, i; + nfs_grace_start_t gsp = { .event = EVENT_JUST_GRACE }; + rados_write_op_t wop; + uint64_t cur, rec; + char *keys[RADOS_KV_STARTING_SLOTS]; + char *vals[RADOS_KV_STARTING_SLOTS]; + size_t lens[RADOS_KV_STARTING_SLOTS]; + struct rados_cluster_kv_pairs kvp = { + .slots = RADOS_KV_STARTING_SLOTS, + .num = 0, + .keys = keys, + .vals = vals, + .lens = lens }; + + + /* Do nothing if we're already enforcing grace period */ + if (nfs_in_grace()) + return; + + /* Fix up the strings */ + ret = rados_grace_epochs(rados_recov_io_ctx, DEFAULT_RADOS_GRACE_OID, + &cur, &rec); + if (ret) { + LogEvent(COMPONENT_CLIENTID, "rados_grace_epochs failed: %d", + ret); + return; + } + + /* No grace period if rec == 0 */ + if (rec == 0) + return; + + /* Start a new grace period */ + nfs_start_grace(&gsp); + + snprintf(rados_recov_oid, sizeof(rados_recov_oid), + "rec-%16.16lx:%s", cur, nodeid); + snprintf(rados_recov_old_oid, sizeof(rados_recov_old_oid), + "rec-%16.16lx:%s", rec, nodeid); + + /* Populate key/val/len arrays from confirmed client hash */ + hashtable_for_each(ht_confirmed_client_id, rados_set_client_cb, &kvp); + + /* Create new write op and package it up for callback */ + wop = rados_create_write_op(); + rados_write_op_create(wop, LIBRADOS_CREATE_IDEMPOTENT, NULL); + rados_write_op_omap_clear(wop); + rados_write_op_omap_set(wop, (char const * const *)keys, + (char const * const *)vals, + (const size_t *)lens, kvp.num); + ret = rados_write_op_operate(wop, rados_recov_io_ctx, + rados_recov_oid, NULL, 0); + if (ret) + LogEvent(COMPONENT_CLIENTID, + "rados_write_op_operate failed: %d", ret); + + rados_release_write_op(wop); + + /* Free copied strings */ + for (i = 0; i < kvp.num; ++i) { + free(kvp.keys[i]); + free(kvp.vals[i]); + } +} + +static void rados_cluster_shutdown(void) +{ + int ret; + uint64_t cur, rec; + + /* + * Request grace on clean shutdown to minimize the chance that we'll + * miss the window and the MDS kills off the old session. + * + * FIXME: only do this if our key is in the omap, and we have a + * non-empty recovery db. + */ + ret = rados_grace_join(rados_recov_io_ctx, DEFAULT_RADOS_GRACE_OID, + nodeid, &cur, &rec, true); + if (ret) + LogEvent(COMPONENT_CLIENTID, + "Failed to start grace period on shutdown: %d", ret); + + ret = rados_unwatch2(rados_recov_io_ctx, rados_watch_cookie); + if (ret) + LogEvent(COMPONENT_CLIENTID, + "Failed to unwatch grace db: %d", ret); + + rados_kv_shutdown(); + gsh_free(nodeid); + nodeid = NULL; +} + +static void rados_cluster_set_enforcing(void) +{ + int ret; + uint64_t cur, rec; + + ret = rados_grace_enforcing_on(rados_recov_io_ctx, + DEFAULT_RADOS_GRACE_OID, nodeid, + &cur, &rec); + if (ret) + LogEvent(COMPONENT_CLIENTID, + "Failed to set enforcing for %s: %d", nodeid, ret); +} + +static bool rados_cluster_grace_enforcing(void) +{ + int ret; + + ret = rados_grace_enforcing_check(rados_recov_io_ctx, + DEFAULT_RADOS_GRACE_OID); + LogEvent(COMPONENT_CLIENTID, "%s: ret=%d", __func__, ret); + return (ret == 0); +} + +struct nfs4_recovery_backend rados_cluster_backend = { + .recovery_init = rados_cluster_init, + .recovery_shutdown = rados_cluster_shutdown, + .recovery_read_clids = rados_cluster_read_clids, + .end_grace = rados_cluster_end_grace, + .add_clid = rados_kv_add_clid, + .rm_clid = rados_kv_rm_clid, + .add_revoke_fh = rados_kv_add_revoke_fh, + .maybe_start_grace = rados_cluster_maybe_start_grace, + .try_lift_grace = rados_cluster_try_lift_grace, + .set_enforcing = rados_cluster_set_enforcing, + .grace_enforcing = rados_cluster_grace_enforcing, +}; + +void rados_cluster_backend_init(struct nfs4_recovery_backend **backend) +{ + *backend = &rados_cluster_backend; +} diff --git a/src/doc/man/ganesha-core-config.rst b/src/doc/man/ganesha-core-config.rst index 6453e800262f..eb8d9bd09f2c 100644 --- a/src/doc/man/ganesha-core-config.rst +++ b/src/doc/man/ganesha-core-config.rst @@ -292,6 +292,7 @@ RecoveryBackend(path, default "fs") - fs_ng: filesystem (better resiliency) - rados_kv : rados key-value - rados_ng : rados key-value (better resiliency) + - rados_cluster: clustered rados backend (active/active) Minor_Versions(enum list, values [0, 1, 2], default [0, 1, 2]) List of supported NFSV4 minor version numbers. diff --git a/src/include/sal_functions.h b/src/include/sal_functions.h index 708290495731..a53fa475c1bb 100644 --- a/src/include/sal_functions.h +++ b/src/include/sal_functions.h @@ -1038,6 +1038,7 @@ void fs_ng_backend_init(struct nfs4_recovery_backend **); int rados_kv_set_param_from_conf(config_file_t, struct config_error_type *); void rados_kv_backend_init(struct nfs4_recovery_backend **); void rados_ng_backend_init(struct nfs4_recovery_backend **); +void rados_cluster_backend_init(struct nfs4_recovery_backend **backend); #endif #endif /* SAL_FUNCTIONS_H */ -- 2.17.0