From: Jeff Layton <jlayton(a)redhat.com&gt; Change-Id: I28dc92a615a4759b3b3b296f5c83ab378e1678d6 Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/MainNFSD/nfs_reaper_thread.c | 8 ++++++++ src/include/nfs_core.h | 1 + 2 files changed, 9 insertions(+) diff --git a/src/MainNFSD/nfs_reaper_thread.c b/src/MainNFSD/nfs_reaper_thread.c index 10ca197baae7..918a3b7f4256 100644 --- a/src/MainNFSD/nfs_reaper_thread.c +++ b/src/MainNFSD/nfs_reaper_thread.c @@ -264,6 +264,14 @@ int reaper_init(void) return 0; } +void reaper_wake(void) +{ + struct fridgethr *frt = reaper_fridge; + + if (frt) + fridgethr_wake(frt); +} + int reaper_shutdown(void) { int rc = fridgethr_sync_command(reaper_fridge, diff --git a/src/include/nfs_core.h b/src/include/nfs_core.h index fa1b78ce56f3..521db7b002b8 100644 --- a/src/include/nfs_core.h +++ b/src/include/nfs_core.h @@ -197,6 +197,7 @@ int nfs_Init_ip_name(void); void nfs_rpc_destroy_chan(rpc_call_channel_t *chan); int reaper_init(void); +void reaper_wake(void); int reaper_shutdown(void); #endif /* !NFS_CORE_H */ -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Change-Id: I3b3bcf6bf1c1100bc6a261030a0e93094a93b27e Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/SAL/recovery/recovery_rados.h | 6 ++++++ src/SAL/recovery/recovery_rados_kv.c | 7 +++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/SAL/recovery/recovery_rados.h b/src/SAL/recovery/recovery_rados.h index 6c068b7fb893..f15c88463845 100644 --- a/src/SAL/recovery/recovery_rados.h +++ b/src/SAL/recovery/recovery_rados.h @@ -28,6 +28,7 @@ extern rados_t rados_recov_cluster; extern rados_ioctx_t rados_recov_io_ctx; extern char rados_recov_oid[NI_MAXHOST + 6]; +extern char rados_recov_old_oid[NI_MAXHOST + 4]; struct rados_kv_parameter { /** Connection to ceph cluster */ @@ -51,10 +52,15 @@ typedef void (*pop_clid_entry_t)(char *, char *, struct pop_args *); int rados_kv_connect(rados_ioctx_t *io_ctx, const char *userid, const char *conf, const char *pool); void rados_kv_shutdown(void); +int rados_kv_put(char *key, char *val, char *object); int rados_kv_get(char *key, char *val, char *object); +void rados_kv_add_clid(nfs_client_id_t *clientid); +void rados_kv_rm_clid(nfs_client_id_t *clientid); +void rados_kv_add_revoke_fh(nfs_client_id_t *delr_clid, nfs_fh4 *delr_handle); void rados_kv_create_key(nfs_client_id_t *clientid, char *key); void rados_kv_create_val(nfs_client_id_t *clientid, char *val); int rados_kv_traverse(pop_clid_entry_t callback, struct pop_args *args, const char *object); void rados_kv_add_revoke_fh(nfs_client_id_t *delr_clid, nfs_fh4 *delr_handle); +void rados_kv_pop_clid_entry(char *key, char *val, struct pop_args *pop_args); #endif /* _RECOVERY_RADOS_H */ diff --git a/src/SAL/recovery/recovery_rados_kv.c b/src/SAL/recovery/recovery_rados_kv.c index e3ca628f80f7..daa6a9c8f6c7 100644 --- a/src/SAL/recovery/recovery_rados_kv.c +++ b/src/SAL/recovery/recovery_rados_kv.c @@ -21,7 +21,7 @@ static rados_t clnt; rados_ioctx_t rados_recov_io_ctx; char rados_recov_oid[NI_MAXHOST + 6]; -static char rados_recov_old_oid[NI_MAXHOST + 4]; +char rados_recov_old_oid[NI_MAXHOST + 4]; struct rados_kv_parameter rados_kv_param; @@ -131,7 +131,7 @@ void rados_kv_create_val(nfs_client_id_t *clientid, char *val) LogDebug(COMPONENT_CLIENTID, "Created client name [%s]", val); } -static int rados_kv_put(char *key, char *val, char *object) +int rados_kv_put(char *key, char *val, char *object) { int ret; char *keys[1]; @@ -448,8 +448,7 @@ void rados_kv_rm_clid(nfs_client_id_t *clientid) clientid->cid_recov_tag = NULL; } -static void rados_kv_pop_clid_entry(char *key, char *val, - struct pop_args *pop_args) +void rados_kv_pop_clid_entry(char *key, char *val, struct pop_args *pop_args) { int ret; char *dupval; -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Add a new maybe_start_grace operation that will check to see if a sibling has requested a grace period and start one if so. Call this function from the reaper thread. Note that we only enforce the grace period and do not reload the recovery db. All of the current node's state is still intact, so we don't need to initiate any reclaim; only enforce the grace period until the other node's reclaim is complete. Change-Id: I85f54907efb220510af0b87baa1798d3f1d7a041 Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/MainNFSD/nfs_reaper_thread.c | 3 +++ src/SAL/nfs4_recovery.c | 15 +++++++++++++++ src/include/sal_functions.h | 2 ++ 3 files changed, 20 insertions(+) diff --git a/src/MainNFSD/nfs_reaper_thread.c b/src/MainNFSD/nfs_reaper_thread.c index 918a3b7f4256..07ef670810fd 100644 --- a/src/MainNFSD/nfs_reaper_thread.c +++ b/src/MainNFSD/nfs_reaper_thread.c @@ -203,6 +203,9 @@ static void reaper_run(struct fridgethr_context *ctx) SetNameFunction("reaper"); + /* see if we need to start a grace period */ + nfs_maybe_start_grace(); + /* * Try to lift the grace period, unless we're shutting down. * Ordinarily, we'd take the mutex to check this, but this is just a diff --git a/src/SAL/nfs4_recovery.c b/src/SAL/nfs4_recovery.c index 120819e621c9..cc4f7cf9facf 100644 --- a/src/SAL/nfs4_recovery.c +++ b/src/SAL/nfs4_recovery.c @@ -198,6 +198,21 @@ bool nfs_in_grace(void) return atomic_fetch_time_t(&current_grace); } +/** + * @brief Enter the grace period if another node in the cluster needs it + * + * Singleton servers generally won't use this operation. Clustered servers + * call this function to check whether another node might need a grace period. + */ +void nfs_maybe_start_grace(void) +{ + if (recovery_backend->maybe_start_grace) { + if (nfs_in_grace()) + return; + recovery_backend->maybe_start_grace(); + } +} + void nfs_try_lift_grace(void) { bool in_grace = true; diff --git a/src/include/sal_functions.h b/src/include/sal_functions.h index 7e30e51eeabf..2829f574c8e7 100644 --- a/src/include/sal_functions.h +++ b/src/include/sal_functions.h @@ -976,6 +976,7 @@ void blocked_lock_polling(struct fridgethr_context *ctx); void nfs_start_grace(nfs_grace_start_t *gsp); bool nfs_in_grace(void); bool simple_try_lift_grace(void); +void nfs_maybe_start_grace(void); void nfs_try_lift_grace(void); void nfs4_add_clid(nfs_client_id_t *); void nfs4_rm_clid(nfs_client_id_t *); @@ -1023,6 +1024,7 @@ struct nfs4_recovery_backend { void (*add_clid)(nfs_client_id_t *); void (*rm_clid)(nfs_client_id_t *); void (*add_revoke_fh)(nfs_client_id_t *, nfs_fh4 *); + void (*maybe_start_grace)(void); bool (*try_lift_grace)(void); }; -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Add a new recovery backend operation to tell whether the grace period is being fully enforced. Singleton backends that don't define this operation will just have nfs_grace_enforcing() return true. Change-Id: I8020723238396efe0949f29e1906c58b0a5e26c9 Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/SAL/nfs4_recovery.c | 14 ++++++++++++++ src/include/sal_functions.h | 1 + 2 files changed, 15 insertions(+) diff --git a/src/SAL/nfs4_recovery.c b/src/SAL/nfs4_recovery.c index debc6cab1bdb..400d8b528d26 100644 --- a/src/SAL/nfs4_recovery.c +++ b/src/SAL/nfs4_recovery.c @@ -227,6 +227,20 @@ void nfs_maybe_start_grace(void) } } +/** + * @brief Are all hosts in cluster enforcing the grace period? + * + * Singleton servers always return true here since the only grace period that + * matters is the local one. Clustered backends should check to make sure that + * the whole cluster is in grace. + */ +bool nfs_grace_enforcing(void) +{ + if (recovery_backend->grace_enforcing) + return recovery_backend->grace_enforcing(); + return true; +} + void nfs_try_lift_grace(void) { bool in_grace = true; diff --git a/src/include/sal_functions.h b/src/include/sal_functions.h index ac01a6eba497..96a4eb40a50b 100644 --- a/src/include/sal_functions.h +++ b/src/include/sal_functions.h @@ -1027,6 +1027,7 @@ struct nfs4_recovery_backend { void (*maybe_start_grace)(void); bool (*try_lift_grace)(void); void (*set_enforcing)(void); + bool (*grace_enforcing)(void); }; void fs_backend_init(struct nfs4_recovery_backend **); -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Add a new support library for coordinating a grace period between multiple nodes. The basic idea is to use a shared RADOS object to mediate requests for a grace period. Typically, each node will perform a read operation on the database and (then possibly) a write operation. In order to ensure atomicity, we assert that the object version has not changed between the read and write ops. If it has, then we start over again from scratch. Change-Id: Idfff61f8d0092fd9095dc14390dfb434f0ad6696 Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/include/rados_grace.h | 82 +++++ src/support/CMakeLists.txt | 4 + src/support/rados_grace.c | 678 +++++++++++++++++++++++++++++++++++++ 3 files changed, 764 insertions(+) create mode 100644 src/include/rados_grace.h create mode 100644 src/support/rados_grace.c diff --git a/src/include/rados_grace.h b/src/include/rados_grace.h new file mode 100644 index 000000000000..5ac26bf9cb33 --- /dev/null +++ b/src/include/rados_grace.h @@ -0,0 +1,82 @@ +#ifndef _RADOS_GRACE_H +#define _RADOS_GRACE_H + +#include <stdbool.h> + +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA + * + * --------------------------------------- + */ +int rados_grace_create(rados_ioctx_t io_ctx, const char *oid); +int rados_grace_dump(rados_ioctx_t io_ctx, const char *oid); +int rados_grace_epochs(rados_ioctx_t io_ctx, const char *oid, + uint64_t *cur, uint64_t *rec); +int rados_grace_enforcing_toggle(rados_ioctx_t io_ctx, const char *oid, + int nodes, const char **nodeids, uint64_t *pcur, + uint64_t *prec, bool start); +int rados_grace_enforcing_check(rados_ioctx_t io_ctx, const char *oid); +int rados_grace_join_bulk(rados_ioctx_t io_ctx, const char *oid, int nodes, + const char **nodeids, uint64_t *pcur, uint64_t *prec, + bool start); +int rados_grace_lift_bulk(rados_ioctx_t io_ctx, const char *oid, int nodes, + const char **nodeids, uint64_t *pcur, uint64_t *prec, + bool remove); + +static inline int +rados_grace_enforcing_on(rados_ioctx_t io_ctx, const char *oid, + const char *nodeid, uint64_t *pcur, uint64_t *prec) +{ + const char *nodeids[1]; + + nodeids[0] = nodeid; + return rados_grace_enforcing_toggle(io_ctx, oid, 1, nodeids, pcur, + prec, true); +} + +static inline int +rados_grace_enforcing_off(rados_ioctx_t io_ctx, const char *oid, + const char *nodeid, uint64_t *pcur, uint64_t *prec) +{ + const char *nodeids[1]; + + nodeids[0] = nodeid; + return rados_grace_enforcing_toggle(io_ctx, oid, 1, nodeids, pcur, + prec, false); +} + +static inline int +rados_grace_join(rados_ioctx_t io_ctx, const char *oid, const char *nodeid, + uint64_t *pcur, uint64_t *prec, bool start) +{ + const char *nodeids[1]; + + nodeids[0] = nodeid; + return rados_grace_join_bulk(io_ctx, oid, 1, nodeids, pcur, prec, + start); +} + +static inline int +rados_grace_lift(rados_ioctx_t io_ctx, const char *oid, const char *nodeid, + uint64_t *pcur, uint64_t *prec) +{ + const char *nodeids[1]; + + nodeids[0] = nodeid; + return rados_grace_lift_bulk(io_ctx, oid, 1, nodeids, pcur, prec, + false); +} +#endif /* _RADOS_GRACE_H */ diff --git a/src/support/CMakeLists.txt b/src/support/CMakeLists.txt index 42dc7ab7b589..380f01d3d83b 100644 --- a/src/support/CMakeLists.txt +++ b/src/support/CMakeLists.txt @@ -41,6 +41,10 @@ set(netgroup_cache_SRCS add_library(netgroup_cache STATIC ${netgroup_cache_SRCS}) add_sanitizers(netgroup_cache) +if (USE_RADOS_RECOV) + add_library(rados_grace STATIC rados_grace.c) +endif(USE_RADOS_RECOV) + ########### next target ############### SET(support_STAT_SRCS diff --git a/src/support/rados_grace.c b/src/support/rados_grace.c new file mode 100644 index 000000000000..1b2d09024db7 --- /dev/null +++ b/src/support/rados_grace.c @@ -0,0 +1,678 @@ +/* + * vim:noexpandtab:shiftwidth=8:tabstop=8: + * + * Copyright 2018 Red Hat, Inc. and/or its affiliates. + * Author: Jeff Layton <jlayton(a)redhat.com&gt; + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * The rados_grace database is a rados object with a well-known name that + * with which all cluster nodes can interact to coordinate grace-period + * enforcement. + * + * It consists of two parts: + * + * 1) 2 uint64_t epoch values (stored LE) that indicate the serial number of + * the current grace period (C) and the serial number of the grace period that + * from which recovery is currently allowed (R). These are stored as object + * data. + * + * 2) An omap containing a key value pair for each cluster node. The key is + * the hostname of the node running ganesha, and the value is a byte with a + * set of flags. + * + * Consider a single server epoch (E) of an individual NFS server to be the + * period between reboots. That consists of an initial grace period and + * a regular operation period. An epoch value of 0 is never valid. + * + * The first value (C) indicates the current server epoch. The client recovery + * db should be tagged with this value on creation, or when updating the db + * after the grace period has been fully lifted. + * + * The second uint64_t value in the data tells the NFS server from what + * recovery db it is allowed to reclaim. A value of 0 in this field means that + * we are out of the cluster-wide grace period and that no recovery is allowed. + * + * The omap contains a key for each host in the cluster. Typically, nodes join + * the cluster by setting their omap key. The value of the omap is a single + * byte that contains a set of flags that indicates their current need for a + * grace period and whether they are locally enforcing one. + * + * The grace period handling engine will update and store the flags, and it + * can be queried to determine whether other nodes may need a grace period or + * are enforcing. + */ +#include "config.h" +#include <stdio.h> +#include <stdint.h> +#include <endian.h> +#include <rados/librados.h> +#include <errno.h> +#include <getopt.h> +#include <stdlib.h> +#include <limits.h> +#include <stdbool.h> + +/* Each cluster node needs a slot here */ +#define MAX_ITEMS 1024 + +/* Flags for the omap value flags field */ + +/* Does this node currently require a grace period? */ +#define RADOS_GRACE_NEED_GRACE 0x1 + +/* Is this node currently enforcing its grace period locally? */ +#define RADOS_GRACE_ENFORCING 0x2 + +static void rados_grace_notify(rados_ioctx_t io_ctx, const char *oid) +{ + static char *buf; + static size_t len; + + /* FIXME: we don't really want or need this to be synchronous */ + rados_notify2(io_ctx, oid, "", 0, 3000, &buf, &len); + rados_buffer_free(buf); +} + +int +rados_grace_create(rados_ioctx_t io_ctx, const char *oid) +{ + int ret; + rados_write_op_t op = NULL; + uint64_t cur = htole64(1); // starting epoch + uint64_t rec = htole64(0); // no recovery yet + char buf[sizeof(uint64_t) * 2]; + + /* + * 2 uint64_t's + * + * The first denotes the current epoch serial number, the epoch serial + * number under which new recovery records should be created. + * + * The second number denotes the epoch from which clients are allowed + * to reclaim. + * + * An epoch of zero is never allowed, so if rec=0, then the grace + * period is no longer in effect, and can't be joined. + */ + memcpy(buf, (char *)&cur, sizeof(cur)); + memcpy(buf + sizeof(cur), (char *)&rec, sizeof(rec)); + + op = rados_create_write_op(); + /* Create the object */ + rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL); + + /* Set serial numbers if we created the object */ + rados_write_op_write_full(op, buf, sizeof(buf)); + + ret = rados_write_op_operate(op, io_ctx, oid, NULL, 0); + rados_release_write_op(op); + return ret; +} + +int +rados_grace_dump(rados_ioctx_t io_ctx, const char *oid) +{ + int ret; + rados_omap_iter_t iter; + rados_read_op_t op; + char *key_out = NULL; + char *val_out = NULL; + unsigned char more = '\0'; + size_t len_out = 0; + char buf[sizeof(uint64_t) * 2]; + uint64_t cur, rec; + + op = rados_create_read_op(); + rados_read_op_read(op, 0, sizeof(buf), buf, &len_out, NULL); + rados_read_op_omap_get_vals2(op, "", "", MAX_ITEMS, &iter, &more, NULL); + ret = rados_read_op_operate(op, io_ctx, oid, 0); + if (ret < 0) { + printf("%s: ret=%d", __func__, ret); + goto out; + } + + if (len_out != sizeof(buf)) { + ret = -ENOTRECOVERABLE; + goto out; + } + + if (more) { + ret = -ENOTRECOVERABLE; + goto out; + } + + cur = le64toh(*(uint64_t *)buf); + rec = le64toh(*(uint64_t *)(buf + sizeof(uint64_t))); + printf("cur=%lu rec=%lu\n", cur, rec); + printf("======================================================\n"); + for (;;) { + char need = ' ', enforcing = ' '; + + rados_omap_get_next(iter, &key_out, &val_out, &len_out); + if (key_out == NULL || val_out == NULL) + break; + if (*val_out & RADOS_GRACE_NEED_GRACE) + need = 'N'; + if (*val_out & RADOS_GRACE_ENFORCING) + enforcing = 'E'; + printf("%s\t%c%c\n", key_out, need, enforcing); + } + rados_omap_get_end(iter); +out: + rados_release_read_op(op); + return ret; +} + +int +rados_grace_epochs(rados_ioctx_t io_ctx, const char *oid, + uint64_t *cur, uint64_t *rec) +{ + int ret; + rados_read_op_t op; + size_t len_out = 0; + char buf[sizeof(uint64_t) * 2]; + + op = rados_create_read_op(); + rados_read_op_read(op, 0, sizeof(buf), buf, &len_out, NULL); + ret = rados_read_op_operate(op, io_ctx, oid, 0); + if (ret < 0) + goto out; + + ret = -ENOTRECOVERABLE; + if (len_out != sizeof(buf)) + goto out; + + *cur = le64toh(*(uint64_t *)buf); + *rec = le64toh(*(uint64_t *)(buf + sizeof(uint64_t))); + ret = 0; +out: + rados_release_read_op(op); + return ret; +} + +int +rados_grace_enforcing_toggle(rados_ioctx_t io_ctx, const char *oid, int nodes, + const char **nodeids, uint64_t *pcur, uint64_t *prec, + bool enable) +{ + int i, ret; + char *flags = NULL; + char **vals = NULL; + size_t *lens = NULL; + uint64_t cur, rec, ver; + + /* allocate an array of flag bytes and set ENFORCING on all of them */ + flags = calloc(nodes, 1); + if (!flags) { + ret = -ENOMEM; + goto out; + } + + /* pointers to each val byte */ + vals = calloc(nodes, sizeof(char *)); + if (!vals) { + ret = -ENOMEM; + goto out; + } + + /* lengths */ + lens = calloc(nodes, sizeof(size_t)); + if (!lens) { + ret = -ENOMEM; + goto out; + } + + /* each val is one flag byte */ + for (i = 0; i < nodes; ++i) { + vals[i] = &flags[i]; + lens[i] = 1; + } + + do { + rados_write_op_t wop; + rados_read_op_t rop; + rados_omap_iter_t iter; + size_t len = 0; + unsigned char more = 0; + char buf[sizeof(uint64_t) * 2]; + + /* read epoch blob */ + rop = rados_create_read_op(); + rados_read_op_read(rop, 0, sizeof(buf), buf, &len, NULL); + rados_read_op_omap_get_vals2(rop, "", "", MAX_ITEMS, &iter, + &more, NULL); + ret = rados_read_op_operate(rop, io_ctx, oid, 0); + if (ret < 0) { + rados_release_read_op(rop); + break; + } + + if (more || (len != sizeof(buf))) { + ret = -ENOTRECOVERABLE; + rados_release_read_op(rop); + break; + } + + ver = rados_get_last_version(io_ctx); + + /* + * Walk the returned kv pairs and flip on any existing flags + * in the matching nodeid (if there is one) + */ + for (;;) { + char *key, *val; + + rados_omap_get_next(iter, &key, &val, &len); + if (!key) + break; + for (i = 0; i < nodes; ++i) { + if (strcmp(key, nodeids[i])) + continue; + flags[i] = *val; + if (enable) + flags[i] |= RADOS_GRACE_ENFORCING; + else + flags[i] &= ~RADOS_GRACE_ENFORCING; + break; + } + } + rados_omap_get_end(iter); + rados_release_read_op(rop); + + /* Get old epoch numbers and version */ + cur = le64toh(*(uint64_t *)buf); + rec = le64toh(*(uint64_t *)(buf + sizeof(uint64_t))); + + /* Attempt to update object */ + wop = rados_create_write_op(); + + /* Ensure that nothing has changed */ + rados_write_op_assert_version(wop, ver); + + /* Set omap values to given ones */ + rados_write_op_omap_set(wop, nodeids, + (const char * const*)vals, lens, nodes); + + ret = rados_write_op_operate(wop, io_ctx, oid, NULL, 0); + rados_release_write_op(wop); + if (ret >= 0) + rados_grace_notify(io_ctx, oid); + } while (ret == -ERANGE); + + if (!ret) { + *pcur = cur; + *prec = rec; + } +out: + free(lens); + free(flags); + free(vals); + return ret; +} + +int +rados_grace_enforcing_check(rados_ioctx_t io_ctx, const char *oid) +{ + int ret; + rados_read_op_t rop; + rados_omap_iter_t iter; + unsigned char more = 0; + + rop = rados_create_read_op(); + rados_read_op_omap_get_vals2(rop, "", "", MAX_ITEMS, &iter, + &more, NULL); + ret = rados_read_op_operate(rop, io_ctx, oid, 0); + if (ret < 0) { + rados_release_read_op(rop); + goto out; + } + + if (more) { + ret = -ENOTRECOVERABLE; + rados_release_read_op(rop); + goto out; + } + + ret = 0; + for (;;) { + char *key, *val; + size_t len = 0; + + rados_omap_get_next(iter, &key, &val, &len); + if (!key) + break; + /* If anyone isn't enforcing, then return an err */ + if (!(*val & RADOS_GRACE_ENFORCING)) { + ret = -EL2NSYNC; + break; + } + } + rados_omap_get_end(iter); + rados_release_read_op(rop); +out: + return ret; +} + +int +rados_grace_join_bulk(rados_ioctx_t io_ctx, const char *oid, int nodes, + const char **nodeids, uint64_t *pcur, uint64_t *prec, + bool start) +{ + int i, ret; + char *flags = NULL; + char **vals = NULL; + size_t *lens = NULL; + uint64_t cur, rec, ver; + + /* flag bytes */ + flags = malloc(nodes); + if (!flags) { + ret = -ENOMEM; + goto out; + } + + /* pointers to each val byte */ + vals = calloc(nodes, sizeof(char *)); + if (!vals) { + ret = -ENOMEM; + goto out; + } + + /* lengths */ + lens = calloc(nodes, sizeof(size_t)); + if (!lens) { + ret = -ENOMEM; + goto out; + } + + /* each val is one flag byte */ + for (i = 0; i < nodes; ++i) { + vals[i] = &flags[i]; + lens[i] = 1; + } + + do { + rados_write_op_t wop; + rados_read_op_t rop; + rados_omap_iter_t iter; + size_t len = 0; + unsigned char more = 0; + char buf[sizeof(uint64_t) * 2]; + + /* read epoch blob */ + rop = rados_create_read_op(); + rados_read_op_read(rop, 0, sizeof(buf), buf, &len, NULL); + rados_read_op_omap_get_vals2(rop, "", "", MAX_ITEMS, &iter, + &more, NULL); + ret = rados_read_op_operate(rop, io_ctx, oid, 0); + if (ret < 0) { + rados_release_read_op(rop); + break; + } + + if (more || (len != sizeof(buf))) { + ret = -ENOTRECOVERABLE; + rados_release_read_op(rop); + break; + } + + ver = rados_get_last_version(io_ctx); + + /* + * Walk the returned kv pairs and flip on any existing flags + * in the matching nodeid (if there is one) + */ + memset(flags, RADOS_GRACE_NEED_GRACE|RADOS_GRACE_ENFORCING, + nodes); + for (;;) { + char *key, *val; + + rados_omap_get_next(iter, &key, &val, &len); + if (!key) + break; + for (i = 0; i < nodes; ++i) { + if (!strcmp(key, nodeids[i])) { + flags[i] |= *val; + break; + } + } + } + rados_omap_get_end(iter); + rados_release_read_op(rop); + + /* Get old epoch numbers and version */ + cur = le64toh(*(uint64_t *)buf); + rec = le64toh(*(uint64_t *)(buf + sizeof(uint64_t))); + + /* Only start a new grace period if start bool is set */ + /* FIXME: do we need this with real membership? */ + if (rec == 0 && !start) + break; + + /* Attempt to update object */ + wop = rados_create_write_op(); + + /* Ensure that nothing has changed */ + rados_write_op_assert_version(wop, ver); + + /* Update the object data iff rec == 0 */ + if (rec == 0) { + uint64_t tc, tr; + + rec = cur; + ++cur; + tc = htole64(cur); + tr = htole64(rec); + memcpy(buf, (char *)&tc, sizeof(tc)); + memcpy(buf + sizeof(tc), (char *)&tr, sizeof(tr)); + rados_write_op_write_full(wop, buf, sizeof(buf)); + } + + /* Set omap values to given ones */ + rados_write_op_omap_set(wop, nodeids, + (const char * const*)vals, lens, nodes); + + ret = rados_write_op_operate(wop, io_ctx, oid, NULL, 0); + rados_release_write_op(wop); + if (ret >= 0) + rados_grace_notify(io_ctx, oid); + } while (ret == -ERANGE); + + if (!ret) { + *pcur = cur; + *prec = rec; + } +out: + free(lens); + free(flags); + free(vals); + return ret; +} + +int +rados_grace_lift_bulk(rados_ioctx_t io_ctx, const char *oid, int nodes, + const char **nodeids, uint64_t *pcur, uint64_t *prec, + bool remove) +{ + int ret; + char *flags = NULL; + char **vals = NULL; + size_t *lens = NULL; + const char **keys = NULL; + uint64_t cur, rec; + + keys = calloc(nodes, sizeof(char *)); + if (!keys) { + ret = -ENOMEM; + goto out; + } + + /* We don't need these arrays if we're just removing the keys */ + if (!remove) { + flags = calloc(nodes, 1); + if (!flags) { + ret = -ENOMEM; + goto out; + } + + /* pointers to each val byte */ + vals = calloc(nodes, sizeof(char *)); + if (!vals) { + ret = -ENOMEM; + goto out; + } + + lens = calloc(nodes, sizeof(size_t)); + if (!lens) { + ret = -ENOMEM; + goto out; + } + } + + do { + int i, k, cnt; + rados_write_op_t wop; + rados_read_op_t rop; + rados_omap_iter_t iter; + char *key, *val; + size_t len; + char buf[sizeof(uint64_t) * 2]; + unsigned char more = '\0'; + uint64_t ver; + + /* read epoch blob and omap keys */ + rop = rados_create_read_op(); + rados_read_op_read(rop, 0, sizeof(buf), buf, &len, NULL); + rados_read_op_omap_get_vals2(rop, "", "", MAX_ITEMS, &iter, + &more, NULL); + ret = rados_read_op_operate(rop, io_ctx, oid, 0); + if (ret < 0) { + rados_release_read_op(rop); + break; + } + + if (more) { + ret = -ENOTRECOVERABLE; + rados_release_read_op(rop); + break; + } + + if (len != sizeof(buf)) { + ret = -ENOTRECOVERABLE; + rados_release_read_op(rop); + break; + } + + /* Get old epoch numbers and version */ + ver = rados_get_last_version(io_ctx); + cur = le64toh(*(uint64_t *)buf); + rec = le64toh(*(uint64_t *)(buf + sizeof(uint64_t))); + + /* + * Walk omap keys, see if it's in nodeids array. Add any that + * are and have NEED_GRACE set to the "keys" array along with + * the the flags field with the NEED_GRACE flag cleared. + * + * If the remove flag is set, then just remove them from the + * omap and don't bother with changing the flags. + */ + cnt = 0; + k = 0; + for (;;) { + ret = rados_omap_get_next(iter, &key, &val, &len); + if (!key) + break; + + /* + * Skip hosts that already have the flag clear, unless + * we're removing them. + */ + if (!remove && !(*val & RADOS_GRACE_NEED_GRACE)) + continue; + + ++cnt; + for (i = 0; i < nodes; ++i) { + if (strcmp(key, nodeids[i])) + continue; + keys[k] = nodeids[i]; + /* + * For the removal case, we just need the + * keys. + */ + if (!remove) { + flags[k] = + *val & ~RADOS_GRACE_NEED_GRACE; + vals[k] = &flags[k]; + lens[k] = 1; + } + ++k; + break; + } + }; + rados_omap_get_end(iter); + rados_release_read_op(rop); + + /* No matching keys? Nothing to do. */ + if (k == 0) + break; + + /* Attempt to update object */ + wop = rados_create_write_op(); + + /* Ensure that nothing has changed */ + rados_write_op_assert_version(wop, ver); + + /* Set or remove any keys we matched earlier */ + if (remove) + rados_write_op_omap_rm_keys(wop, keys, k); + else + rados_write_op_omap_set(wop, keys, + (const char * const *)vals, + lens, k); + + /* + * If number of omap records we're setting is the same as the + * number of hosts that have NEED_GRACE set, then fully lift + * the grace period. + */ + if (cnt == k) { + uint64_t tc, tr; + + rec = 0; + tr = htole64(rec); + tc = htole64(cur); + memcpy(buf, (char *)&tc, sizeof(tc)); + memcpy(buf + sizeof(tc), (char *)&tr, sizeof(tr)); + rados_write_op_write_full(wop, buf, sizeof(buf)); + } + + ret = rados_write_op_operate(wop, io_ctx, oid, NULL, 0); + rados_release_write_op(wop); + if (ret >= 0) + rados_grace_notify(io_ctx, oid); + } while (ret == -ERANGE); + if (!ret) { + *pcur = cur; + *prec = rec; + } +out: + free(vals); + free(lens); + free(flags); + free(keys); + return ret; +} -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Add a new command-line tool for manipulating and querying the rados_grace database. It will dump out the state of the epochs and then a list of omap keys and their flags. Change-Id: If57591265ce736cdcebab749651d5ab6982341d8 Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/nfs-ganesha.spec-in.cmake | 2 + src/tools/CMakeLists.txt | 4 + src/tools/rados_grace_tool.c | 178 ++++++++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+) create mode 100644 src/tools/rados_grace_tool.c diff --git a/src/nfs-ganesha.spec-in.cmake b/src/nfs-ganesha.spec-in.cmake index e93973eda1f7..c55fad58c083 100644 --- a/src/nfs-ganesha.spec-in.cmake +++ b/src/nfs-ganesha.spec-in.cmake @@ -508,6 +508,7 @@ install -m 644 config_samples/xfs.conf %{buildroot}%{_sysconfdir}/ganesha %if %{with ceph} install -m 644 config_samples/ceph.conf %{buildroot}%{_sysconfdir}/ganesha +install -m 755 tools/rados_grace_tool %{buildroot}%{_sbindir}/rados_grace_tool %endif %if %{with rgw} @@ -690,6 +691,7 @@ exit 0 %defattr(-,root,root,-) %{_libdir}/ganesha/libfsalceph* %config(noreplace) %{_sysconfdir}/ganesha/ceph.conf +%{_sbindir}/rados_grace_tool %if %{with man_page} %{_mandir}/*/ganesha-ceph-config.8.gz %endif diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 7b536193568a..3856571f3ba1 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -1,3 +1,7 @@ +if (USE_RADOS_RECOV) + add_executable(rados_grace_tool rados_grace_tool.c) + target_link_libraries(rados_grace_tool rados_grace ${RADOS_LIBRARIES}) +endif(USE_RADOS_RECOV) ########### install files ############### diff --git a/src/tools/rados_grace_tool.c b/src/tools/rados_grace_tool.c new file mode 100644 index 000000000000..6cc15e8f3b38 --- /dev/null +++ b/src/tools/rados_grace_tool.c @@ -0,0 +1,178 @@ +/* + * vim:noexpandtab:shiftwidth=8:tabstop=8: + * + * Copyright 2017 Red Hat, Inc. and/or its affiliates. + * Author: Jeff Layton <jlayton(a)redhat.com&gt; + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * rados-grace: tool for managing coordinated grace period database + * + * This tool allows an administrator to make direct changes to the rados_grace + * database. See the rados_grace support library sources for more info about + * the internals. + */ +#include "config.h" +#include <stdio.h> +#include <stdint.h> +#include <endian.h> +#include <rados/librados.h> +#include <errno.h> +#include <getopt.h> +#include <stdlib.h> +#include <limits.h> +#include <stdbool.h> +#include <rados_grace.h> + +#define POOL_ID "nfs-ganesha" +#define RADOS_GRACE_OID "grace" + +static int +cluster_connect(rados_ioctx_t *io_ctx, const char *pool) +{ + int ret; + rados_t clnt; + + ret = rados_create(&clnt, NULL); + if (ret < 0) { + fprintf(stderr, "rados_create: %d\n", ret); + return ret; + } + + ret = rados_conf_read_file(clnt, NULL); + if (ret < 0) { + fprintf(stderr, "rados_conf_read_file: %d\n", ret); + return ret; + } + + ret = rados_connect(clnt); + if (ret < 0) { + fprintf(stderr, "rados_connect: %d\n", ret); + return ret; + } + + ret = rados_pool_create(clnt, pool); + if (ret < 0 && ret != -EEXIST) { + fprintf(stderr, "rados_pool_create: %d\n", ret); + return ret; + } + + ret = rados_ioctx_create(clnt, pool, io_ctx); + if (ret < 0) { + fprintf(stderr, "rados_ioctx_create: %d\n", ret); + return ret; + } + return 0; +} + +static void usage(const char **argv) +{ + fprintf(stderr, + "Usage:\n%s [dump|start|join|lift|remove|enforce|noenforce] [ nodeid ... ]\n", + argv[0]); +} + +int main(int argc, const char **argv) +{ + int ret, nodes = 0; + rados_ioctx_t io_ctx; + const char *cmd = "dump"; + uint64_t cur, rec; + + if (argc > 1) { + cmd = argv[1]; + nodes = argc - 2; + } + + ret = cluster_connect(&io_ctx, POOL_ID); + if (ret) { + fprintf(stderr, "Can't connect to cluster: %d\n", ret); + return 1; + } + + ret = rados_grace_create(io_ctx, RADOS_GRACE_OID); + if (ret < 0 && ret != -EEXIST) { + fprintf(stderr, "Can't create grace db: %d\n", ret); + return 1; + } + + if (!strcmp(cmd, "dump")) { + ret = rados_grace_dump(io_ctx, RADOS_GRACE_OID); + } else if (!strcmp(cmd, "start")) { + if (!nodes) { + fprintf(stderr, "Need at least one nodeid.\n"); + ret = -EINVAL; + } else { + ret = rados_grace_join_bulk(io_ctx, RADOS_GRACE_OID, + nodes, &argv[2], &cur, &rec, + true); + } + } else if (!strcmp(cmd, "join")) { + uint64_t cur, rec; + + if (!nodes) { + fprintf(stderr, "Need at least one nodeid.\n"); + ret = -EINVAL; + } else { + ret = rados_grace_join_bulk(io_ctx, RADOS_GRACE_OID, + nodes, &argv[2], &cur, &rec, + false); + } + } else if (!strcmp(cmd, "lift")) { + if (!nodes) { + fprintf(stderr, "Need at least one nodeid.\n"); + ret = -EINVAL; + } else { + ret = rados_grace_lift_bulk(io_ctx, RADOS_GRACE_OID, + nodes, &argv[2], &cur, &rec, + false); + } + } else if (!strcmp(cmd, "remove")) { + if (!nodes) { + fprintf(stderr, "Need at least one nodeid.\n"); + ret = -EINVAL; + } else { + ret = rados_grace_lift_bulk(io_ctx, RADOS_GRACE_OID, + nodes, &argv[2], &cur, &rec, + true); + } + } else if (!strcmp(cmd, "enforce")) { + if (!nodes) { + fprintf(stderr, "Need at least one nodeid.\n"); + ret = -EINVAL; + } else { + ret = rados_grace_enforcing_toggle(io_ctx, + RADOS_GRACE_OID, nodes, &argv[2], + &cur, &rec, true); + } + } else if (!strcmp(cmd, "noenforce")) { + if (!nodes) { + fprintf(stderr, "Need at least one nodeid.\n"); + ret = -EINVAL; + } else { + ret = rados_grace_enforcing_toggle(io_ctx, + RADOS_GRACE_OID, nodes, &argv[2], + &cur, &rec, false); + } + } else { + usage(argv); + ret = -EINVAL; + } + + if (ret) + return 1; + return 0; +} -- 2.17.0

From: Jeff Layton <jlayton(a)redhat.com&gt; Add a test for the new cephfs reclaim infrastructure. We don't actually try to reclaim at this point, only ask the MDS to destroy the old session. For now, we just use the hostname as the uuid here. The libcephfs interface is still under debate, so this may look quite different in its final form. Change-Id: I70e93cbff2e2ae80989aaa9ee9c96d3031f5e919 Signed-off-by: Jeff Layton <jlayton(a)redhat.com&gt; --- src/CMakeLists.txt | 1 + src/FSAL/FSAL_CEPH/main.c | 39 ++++++++++++++++++++++++++++++ src/cmake/modules/FindCEPHFS.cmake | 8 ++++++ src/include/config-h.in.cmake | 1 + 4 files changed, 49 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fa1f6bc2e54d..845ac46d1f14 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1101,6 +1101,7 @@ message(STATUS "USE_FSAL_CEPH_LL_LOOKUP_ROOT = ${USE_FSAL_CEPH_LL_LOOKUP_ROOT}") message(STATUS "USE_FSAL_CEPH_STATX = ${USE_FSAL_CEPH_STATX}") message(STATUS "USE_FSAL_CEPH_LL_DELEGATION = ${USE_FSAL_CEPH_LL_DELEGATION}") message(STATUS "USE_FSAL_CEPH_LL_SYNC_INODE = ${USE_FSAL_CEPH_LL_SYNC_INODE}") +message(STATUS "USE_FSAL_CEPH_RECLAIM_RESET = ${USE_FSAL_CEPH_RECLAIM_RESET}") message(STATUS "USE_FSAL_RGW = ${USE_FSAL_RGW}") message(STATUS "USE_FSAL_XFS = ${USE_FSAL_XFS}") message(STATUS "USE_FSAL_PANFS = ${USE_FSAL_PANFS}") diff --git a/src/FSAL/FSAL_CEPH/main.c b/src/FSAL/FSAL_CEPH/main.c index e78a4162c754..8d7039722fae 100644 --- a/src/FSAL/FSAL_CEPH/main.c +++ b/src/FSAL/FSAL_CEPH/main.c @@ -248,6 +248,10 @@ static fsal_status_t create_export(struct fsal_module *module_in, int rc; /* Return code from Ceph calls */ int ceph_status; +#ifdef USE_FSAL_CEPH_RECLAIM_RESET + long maxlen = sysconf(_SC_HOST_NAME_MAX); + char *nodeid; +#endif fsal_export_init(&export->export); export_ops_init(&export->export.exp_ops); @@ -311,6 +315,41 @@ static fsal_status_t create_export(struct fsal_module *module_in, goto error; } + ceph_status = ceph_init(export->cmount); + if (ceph_status != 0) { + status.major = ERR_FSAL_SERVERFAULT; + LogCrit(COMPONENT_FSAL, + "Unable to init Ceph handle for %s.", + op_ctx->ctx_export->fullpath); + goto error; + } + +#ifdef USE_FSAL_CEPH_RECLAIM_RESET + /* + * Set long timeout for the session to ensure that MDS doesn't lose + * state before server can come back and do recovery. + */ + ceph_set_timeout(export->cmount, 300); + + nodeid = gsh_malloc(maxlen); + rc = gethostname(nodeid, maxlen); + if (rc) { + status.major = ERR_FSAL_SERVERFAULT; + LogEvent(COMPONENT_FSAL, "gethostname failed: %d", errno); + free(nodeid); + goto error; + } + + ceph_status = ceph_start_reclaim(export->cmount, nodeid, + CEPH_RECLAIM_RESET); + if (ceph_status) + LogEvent(COMPONENT_FSAL, "start_reclaim failed: %d", + ceph_status); + ceph_finish_reclaim(export->cmount); + ceph_set_uuid(export->cmount, nodeid); + free(nodeid); +#endif /* USE_FSAL_CEPH_RECLAIM_RESET */ + ceph_status = ceph_mount(export->cmount, op_ctx->ctx_export->fullpath); if (ceph_status != 0) { status.major = ERR_FSAL_SERVERFAULT; diff --git a/src/cmake/modules/FindCEPHFS.cmake b/src/cmake/modules/FindCEPHFS.cmake index 4705d5d3135c..2f08f1f2dc5f 100644 --- a/src/cmake/modules/FindCEPHFS.cmake +++ b/src/cmake/modules/FindCEPHFS.cmake @@ -94,6 +94,14 @@ else (NOT CEPH_FS) set(USE_FSAL_CEPH_LL_SYNC_INODE ON) endif(NOT CEPH_FS_SYNC_INODE) + check_library_exists(cephfs ceph_start_reclaim ${CEPHFS_LIBRARY_DIR} CEPH_FS_RECLAIM_RESET) + if(NOT CEPH_FS_RECLAIM_RESET) + message("Cannot find ceph_start_reclaim. FSAL_CEPH will not kill off old sessions.") + set(USE_FSAL_CEPH_RECLAIM_RESET OFF) + else(NOT CEPH_FS_RECLAIM_RESET) + set(USE_FSAL_CEPH_RECLAIM_RESET ON) + endif(NOT CEPH_FS_RECLAIM_RESET) + set(CMAKE_REQUIRED_INCLUDES ${CEPHFS_INCLUDE_DIR}) check_symbol_exists(CEPH_STATX_INO "cephfs/libcephfs.h" CEPH_FS_CEPH_STATX) if(NOT CEPH_FS_CEPH_STATX) diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake index 5f5685307c19..a7c8a80cd80a 100644 --- a/src/include/config-h.in.cmake +++ b/src/include/config-h.in.cmake @@ -63,6 +63,7 @@ #cmakedefine USE_FSAL_CEPH_STATX 1 #cmakedefine USE_FSAL_CEPH_LL_DELEGATION 1 #cmakedefine USE_FSAL_CEPH_LL_SYNC_INODE 1 +#cmakedefine USE_FSAL_CEPH_RECLAIM_RESET 1 #cmakedefine USE_FSAL_RGW_MOUNT2 1 #cmakedefine ENABLE_LOCKTRACE 1 #cmakedefine SANITIZE_ADDRESS 1 -- 2.17.0