The test was trying a NFSv4.0 mount and first SETCLIENTID_CONFIRM fails because "gssd_refresh_krb5_machine_credential" fails with Unknown error -1765328203 <ganesha,log> 2020-05-12T05:15:33Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_3] 115 :nfs4_op_setclientid_confirm :CLIENT ID :SETCLIENTID_CONFIRM client addr=::ffff:10.206.211.195 clientid=Epoch=0x5eba30ea Counter=0x00000001 setclientid_confirm=0100000000000000 2020-05-12T05:15:33Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_3] 114 :fs_create_clid_name :CLIENT ID :Created client name [::ffff:10.206.211.195-(94:0x4c696e7578204e465376342e302031302e3230362e3231312e3139352f31302e3230362e3230392e363820746370)] 2020-05-12T05:15:33Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_3] 209 :fs_add_clid :CLIENT ID :Created client dir [/var/lib/nfs/ganesha/v4recov/node0/::ffff:10.206.211.195-(94:0x4c696e7578204e465376342e302031302e3230362e3231312e3139352f31302e3230362e3230392e363820746370)] 2020-05-12T05:15:33Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_3] 391 :nfs4_chk_clid_impl :CLIENT ID :chk for 6825821966813888513 2020-05-12T05:15:33Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_3] 453 :nfs4_op_setclientid_confirm :CLIENT ID :Confirmed 0x7fd64c0025a0 ClientID={Epoch=0x5eba30ea Counter=0x00000001} CONFIRMED Client={0x7fd64c0024a0 name=(46:Linux NFSv4.0 10.206.211.195/10.206.209.68 tcp) refcount=2} t_delta=0 reservations=0 refcount=2 cb_prog=1073741824 r_addr=10.206.211.195.179.218 r_netid=tcp 2020-05-12T05:15:33Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_3] 477 :nfs_rpc_callback_setup_gss :NFS CB :gssd_refresh_krb5_machine_credential failed (-1765328203:0) 2020-05-12T05:15:33Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_3] 599 :nfs_rpc_create_chan_v40 :NFS CB :failed: RPC: Remote system error; errno = Unknown error -1765328203 2020-05-12T05:15:33Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_3] 1308 :nfs_test_cb_chan :NFS CB :nfs_rpc_get_chan failed 2020-05-12T05:15:33Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_3] 461 :nfs4_op_setclientid_confirm :CLIENT ID :setclid confirm: Callback channel is down <ganesha,log> Next when client resends the SETCLIENTID_CONFIRM, ganesha.log ends abruptly at this line and ganesha panics with the core. <ganesha,log> 2020-05-12T05:15:47Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_13] 687 :nfs4_Compound :NFS4 :COMPOUND: There are 1 operations, res = 0x7fd62c001f60, tag = NO TAG 2020-05-12T05:15:47Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_13] 801 :nfs4_Compound :NFS4 :Request 0: opcode 36 is OP_SETCLIENTID_CONFIRM 2020-05-12T05:15:47Z : epoch 5eba30ea : h10-206-209-68.vsanfs-sh.prv : ganesha.nfsd-99[::ffff:10.206.211.195] [svc_13] 115 :nfs4_op_setclientid_confirm :CLIENT ID :SETCLIENTID_CONFIRM client addr=::ffff:10.206.211.195 clientid=Epoch=0x5eba30ea Counter=0x00000001 setclientid_confirm=0200000000000000 ~ <ganesha,log> (gdb) bt #0 0x0000000000000000 in ?? () #1 0x00007fd66badf72e in svc_release_it (xprt=0x7fd658002e90, flags=0, tag=0x7fd66bb06fd0 "clnt_vc_destroy", line=462) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/ntirpc/rpc/svc.h:433 #2 0x00007fd66bae04fb in clnt_vc_destroy (clnt=0x7fd658002ba0) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/src/clnt_vc.c:462 #3 0x000000000043b4e1 in clnt_release_it (clnt=0x7fd658002ba0, flags=0, tag=0x55e550 <__func__.21824> "_nfs_rpc_destroy_chan", line=628) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/ntirpc/rpc/clnt.h:319 #4 0x000000000043b577 in clnt_destroy_it (clnt=0x7fd658002ba0, tag=0x55e550 <__func__.21824> "_nfs_rpc_destroy_chan", line=628) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/ntirpc/rpc/clnt.h:341 #5 0x000000000043eb97 in _nfs_rpc_destroy_chan (chan=0x7fd64c002648) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/MainNFSD/nfs_rpc_callback.c:628 #6 0x000000000043f800 in nfs_rpc_destroy_chan (chan=0x7fd64c002648) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/MainNFSD/nfs_rpc_callback.c:864 #7 0x000000000048011c in nfs4_op_setclientid_confirm (op=0x7fd62c001d90, ---Type <return> to continue, or q <return> to quit--- data=0x7fd6607dff70, resp=0x7fd62c002070) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/Protocols/NFS/nfs4_op_setclientid_confirm.c:382 #8 0x000000000045b4b1 in nfs4_Compound (arg=0x7fd62c0011a8, req=0x7fd62c000aa0, res=0x7fd62c001f60) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/Protocols/NFS/nfs4_Compound.c:946 #9 0x000000000045813e in nfs_rpc_process_request (reqdata=0x7fd62c000aa0) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/MainNFSD/nfs_worker_thread.c:1328 #10 0x0000000000458885 in nfs_rpc_valid_NFS (req=0x7fd62c000aa0) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/MainNFSD/nfs_worker_thread.c:1538 #11 0x00007fd66baf9752 in svc_vc_decode (req=0x7fd62c000aa0) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/src/svc_vc.c:827 #12 0x000000000044b2ef in nfs_rpc_decode_request (xprt=0x7fd63c0008c0, xdrs=0x7fd62c0008c0) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/MainNFSD/nfs_rpc_dispatcher_thread.c:1345 #13 0x00007fd66baf9663 in svc_vc_recv (xprt=0x7fd63c0008c0) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/src/svc_vc.c:800 ---Type <return> to continue, or q <return> to quit--- #14 0x00007fd66baf5db1 in svc_rqst_xprt_task (wpe=0x7fd63c000ad8) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/src/svc_rqst.c:769 #15 0x00007fd66baf620e in svc_rqst_epoll_events (sr_rec=0x21338f0, n_events=1) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/src/svc_rqst.c:941 #16 0x00007fd66baf64a3 in svc_rqst_epoll_loop (sr_rec=0x21338f0) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/src/svc_rqst.c:1014 #17 0x00007fd66baf6556 in svc_rqst_run_task (wpe=0x21338f0) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/src/svc_rqst.c:1050 #18 0x00007fd66bafeef6 in work_pool_thread (arg=0x7fd648000a90) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/src/work_pool.c:181 #19 0x00007fd66a6ab3d4 in start_thread (arg=0x7fd6607e1700) at pthread_create.c:334 #20 0x00007fd669fcaebd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 (gdb) f 1 #1 0x00007fd66badf72e in svc_release_it (xprt=0x7fd658002e90, flags=0, tag=0x7fd66bb06fd0 "clnt_vc_destroy", line=462) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/ntirpc/rpc/svc.h:433 433 /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/libntirpc/ntirpc/rpc/svc.h: No such file or directory. (gdb) p clnt No symbol "clnt" in current context. (gdb) p xprt $10 = (SVCXPRT *) 0x7fd658002e90 (gdb) p *$ $11 = {xp_ops = 0x7fd658000e20, xp_dispatch = {process_cb = 0x7fd658000078, rendezvous_cb = 0x7fd658000078}, xp_parent = 0x0, xp_tp = 0x0, xp_netid = 0x0, xp_p1 = 0x0, xp_p2 = 0x0, xp_p3 = 0x0, xp_u1 = 0x0, xp_u2 = 0x0, xp_local = {nb = {maxlen = 483619223, len = 1, buf = 0x2}, ss = {ss_family = 0, __ss_align = 0, __ss_padding = "\313)\260k\326\177\000\000\020\320\236b\326\177\000\000\006\000\000\000\034\000\000\000\004\004\005\377\377\377\377\377\000\000\000\000\020\373\364\310\333c\335\363\245\332\362b\324.M\332", '\000' <repeats 59 times>}}, xp_remote = {nb = {maxlen = 0, len = 0, buf = 0x0}, ss = {ss_family = 0, __ss_align = 0, __ss_padding = '\000' <repeats 111 times>}}, xp_lock = { __data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}, xp_fd = 0, xp_ifindex = 0, xp_si_type = 0, xp_type = 0, xp_refcnt = -1, xp_flags = 64} (gdb) f 6 #6 0x000000000043f800 in nfs_rpc_destroy_chan (chan=0x7fd64c002648) at /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/MainNFSD/nfs_rpc_callback.c:864 864 /build/mts/release/bora-16138726/cayman_nfs-ganesha/nfs-ganesha/src/src/MainNFSD/nfs_rpc_callback.c: No such file or directory. (gdb) p chan $12 = (rpc_call_channel_t *) 0x7fd64c002648 (gdb) p *$ $13 = {type = RPC_CHAN_V40, mtx = {__data = {__lock = 1, __count = 0, __owner = 163, __nusers = 1, __kind = 0, __spins = 0, __list = { __prev = 0x0, __next = 0x0}}, __size = "\001\000\000\000\000\000\000\000\243\000\000\000\001", '\000' <repeats 26 times>, __align = 1}, states = 0, source = {clientid = 0x7fd64c0025a0, session = 0x7fd64c0025a0}, last_called = 0, clnt = 0x7fd658002ba0, auth = 0x0, gss_sec = {mech = 0x0, qop = 0, svc = RPCSEC_GSS_SVC_INTEGRITY, cred = 0x0, req_flags = 0}} (gdb) p chan->client There is no member named client. (gdb) p chan->clnt $14 = (CLIENT *) 0x7fd658002ba0 (gdb) p *$ $15 = {cl_ops = 0x7fd66bd192e0, cl_netid = 0x0, cl_tp = 0x0, cl_u1 = 0x0, cl_u2 = 0x0, cl_lock = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 3, __spins = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 16 times>, "\003", '\000' <repeats 22 times>, __align = 0}, cl_error = {ru = {RE_errno = 0, RE_why = AUTH_OK, RE_vers = { low = 0, high = 0}, RE_lb = {s1 = 0, s2 = 0}}, re_status = RPC_SUCCESS}, cl_refcnt = 0, cl_flags = 96} (gdb) Do we know what is happening here? Need help here.