[NFS-Ganesha-Devel] Crash in cih_latch_entry

We are getting this crash with ganesha2.6.3.There are too many readers(0xFFFFFFFF) waiting on cih_fhcache.partition[0].lock, so pthread_rwlock_rdlock returns EAGAIN.Did anyone see this backtrace before, if so what fixed it? (gdb) bt#0 0x00007fe5a909c1f7 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56#1 0x00007fe5a909d8e8 in __GI_abort () at abort.c:90#2 0x000000000050c86b in cih_latch_entry (key=0x7fe58300a440, latch=0x7fe58300a2c0, flags=5, func=0x563cd0 <__func__.20115> "mdcache_find_keyed", line=888) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_hash.h:262#3 0x000000000050c8d4 in cih_get_by_key_latch (key=0x7fe58300a440, latch=0x7fe58300a2c0, flags=5, func=0x563cd0 <__func__.20115> "mdcache_find_keyed", line=888) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_hash.h:293#4 0x000000000050f6c1 in mdcache_find_keyed (key=0x7fe58300a440, entry=0x7fe58300a4f0) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:886#5 0x000000000050f9af in mdcache_locate_host (fh_desc=0x7fe58300a970, export=0xd97820, entry=0x7fe58300a4f0, attrs_out=0x0) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:974#6 0x0000000000508c94 in mdcache_create_handle (exp_hdl=0xd97820, fh_desc=0x7fe58300a970, handle=0x7fe58300a968, attrs_out=0x0) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_handle.c:1759#7 0x000000000046b4a4 in nfs4_mds_putfh (data=0x7fe58300aa70) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/Protocols/NFS/nfs4_op_putfh.c:211#8 0x000000000046b691 in nfs4_op_putfh (op=0x7fe5700022f0, data=0x7fe58300aa70, resp=0x7fe570003830) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/Protocols/NFS/nfs4_op_putfh.c:281#9 0x0000000000454494 in nfs4_Compound (arg=0x7fe570004498, req=0x7fe570003d90, res=0x7fe570003720) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/Protocols/NFS/nfs4_Compound.c:752#10 0x000000000045157a in nfs_rpc_process_request (reqdata=0x7fe570003d90) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/MainNFSD/nfs_worker_thread.c:1329#11 0x0000000000451cc1 in nfs_rpc_valid_NFS (req=0x7fe570003d90) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/MainNFSD/nfs_worker_thread.c:1554#12 0x00007fe5a966c4d6 in svc_vc_decode (req=0x7fe570003d90) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_vc.c:815#13 0x0000000000445816 in nfs_rpc_decode_request (xprt=0x7fe574000c10, xdrs=0x7fe570002110) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/MainNFSD/nfs_rpc_dispatcher_thread.c:1341#14 0x00007fe5a966c3e7 in svc_vc_recv (xprt=0x7fe574000c10) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_vc.c:788#15 0x00007fe5a9668bc5 in svc_rqst_xprt_task (wpe=0x7fe574000e28) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:751#16 0x00007fe5a966903a in svc_rqst_epoll_events (sr_rec=0xda67b0, n_events=1) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:923#17 0x00007fe5a96692dc in svc_rqst_epoll_loop (sr_rec=0xda67b0) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:996#18 0x00007fe5a966938f in svc_rqst_run_task (wpe=0xda67b0) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:1032#19 0x00007fe5a967249f in work_pool_thread (arg=0x7fe56c0008c0) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/work_pool.c:176#20 0x00007fe5a9a94e25 in start_thread (arg=0x7fe58300c700) at pthread_create.c:308#21 0x00007fe5a915f34d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113(gdb) frame 2#2 0x000000000050c86b in cih_latch_entry (key=0x7fe58300a440, latch=0x7fe58300a2c0, flags=5, func=0x563cd0 <__func__.20115> "mdcache_find_keyed", line=888) at /export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_hash.h:262262 PTHREAD_RWLOCK_rdlock(&cp->lock); /* SUBTREE_RLOCK */(gdb) p cp->lock$1 = {__data = {__lock = 0, __nr_readers = 4294967295, __readers_wakeup = 0, __writer_wakeup = 0, __nr_readers_queued = 0, __nr_writers_queued = 0, __writer = 0, __shared = 0, __pad1 = 0, __pad2 = 0, __flags = 0}, __size = "\000\000\000\000\377\377\377\377", '\000' <repeats 47 times>, __align = -4294967296}(gdb) p rc$2 = 11