9:18 p.m.
We are getting this crash with ganesha2.6.3.There are too many readers(0xFFFFFFFF) waiting
on cih_fhcache.partition[0].lock, so pthread_rwlock_rdlock returns EAGAIN.Did anyone see
this backtrace before, if so what fixed it?
(gdb) bt#0 0x00007fe5a909c1f7 in __GI_raise (sig=sig@entry=6) at
../nptl/sysdeps/unix/sysv/linux/raise.c:56#1 0x00007fe5a909d8e8 in __GI_abort () at
abort.c:90#2 0x000000000050c86b in cih_latch_entry (key=0x7fe58300a440,
latch=0x7fe58300a2c0, flags=5, func=0x563cd0 <__func__.20115>
"mdcache_find_keyed", line=888) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_hash.h:262#3
0x000000000050c8d4 in cih_get_by_key_latch (key=0x7fe58300a440, latch=0x7fe58300a2c0,
flags=5, func=0x563cd0 <__func__.20115> "mdcache_find_keyed", line=888)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_hash.h:293#4
0x000000000050f6c1 in mdcache_find_keyed (key=0x7fe58300a440, entry=0x7fe58300a4f0) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:886#5
0x000000000050f9af in mdcache_locate_host (fh_desc=0x7fe58300a970, export=0xd97820,
entry=0x7fe58300a4f0, attrs_out=0x0) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:974#6
0x0000000000508c94 in mdcache_create_handle (exp_hdl=0xd97820, fh_desc=0x7fe58300a970,
handle=0x7fe58300a968, attrs_out=0x0) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_handle.c:1759#7
0x000000000046b4a4 in nfs4_mds_putfh (data=0x7fe58300aa70) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/Protocols/NFS/nfs4_op_putfh.c:211#8
0x000000000046b691 in nfs4_op_putfh (op=0x7fe5700022f0, data=0x7fe58300aa70,
resp=0x7fe570003830) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/Protocols/NFS/nfs4_op_putfh.c:281#9
0x0000000000454494 in nfs4_Compound (arg=0x7fe570004498, req=0x7fe570003d90,
res=0x7fe570003720) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/Protocols/NFS/nfs4_Compound.c:752#10
0x000000000045157a in nfs_rpc_process_request (reqdata=0x7fe570003d90) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/MainNFSD/nfs_worker_thread.c:1329#11
0x0000000000451cc1 in nfs_rpc_valid_NFS (req=0x7fe570003d90) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/MainNFSD/nfs_worker_thread.c:1554#12
0x00007fe5a966c4d6 in svc_vc_decode (req=0x7fe570003d90) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_vc.c:815#13
0x0000000000445816 in nfs_rpc_decode_request (xprt=0x7fe574000c10, xdrs=0x7fe570002110)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/MainNFSD/nfs_rpc_dispatcher_thread.c:1341#14
0x00007fe5a966c3e7 in svc_vc_recv (xprt=0x7fe574000c10) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_vc.c:788#15
0x00007fe5a9668bc5 in svc_rqst_xprt_task (wpe=0x7fe574000e28) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:751#16
0x00007fe5a966903a in svc_rqst_epoll_events (sr_rec=0xda67b0, n_events=1) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:923#17
0x00007fe5a96692dc in svc_rqst_epoll_loop (sr_rec=0xda67b0) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:996#18
0x00007fe5a966938f in svc_rqst_run_task (wpe=0xda67b0) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:1032#19
0x00007fe5a967249f in work_pool_thread (arg=0x7fe56c0008c0) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/work_pool.c:176#20
0x00007fe5a9a94e25 in start_thread (arg=0x7fe58300c700) at pthread_create.c:308#21
0x00007fe5a915f34d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113(gdb) frame
2#2 0x000000000050c86b in cih_latch_entry (key=0x7fe58300a440, latch=0x7fe58300a2c0,
flags=5, func=0x563cd0 <__func__.20115> "mdcache_find_keyed", line=888)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_hash.h:262262 PTHREAD_RWLOCK_rdlock(&cp->lock); /*
SUBTREE_RLOCK */(gdb) p cp->lock$1 = {__data = {__lock = 0, __nr_readers = 4294967295,
__readers_wakeup = 0, __writer_wakeup = 0, __nr_readers_queued = 0, __nr_writers_queued =
0, __writer = 0, __shared = 0, __pad1 = 0, __pad2 = 0, __flags = 0}, __size =
"\000\000\000\000\377\377\377\377", '\000' <repeats 47 times>,
__align = -4294967296}(gdb) p rc$2 = 11
7:27 a.m.
This is a new one, and very strange. The issue is that the lock is
pointing to bad memory, likely uninitialized. There should be values
other than zero in those fields.
The problem is that the cih_fhcache is initialized when Ganesha starts
up, and cleaned up when it shuts down, so there should never be a case
where the memory is invalid.
If this reproduces, can you look at result of cih_partition_of_scaler(),
and look at cih_fhcache to see if it looks okay, and post those to the
list? I'm wondering if there's memory corruption somewhere.
Daniel
On 1/19/19 10:18 PM, QR wrote:
We are getting this crash with ganesha2.6.3.
There are too many readers(0xFFFFFFFF) waiting on
cih_fhcache.partition[0].lock, so pthread_rwlock_rdlock returns EAGAIN.
Did anyone see this backtrace before, if so what fixed it?
(gdb) bt
#0 0x00007fe5a909c1f7 in __GI_raise (sig=sig@entry=6) at
../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1 0x00007fe5a909d8e8 in __GI_abort () at abort.c:90
#2 0x000000000050c86b in cih_latch_entry (key=0x7fe58300a440,
latch=0x7fe58300a2c0, flags=5, func=0x563cd0 <__func__.20115>
"mdcache_find_keyed", line=888)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_hash.h:262
#3 0x000000000050c8d4 in cih_get_by_key_latch (key=0x7fe58300a440,
latch=0x7fe58300a2c0, flags=5, func=0x563cd0 <__func__.20115>
"mdcache_find_keyed", line=888)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_hash.h:293
#4 0x000000000050f6c1 in mdcache_find_keyed (key=0x7fe58300a440,
entry=0x7fe58300a4f0)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:886
#5 0x000000000050f9af in mdcache_locate_host (fh_desc=0x7fe58300a970,
export=0xd97820, entry=0x7fe58300a4f0, attrs_out=0x0)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:974
#6 0x0000000000508c94 in mdcache_create_handle (exp_hdl=0xd97820,
fh_desc=0x7fe58300a970, handle=0x7fe58300a968, attrs_out=0x0)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_handle.c:1759
#7 0x000000000046b4a4 in nfs4_mds_putfh (data=0x7fe58300aa70) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/Protocols/NFS/nfs4_op_putfh.c:211
#8 0x000000000046b691 in nfs4_op_putfh (op=0x7fe5700022f0,
data=0x7fe58300aa70, resp=0x7fe570003830)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/Protocols/NFS/nfs4_op_putfh.c:281
#9 0x0000000000454494 in nfs4_Compound (arg=0x7fe570004498,
req=0x7fe570003d90, res=0x7fe570003720)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/Protocols/NFS/nfs4_Compound.c:752
#10 0x000000000045157a in nfs_rpc_process_request
(reqdata=0x7fe570003d90) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/MainNFSD/nfs_worker_thread.c:1329
#11 0x0000000000451cc1 in nfs_rpc_valid_NFS (req=0x7fe570003d90) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/MainNFSD/nfs_worker_thread.c:1554
#12 0x00007fe5a966c4d6 in svc_vc_decode (req=0x7fe570003d90) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_vc.c:815
#13 0x0000000000445816 in nfs_rpc_decode_request (xprt=0x7fe574000c10,
xdrs=0x7fe570002110)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/MainNFSD/nfs_rpc_dispatcher_thread.c:1341
#14 0x00007fe5a966c3e7 in svc_vc_recv (xprt=0x7fe574000c10) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_vc.c:788
#15 0x00007fe5a9668bc5 in svc_rqst_xprt_task (wpe=0x7fe574000e28) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:751
#16 0x00007fe5a966903a in svc_rqst_epoll_events (sr_rec=0xda67b0,
n_events=1) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:923
#17 0x00007fe5a96692dc in svc_rqst_epoll_loop (sr_rec=0xda67b0) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:996
#18 0x00007fe5a966938f in svc_rqst_run_task (wpe=0xda67b0) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/svc_rqst.c:1032
#19 0x00007fe5a967249f in work_pool_thread (arg=0x7fe56c0008c0) at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/libntirpc/src/work_pool.c:176
#20 0x00007fe5a9a94e25 in start_thread (arg=0x7fe58300c700) at
pthread_create.c:308
#21 0x00007fe5a915f34d in clone () at
../sysdeps/unix/sysv/linux/x86_64/clone.S:113
(gdb) frame 2
#2 0x000000000050c86b in cih_latch_entry (key=0x7fe58300a440,
latch=0x7fe58300a2c0, flags=5, func=0x563cd0 <__func__.20115>
"mdcache_find_keyed", line=888)
at
/export/jcloud-zbs/src/jd.com/zfs/FSAL_SkyFS/nfs-ganesha/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_hash.h:262
262PTHREAD_RWLOCK_rdlock(&cp->lock);/* SUBTREE_RLOCK */
(gdb) p cp->lock
$1 = {__data = {__lock = 0, __nr_readers = 4294967295, __readers_wakeup
= 0, __writer_wakeup = 0, __nr_readers_queued = 0, __nr_writers_queued =
0, __writer = 0, __shared = 0, __pad1 = 0,
__pad2 = 0, __flags = 0}, __size =
"\000\000\000\000\377\377\377\377", '\000' <repeats 47 times>,
__align =
-4294967296}
(gdb) p rc
$2 = 11
_______________________________________________
Devel mailing list -- devel(a)lists.nfs-ganesha.org
To unsubscribe send an email to devel-leave(a)lists.nfs-ganesha.org
2127
days inactive
2129
days old
1 comments
2 participants
participants (2)
-
Daniel Gryniewicz -
QR