NFS GANESHA 2.7.6
I am seeing the following crash when the FSAL returns a ERR_FSAL_DELAY and mdcache_read_cb calls mdcache_kill_entry. The crash is in
mdcache_lru_cleanup_push while trying to do a
LRU_DQ_SAFE(lru, q);
I am happy to provide any additional debug information needed from the core.
(gdb) bt
#0
0x00000000005280e9 in mdcache_lru_cleanup_push (entry=0x1e2da00) at /src/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_lru.c:969
#1
0x000000000054585d in _mdcache_kill_entry (entry=0x1e2da00, file=0x598080 "/src/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_file.c", line=554,
function=0x598220 <__func__.20126> "mdc_read_cb") at /src/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:3453
#2
0x0000000000536f7c in mdc_read_cb (obj=0x1d10900, ret=..., obj_data=0x7f60be49ab60, caller_data=0x1e2dfc0)
at /src/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_file.c:554
#3
0x00007f60c65bb047 in foo_read2 (obj_hdl=0x1d10900, bypass=true, done_cb=0x536e7b <mdc_read_cb>, read_arg=0x7f60be49ab60, caller_arg=0x1e2dfc0)
at /opt/src/src/handle.c:1576
#4
0x000000000053706e in mdcache_read2 (obj_hdl=0x1e2da38, bypass=true, done_cb=0x490cd7 <nfs3_read_cb>, read_arg=0x7f60be49ab60, caller_arg=0x7f60be49afc0)
at /src/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_file.c:589
#5
0x00000000004914a7 in nfs3_read (arg=0x2f35728, req=0x2f35020, res=0x1cd3d20) at /src/src/Protocols/NFS/nfs3_read.c:311
#6
0x0000000000457e56 in nfs_rpc_process_request (reqdata=0x2f35020) at /src/src/MainNFSD/nfs_worker_thread.c:1328
#7
0x0000000000458615 in nfs_rpc_valid_NFS (req=0x2f35020) at /src/src/MainNFSD/nfs_worker_thread.c:1548
#8
0x00007f60ca6d9034 in svc_vc_decode (req=0x2f35020) at /src/src/libntirpc/src/svc_vc.c:829
#9
0x000000000044b005 in nfs_rpc_decode_request (xprt=0x1d25fc0, xdrs=0x1d2be90) at /src/src/MainNFSD/nfs_rpc_dispatcher_thread.c:1345
#10 0x00007f60ca6d8f45 in svc_vc_recv (xprt=0x1d25fc0) at /src/src/libntirpc/src/svc_vc.c:802
#11 0x00007f60ca6d5689 in svc_rqst_xprt_task (wpe=0x1d261d8) at /src/src/libntirpc/src/svc_rqst.c:769
#12 0x00007f60ca6d5ae6 in svc_rqst_epoll_events (sr_rec=0x1c92990, n_events=2) at /src/src/libntirpc/src/svc_rqst.c:941
#13 0x00007f60ca6d5d7b in svc_rqst_epoll_loop (sr_rec=0x1c92990) at /src/src/libntirpc/src/svc_rqst.c:1014
#14 0x00007f60ca6d5e2e in svc_rqst_run_task (wpe=0x1c92990) at /src/src/libntirpc/src/svc_rqst.c:1050
#15 0x00007f60ca6de7f6 in work_pool_thread (arg=0x1cec1c0) at /src/src/libntirpc/src/work_pool.c:181
#16 0x00007f60c96fcde5 in start_thread () from /lib64/libpthread.so.0
#17 0x00007f60c9003f1d in clone () from /lib64/libc.so.6
(gdb) print entry->lru
$4 = {q = {next = 0x0, prev = 0x0}, qid = LRU_ENTRY_NONE, refcnt = 3, flags = 0, lane = 12, cf = 0}
(gdb) print *entry
$5 = {attr_lock = {__data = {__lock = 0, __nr_readers = 0, __readers_wakeup = 0, __writer_wakeup = 0, __nr_readers_queued = 0, __nr_writers_queued = 0,
__writer = 0, __shared = 0, __pad1 = 0, __pad2 = 0, __flags = 0}, __size = '\000' <repeats 55 times>, __align = 0}, obj_handle = {handles = {
next = 0x212fb88, prev = 0x7d6460 <MDCACHE+32>}, fs = 0x0, fsal = 0x7d6440 <MDCACHE>, obj_ops = 0x7d6598 <MDCACHE+344>, obj_lock = {__data = {__lock = 0,
__nr_readers = 0, __readers_wakeup = 0, __writer_wakeup = 0, __nr_readers_queued = 0, __nr_writers_queued = 0, __writer = 0, __shared = 0, __pad1 = 0,
__pad2 = 0, __flags = 0}, __size = '\000' <repeats 55 times>, __align = 0}, type = REGULAR_FILE, fsid = {major = 0, minor = 0}, fileid = 20004,
state_hdl = 0x1e2dca0}, sub_handle = 0x1d10900, attrs = {request_mask = 1433550, valid_mask = 1433550, supported = 1433582, type = REGULAR_FILE,
filesize = 10737418240, fsid = {major = 0, minor = 0}, acl = 0x0, fileid = 20004, mode = 420, numlinks = 1, owner = 65534, group = 65534, rawdev = {major = 0,
minor = 0}, atime = {tv_sec = 1573241243, tv_nsec = 33000000}, creation = {tv_sec = 0, tv_nsec = 0}, ctime = {tv_sec = 1573241243, tv_nsec = 33000000},
mtime = {tv_sec = 1573241243, tv_nsec = 33000000}, chgtime = {tv_sec = 1573241243, tv_nsec = 33000000}, spaceused = 10737418240, change = 1573241243033000000,
generation = 0, expire_time_attr = 60, fs_locations = 0x0, sec_label = {slai_lfs = {lfs_lfs = 0, lfs_pi = 0}, slai_data = {slai_data_len = 0,
slai_data_val = 0x0}}}, fh_hk = {node_k = {left = 0x0, right = 0x0, parent = 2}, key = {hk = 12628871282545812592, fsal = 0x7f60c6801100 <FOO>, kv = {
addr = 0x1d0c930, len = 10}}, inavl = false}, mde_flags = 1, attr_time = 1573511991, acl_time = 0, fs_locations_time = 0, lru = {q = {next = 0x0,
prev = 0x0}, qid = LRU_ENTRY_NONE, refcnt = 3, flags = 0, lane = 12, cf = 0}, export_list = {next = 0x1d00a80, prev = 0x1d00a80}, first_export_id = 2,
content_lock = {__data = {__lock = 0, __nr_readers = 0, __readers_wakeup = 0, __writer_wakeup = 0, __nr_readers_queued = 0, __nr_writers_queued = 0,
__writer = 0, __shared = 0, __pad1 = 0, __pad2 = 0, __flags = 0}, __size = '\000' <repeats 55 times>, __align = 0}, fsobj = {hdl = {state_lock = {__data = {
__lock = 0, __nr_readers = 0, __readers_wakeup = 0, __writer_wakeup = 0, __nr_readers_queued = 0, __nr_writers_queued = 0, __writer = 0, __shared = 0,
__pad1 = 0, __pad2 = 0, __flags = 0}, __size = '\000' <repeats 55 times>, __align = 0}, no_cleanup = false, {file = {obj = 0x1e2da38, list_of_states = {
next = 0x1e2dce8, prev = 0x1e2dce8}, layoutrecall_list = {next = 0x1e2dcf8, prev = 0x1e2dcf8}, lock_list = {next = 0x1e2dd08, prev = 0x1e2dd08},
nlm_share_list = {next = 0x1e2dd18, prev = 0x1e2dd18}, write_delegated = false, fdeleg_stats = {fds_curr_delegations = 0,
fds_deleg_type = OPEN_DELEGATE_NONE, fds_delegation_count = 0, fds_recall_count = 0, fds_avg_hold = 0, fds_last_delegation = 0, fds_last_recall = 0,
fds_num_opens = 0, fds_first_open = 0}, anon_ops = 0}, dir = {junction_export = 0x1e2da38, export_roots = {next = 0x1e2dce8, prev = 0x1e2dce8},
exp_root_refcount = 31644920}}}, fsdir = {chunks = {next = 0x0, prev = 0x0}, detached = {next = 0x0, prev = 0x0}, spin = 0, detached_count = 0, dhdl = {
state_lock = {__data = {__lock = 0, __nr_readers = 0, __readers_wakeup = 0, __writer_wakeup = 0, __nr_readers_queued = 0, __nr_writers_queued = 0,
__writer = 31644216, __shared = 0, __pad1 = 31644904, __pad2 = 31644904, __flags = 31644920},
__size = '\000' <repeats 24 times>, "\070\332\342\001\000\000\000\000\350\334\342\001\000\000\000\000\350\334\342\001\000\000\000\000\370\334\342\001\000\000\000", __align =
0}, no_cleanup = 248, {file = {obj = 0x1e2dd08, list_of_states = {next = 0x1e2dd08, prev = 0x1e2dd18}, layoutrecall_list = {next = 0x1e2dd18,
prev = 0x0}, lock_list = {next = 0x0, prev = 0x0}, nlm_share_list = {next = 0x0, prev = 0x0}, write_delegated = false, fdeleg_stats = {
fds_curr_delegations = 0, fds_deleg_type = OPEN_DELEGATE_NONE, fds_delegation_count = 0, fds_recall_count = 0, fds_avg_hold = 0,
fds_last_delegation = 0, fds_last_recall = 0, fds_num_opens = 0, fds_first_open = 0}, anon_ops = 0}, dir = {junction_export = 0x1e2dd08,
export_roots = {next = 0x1e2dd08, prev = 0x1e2dd18}, exp_root_refcount = 31644952}}}, parent = {addr = 0x0, len = 0}, parent_time = 0, first_ck = 0,
avl = {t = {root = 0x0, cmp_fn = 0x0, height = 0, first = 0x0, last = 0x0, size = 0}, ck = {root = 0x0, cmp_fn = 0x0, height = 0, first = 0x0, last = 0x0,
size = 0}, sorted = {root = 0x0, cmp_fn = 0x0, height = 0, first = 0x0, last = 0x0, size = 0}, collisions = 0}}}}
(gdb)