In this code path has_write is false, so the entry was found in cache - mdcache_avl_lookup_ck successfully found dirent and mdcache_find_keyed_reason successfully returned entry and should have increased the refcount. Current
refcount is 0. We crashed because obj_ops is 0 when trying to call gettarrs “status = entry->obj_handle.obj_ops->getattrs()”
Crash is reproducible.
Unfortunately I can’t reproduce with debug flags for COMPONENT_CACHE_INODE and COMPONENT_NFS_READDIR enabled
Test conditions:
Windows client using robocopy. The test creates a set of local files. Uses robocopy to sync the local directory to the NFS file share. Deletes the folder from the file share and then uses robocopy to sync to a different folder on the NFS file
share.
Ganesha Version 2.7.1 + commits:
https://github.com/nfs-ganesha/nfs-ganesha/commit/25320e6544f6c5a045f20c51446f57c9dc036412
https://github.com/nfs-ganesha/nfs-ganesha/commit/03ee21eae53f33e49a993f14309fadcb271a0cd
(gdb) bt
#0
0x00000000005418a1 in mdcache_readdir_chunked (directory=0x32ce1490, whence=121480190, dir_state=0x7f237b2a2af0,
cb=0x43217c <populate_dirent>, attrmask=0, eod_met=0x7f237b2a2feb)
at /src/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:3136
#1
0x000000000052e8c3 in mdcache_readdir (dir_hdl=0x32ce14c8, whence=0x7f237b2a2ad0, dir_state=0x7f237b2a2af0,
cb=0x43217c <populate_dirent>, attrmask=0, eod_met=0x7f237b2a2feb)
at /src/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_handle.c:559
#2
0x0000000000432a76 in fsal_readdir (directory=0x32ce14c8, cookie=121480190, nbfound=0x7f237b2a2fec,
eod_met=0x7f237b2a2feb, attrmask=0, cb=0x4912a2 <nfs3_readdir_callback>, opaque=0x7f237b2a2fa0)
at /src/src/FSAL/fsal_helper.c:1158
#3
0x000000000049108a in nfs3_readdir (arg=0x6cf58738, req=0x6cf58030, res=0x6cc27720)
at /src/src/Protocols/NFS/nfs3_readdir.c:289
#4
0x00000000004574d1 in nfs_rpc_process_request (reqdata=0x6cf58030) at /src/src/MainNFSD/nfs_worker_thread.c:1329
#5
0x0000000000457c90 in nfs_rpc_valid_NFS (req=0x6cf58030) at /src/src/MainNFSD/nfs_worker_thread.c:1549
#6
0x00007f238335ae75 in svc_vc_decode (req=0x6cf58030) at /src/src/libntirpc/src/svc_vc.c:825
#7
0x000000000044a688 in nfs_rpc_decode_request (xprt=0x1c28880, xdrs=0x6cf92980)
at /src/src/MainNFSD/nfs_rpc_dispatcher_thread.c:1341
#8
0x00007f238335ad86 in svc_vc_recv (xprt=0x1c28880) at /src/src/libntirpc/src/svc_vc.c:798
#9
0x00007f23833574d3 in svc_rqst_xprt_task (wpe=0x1c28a98) at /src/src/libntirpc/src/svc_rqst.c:767
#10 0x00007f238335794d in svc_rqst_epoll_events (sr_rec=0x1bfb260, n_events=1) at /src/src/libntirpc/src/svc_rqst.c:939
#11 0x00007f2383357be2 in svc_rqst_epoll_loop (sr_rec=0x1bfb260) at /src/src/libntirpc/src/svc_rqst.c:1012
#12 0x00007f2383357c95 in svc_rqst_run_task (wpe=0x1bfb260) at /src/src/libntirpc/src/svc_rqst.c:1048
#13 0x00007f23833605f6 in work_pool_thread (arg=0x6cc0580) at /src/src/libntirpc/src/work_pool.c:181
#14 0x00007f2382367de5 in start_thread () from /lib64/libpthread.so.0
#15 0x00007f2381c6fbad in clone () from /lib64/libc.so.6
(gdb) print *entry
$1 = {attr_lock = {__data = {__lock = 0, __nr_readers = 0, __readers_wakeup = 848659816, __writer_wakeup = 0,
__nr_readers_queued = 8205728, __nr_writers_queued = 0, __writer = 0, __shared = 0, __pad1 = 8205696,
__pad2 = 8206032, __flags = 0},
__size = "\000\000\000\000\000\000\000\000h\205\225\062\000\000\000\000\240\065}", '\000' <repeats 13 times>, "\200\065}\000\000\000\000\000\320\066}", '\000' <repeats 12 times>,
__align = 0}, obj_handle = {handles = {next = 0x0,
prev = 0x0}, fs = 0x0, fsal = 0x0, obj_ops = 0x0, obj_lock = {__data = {__lock = 0, __nr_readers = 0,
__readers_wakeup = 1, __writer_wakeup = 0, __nr_readers_queued = 0, __nr_writers_queued = 0, __writer = 0,
__shared = 0, __pad1 = 4542671, __pad2 = 1812466792, __flags = 1753052544},
__size = "\000\000\000\000\000\000\000\000\001", '\000' <repeats 23 times>, "\317PE\000\000\000\000\000h\f\bl\000\000\000\000\200u}h\000\000\000", __align = 0}, type = 1433550,
fsid = {major = 1433550, minor = 1433582}, fileid = 1,
state_hdl = 0x400}, sub_handle = 0x0, attrs = {request_mask = 0, valid_mask = 0, supported = 4542671, type = 438,
filesize = 65534, fsid = {major = 65534, minor = 0}, acl = 0x0, fileid = 1549686770, mode = 225000000, numlinks = 0,
owner = 0, group = 0, rawdev = {major = 1549686770, minor = 225000000}, atime = {tv_sec = 1549686770,
tv_nsec = 225000000}, creation = {tv_sec = 1549686770, tv_nsec = 225000000}, ctime = {tv_sec = 1024,
tv_nsec = 1549686770225}, mtime = {tv_sec = 0, tv_nsec = 60}, chgtime = {tv_sec = 0, tv_nsec = 0}, spaceused = 0,
change = 697563970, generation = 10661591424062854996, expire_time_attr = 2142117152, fs_locations = 0x6cf4a550},
fh_hk = {node_k = {left = 0xa, right = 0x1, parent = 1}, key = {hk = 1550089231, fsal = 0x0, kv = {addr = 0x0,
len = 933111888}}, inavl = 96}, mde_flags = 1, attr_time = 8589934592, acl_time = 0,
fs_locations_time = 1828650080, lru = {q = {next = 0x6cfefc60, prev = 0x1}, qid = LRU_ENTRY_NONE, refcnt = 0,
flags = 0, lane = 0, cf = 0}, export_list = {next = 0x0, prev = 0x0}, first_export_id = 0, content_lock = {__data = {
__lock = 0, __nr_readers = 0, __readers_wakeup = 0, __writer_wakeup = 0, __nr_readers_queued = 0,
__nr_writers_queued = 0, __writer = 0, __shared = 0, __pad1 = 0, __pad2 = 0, __flags = 0},
__size = '\000' <repeats 55 times>, __align = 0}, fsobj = {hdl = {state_lock = {__data = {__lock = 0,
__nr_readers = 0, __readers_wakeup = 0, __writer_wakeup = 0, __nr_readers_queued = 1812466200,
__nr_writers_queued = 0, __writer = 1812466864, __shared = 0, __pad1 = 1812466864, __pad2 = 1812466880,
__flags = 1812466880},
__size = '\000' <repeats 16 times>, "\030\n\bl\000\000\000\000\260\f\bl\000\000\000\000\260\f\bl\000\000\000\000\300\f\bl\000\000\000\000\300\f\bl\000\000\000", __align = 0},
no_cleanup = 208, {file = {obj = 0x6c080cd0,
list_of_states = {next = 0x6c080ce0, prev = 0x6c080ce0}, layoutrecall_list = {next = 0x0, prev = 0x0},
lock_list = {next = 0x0, prev = 0x0}, nlm_share_list = {next = 0x0, prev = 0x0}, write_delegated = false,
fdeleg_stats = {fds_curr_delegations = 0, fds_deleg_type = OPEN_DELEGATE_NONE, fds_delegation_count = 0,
---Type <return> to continue, or q <return> to quit---
fds_recall_count = 0, fds_avg_hold = 0, fds_last_delegation = 0, fds_last_recall = 0, fds_num_opens = 0,
fds_first_open = 0}, anon_ops = 0}, dir = {junction_export = 0x6c080cd0, export_roots = {next = 0x6c080ce0,
prev = 0x6c080ce0}, exp_root_refcount = 0}}}, fsdir = {chunks = {next = 0x0, prev = 0x0}, detached = {
next = 0x6c080a18, prev = 0x6c080cb0}, spin = 1812466864, detached_count = 0, dhdl = {state_lock = {__data = {
__lock = 1812466880, __nr_readers = 0, __readers_wakeup = 1812466880, __writer_wakeup = 0,
__nr_readers_queued = 1812466896, __nr_writers_queued = 0, __writer = 1812466896, __shared = 0,
__pad1 = 1812466912, __pad2 = 1812466912, __flags = 0},
__size = "\300\f\bl\000\000\000\000\300\f\bl\000\000\000\000\320\f\bl\000\000\000\000\320\f\bl\000\000\000\000\340\f\bl\000\000\000\000\340\f\bl", '\000' <repeats 11 times>,
__align = 1812466880}, no_cleanup = false, {file = {
obj = 0x0, list_of_states = {next = 0x0, prev = 0x0}, layoutrecall_list = {next = 0x0, prev = 0x0},
lock_list = {next = 0x0, prev = 0x0}, nlm_share_list = {next = 0x0, prev = 0x0}, write_delegated = false,
fdeleg_stats = {fds_curr_delegations = 0, fds_deleg_type = OPEN_DELEGATE_NONE, fds_delegation_count = 0,
fds_recall_count = 0, fds_avg_hold = 0, fds_last_delegation = 0, fds_last_recall = 0, fds_num_opens = 0,
fds_first_open = 0}, anon_ops = 0}, dir = {junction_export = 0x0, export_roots = {next = 0x0, prev = 0x0},
exp_root_refcount = 0}}}, parent = {addr = 0x0, len = 0}, first_ck = 0, avl = {t = {root = 0x0,
cmp_fn = 0x0, height = 0, first = 0x0, last = 0x0, size = 0}, ck = {root = 0x0, cmp_fn = 0x0, height = 0,
first = 0x0, last = 0x0, size = 0}, sorted = {root = 0x0, cmp_fn = 0x0, height = 49, first = 0x6bf94870,
last = 0x7f2381f377d8 <main_arena+120>, size = 0}, collisions = 0}}}}
(gdb) info locals
status = {major = ERR_FSAL_NO_ERROR, minor = 0}
cb_result = DIR_CONTINUE
entry = 0x6c080a10
attrs = {request_mask = 0, valid_mask = 0, supported = 0, type = NO_FILE_TYPE, filesize = 0, fsid = {major = 0,
minor = 0}, acl = 0x0, fileid = 0, mode = 0, numlinks = 0, owner = 0, group = 0, rawdev = {major = 0, minor = 0},
atime = {tv_sec = 0, tv_nsec = 0}, creation = {tv_sec = 0, tv_nsec = 0}, ctime = {tv_sec = 0, tv_nsec = 0}, mtime = {
tv_sec = 0, tv_nsec = 0}, chgtime = {tv_sec = 0, tv_nsec = 0}, spaceused = 0, change = 0, generation = 0,
expire_time_attr = 0, fs_locations = 0x0}
dirent = 0x6aef2150
has_write = false
set_first_ck = false
next_ck = 121480231
look_ck = 121480190
chunk = 0x6c96c8b0
first_pass = true
eod = false
reload_chunk = false
__func__ = "mdcache_readdir_chunked"
__PRETTY_FUNCTION__ = "mdcache_readdir_chunked"
(gdb) print *dirent
$2 = {chunk_list = {next = 0x6be81080, prev = 0x6b1af310}, chunk = 0x6c96c8b0, node_name = {left = 0x68d85358,
right = 0x685443e8, parent = 1817635595}, node_ck = {left = 0x0, right = 0x0, parent = 1810370738}, node_sorted = {
left = 0x0, right = 0x0, parent = 0}, ck = 121480231, eod = false, namehash = 13944437367817932926, ckey = {
hk = 13666917134750151872, fsal = 0x7f237fae1d20 <FOO>, kv = {addr = 0x6ce752b0, len = 10}}, flags = 0,
name = 0x6aef21f8 "random.348", name_buffer = 0x6aef21f8 "random.348"}
(gdb) print *chunk
$3 = {chunks = {next = 0x32ce1718, prev = 0x32ce1718}, dirents = {next = 0x5ee461c0, prev = 0x6ce55210},
parent = 0x32ce1490, chunk_lru = {q = {next = 0x7e1920 <CHUNK_LRU+672>, prev = 0x6bae17c8}, qid = LRU_ENTRY_L1,
refcnt = 0, flags = 0, lane = 3, cf = 0}, reload_ck = 121480068, next_ck = 0, num_entries = 480}
(gdb)