12:54 p.m.
In this code path has_write is false, so the entry was found in cache -
mdcache_avl_lookup_ck successfully found dirent and mdcache_find_keyed_reason successfully
returned entry and should have increased the refcount. Current refcount is 0. We crashed
because obj_ops is 0 when trying to call gettarrs “status =
entry->obj_handle.obj_ops->getattrs()”
Crash is reproducible.
Unfortunately I can’t reproduce with debug flags for COMPONENT_CACHE_INODE and
COMPONENT_NFS_READDIR enabled
Test conditions:
Windows client using robocopy. The test creates a set of local files. Uses robocopy to
sync the local directory to the NFS file share. Deletes the folder from the file share
and then uses robocopy to sync to a different folder on the NFS file share.
Ganesha Version 2.7.1 + commits:
https://github.com/nfs-ganesha/nfs-ganesha/commit/25320e6544f6c5a045f20c5...
https://github.com/nfs-ganesha/nfs-ganesha/commit/03ee21eae53f33e49a993f1...
(gdb) bt
#0 0x00000000005418a1 in mdcache_readdir_chunked (directory=0x32ce1490, whence=121480190,
dir_state=0x7f237b2a2af0,
cb=0x43217c <populate_dirent>, attrmask=0, eod_met=0x7f237b2a2feb)
at /src/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:3136
#1 0x000000000052e8c3 in mdcache_readdir (dir_hdl=0x32ce14c8, whence=0x7f237b2a2ad0,
dir_state=0x7f237b2a2af0,
cb=0x43217c <populate_dirent>, attrmask=0, eod_met=0x7f237b2a2feb)
at /src/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_handle.c:559
#2 0x0000000000432a76 in fsal_readdir (directory=0x32ce14c8, cookie=121480190,
nbfound=0x7f237b2a2fec,
eod_met=0x7f237b2a2feb, attrmask=0, cb=0x4912a2 <nfs3_readdir_callback>,
opaque=0x7f237b2a2fa0)
at /src/src/FSAL/fsal_helper.c:1158
#3 0x000000000049108a in nfs3_readdir (arg=0x6cf58738, req=0x6cf58030, res=0x6cc27720)
at /src/src/Protocols/NFS/nfs3_readdir.c:289
#4 0x00000000004574d1 in nfs_rpc_process_request (reqdata=0x6cf58030) at
/src/src/MainNFSD/nfs_worker_thread.c:1329
#5 0x0000000000457c90 in nfs_rpc_valid_NFS (req=0x6cf58030) at
/src/src/MainNFSD/nfs_worker_thread.c:1549
#6 0x00007f238335ae75 in svc_vc_decode (req=0x6cf58030) at
/src/src/libntirpc/src/svc_vc.c:825
#7 0x000000000044a688 in nfs_rpc_decode_request (xprt=0x1c28880, xdrs=0x6cf92980)
at /src/src/MainNFSD/nfs_rpc_dispatcher_thread.c:1341
#8 0x00007f238335ad86 in svc_vc_recv (xprt=0x1c28880) at
/src/src/libntirpc/src/svc_vc.c:798
#9 0x00007f23833574d3 in svc_rqst_xprt_task (wpe=0x1c28a98) at
/src/src/libntirpc/src/svc_rqst.c:767
#10 0x00007f238335794d in svc_rqst_epoll_events (sr_rec=0x1bfb260, n_events=1) at
/src/src/libntirpc/src/svc_rqst.c:939
#11 0x00007f2383357be2 in svc_rqst_epoll_loop (sr_rec=0x1bfb260) at
/src/src/libntirpc/src/svc_rqst.c:1012
#12 0x00007f2383357c95 in svc_rqst_run_task (wpe=0x1bfb260) at
/src/src/libntirpc/src/svc_rqst.c:1048
#13 0x00007f23833605f6 in work_pool_thread (arg=0x6cc0580) at
/src/src/libntirpc/src/work_pool.c:181
#14 0x00007f2382367de5 in start_thread () from /lib64/libpthread.so.0
#15 0x00007f2381c6fbad in clone () from /lib64/libc.so.6
(gdb) print *entry
$1 = {attr_lock = {__data = {__lock = 0, __nr_readers = 0, __readers_wakeup = 848659816,
__writer_wakeup = 0,
__nr_readers_queued = 8205728, __nr_writers_queued = 0, __writer = 0, __shared = 0,
__pad1 = 8205696,
__pad2 = 8206032, __flags = 0},
__size =
"\000\000\000\000\000\000\000\000h\205\225\062\000\000\000\000\240\065}",
'\000' <repeats 13 times>,
"\200\065}\000\000\000\000\000\320\066}", '\000' <repeats 12
times>, __align = 0}, obj_handle = {handles = {next = 0x0,
prev = 0x0}, fs = 0x0, fsal = 0x0, obj_ops = 0x0, obj_lock = {__data = {__lock = 0,
__nr_readers = 0,
__readers_wakeup = 1, __writer_wakeup = 0, __nr_readers_queued = 0,
__nr_writers_queued = 0, __writer = 0,
__shared = 0, __pad1 = 4542671, __pad2 = 1812466792, __flags = 1753052544},
__size = "\000\000\000\000\000\000\000\000\001", '\000'
<repeats 23 times>,
"\317PE\000\000\000\000\000h\f\bl\000\000\000\000\200u}h\000\000\000", __align =
0}, type = 1433550, fsid = {major = 1433550, minor = 1433582}, fileid = 1,
state_hdl = 0x400}, sub_handle = 0x0, attrs = {request_mask = 0, valid_mask = 0,
supported = 4542671, type = 438,
filesize = 65534, fsid = {major = 65534, minor = 0}, acl = 0x0, fileid = 1549686770,
mode = 225000000, numlinks = 0,
owner = 0, group = 0, rawdev = {major = 1549686770, minor = 225000000}, atime =
{tv_sec = 1549686770,
tv_nsec = 225000000}, creation = {tv_sec = 1549686770, tv_nsec = 225000000}, ctime =
{tv_sec = 1024,
tv_nsec = 1549686770225}, mtime = {tv_sec = 0, tv_nsec = 60}, chgtime = {tv_sec = 0,
tv_nsec = 0}, spaceused = 0,
change = 697563970, generation = 10661591424062854996, expire_time_attr = 2142117152,
fs_locations = 0x6cf4a550},
fh_hk = {node_k = {left = 0xa, right = 0x1, parent = 1}, key = {hk = 1550089231, fsal =
0x0, kv = {addr = 0x0,
len = 933111888}}, inavl = 96}, mde_flags = 1, attr_time = 8589934592, acl_time =
0,
fs_locations_time = 1828650080, lru = {q = {next = 0x6cfefc60, prev = 0x1}, qid =
LRU_ENTRY_NONE, refcnt = 0,
flags = 0, lane = 0, cf = 0}, export_list = {next = 0x0, prev = 0x0}, first_export_id
= 0, content_lock = {__data = {
__lock = 0, __nr_readers = 0, __readers_wakeup = 0, __writer_wakeup = 0,
__nr_readers_queued = 0,
__nr_writers_queued = 0, __writer = 0, __shared = 0, __pad1 = 0, __pad2 = 0, __flags
= 0},
__size = '\000' <repeats 55 times>, __align = 0}, fsobj = {hdl =
{state_lock = {__data = {__lock = 0,
__nr_readers = 0, __readers_wakeup = 0, __writer_wakeup = 0, __nr_readers_queued
= 1812466200,
__nr_writers_queued = 0, __writer = 1812466864, __shared = 0, __pad1 =
1812466864, __pad2 = 1812466880,
__flags = 1812466880},
__size = '\000' <repeats 16 times>,
"\030\n\bl\000\000\000\000\260\f\bl\000\000\000\000\260\f\bl\000\000\000\000\300\f\bl\000\000\000\000\300\f\bl\000\000\000",
__align = 0}, no_cleanup = 208, {file = {obj = 0x6c080cd0,
list_of_states = {next = 0x6c080ce0, prev = 0x6c080ce0}, layoutrecall_list =
{next = 0x0, prev = 0x0},
lock_list = {next = 0x0, prev = 0x0}, nlm_share_list = {next = 0x0, prev = 0x0},
write_delegated = false,
fdeleg_stats = {fds_curr_delegations = 0, fds_deleg_type = OPEN_DELEGATE_NONE,
fds_delegation_count = 0,
---Type <return> to continue, or q <return> to quit---
fds_recall_count = 0, fds_avg_hold = 0, fds_last_delegation = 0,
fds_last_recall = 0, fds_num_opens = 0,
fds_first_open = 0}, anon_ops = 0}, dir = {junction_export = 0x6c080cd0,
export_roots = {next = 0x6c080ce0,
prev = 0x6c080ce0}, exp_root_refcount = 0}}}, fsdir = {chunks = {next = 0x0,
prev = 0x0}, detached = {
next = 0x6c080a18, prev = 0x6c080cb0}, spin = 1812466864, detached_count = 0, dhdl
= {state_lock = {__data = {
__lock = 1812466880, __nr_readers = 0, __readers_wakeup = 1812466880,
__writer_wakeup = 0,
__nr_readers_queued = 1812466896, __nr_writers_queued = 0, __writer =
1812466896, __shared = 0,
__pad1 = 1812466912, __pad2 = 1812466912, __flags = 0},
__size =
"\300\f\bl\000\000\000\000\300\f\bl\000\000\000\000\320\f\bl\000\000\000\000\320\f\bl\000\000\000\000\340\f\bl\000\000\000\000\340\f\bl",
'\000' <repeats 11 times>, __align = 1812466880}, no_cleanup = false, {file
= {
obj = 0x0, list_of_states = {next = 0x0, prev = 0x0}, layoutrecall_list =
{next = 0x0, prev = 0x0},
lock_list = {next = 0x0, prev = 0x0}, nlm_share_list = {next = 0x0, prev =
0x0}, write_delegated = false,
fdeleg_stats = {fds_curr_delegations = 0, fds_deleg_type = OPEN_DELEGATE_NONE,
fds_delegation_count = 0,
fds_recall_count = 0, fds_avg_hold = 0, fds_last_delegation = 0,
fds_last_recall = 0, fds_num_opens = 0,
fds_first_open = 0}, anon_ops = 0}, dir = {junction_export = 0x0,
export_roots = {next = 0x0, prev = 0x0},
exp_root_refcount = 0}}}, parent = {addr = 0x0, len = 0}, first_ck = 0, avl =
{t = {root = 0x0,
cmp_fn = 0x0, height = 0, first = 0x0, last = 0x0, size = 0}, ck = {root = 0x0,
cmp_fn = 0x0, height = 0,
first = 0x0, last = 0x0, size = 0}, sorted = {root = 0x0, cmp_fn = 0x0, height =
49, first = 0x6bf94870,
last = 0x7f2381f377d8 <main_arena+120>, size = 0}, collisions = 0}}}}
(gdb) info locals
status = {major = ERR_FSAL_NO_ERROR, minor = 0}
cb_result = DIR_CONTINUE
entry = 0x6c080a10
attrs = {request_mask = 0, valid_mask = 0, supported = 0, type = NO_FILE_TYPE, filesize =
0, fsid = {major = 0,
minor = 0}, acl = 0x0, fileid = 0, mode = 0, numlinks = 0, owner = 0, group = 0,
rawdev = {major = 0, minor = 0},
atime = {tv_sec = 0, tv_nsec = 0}, creation = {tv_sec = 0, tv_nsec = 0}, ctime = {tv_sec
= 0, tv_nsec = 0}, mtime = {
tv_sec = 0, tv_nsec = 0}, chgtime = {tv_sec = 0, tv_nsec = 0}, spaceused = 0, change =
0, generation = 0,
expire_time_attr = 0, fs_locations = 0x0}
dirent = 0x6aef2150
has_write = false
set_first_ck = false
next_ck = 121480231
look_ck = 121480190
chunk = 0x6c96c8b0
first_pass = true
eod = false
reload_chunk = false
__func__ = "mdcache_readdir_chunked"
__PRETTY_FUNCTION__ = "mdcache_readdir_chunked"
(gdb) print *dirent
$2 = {chunk_list = {next = 0x6be81080, prev = 0x6b1af310}, chunk = 0x6c96c8b0, node_name =
{left = 0x68d85358,
right = 0x685443e8, parent = 1817635595}, node_ck = {left = 0x0, right = 0x0, parent =
1810370738}, node_sorted = {
left = 0x0, right = 0x0, parent = 0}, ck = 121480231, eod = false, namehash =
13944437367817932926, ckey = {
hk = 13666917134750151872, fsal = 0x7f237fae1d20 <FOO>, kv = {addr = 0x6ce752b0,
len = 10}}, flags = 0,
name = 0x6aef21f8 "random.348", name_buffer = 0x6aef21f8
"random.348"}
(gdb) print *chunk
$3 = {chunks = {next = 0x32ce1718, prev = 0x32ce1718}, dirents = {next = 0x5ee461c0, prev
= 0x6ce55210},
parent = 0x32ce1490, chunk_lru = {q = {next = 0x7e1920 <CHUNK_LRU+672>, prev =
0x6bae17c8}, qid = LRU_ENTRY_L1,
refcnt = 0, flags = 0, lane = 3, cf = 0}, reload_ck = 121480068, next_ck = 0,
num_entries = 480}
(gdb)