Multiple core related to state owner refcount being negative in state reclaim ops.
by Sagar Singh
Are we not guarding all the places by mutex where we do ref/unref?
(gdb) bt
#0 0x00007f58a18dc4fb in raise () from /lib64/libpthread.so.0
#1 0x00007f58a31e8a36 in crash_handler (signo=6, info=0x7f588e3f8470,
ctx=0x7f588e3f8340) at
/usr/src/debug/nfs-ganesha-4.0.2/MainNFSD/nfs_init.c:261
#2 <signal handler called>
#3 0x00007f58a1113387 in raise () from /lib64/libc.so.6
#4 0x00007f58a1114a78 in abort () from /lib64/libc.so.6
#5 0x00007f58a110c1a6 in __assert_fail_base () from /lib64/libc.so.6
#6 0x00007f58a110c252 in __assert_fail () from /lib64/libc.so.6
#7 0x00007f58a3234338 in dec_state_owner_ref (owner=0x7f57ed99bc00)
at /usr/src/debug/nfs-ganesha-4.0.2/SAL/state_misc.c:933
#8 0x00007f58a3241065 in nfs_client_id_expire
(clientid=0x7f57e3929c00, make_stale=false) at
/usr/src/debug/nfs-ganesha-4.0.2/SAL/nfs4_clientid.c:1093
#9 0x00007f58a32a4441 in nfs4_op_create_session (op=0x7f57f30f5aa0,
data=0x7f5840d16e80, resp=0x7f57f30f5960) at
/usr/src/debug/nfs-ganesha-4.0.2/Protocols/NFS/nfs4_op_create_session.c:444
#10 0x00007f58a329dfc6 in process_one_op (data=0x7f5840d16e80,
status=0x7f588e3fc87c) at
/usr/src/debug/nfs-ganesha-4.0.2/Protocols/NFS/nfs4_Compound.c:924
#11 0x00007f58a329f04e in nfs4_Compound (arg=0x7f57f30e9680,
req=0x7f57f30e8e00, res=0x7f588408ac00) at
/usr/src/debug/nfs-ganesha-4.0.2/Protocols/NFS/nfs4_Compound.c:1339
#12 0x00007f58a31e3664 in nfs_rpc_process_request
(reqdata=0x7f57f30e8e00, retry=false) at
/usr/src/debug/nfs-ganesha-4.0.2/MainNFSD/nfs_worker_thread.c:2079
#13 0x00007f58a31e3cdd in nfs_rpc_valid_NFS (req=0x7f57f30e8e00) at
/usr/src/debug/nfs-ganesha-4.0.2/MainNFSD/nfs_worker_thread.c:2317
#14 0x00007f58a35bdba1 in svc_vc_decode (req=0x7f57f30e8e00) at
/usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/svc_vc.c:1125
#15 0x00007f58a35b8cd3 in svc_request (xprt=0x7f57eab7f600,
xdrs=0x7f588d7cc340) at
/usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/svc_rqst.c:1229
#16 0x00007f58a35bdaa6 in svc_vc_recv (xprt=0x7f57eab7f600) at
/usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/svc_vc.c:1098
#17 0x00007f58a35b8c37 in svc_rqst_xprt_task_recv (wpe=0x7f57eab7f8f0)
at /usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/svc_rqst.c:1209
#18 0x00007f58a35b98ba in svc_rqst_epoll_loop (wpe=0x7f589e2e5030) at
/usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/svc_rqst.c:1608
#19 0x00007f58a35c6dc5 in work_pool_thread (arg=0x7f588ea001e0) at
/usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/work_pool.c:190
#20 0x00007f58a18d4ea5 in start_thread () from /lib64/libpthread.so.0
#21 0x00007f58a11dbb0d in clone () from /lib64/libc.so.6
#0 0x00007f58a18dc4fb in raise () from /lib64/libpthread.so.0
#1 0x00007f58a31e8a36 in crash_handler (signo=6, info=0x7f588e3f8470,
ctx=0x7f588e3f8340) at
/usr/src/debug/nfs-ganesha-4.0.2/MainNFSD/nfs_init.c:261
#2 <signal handler called>
#3 0x00007f58a1113387 in raise () from /lib64/libc.so.6
#4 0x00007f58a1114a78 in abort () from /lib64/libc.so.6
#5 0x00007f58a110c1a6 in __assert_fail_base () from /lib64/libc.so.6
#6 0x00007f58a110c252 in __assert_fail () from /lib64/libc.so.6
#7 0x00007f58a3234338 in dec_state_owner_ref (owner=0x7f57ed99bc00)
at /usr/src/debug/nfs-ganesha-4.0.2/SAL/state_misc.c:933
#8 0x00007f58a3241065 in nfs_client_id_expire
(clientid=0x7f57e3929c00, make_stale=false) at
/usr/src/debug/nfs-ganesha-4.0.2/SAL/nfs4_clientid.c:1093
#9 0x00007f58a32a4441 in nfs4_op_create_session (op=0x7f57f30f5aa0,
data=0x7f5840d16e80, resp=0x7f57f30f5960) at
/usr/src/debug/nfs-ganesha-4.0.2/Protocols/NFS/nfs4_op_create_session.c:444
#10 0x00007f58a329dfc6 in process_one_op (data=0x7f5840d16e80,
status=0x7f588e3fc87c) at
/usr/src/debug/nfs-ganesha-4.0.2/Protocols/NFS/nfs4_Compound.c:924
#11 0x00007f58a329f04e in nfs4_Compound (arg=0x7f57f30e9680,
req=0x7f57f30e8e00, res=0x7f588408ac00) at
/usr/src/debug/nfs-ganesha-4.0.2/Protocols/NFS/nfs4_Compound.c:1339
#12 0x00007f58a31e3664 in nfs_rpc_process_request
(reqdata=0x7f57f30e8e00, retry=false) at
/usr/src/debug/nfs-ganesha-4.0.2/MainNFSD/nfs_worker_thread.c:2079
#13 0x00007f58a31e3cdd in nfs_rpc_valid_NFS (req=0x7f57f30e8e00) at
/usr/src/debug/nfs-ganesha-4.0.2/MainNFSD/nfs_worker_thread.c:2317
#14 0x00007f58a35bdba1 in svc_vc_decode (req=0x7f57f30e8e00) at
/usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/svc_vc.c:1125
#15 0x00007f58a35b8cd3 in svc_request (xprt=0x7f57eab7f600,
xdrs=0x7f588d7cc340) at
/usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/svc_rqst.c:1229
#16 0x00007f58a35bdaa6 in svc_vc_recv (xprt=0x7f57eab7f600) at
/usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/svc_vc.c:1098
#17 0x00007f58a35b8c37 in svc_rqst_xprt_task_recv (wpe=0x7f57eab7f8f0)
at /usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/svc_rqst.c:1209
#18 0x00007f58a35b98ba in svc_rqst_epoll_loop (wpe=0x7f589e2e5030) at
/usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/svc_rqst.c:1608
#19 0x00007f58a35c6dc5 in work_pool_thread (arg=0x7f588ea001e0) at
/usr/src/debug/nfs-ganesha-4.0.2/libntirpc/src/work_pool.c:190
#20 0x00007f58a18d4ea5 in start_thread () from /lib64/libpthread.so.0
#21 0x00007f58a11dbb0d in clone () from /lib64/libc.so.6
(gdb) f 7
#7 0x00007f58a3234338 in dec_state_owner_ref (owner=0x7f57ed99bc00)
at /usr/src/debug/nfs-ganesha-4.0.2/SAL/state_misc.c:933
933 assert(refcount > 0);
(gdb) l
928 if (str_valid)
929 LogFullDebug(COMPONENT_STATE,
930 "Decrement refcount now=%" PRId32 " {%s}",
931 refcount, str);
932
933 assert(refcount > 0);
934
935 return;
936 }
937
(gdb) l -
918 hash_table_t *ht_owner;
919
920 if (isDebug(COMPONENT_STATE)) {
921 display_owner(&dspbuf, owner);
922 str_valid = true;
923 }
924
925 refcount = atomic_dec_int32_t(&owner->so_refcount);
926
927 if (refcount != 0) {
(gdb) p owner->so_refcount
$1 = -1
7 hours, 41 minutes
FW: Announcing the Spring 2023 NFS bake-a-thon
by Frank Filz
-----Original Message-----
From: Steve Dickson [mailto:steved@redhat.com]
Sent: Wednesday, March 15, 2023 11:46 AM
To: nfsv4(a)ietf.org; linux-nfs(a)vger.kernel.org
Subject: Announcing the Spring 2023 NFS bake-a-thon
Hello,
I am pleased to announce the Spring 2023 NFS bake-a-thon, April 24-28, 20223. You can find information on the nfsvbat.org website [1].
Like last spring, we will be running this event in virtual space. Please note the VPN registration and setup instructions so that you can punch into the BAT network.
Look forward to seeing you (virtually) at the next BAT.
steved.
[1] http://nfsv4bat.org/Events/2023/Apr/BAT/index.html
1 week, 1 day
[L] Change in ...nfs-ganesha[next]: free_state may not always be able to be called with a valid export
by Frank Filz (GerritHub)
Frank Filz has uploaded this change for review. ( https://review.gerrithub.io/c/ffilz/nfs-ganesha/+/551045 )
Change subject: free_state may not always be able to be called with a valid export
......................................................................
free_state may not always be able to be called with a valid export
Instead of trying to use the export attached to a state to call
the FSAL free_state, use a function pointer which if non-NULL
is used, otherwise we just gsh_free the state (which works for all
in-tree FSALs).
Change-Id: I1aa7cccc144a50c020ee0f5d450b110686ac6647
Signed-off-by: Frank S. Filz <ffilzlnx(a)mindspring.com>
---
M src/FSAL/FSAL_CEPH/export.c
M src/FSAL/FSAL_CEPH/handle.c
M src/FSAL/FSAL_CEPH/internal.h
M src/FSAL/FSAL_GLUSTER/export.c
M src/FSAL/FSAL_GLUSTER/gluster_internal.h
M src/FSAL/FSAL_GPFS/export.c
M src/FSAL/FSAL_GPFS/fsal_internal.h
M src/FSAL/FSAL_KVSFS/kvsfs_export.c
M src/FSAL/FSAL_KVSFS/kvsfs_methods.h
M src/FSAL/FSAL_LIZARDFS/export.c
M src/FSAL/FSAL_LIZARDFS/lzfs_internal.h
M src/FSAL/FSAL_MEM/mem_export.c
M src/FSAL/FSAL_MEM/mem_int.h
M src/FSAL/FSAL_PROXY_V4/export.c
M src/FSAL/FSAL_PROXY_V4/handle.c
M src/FSAL/FSAL_PROXY_V4/proxyv4_fsal_methods.h
M src/FSAL/FSAL_RGW/handle.c
M src/FSAL/FSAL_VFS/export.c
M src/FSAL/FSAL_VFS/file.c
M src/FSAL/FSAL_VFS/vfs_methods.h
M src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_export.c
M src/FSAL/Stackable_FSALs/FSAL_NULL/export.c
M src/FSAL/default_methods.c
M src/Protocols/9P/9p_proto_tools.c
M src/Protocols/NFS/nfs4_op_open.c
M src/SAL/nfs4_state.c
M src/SAL/nfs4_state_id.c
M src/SAL/nlm_state.c
M src/gtest/fsal_api/test_close2_latency.cc
M src/gtest/fsal_api/test_commit2_latency.cc
M src/gtest/fsal_api/test_open2_latency.cc
M src/gtest/fsal_api/test_read2_latency.cc
M src/gtest/fsal_api/test_reopen2_latency.cc
M src/gtest/fsal_api/test_write2_latency.cc
M src/include/FSAL/fsal_commonlib.h
M src/include/fsal_api.h
M src/include/sal_data.h
37 files changed, 63 insertions(+), 201 deletions(-)
git pull ssh://review.gerrithub.io:29418/ffilz/nfs-ganesha refs/changes/45/551045/1
--
To view, visit https://review.gerrithub.io/c/ffilz/nfs-ganesha/+/551045
To unsubscribe, or for help writing mail filters, visit https://review.gerrithub.io/settings
Gerrit-Project: ffilz/nfs-ganesha
Gerrit-Branch: next
Gerrit-Change-Id: I1aa7cccc144a50c020ee0f5d450b110686ac6647
Gerrit-Change-Number: 551045
Gerrit-PatchSet: 1
Gerrit-Owner: Frank Filz <ffilzlnx(a)mindspring.com>
Gerrit-MessageType: newchange
1 week, 3 days
Announce Push of V4.4.1
by Frank Filz
Branch next
Tag:V4.4.1
Merge Highlights
* COMMIT: fsal_reopen_obj() called with FSAL_O_ANY to try avoid opening
globalfd
* Fix Failed to carry the correct error code when call ceph_fsal_setattr2()
Signed-off-by: Frank S. Filz <ffilzlnx(a)mindspring.com>
Contents:
8debf3b31 Frank S. Filz V4.4.1
138da318b hnkboy Fix Failed to carry the correct error code when call
ceph_fsal_setattr2()
106d5ad10 Arnab Tah COMMIT: fsal_reopen_obj() called with FSAL_O_ANY to try
avoid opening globalfd
2 weeks
Announce Push of V5-dev.4
by Frank Filz
Branch next
Tag:V5-dev.4
NOTE: Major update to LRU that should solve many of the unbounded memory
use issues due to inability to recycle mdcache entries.
Merge Highlights
* Ceph aysnc/non-blocking (dormant feature waiting for ceph changes)
* fsal_fd rework to utilize a locking mechanism that can be held across
threads
* Global fd closing removed from MDCACHE LRU into its own LRU
* LRU revamp to remove long term references from the LRU
* Clean up use of obj_lock now that it isn't used to protect global fd
* Allow read and write FSAL calls to resume to make it possible to avoid
calls into the filesystem on callbacks from the filesystem.
* Some other cleanup related to all the above
* COMMIT: fsal_reopen_obj() called with FSAL_O_ANY to try avoid opening
globalfd
* Fix Failed to carry the correct error code when call ceph_fsal_setattr2()
Signed-off-by: Frank S. Filz <ffilzlnx(a)mindspring.com>
Contents:
24e45d147 Frank S. Filz V5-dev.4
ebfd06bf9 Frank S. Filz FSAL: Remove old fsal_fd management
2cd524697 Frank S. Filz FSAL: Remove seek method
7b1ff6e00 Frank S. Filz 9P: Check file open mode for locks
9627626d8 Frank S. Filz nfs4_op_write.c - fix annoying typo
83fb4ed15 Frank S. Filz FSAL_MEM: Convert to new fsal_fd handling and new
async I/O
f2f6c280a Frank S. Filz FSAL_MEM: Remove non-functional lock_op2
ac7bcdf73 Frank S. Filz MDCACHE: export roots aren't in LRU anymore
8e6637ecb Frank S. Filz MDCACHE: Don't need DQ_SAFE any more
fb7bfe9c6 Frank S. Filz MDCACHE: Strip out old fd management
aa00d4136 Frank S. Filz Use long term references at the protocol layer
7a62f317e Frank S. Filz MDCACHE: Add long term obj handle (ultimately
mdcache) references
c0ae4b4dd Frank S. Filz MDCACHE: Clean up refcounting and LRU
8ae1c343b Frank S. Filz FSAL_CEPH: Implement async
a1ba06f18 Frank S. Filz Protocols: Add support for read2 and write2 that ask
for resume
d6816609b Frank S. Filz MDCACHE: Add support for async FSAL subcall and
supercall
310fee636 Frank S. Filz FSAL_GLUSTER - Convert to new fsal_fd management
5bf71d8ea Frank S. Filz FSAL_LIZARDFS - Convert to new fsal_fd management
9e458e166 Frank S. Filz FSAL_KVSFS: Use new fsal_fd management
31698f39d Frank S. Filz FSAL_GPFS: Use new fsal_fd management
065015da2 Frank S. Filz FSAL_RGW: Don't use old fsal_find_fd just do share
stuff
7c2374e78 Frank S. Filz FSAL_CEPH: Use new fsal_fd management
9a8136292 Frank S. Filz FSAL_VFS: Use new fsal_fd management
70443fbe0 Frank S. Filz FSAL: Add global fd LRU to FSAL layer
138cb514c Frank S. Filz Synchonizing I/O and file open/close/reopen without
holding rwlock
231a86975 Frank S. Filz Make FSAL_VFS compile first
be0d8cacb Frank S. Filz Add atomic_add_unless and
PTHREAD_MUTEX_decrement_XXX_t_and_lock
2fd7359c0 hnkboy Fix Failed to carry the correct error code when call
ceph_fsal_setattr2()
c2363dce8 Arnab Tah COMMIT: fsal_reopen_obj() called with FSAL_O_ANY to try
avoid opening globalfd
2 weeks
Announce Push of V5-dev.3
by Frank Filz
Branch next
Tag:V5-dev.3
NOTE: whatever problem I was having with signed tags seems to be fixed now.
Merge Highlights
* Minor EXPORT CLIENT documentation fix
* Split CLIENT list out to separate handling
* Add support for HA Proxy - NOTE - includes nitirpc pullup
* glusterfs-api-devel -> libgfapi-devel
* config: Fix copy/paste error in %include handling
* recovery: Fix typo in log messages
* docs: Switch GitHub git URLs to https
Signed-off-by: Frank S. Filz <ffilzlnx(a)mindspring.com>
Contents:
91dd6865b Frank S. Filz V5-dev.3
2e24722c6 Martin Schwenke docs: Switch GitHub git URLs to https
5c75a0a5a Martin Schwenke recovery: Fix typo in log messages
b56d4443e Martin Schwenke config: Fix copy/paste error in %include handling
16daf4929 Kaleb S. KEITHLEY glusterfs-api-devel -> libgfapi-devel
37587ad00 Frank S. Filz Add support for HA Proxy - NOTE - includes nitirpc
pullup
ba4ef255a Frank S. Filz Split CLIENT list out to separate handling
f0b2d333d Frank S. Filz Minor EXPORT CLIENT documentation fix
2 weeks, 2 days
[S] Change in ...nfs-ganesha[next]: COMMIT: fsal_reopen_obj() called with FSAL_O_ANY to try avoid opening...
by Arnab Tah (GerritHub)
Arnab Tah has uploaded this change for review. ( https://review.gerrithub.io/c/ffilz/nfs-ganesha/+/550651 )
Change subject: COMMIT: fsal_reopen_obj() called with FSAL_O_ANY to try avoid opening globalfd
......................................................................
COMMIT: fsal_reopen_obj() called with FSAL_O_ANY to try avoid opening globalfd
Fixes: https://github.com/nfs-ganesha/nfs-ganesha/issues/902
FSAL commit2 functions call fsal_reopen_obj() with openflags argument as
FSAL_O_WRITE. Which causes it to generate a global FD (unless there is
already a compatible one open).Since, fsync system call doesn't require
a fd to be open for write; it is adequate to call fsal_reopen_obj() with
openflags as FSAL_O_ANY.
Change-Id: I0a469b6001f911127a0ac34e568a6205fbcd3b2a
Signed-off-by: Arnab Tah <atah(a)ddn.com>
---
M src/FSAL/FSAL_CEPH/handle.c
M src/FSAL/FSAL_GLUSTER/handle.c
M src/FSAL/FSAL_GPFS/file.c
M src/FSAL/FSAL_LIZARDFS/handle.c
M src/FSAL/FSAL_VFS/file.c
5 files changed, 23 insertions(+), 5 deletions(-)
git pull ssh://review.gerrithub.io:29418/ffilz/nfs-ganesha refs/changes/51/550651/1
--
To view, visit https://review.gerrithub.io/c/ffilz/nfs-ganesha/+/550651
To unsubscribe, or for help writing mail filters, visit https://review.gerrithub.io/settings
Gerrit-Project: ffilz/nfs-ganesha
Gerrit-Branch: next
Gerrit-Change-Id: I0a469b6001f911127a0ac34e568a6205fbcd3b2a
Gerrit-Change-Number: 550651
Gerrit-PatchSet: 1
Gerrit-Owner: Arnab Tah <atah(a)ddn.com>
Gerrit-MessageType: newchange
2 weeks, 2 days
NOTE - UNOFFICIAL RELEASE - Announce Push of unofficial-V5-dev.3
by Frank Filz
NOTE: I'm having difficulty getting a proper signed tag for V5-dev.3 but I
didn't want to deprive folks of the merge, so here it is with an unofficial
tag. Next week when I can get some help I will get a proper signed tag put
on (and re-send this announcement).
Thanks for your patience,
Frank Filz
Branch next
Tag:V5-dev.3
Merge Highlights
* Minor EXPORT CLIENT documentation fix
* Split CLIENT list out to separate handling
* Add support for HA Proxy - NOTE - includes nitirpc pullup
* glusterfs-api-devel -> libgfapi-devel
* config: Fix copy/paste error in %include handling
* recovery: Fix typo in log messages
* docs: Switch GitHub git URLs to https
Signed-off-by: Frank S. Filz <ffilzlnx(a)mindspring.com>
Contents:
91dd6865b Frank S. Filz V5-dev.3
2e24722c6 Martin Schwenke docs: Switch GitHub git URLs to https
5c75a0a5a Martin Schwenke recovery: Fix typo in log messages
b56d4443e Martin Schwenke config: Fix copy/paste error in %include handling
16daf4929 Kaleb S. KEITHLEY glusterfs-api-devel -> libgfapi-devel
37587ad00 Frank S. Filz Add support for HA Proxy - NOTE - includes nitirpc
pullup
ba4ef255a Frank S. Filz Split CLIENT list out to separate handling
f0b2d333d Frank S. Filz Minor EXPORT CLIENT documentation fix
3 weeks