Hi all, We have seen multiple crashes (with and without ASAN) Posting the stack traces for reference... Crash 1: (gdb) bt #0  0x00007f6d41f3523b in raise () from /lib64/libpthread.so.0 #1  0x000000000047a9f0 in crash_handler (signo=6, info=0x7f6d1db0a030, ctx=0x7f6d1db09f00) at /usr/src/debug/nfs-ganesha-2.7.5-ibm056.02_asan/MainNFSD/nfs_init.c:244 #2  <signal handler called> #3  0x00007f6d417791d7 in raise () from /lib64/libc.so.6 #4  0x00007f6d4177a8c8 in abort () from /lib64/libc.so.6 #5  0x00007f6d43cb8e6e in __sanitizer::Abort() () from /lib64/libasan.so.5 #6  0x00007f6d43cc1558 in __sanitizer::Die() () from /lib64/libasan.so.5 #7  0x00007f6d43ca4b15 in __asan::ReportGenericError(unsigned long, unsigned long, unsigned long, unsigned long, bool, unsigned long, unsigned int, bool) () from /lib64/libasan.so.5 #8  0x00007f6d43ca5567 in __asan_report_load8 () from /lib64/libasan.so.5 #9  0x00000000005c63ca in process_blocked_lock_upcall (block_data=0x60c0007feac0) at /usr/src/debug/nfs-ganesha-2.7.5-ibm056.02_asan/SAL/state_lock.c:1809 #10 0x00000000005ba962 in state_blocked_lock_caller (ctx=0x616000a6fe80)     at /usr/src/debug/nfs-ganesha-2.7.5-ibm056.02_asan/SAL/state_async.c:81 #11 0x0000000000688479 in fridgethr_start_routine (arg=0x616000a6fe80) at /usr/src/debug/nfs-ganesha-2.7.5-ibm056.02_asan/support/fridgethr.c:554 #12 0x00007f6d41f2ddc5 in start_thread () from /lib64/libpthread.so.0 #13 0x00007f6d4183b73d in clone () from /lib64/libc.so.6 (gdb) f 9 #9  0x00000000005c63ca in process_blocked_lock_upcall (block_data=0x60c0007feac0) at /usr/src/debug/nfs-ganesha-2.7.5-ibm056.02_asan/SAL/state_lock.c:1809 1809            PTHREAD_RWLOCK_unlock(&lock_entry->sle_obj->state_hdl-> state_lock); (gdb) print *block_data $1 = {sbd_list = {next = 0x20000011, prev = 0x0}, sbd_grant_type = STATE_GRANT_POLL,   sbd_block_type = STATE_BLOCK_POLL, sbd_granted_callback = 0x57afd2 <nlm_granted_callback>,   sbd_blocked_cookie = 0x0, sbd_lock_entry = 0x6110007a1240, sbd_prot = {sbd_nlm = {       sbd_nlm_fh = {n_len = 56, n_bytes = 0x60c0007feb00 "C"},       sbd_nlm_fh_buf = "C\000\000\004\060\060\000\n\000\002\000(\000\300 \250z\254^%j\365\002\000\000\002\000\000\000\000\251\344\000\000\000\000 \000\000\000\000\000\000=k@y\273\344", '\000' <repeats 16 times>}, sbd_v4 = 0x38}} (gdb) print *(state_lock_entry_t *)0x6110007a1240 $2 = {sle_list = {next = 0x7400001a, prev = 0x0}, sle_owner_locks = {next = 0x0, prev = 0x0},   sle_client_locks = {next = 0x0, prev = 0x0}, sle_state_locks = {next = 0x0, prev = 0x0},   sle_export_locks = {next = 0x0, prev = 0x0}, sle_export = 0x615000000348,   sle_obj = 0x6190013ce5b8, sle_block_data = 0x60c0007feac0, sle_owner = 0x0,   sle_state = 0x61200159cc40, sle_blocked = STATE_CANCELED, sle_ref_count = 0, sle_lock = {                 <<<  cancelled     lock_sle_type = FSAL_POSIX_LOCK, lock_type = FSAL_LOCK_W, lock_start = 527338800,     lock_length = 517175, lock_reclaim = false}, sle_mutex = {__data = {__lock = 0, __count = 0,       __owner = 0, __nusers = 0, __kind = -1, __spins = 0, __elision = 0, __list = {         __prev = 0x0, __next = 0x0}},     __size = '\000' <repeats 16 times>, "\377\377\377\377", '\000' <repeats 19 times>,     __align = 0}} (gdb) The problem is seen when doing unlock using lock_entry and ASAN reports heap-use-after-free. try_to_grant_lock() finds cancelled block entry and invokes remove_from_locklist() which calls lock_entry_dec_ref() causing free. ************************************************************************************************************************************************************************************ Crash 2: (gdb) bt #0  0x00007f212862923b in raise () from /lib64/libpthread.so.0 #1  0x000000000047a9f0 in crash_handler (signo=6, info=0x7f210bf6d070, ctx=0x7f210bf6cf40) at /usr/src/debug/nfs-ganesha-2.7.5-ibm056.02_asan/MainNFSD/nfs_init.c:244 #2  <signal handler called> #3  0x00007f2127e6d1d7 in raise () from /lib64/libc.so.6 #4  0x00007f2127e6e8c8 in abort () from /lib64/libc.so.6 #5  0x00007f212a3ace6e in __sanitizer::Abort() () from /lib64/libasan.so.5 #6  0x00007f212a3b5558 in __sanitizer::Die() () from /lib64/libasan.so.5 #7  0x00007f212a398b15 in __asan::ReportGenericError(unsigned long, unsigned long, unsigned long, unsigned long, bool, unsigned long, unsigned int, bool) () from /lib64/libasan.so.5 #8  0x00007f212a399567 in __asan_report_load8 () from /lib64/libasan.so.5 #9  0x00000000005ba881 in state_blocked_lock_caller (ctx=0x616000530480)     at /usr/src/debug/nfs-ganesha-2.7.5-ibm056.02_asan/SAL/state_async.c:72 #10 0x0000000000688479 in fridgethr_start_routine (arg=0x616000530480) at /usr/src/debug/nfs-ganesha-2.7.5-ibm056.02_asan/support/fridgethr.c:554 #11 0x00007f2128621dc5 in start_thread () from /lib64/libpthread.so.0 #12 0x00007f2127f2f73d in clone () from /lib64/libc.so.6 (gdb) (gdb) f 9 #9 0x00000000005ba881 in state_blocked_lock_caller (ctx=0x616000530480) at /usr/src/debug/nfs-ganesha-2.7.5-ibm056.02_asan/SAL/state_async.c:72 72 export = block->sbd_lock_entry->sle_export; (gdb) print *block $1 = {sbd_list = {next = 0x72800009, prev = 0x613000a8e128}, sbd_grant_type = STATE_GRANT_NONE, sbd_block_type = STATE_BLOCK_NONE, sbd_granted_callback = 0x0, sbd_blocked_cookie = 0x0, sbd_lock_entry = 0x0, sbd_prot = {sbd_nlm = {sbd_nlm_fh = {n_len = 0, n_bytes = 0x0}, <<<<<<lock_entry is freed, we did state_add_grant_cookie just before sbd_nlm_fh_buf = '\000' <repeats 16 times>, "\002\000\000\000\000\000 \000\000\000\341a\004Pb\000\000\000\341a\004Pb\000\000\334\341a\004Pb\000 \000\000\001b\004Pb\000\000\000\000\000\000\000\000\000"}, sbd_v4 = 0x0}} (gdb) *********************************************************************************************************************************************************************************** Crash 3: (gdb) bt #0  0x00007fd4324824fb in raise (sig=11) at ../nptl/sysdeps/unix/sysv/linux/pt-raise.c:36 #1  0x0000000000443ae5 in crash_handler (signo=11, info=0x7fd00b8049f0, ctx=0x7fd00b8048c0)     at /usr/src/debug/nfs-ganesha-2.7.5-ibm055.00/MainNFSD/nfs_init.c:244 #2  <signal handler called> #3  0x00000000004a3bf1 in nlm_granted_callback (obj=0x7fd36c002ba8, lock_entry=0x7fd304102d80) at /usr/src/debug/nfs-ganesha-2.7.5-ibm055.00/Protocols/NLM/nlm_util.c:609 #4  0x00000000004c217a in try_to_grant_lock (lock_entry=0x7fd304102d80) at /usr/src/debug/nfs-ganesha-2.7.5-ibm055.00/SAL/state_lock.c:1758 #5  0x00000000004c252e in process_blocked_lock_upcall (block_data=0x7fd304102320) at /usr/src/debug/nfs-ganesha-2.7.5-ibm055.00/SAL/state_lock.c:1807 #6  0x00000000004bd43f in state_blocked_lock_caller (ctx=0x7fd0400028d0) at /usr/src/debug/nfs-ganesha-2.7.5-ibm055.00/SAL/state_async.c:81 #7  0x0000000000514b92 in fridgethr_start_routine (arg=0x7fd0400028d0) at /usr/src/debug/nfs-ganesha-2.7.5-ibm055.00/support/fridgethr.c:554 #8  0x00007fd43247aea5 in start_thread (arg=0x7fd00b806700) at pthread_create.c:307 #9  0x00007fd431d818cd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 (gdb) f 3 #3  0x00000000004a3bf1 in nlm_granted_callback (obj=0x7fd36c002ba8, lock_entry=0x7fd304102d80) at /usr/src/debug/nfs-ganesha-2.7.5-ibm055.00/Protocols/NLM/nlm_util.c:609 609             state_nlm_client_t *nlm_grant_client = nlm_grant_owner-> so_client; The nlm_grant_owner comes from lock entry's sle_owner which is NULL. Thanks and regards, Trishali. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Trishali Nayar IBM Systems ETZ, Pune. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~