Good morning everybody. Does anybody can help me in case on memory leak of nfs-ganesha? After 4-7 hours of copy files from local disk to nfs share I catch OOM killer because of nfs-ganesha use all hosts memory. I've changed all parameters in my ganesha config file in MDCACHE block - nothing is changed - memory leak is present. Please help!!! My host CPU - 8 RAM - 16 GB My os: cat /etc/os-release NAME="CentOS Stream" VERSION="8" kernel 4.18.0-383.el8.x86_64 Version nfs-ganesha-4.0-1.el8s.x86_64 my config is: cat /etc/ganesha/ganesha.conf NFS_CORE_PARAM { # possible to mount with NFSv3 to NFSv4 Pseudo path mount_path_pseudo = true; # NFS protocol Protocols = "3","4"; # NFS protocols supported #Use a non-privileged port for RQuota Rquota_Port = 875; } MDCACHE { Entries_HWMark = 500; Chunks_HWMark = 500; Entries_Release_Size = 500; FD_HWMark_Percent = 10; FD_LWMark_Percent = 5; FD_Limit_Percent = 2; } NFSv4 { Lease_Lifetime = 30; Grace_Period = 40; } EXPORT_DEFAULTS { # default access mode Access_Type = RW; } LOG { Default_log_level = WARN; COMPONENTS { ALL = WARN; } FACILITY { name = FILE; destination = "/var/log/ganesha/ganesha.log"; max_level = WARN; enable = active; } } %include "/etc/ganesha/conf.d/ERPAS2_01.conf" cat /etc/ganesha/conf.d/ERPAS2_01.conf EXPORT{ Export_Id = 1001 ; # Unique identifier for each EXPORT (share) Path = "/mnt/gluster/nfs/ERPAS2_01"; # Export path of our NFS share FSAL { name = GLUSTER; # Backing type is Gluster hostname = "localhost"; # Hostname of Gluster server volume = "ERPAS2_01"; # The name of our Gluster volume } Squash = No_root_squash; # Control NFS root squashing Disable_ACL = FALSE; # Enable NFSv4 ACLs Pseudo = "/ERPAS2_01"; # NFSv4 pseudo path for our NFS share Transports = "UDP","TCP" ; # Transport protocols supported SecType = "sys"; # NFS Security flavors supported } Error is: [99062.180392] ip invoked oom-killer: gfp_mask=0x6200ca(GFP_HIGHUSER_MOVABLE), order=0, oom_score_adj=0 [99062.180399] CPU: 4 PID: 1104454 Comm: ip Not tainted 4.18.0-383.el8.x86_64 #1 [99062.180402] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018 [99062.180404] Call Trace: [99062.180407] dump_stack+0x41/0x60 [99062.180414] dump_header+0x4a/0x1db [99062.180420] oom_kill_process.cold.32+0xb/0x10 [99062.180424] out_of_memory+0x1bd/0x4e0 [99062.180427] __alloc_pages_slowpath+0xbdc/0xcc0 [99062.180434] __alloc_pages_nodemask+0x2db/0x310 [99062.180437] alloc_pages_vma+0x74/0x1d0 [99062.180440] do_fault+0x6f/0x3c0 [99062.180444] __handle_mm_fault+0x4a3/0x7e0 [99062.180447] handle_mm_fault+0xc1/0x1e0 [99062.180449] do_user_addr_fault+0x1b5/0x440 [99062.180454] do_page_fault+0x37/0x130 [99062.180457] ? page_fault+0x8/0x30 [99062.180461] page_fault+0x1e/0x30 [99062.180464] RIP: 0033:0x7f152c145f23 [99062.180466] Code: 0f 7f 44 17 f0 f3 0f 7f 07 c3 48 83 fa 40 77 16 f3 0f 7f 07 f3 0f 7f 47 10 f3 0f 7f 44 17 f0 f3 0f 7f 44 17 e0 c3 48 8d 4f 40 <f3> 0f 7f 07 48 83 e1 c0 f3 0f 7f 44 17 f0 f3 0f 7f 47 10 f3 0f 7f [99062.180469] RSP: 002b:00007ffcc4cc6e98 EFLAGS: 00010206 [99062.180471] RAX: 00007f152b0e0160 RBX: 00007f152c3634f0 RCX: 00007f152b0e01a0 [99062.180472] RDX: 00000000000000c8 RSI: 0000000000000000 RDI: 00007f152b0e0160 [99062.180474] RBP: 00007ffcc4cc7190 R08: 00007f152b0e0160 R09: 0000000000082000 [99062.180475] R10: 0000000000000002 R11: 00007f152b0e0228 R12: 00007ffcc4cc6ed0 [99062.180476] R13: 00007ffcc4cc7278 R14: 00007ffcc4cc7230 R15: 000000000000fd00 [99062.180479] Mem-Info: [99062.180480] active_anon:3490970 inactive_anon:388300 isolated_anon:0 active_file:294 inactive_file:499 isolated_file:0 unevictable:51586 dirty:2 writeback:0 slab_reclaimable:40628 slab_unreclaimable:42036 mapped:14000 shmem:8598 pagetables:13124 bounce:0 free:33881 free_pcp:314 free_cma:0 [99062.180484] Node 0 active_anon:13963880kB inactive_anon:1553200kB active_file:1176kB inactive_file:1996kB unevictable:206344kB isolated(anon):0kB isolated(file):0kB mapped:56000kB dirty:8kB writeback:0kB shmem:34392kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 438272kB writeback_tmp:0kB kernel_stack:11280kB pagetables:52496kB all_unreclaimable? no [99062.180489] Node 0 DMA free:13296kB min:60kB low:72kB high:84kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [99062.180493] lowmem_reserve[]: 0 2960 15964 15964 15964 [99062.180496] Node 0 DMA32 free:65668kB min:12520kB low:15648kB high:18776kB active_anon:2800952kB inactive_anon:122084kB active_file:612kB inactive_file:1352kB unevictable:3072kB writepending:0kB present:3129216kB managed:3063680kB mlocked:3072kB bounce:0kB free_pcp:1348kB local_pcp:436kB free_cma:0kB [99062.180501] lowmem_reserve[]: 0 0 13003 13003 13003 [99062.180504] Node 0 Normal free:56560kB min:57044kB low:70792kB high:84540kB active_anon:11162928kB inactive_anon:1431116kB active_file:684kB inactive_file:0kB unevictable:203272kB writepending:0kB present:13631488kB managed:13323868kB mlocked:203272kB bounce:0kB free_pcp:60kB local_pcp:28kB free_cma:0kB [99062.180525] lowmem_reserve[]: 0 0 0 0 0 [99062.180528] Node 0 DMA: 0*4kB 0*8kB 1*16kB (U) 1*32kB (U) 1*64kB (U) 1*128kB (U) 1*256kB (U) 1*512kB (U) 0*1024kB 2*2048kB (UM) 2*4096kB (M) = 13296kB [99062.180540] Node 0 DMA32: 64*4kB (UMEH) 309*8kB (UMEH) 288*16kB (UMEH) 277*32kB (UMEH) 210*64kB (UMEH) 123*128kB (UMEH) 48*256kB (UME) 14*512kB (UMEH) 1*1024kB (H) 0*2048kB 0*4096kB = 65864kB [99062.180554] Node 0 Normal: 1267*4kB (UME) 1715*8kB (UMEH) 1186*16kB (UMEH) 362*32kB (UMEH) 95*64kB (UMH) 2*128kB (H) 2*256kB (H) 2*512kB (H) 0*1024kB 0*2048kB 0*4096kB = 57220kB [99062.180566] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [99062.180567] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [99062.180569] 22686 total pagecache pages [99062.180570] 8450 pages in swap cache [99062.180571] Swap cache stats: add 1539241, delete 1530735, find 450286/879391 [99062.180572] Free swap = 0kB [99062.180573] Total swap = 1048572kB [99062.180573] 4194174 pages RAM [99062.180574] 0 pages HighMem/MovableOnly [99062.180575] 93447 pages reserved [99062.180575] 0 pages hwpoisoned [99062.180576] Tasks state (memory values in pages): [99062.180577] [ pid ] uid tgid total_vm rss pgtables_bytes swapents oom_score_adj name [99062.180584] [ 1247] 0 1247 36819 583 278528 356 0 systemd-journal [99062.180587] [ 1292] 0 1292 28913 583 229376 540 -1000 systemd-udevd [99062.180591] [ 1562] 0 1562 34363 6702 212992 0 -1000 dmeventd [99062.180595] [ 1663] 32 1663 16806 836 180224 191 0 rpcbind [99062.180597] [ 1665] 0 1665 16996 61 135168 128 -1000 auditd [99062.180599] [ 1687] 0 1687 31249 633 151552 128 0 irqbalance [99062.180602] [ 1688] 0 1688 21554 586 192512 423 0 VGAuthService [99062.180604] [ 1689] 0 1689 108649 190 327680 333 0 vmtoolsd [99062.180606] [ 1693] 0 1693 48584 827 380928 438 0 sssd [99062.180608] [ 1694] 81 1694 19260 645 167936 89 -900 dbus-daemon [99062.180610] [ 1698] 0 1698 74837 4194 491520 3293 0 firewalld [99062.180612] [ 1702] 995 1702 37792 643 188416 131 0 chronyd [99062.180614] [ 1723] 0 1723 50567 437 405504 556 0 sssd_be [99062.180616] [ 1724] 0 1724 81574 1582 532480 447 0 sssd_be [99062.180617] [ 1731] 0 1731 48705 897 425984 263 0 sssd_nss [99062.180619] [ 1732] 0 1732 43458 1038 376832 312 0 sssd_pam [99062.180621] [ 1733] 0 1733 41696 779 368640 314 0 sssd_ssh [99062.180623] [ 1745] 0 1745 22633 848 200704 164 0 systemd-logind [99062.180625] [ 1792] 0 1792 97932 785 372736 435 0 NetworkManager [99062.180626] [ 1810] 0 1810 59491 5083 471040 8707 0 puppet [99062.180628] [ 1812] 0 1812 12849 331 139264 115 0 oddjobd [99062.180630] [ 1813] 0 1813 10448 330 106496 106 0 rhsmcertd [99062.180632] [ 1815] 0 1815 23096 810 196608 195 -1000 sshd [99062.180634] [ 1819] 0 1819 42373 306 196608 449 0 gssproxy [99062.180636] [ 1839] 0 1839 13953 168 126976 141 0 rpc.gssd [99062.180638] [ 1844] 0 1844 9240 478 98304 159 0 crond [99062.180640] [ 1849] 0 1849 3416 304 65536 30 0 agetty [99062.180642] [ 1917] 998 1917 509185 431 389120 2164 0 polkitd [99062.180644] [ 2075] 0 2075 167591 49448 622592 0 0 corosync [99062.180646] [ 2077] 0 2077 500773 8644 765952 22760 0 pcsd [99062.180650] [ 2079] 0 2079 182806 426 1015808 753 0 rsyslogd [99062.180652] [ 2138] 0 2138 33747 1103 286720 250 0 pacemakerd [99062.180653] [ 2139] 0 2139 28235 2551 241664 554 0 corosync-qdevic [99062.180655] [ 2140] 0 2140 15745 285 163840 105 0 corosync-qdevic [99062.180657] [ 2142] 189 2142 39236 1390 327680 2046 0 pacemaker-based [99062.180659] [ 2143] 0 2143 38434 960 327680 983 0 pacemaker-fence [99062.180661] [ 2144] 0 2144 29008 1099 241664 185 0 pacemaker-execd [99062.180662] [ 2145] 189 2145 36482 971 311296 812 0 pacemaker-attrd [99062.180664] [ 2146] 189 2146 33508 909 274432 524 0 pacemaker-sched [99062.180666] [ 2147] 189 2147 42562 1608 352256 614 0 pacemaker-contr [99062.180667] [ 2355] 0 2355 53990 3288 434176 2991 0 pcsd [99062.180669] [ 2463] 994 2463 15215 307 114688 85 0 zabbix_agentd [99062.180671] [ 2464] 994 2464 15215 346 114688 71 0 zabbix_agentd [99062.180673] [ 2465] 994 2465 15215 5 114688 86 0 zabbix_agentd [99062.180675] [ 2466] 994 2466 15215 5 114688 86 0 zabbix_agentd [99062.180677] [ 2467] 994 2467 15215 5 114688 86 0 zabbix_agentd [99062.180679] [ 2468] 994 2468 17719 226 118784 62 0 zabbix_agentd [99062.180681] [ 2519] 0 2519 51925 2682 311296 3107 0 glusterd [99062.180683] [ 2529] 0 2529 73517 2718 401408 2010 0 glusterfsd [99062.180684] [ 2548] 0 2548 134829 54153 917504 12465 0 glusterfsd [99062.180687] [ 2566] 0 2566 73516 3168 409600 2109 0 glusterfsd [99062.180688] [ 2599] 0 2599 74030 2680 401408 1962 0 glusterfsd [99062.180690] [ 2618] 0 2618 158724 2394 544768 8404 0 glusterfs [99062.180692] [ 2907] 29 2907 17963 358 180224 6461 0 rpc.statd [99062.180694] [ 140707] 0 140707 73516 2153 409600 2116 0 glusterfsd [99062.180696] [ 259891] 994 259891 475267 4331 524288 1006 0 zabbix_agent2 [99062.180699] [ 565190] 0 565190 4665647 3754190 32448512 165004 0 ganesha.nfsd [99062.180701] [ 998412] 1000 998412 22410 1222 208896 0 0 systemd [99062.180703] [ 998416] 1000 998416 81160 188 327680 763 0 (sd-pam) [99062.180705] [1007136] 0 1007136 44579 915 331776 115 0 sshd [99062.180708] [1007138] 1000 1007138 44662 501 319488 122 0 sshd [99062.180710] [1007139] 1000 1007139 6262 568 81920 265 0 bash [99062.180711] [1007501] 1000 1007501 35836 796 274432 2 0 sudo [99062.180714] [1007502] 0 1007502 6828 896 81920 0 0 bash [99062.180716] [1084448] 28 1084448 124401 86 188416 37 0 nscd [99062.180719] [1103885] 0 1103885 133802 14911 528384 1451 0 puppet [99062.180721] [1104142] 0 1104142 9321 519 102400 0 0 vi [99062.180722] [1104188] 994 1104188 35836 778 282624 0 0 sudo [99062.180724] [1104189] 0 1104189 17150 2620 176128 0 0 python3 [99062.180726] [1104230] 0 1104230 35394 1898 229376 0 0 gluster [99062.180727] [1104236] 0 1104236 58344 2528 352256 0 0 glfsheal [99062.180729] [1104429] 0 1104429 49186 5027 393216 0 0 firewall-cmd [99062.180731] [1104437] 0 1104437 6794 653 86016 0 0 IPaddr2 [99062.180733] [1104450] 0 1104450 6794 376 73728 0 0 IPaddr2 [99062.180734] [1104453] 0 1104453 6794 507 73728 0 0 IPaddr2 [99062.180737] [1104454] 0 1104454 5166 141 73728 0 0 ip [99062.180738] [1104455] 0 1104455 266 1 45056 0 0 awk [99062.180740] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/system.slice/nfs-ganesha.service,task=ganesha.nfsd,pid=565190,uid=0 [99062.180949] Out of memory: Killed process 565190 (ganesha.nfsd) total-vm:18662588kB, anon-rss:15016760kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:31688kB oom_score_adj:0 [99063.227775] oom_reaper: reaped process 565190 (ganesha.nfsd), now anon-rss:0kB, file-rss:0kB, shmem-rss:0kB