Adaptive mutex hold: 17362721 events in 60.484 seconds (287064 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 544657 14% 14% 0.00 14444 sellck _cv_timedwait_sig_sbt+0x1bf nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@ 435036 seltdwait+0xba 32768 |@@@@@ 107654 kern_select+0x96f 65536 | 642 sys_select+0x54 131072 | 268 amd64_syscall+0x508 262144 | 158 0xffffffff80f096ab 524288 | 2 1048576 | 1 2097152 | 0 4194304 | 0 8388608 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 219889 11% 25% 0.00 27998 so_rcv tcp_do_segment+0x2385 nsec ------ Time Distribution ------ count Stack 2048 | 13 tcp_input+0x121b 4096 | 3 ip_input+0x15b 8192 | 7259 netisr_dispatch_src+0x61 16384 |@@@@@@@@@@@ 87679 ether_demux+0x13e 32768 |@@@@@@@@@@@@@ 96580 ether_nh_input+0x32e 65536 |@@ 14967 netisr_dispatch_src+0x61 131072 | 2801 ether_input+0x26 262144 |@ 10206 vlan_input+0x18a 524288 | 370 ether_demux+0x91 1048576 | 1 ether_nh_input+0x32e 2097152 | 4 netisr_dispatch_src+0x61 4194304 | 0 ether_input+0x26 8388608 | 6 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 672829 8% 32% 0.00 6516 pipe mutex kern_select+0x7df nsec ------ Time Distribution ------ count Stack 1024 | 877 sys_select+0x54 2048 | 1615 amd64_syscall+0x508 4096 | 1738 0xffffffff80f096ab 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 632140 16384 |@ 24100 32768 | 2749 65536 | 2607 131072 | 3513 262144 | 2145 524288 | 53 1048576 | 2 2097152 | 1 4194304 | 0 8388608 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 207586 6% 39% 0.00 17743 sg_lock svc_vc_soupcall+0x35 nsec ------ Time Distribution ------ count Stack 1024 | 6128 sowakeup+0x83 2048 | 5652 tcp_do_segment+0x2385 4096 | 27 tcp_input+0x121b 8192 | 4 ip_input+0x15b 16384 |@@@@@@@@@@@@@@@@@@@@ 141586 netisr_dispatch_src+0x61 32768 |@@@@@@ 45808 ether_demux+0x13e 65536 | 858 ether_nh_input+0x32e 131072 | 3303 netisr_dispatch_src+0x61 262144 | 4172 ether_input+0x26 524288 | 40 vlan_input+0x18a 1048576 | 0 ether_demux+0x91 2097152 | 4 ether_nh_input+0x32e 4194304 | 0 netisr_dispatch_src+0x61 8388608 | 4 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 262032 3% 42% 0.00 6712 rtentry ip_output+0x447 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 234107 tcp_output+0x1a21 16384 |@@ 24289 tcp_do_segment+0x27dc 32768 | 1992 tcp_input+0x121b 65536 | 787 ip_input+0x15b 131072 | 638 netisr_dispatch_src+0x61 262144 | 213 ether_demux+0x13e 524288 | 1 ether_nh_input+0x32e 1048576 | 1 netisr_dispatch_src+0x61 2097152 | 1 ether_input+0x26 4194304 | 0 vlan_input+0x18a 8388608 | 3 ether_demux+0x91 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1133915 3% 45% 0.00 1527 igb3:rx(7) igb_msix_que+0x167 nsec ------ Time Distribution ------ count Stack 1024 |@@@@ 181018 intr_event_execute_handlers+0x1b6 2048 |@@@@@@@@@@@@@@@@@@@@@@ 830398 ithread_loop+0xa6 4096 |@@ 110933 fork_exit+0x9c 8192 | 6339 0xffffffff80f098fe 16384 | 1421 32768 | 680 65536 | 551 131072 | 3 262144 | 0 524288 | 0 1048576 | 0 2097152 | 0 4194304 | 0 8388608 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 701724 2% 47% 0.00 1928 so_snd tcp_do_segment+0x27dc nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@ 162867 tcp_input+0x121b 2048 |@@@@@@@@@@@@@@@ 368212 ip_input+0x15b 4096 |@@@@@@ 145082 netisr_dispatch_src+0x61 8192 | 17804 ether_demux+0x13e 16384 | 4292 ether_nh_input+0x32e 32768 | 536 netisr_dispatch_src+0x61 65536 | 348 ether_input+0x26 131072 | 316 vlan_input+0x18a 262144 | 202 ether_demux+0x91 524288 | 0 ether_nh_input+0x32e 1048576 | 1 netisr_dispatch_src+0x61 2097152 | 1 ether_input+0x26 4194304 | 0 igb_rxeof+0x755 8388608 | 1 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 77200 2% 50% 0.00 17017 so_snd tcp_do_segment+0x12ae nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@ 19901 tcp_input+0x121b 16384 |@@@@@@@@@@ 28133 ip_input+0x15b 32768 |@@@@@@@@@ 24125 netisr_dispatch_src+0x61 65536 |@ 4243 ether_demux+0x13e 131072 | 386 ether_nh_input+0x32e 262144 | 402 netisr_dispatch_src+0x61 524288 | 9 ether_input+0x26 1048576 | 0 vlan_input+0x18a 2097152 | 1 ether_demux+0x91 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 56453 2% 52% 0.00 22933 so_rcv tcp_do_segment+0x272c nsec ------ Time Distribution ------ count Stack 2048 | 34 tcp_input+0x121b 4096 | 7 ip_input+0x15b 8192 |@@@@@ 10201 netisr_dispatch_src+0x61 16384 |@@@@@@@@@@@ 21692 ether_demux+0x13e 32768 |@@@@@@@@@ 18122 ether_nh_input+0x32e 65536 |@@ 3772 netisr_dispatch_src+0x61 131072 | 499 ether_input+0x26 262144 |@ 1907 vlan_input+0x18a 524288 | 59 ether_demux+0x91 1048576 | 3 ether_nh_input+0x32e 2097152 | 0 netisr_dispatch_src+0x61 4194304 | 0 ether_input+0x26 8388608 | 2 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 75422 2% 54% 0.00 16207 so_snd tcp_do_segment+0x2cbc nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@ 19540 tcp_input+0x121b 16384 |@@@@@@@@@@@ 29652 ip_input+0x15b 32768 |@@@@@@@@ 21940 netisr_dispatch_src+0x61 65536 |@ 3457 ether_demux+0x13e 131072 | 375 ether_nh_input+0x32e 262144 | 94 netisr_dispatch_src+0x61 524288 | 8 ether_input+0x26 1048576 | 1 vlan_input+0x18a 2097152 | 0 ether_demux+0x91 4194304 | 0 ether_nh_input+0x32e 8388608 | 2 netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Spin lock hold: 20733975 events in 60.484 seconds (342802 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2604 3% 3% 0.00 783131015 sched lock 26 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 40 taskqueue_thread_loop+0x131 32768 | 57 fork_exit+0x9c 65536 | 51 0xffffffff80f098fe 131072 |@ 100 262144 |@@ 220 524288 |@ 131 1048576 |@ 118 2097152 |@ 112 4194304 |@ 159 8388608 |@ 152 16777216 |@@ 191 33554432 |@ 157 67108864 |@ 118 134217728 |@ 139 268435456 |@@ 178 536870912 |@ 159 1073741824 |@ 162 2147483648 |@ 95 4294967296 | 35 8589934592 | 78 17179869184 | 47 34359738368 | 16 68719476736 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3048 3% 6% 0.00 614444805 sched lock 29 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 33 taskqueue_thread_loop+0x131 32768 | 77 fork_exit+0x9c 65536 | 90 0xffffffff80f098fe 131072 |@ 119 262144 |@@ 211 524288 |@ 181 1048576 |@ 114 2097152 | 100 4194304 |@ 160 8388608 |@ 181 16777216 |@@ 244 33554432 |@@ 212 67108864 |@ 185 134217728 |@ 165 268435456 |@@ 210 536870912 |@@ 248 1073741824 |@@ 224 2147483648 |@ 122 4294967296 | 49 8589934592 | 70 17179869184 | 40 34359738368 | 8 68719476736 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2964 3% 9% 0.00 608141359 sched lock 27 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 52 taskqueue_thread_loop+0x131 32768 | 77 fork_exit+0x9c 65536 | 82 0xffffffff80f098fe 131072 |@ 117 262144 |@@ 290 524288 |@@ 203 1048576 |@ 133 2097152 |@ 121 4194304 |@ 159 8388608 |@ 189 16777216 |@@ 233 33554432 |@@ 199 67108864 |@ 184 134217728 |@ 151 268435456 |@ 153 536870912 |@ 186 1073741824 |@ 174 2147483648 |@ 105 4294967296 | 39 8589934592 | 66 17179869184 | 31 34359738368 | 14 68719476736 | 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3293 3% 12% 0.00 536747128 sched lock 31 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 34 taskqueue_thread_loop+0x131 32768 | 101 fork_exit+0x9c 65536 | 76 0xffffffff80f098fe 131072 |@ 125 262144 |@@ 270 524288 |@ 190 1048576 |@ 143 2097152 |@ 144 4194304 |@ 185 8388608 |@@ 230 16777216 |@@ 233 33554432 |@@ 274 67108864 |@ 172 134217728 |@ 169 268435456 |@ 202 536870912 |@@ 258 1073741824 |@ 209 2147483648 |@ 110 4294967296 | 47 8589934592 | 73 17179869184 | 33 34359738368 | 12 68719476736 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 836 3% 14% 0.00 2058154643 sched lock 15 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 1 taskqueue_thread_loop+0x131 32768 | 14 fork_exit+0x9c 65536 | 7 0xffffffff80f098fe 131072 | 21 262144 |@ 43 524288 |@ 34 1048576 | 23 2097152 | 27 4194304 |@ 28 8388608 |@ 34 16777216 |@ 35 33554432 |@ 41 67108864 |@ 46 134217728 |@ 43 268435456 |@ 50 536870912 |@@ 67 1073741824 |@@@ 89 2147483648 |@@ 72 4294967296 |@ 54 8589934592 |@ 53 17179869184 |@ 28 34359738368 | 17 68719476736 | 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2296 3% 17% 0.00 746227524 sched lock 5 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 18 taskqueue_thread_loop+0x131 32768 | 55 fork_exit+0x9c 65536 | 51 0xffffffff80f098fe 131072 | 70 262144 |@@ 174 524288 |@ 98 1048576 |@ 81 2097152 |@ 78 4194304 |@ 84 8388608 |@ 108 16777216 |@ 128 33554432 |@@ 178 67108864 |@ 136 134217728 |@ 130 268435456 |@@ 141 536870912 |@@ 159 1073741824 |@@ 147 2147483648 |@ 105 4294967296 | 48 8589934592 |@ 79 17179869184 | 25 34359738368 | 13 68719476736 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1793 2% 20% 0.00 865934323 sched lock 9 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 12 taskqueue_thread_loop+0x131 32768 | 26 fork_exit+0x9c 65536 | 41 0xffffffff80f098fe 131072 | 41 262144 |@ 96 524288 |@ 88 1048576 |@ 70 2097152 |@ 74 4194304 |@ 69 8388608 |@ 80 16777216 |@ 98 33554432 |@@ 139 67108864 |@ 111 134217728 |@ 110 268435456 |@ 101 536870912 |@@ 162 1073741824 |@@ 174 2147483648 |@@ 133 4294967296 | 59 8589934592 |@ 68 17179869184 | 34 34359738368 | 5 68719476736 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2334 2% 22% 0.00 656132491 sched lock 25 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 26 taskqueue_thread_loop+0x131 32768 | 58 fork_exit+0x9c 65536 | 54 0xffffffff80f098fe 131072 | 68 262144 |@@ 161 524288 |@ 115 1048576 |@ 80 2097152 |@ 105 4194304 |@ 104 8388608 |@ 154 16777216 |@@ 174 33554432 |@@ 196 67108864 |@ 131 134217728 |@ 142 268435456 |@ 151 536870912 |@@ 168 1073741824 |@@ 190 2147483648 |@ 112 4294967296 | 43 8589934592 | 64 17179869184 | 25 34359738368 | 9 68719476736 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1744 2% 24% 0.00 874875463 sched lock 6 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 17 taskqueue_thread_loop+0x131 32768 | 48 fork_exit+0x9c 65536 | 44 0xffffffff80f098fe 131072 |@ 71 262144 |@@ 137 524288 |@ 93 1048576 |@ 72 2097152 | 46 4194304 |@ 83 8388608 |@ 70 16777216 |@ 111 33554432 |@@ 120 67108864 |@ 106 134217728 |@ 101 268435456 |@ 113 536870912 |@@ 151 1073741824 |@@ 132 2147483648 |@ 97 4294967296 | 41 8589934592 | 46 17179869184 | 25 34359738368 | 14 68719476736 | 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2442 2% 27% 0.00 619949011 sched lock 20 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 19 taskqueue_thread_loop+0x131 32768 | 79 fork_exit+0x9c 65536 | 63 0xffffffff80f098fe 131072 |@ 98 262144 |@@ 206 524288 |@ 162 1048576 |@ 130 2097152 |@ 131 4194304 |@ 146 8388608 |@ 146 16777216 |@@ 180 33554432 |@@ 174 67108864 |@ 136 134217728 |@ 132 268435456 |@ 151 536870912 |@ 144 1073741824 |@ 148 2147483648 | 69 4294967296 | 36 8589934592 | 50 17179869184 | 25 34359738368 | 12 68719476736 | 5 ------------------------------------------------------------------------------- R/W writer hold: 2225471 events in 60.484 seconds (36794 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 645513 83% 83% 0.00 71672 tcpinp tcp_input+0x121b nsec ------ Time Distribution ------ count Stack 8192 | 2167 ip_input+0x15b 16384 |@ 35409 netisr_dispatch_src+0x61 32768 |@@@ 75246 ether_demux+0x13e 65536 |@@@@@@@@@@@ 255820 ether_nh_input+0x32e 131072 |@@@@@@@@@@ 216240 netisr_dispatch_src+0x61 262144 |@@ 51601 ether_input+0x26 524288 | 8143 vlan_input+0x18a 1048576 | 34 ether_demux+0x91 2097152 | 34 ether_nh_input+0x32e 4194304 | 0 netisr_dispatch_src+0x61 8388608 | 59 ether_input+0x26 16777216 | 3 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 75718 8% 91% 0.00 58976 tcpinp sosend_generic+0x570 nsec ------ Time Distribution ------ count Stack 16384 | 1 svc_vc_reply+0x16a 32768 | 620 svc_sendreply_common+0x9a 65536 |@@@@@@@@@@@@@@@@@@@@@@@@ 61389 svc_sendreply_mbuf+0x5a 131072 |@@@ 8483 nfssvc_program+0x879 262144 |@ 3772 svc_run_internal+0xf69 524288 | 809 svc_thread_start+0xb 1048576 | 13 fork_exit+0x9c 2097152 | 1 0xffffffff80f098fe 4194304 | 0 8388608 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1074713 3% 94% 0.00 1348 vm object sysctl_root_handler_locked+0xb1 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@ 334753 sysctl_root+0x1e1 2048 |@@@@@@@@@@@@@@@@@@@@ 728825 userland_sysctl+0x1b6 4096 | 8146 sys___sysctl+0x74 8192 | 175 amd64_syscall+0x508 16384 | 994 0xffffffff80f096ab 32768 | 64 65536 | 278 131072 | 1193 262144 | 280 524288 | 2 1048576 | 0 2097152 | 0 4194304 | 1 8388608 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 149601 2% 96% 0.00 6955 tcpinp soreceive_generic+0xfe1 nsec ------ Time Distribution ------ count Stack 4096 | 7 svc_vc_recv+0x14f 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@ 125834 svc_run_internal+0x4e6 16384 |@@@@ 21552 svc_thread_start+0xb 32768 | 658 fork_exit+0x9c 65536 | 298 0xffffffff80f098fe 131072 | 239 262144 | 566 524288 | 21 1048576 | 0 2097152 | 0 4194304 | 0 8388608 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 59498 2% 97% 0.00 14917 tcpinp sosend_generic+0x570 nsec ------ Time Distribution ------ count Stack 8192 | 882 soo_write+0x49 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 53753 dofilewrite+0x89 32768 | 1437 kern_writev+0x68 65536 | 1779 sys_write+0x60 131072 | 922 amd64_syscall+0x508 262144 | 526 0xffffffff80f096ab 524288 | 31 1048576 | 0 2097152 | 0 4194304 | 0 8388608 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4792 1% 98% 0.00 59276 tcpinp tcp_input+0x121b nsec ------ Time Distribution ------ count Stack 8192 | 1 ip_input+0x15b 16384 |@@@@@@@ 1218 netisr_dispatch_src+0x61 32768 |@@@ 521 ether_demux+0x13e 65536 |@@@@@@@@@@@ 1777 ether_nh_input+0x32e 131072 |@@@@@ 957 netisr_dispatch_src+0x61 262144 |@ 198 ether_input+0x26 524288 | 118 vlan_input+0x18a 1048576 | 0 ether_demux+0x91 2097152 | 1 ether_nh_input+0x32e 4194304 | 0 netisr_dispatch_src+0x61 8388608 | 1 ether_input+0x26 igb_rxeof+0x755 igb_handle_que+0x39 taskqueue_run_locked+0x13b taskqueue_thread_loop+0xd8 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 16236 0% 98% 0.00 13403 tcpinp sosend_generic+0x570 nsec ------ Time Distribution ------ count Stack 8192 | 127 soo_write+0x49 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 15113 dofilewrite+0x89 32768 | 474 kern_writev+0x68 65536 | 182 sys_writev+0x36 131072 | 177 amd64_syscall+0x508 262144 | 130 0xffffffff80f096ab 524288 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 136135 0% 99% 0.00 1460 vm object zfs_fhtovp+0x459 nsec ------ Time Distribution ------ count Stack 1024 |@ 7522 nfsvno_fhtovp+0x72 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 125208 nfsd_fhtovp+0xc1 4096 | 3017 nfsrvd_dorpc+0xe34 8192 | 35 nfssvc_program+0x58e 16384 | 118 svc_run_internal+0xf69 32768 | 46 svc_thread_start+0xb 65536 | 49 fork_exit+0x9c 131072 | 93 0xffffffff80f098fe 262144 | 45 524288 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2710 0% 99% 0.00 52138 tcpinp tcp_input+0x121b nsec ------ Time Distribution ------ count Stack 16384 |@@@ 317 ip_input+0x15b 32768 |@@@@@@@@@@ 958 swi_net+0x1b3 65536 |@@@@@@@@@ 900 intr_event_execute_handlers+0x1b6 131072 |@@@ 331 ithread_loop+0xa6 262144 |@ 168 fork_exit+0x9c 524288 | 35 0xffffffff80f098fe 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8929 0% 99% 0.00 13835 vm object vm_fault_hold+0x1ea8 nsec ------ Time Distribution ------ count Stack 8192 |@ 533 vm_fault+0x78 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 7899 trap_pfault+0xfa 32768 | 253 trap+0x518 65536 | 53 0xffffffff80f093c7 131072 | 161 262144 | 29 524288 | 0 1048576 | 0 2097152 | 0 4194304 | 0 8388608 | 0 16777216 | 1 ------------------------------------------------------------------------------- R/W reader hold: 2121478 events in 60.484 seconds (35075 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 649843 26% 26% 0.00 1997 pcbinfohash in_pcblookup_hash+0x237 nsec ------ Time Distribution ------ count Stack 1024 | 13 tcp_input+0x764 2048 |@@@@@@@@@@@@@@@@@@@@@@ 477895 ip_input+0x15b 4096 |@@@@@@@ 168851 netisr_dispatch_src+0x61 8192 | 343 ether_demux+0x13e 16384 | 965 ether_nh_input+0x32e 32768 | 219 netisr_dispatch_src+0x61 65536 | 186 ether_input+0x26 131072 | 872 vlan_input+0x18a 262144 | 339 ether_demux+0x91 524288 | 2 ether_nh_input+0x32e 1048576 | 0 netisr_dispatch_src+0x61 2097152 | 2 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 262281 15% 41% 0.00 2826 rib head lock rtalloc1_fib+0x12a nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 134011 rtalloc_ign_fib+0x117 4096 |@@@@@@@@@@@@ 112815 ip_output+0x447 8192 | 8463 tcp_output+0x1a21 16384 | 5131 tcp_do_segment+0x27dc 32768 | 794 tcp_input+0x121b 65536 | 384 ip_input+0x15b 131072 | 281 netisr_dispatch_src+0x61 262144 | 145 ether_demux+0x13e 524288 | 6 ether_nh_input+0x32e 1048576 | 0 netisr_dispatch_src+0x61 2097152 | 1 ether_input+0x26 4194304 | 0 vlan_input+0x18a 8388608 | 1 ether_demux+0x91 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 177084 11% 52% 0.00 3022 rib head lock rtalloc1_fib+0x12a nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 89769 rtalloc_ign_fib+0x117 4096 |@@@@@@@@@@@@ 74798 ip_findroute+0x3c 8192 |@ 7482 ip_tryforward+0x1ed 16384 | 3524 ip_input+0x351 32768 | 592 netisr_dispatch_src+0x61 65536 | 392 ether_demux+0x13e 131072 | 353 ether_nh_input+0x32e 262144 | 165 netisr_dispatch_src+0x61 524288 | 3 ether_input+0x26 1048576 | 0 vlan_input+0x18a 2097152 | 0 ether_demux+0x91 4194304 | 0 ether_nh_input+0x32e 8388608 | 1 netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 262187 9% 60% 0.00 1644 if_afdata arpresolve+0x1f7 nsec ------ Time Distribution ------ count Stack 1024 | 3 ether_output+0x1b5 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@ 222930 ip_output+0x12e3 4096 |@@@@ 38464 tcp_output+0x1a21 8192 | 71 tcp_do_segment+0x27dc 16384 | 385 tcp_input+0x121b 32768 | 125 ip_input+0x15b 65536 | 102 netisr_dispatch_src+0x61 131072 | 83 ether_demux+0x13e 262144 | 24 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 vlan_input+0x18a ether_demux+0x91 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 82448 6% 66% 0.00 3465 rib head lock rtalloc1_fib+0x12a nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@ 20416 rtalloc_ign_fib+0x117 4096 |@@@@@@@@@@@@@@@@@@@@ 55715 ip_output+0x447 8192 | 2389 tcp_output+0x1a21 16384 | 1256 tcp_usr_send+0x417 32768 | 146 sosend_generic+0x570 65536 | 164 svc_vc_reply+0x16a 131072 | 273 svc_sendreply_common+0x9a 262144 | 211 svc_sendreply_mbuf+0x5a 524288 | 9 nfssvc_program+0x879 svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 153042 5% 72% 0.00 1772 radix node head vfs_stdcheckexp+0xd1 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@ 128614 nfsvno_v4rootexport+0x2c 4096 |@@@@ 22946 nfssvc_program+0x397 8192 | 29 svc_run_internal+0xf69 16384 | 149 svc_thread_start+0xb 32768 | 51 fork_exit+0x9c 65536 | 42 0xffffffff80f098fe 131072 | 98 262144 | 71 524288 | 2 1048576 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 177070 5% 77% 0.00 1501 if_afdata arpresolve+0x1f7 nsec ------ Time Distribution ------ count Stack 1024 | 195 ether_output+0x1b5 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 160821 ip_tryforward+0x689 4096 |@@ 15635 ip_input+0x351 8192 | 61 netisr_dispatch_src+0x61 16384 | 198 ether_demux+0x13e 32768 | 102 ether_nh_input+0x32e 65536 | 54 netisr_dispatch_src+0x61 131072 | 1 ether_input+0x26 262144 | 3 vlan_input+0x18a ether_demux+0x91 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 137023 5% 82% 0.00 1690 radix node head vfs_stdcheckexp+0xd1 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@ 116728 nfsvno_fhtovp+0xd5 4096 |@@@@ 18816 nfsd_fhtovp+0xc1 8192 | 28 nfsrvd_dorpc+0xe34 16384 | 113 nfssvc_program+0x58e 32768 | 47 svc_run_internal+0xf69 65536 | 57 svc_thread_start+0xb 131072 | 95 fork_exit+0x9c 262144 | 41 0xffffffff80f098fe 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 54500 5% 86% 0.00 4175 Name Cache cache_lookup+0x648 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@ 35644 vfs_cache_lookup+0xac 4096 |@@@ 7049 VOP_LOOKUP_APV+0xa0 8192 | 76 lookup+0x59e 16384 |@@@@@ 10315 nfsvno_namei+0x243 32768 | 553 nfsrvd_lookup+0x13b 65536 | 91 nfsrvd_dorpc+0xbe5 131072 | 147 nfssvc_program+0x58e 262144 | 110 svc_run_internal+0xf69 524288 | 6 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 78780 3% 89% 0.00 1686 if_afdata arpresolve+0x1f7 nsec ------ Time Distribution ------ count Stack 1024 | 3 ether_output+0x1b5 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 67681 ip_output+0x12e3 4096 |@@@ 8738 tcp_output+0x1a21 8192 | 11 tcp_usr_send+0x417 16384 | 86 sosend_generic+0x570 32768 | 24 svc_vc_reply+0x16a 65536 | 37 svc_sendreply_common+0x9a 131072 | 53 svc_sendreply_mbuf+0x5a 262144 | 38 nfssvc_program+0x879 524288 | 1 svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- SX shared hold: 6171642 events in 60.484 seconds (102038 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 138870 23% 23% 0.00 102476 zfsvfs->z_hold_mtx[i] zfs_zget+0x1aa nsec ------ Time Distribution ------ count Stack 32768 | 2 zfs_fhtovp+0x30a 65536 | 2473 nfsvno_fhtovp+0x72 131072 |@@@@@@@@@@@@@@@@@@@@@@@@ 114380 nfsd_fhtovp+0xc1 262144 |@@@ 17045 nfsrvd_dorpc+0xe34 524288 | 3719 nfssvc_program+0x58e 1048576 | 42 svc_run_internal+0xf69 2097152 | 16 svc_thread_start+0xb 4194304 | 1 fork_exit+0x9c 8388608 | 16 0xffffffff80f098fe 16777216 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 152177 14% 37% 0.00 56710 xprt->xp_lock svc_run_internal+0x4e6 nsec ------ Time Distribution ------ count Stack 2048 | 17 svc_thread_start+0xb 4096 | 40 fork_exit+0x9c 8192 | 921 0xffffffff80f098fe 16384 | 302 32768 |@@@@@@@@@@@ 59566 65536 |@@@@@@@@@@@ 59954 131072 |@@@ 17354 262144 |@@ 10410 524288 | 2114 1048576 | 42 2097152 | 14 4194304 | 1 8388608 | 13 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 149980 11% 48% 0.00 45604 so_rcv_sx soreceive_generic+0x1358 nsec ------ Time Distribution ------ count Stack 8192 | 18 svc_vc_recv+0x14f 16384 | 1281 svc_run_internal+0x4e6 32768 |@@@@@@@@@@@@@@@@@@ 91553 svc_thread_start+0xb 65536 |@@@@@@ 33075 fork_exit+0x9c 131072 |@@ 12473 0xffffffff80f098fe 262144 |@ 9033 524288 | 1524 1048576 | 37 2097152 | 8 4194304 | 0 8388608 | 12 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 75710 10% 58% 0.00 80809 so_snd_sx sosend_generic+0x99c nsec ------ Time Distribution ------ count Stack 32768 | 2 svc_vc_reply+0x16a 65536 |@@@@@@@@@@@@@@@@@@ 46611 svc_sendreply_common+0x9a 131072 |@@@@@@@ 18667 svc_sendreply_mbuf+0x5a 262144 |@@ 7090 nfssvc_program+0x879 524288 | 1869 svc_run_internal+0xf69 1048576 | 36 svc_thread_start+0xb 2097152 | 3 fork_exit+0x9c 4194304 | 0 0xffffffff80f098fe 8388608 | 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 100801 5% 63% 0.00 30536 so_snd_sx sosend_generic+0x99c nsec ------ Time Distribution ------ count Stack 4096 | 153 soo_write+0x49 8192 |@@@@@@@@@@@ 39526 dofilewrite+0x89 16384 | 1609 kern_writev+0x68 32768 |@@@@@@@@ 28745 sys_write+0x60 65536 |@@@@@@ 20552 amd64_syscall+0x508 131072 |@ 6638 0xffffffff80f096ab 262144 | 2557 524288 | 432 1048576 | 7 2097152 | 2 4194304 | 0 8388608 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 59607 3% 65% 0.00 28149 db->db_mtx dbuf_hold_impl+0x4b8 nsec ------ Time Distribution ------ count Stack 8192 | 478 dbuf_hold+0x25 16384 | 93 dmu_buf_hold_array_by_dnode+0x130 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 53710 dmu_read_uio_dnode+0x40 65536 |@ 2353 dmu_read_uio_dbuf+0x3b 131072 |@ 2173 zfs_freebsd_read+0x5fd 262144 | 727 VOP_READ_APV+0xa0 524288 | 67 vn_read+0x15b 1048576 | 1 vn_io_fault+0x10b 2097152 | 3 dofileread+0x98 4194304 | 0 kern_readv+0x68 8388608 | 2 sys_read+0x60 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 59194 3% 68% 0.00 27914 db->db_mtx dmu_read_uio_dnode+0xe4 nsec ------ Time Distribution ------ count Stack 1024 | 108 dmu_read_uio_dbuf+0x3b 2048 | 385 zfs_freebsd_read+0x5fd 4096 | 13 VOP_READ_APV+0xa0 8192 | 0 vn_read+0x15b 16384 | 6 vn_io_fault+0x10b 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 53378 dofileread+0x98 65536 | 1769 kern_readv+0x68 131072 |@ 2125 sys_read+0x60 262144 | 670 amd64_syscall+0x508 524288 | 71 0xffffffff80f096ab 1048576 | 0 2097152 | 2 4194304 | 0 8388608 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15772 2% 70% 0.00 95931 zfsvfs->z_hold_mtx[i] zfs_zget+0x1aa nsec ------ Time Distribution ------ count Stack 65536 |@ 693 zfs_root+0x61 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@ 13220 lookup+0x82d 262144 |@@ 1430 nfsvno_namei+0x243 524288 | 403 nfsrvd_lookup+0x13b 1048576 | 1 nfsrvd_dorpc+0xbe5 2097152 | 1 nfssvc_program+0x58e 4194304 | 0 svc_run_internal+0xf69 8388608 | 4 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 139907 2% 72% 0.00 8778 zp->z_lock zfs_zget+0x188 nsec ------ Time Distribution ------ count Stack 1024 | 1960 zfs_fhtovp+0x30a 2048 |@@@@ 19618 nfsvno_fhtovp+0x72 4096 | 552 nfsd_fhtovp+0xc1 8192 |@@@@@@@@@@@@@@@ 71399 nfsrvd_dorpc+0xe34 16384 |@@@@@@@ 35890 nfssvc_program+0x58e 32768 |@ 7308 svc_run_internal+0xf69 65536 | 602 svc_thread_start+0xb 131072 | 978 fork_exit+0x9c 262144 | 580 0xffffffff80f098fe 524288 | 35 1048576 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 150699 2% 74% 0.00 6883 db->db_mtx dbuf_hold_impl+0x4b8 nsec ------ Time Distribution ------ count Stack 4096 | 44 dbuf_hold+0x25 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@ 125898 dnode_hold_impl+0x137 16384 |@@@@ 20842 dmu_bonus_hold+0x1d 32768 | 635 zfs_zget+0xdf 65536 | 449 zfs_fhtovp+0x30a 131072 | 573 nfsvno_fhtovp+0x72 262144 | 448 nfsd_fhtovp+0xc1 524288 | 8 nfsrvd_dorpc+0xe34 1048576 | 1 nfssvc_program+0x58e 2097152 | 1 svc_run_internal+0xf69 4194304 | 0 svc_thread_start+0xb 8388608 | 2 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- SX exclusive hold: 1076133 events in 60.484 seconds (17792 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7618 25% 25% 0.00 452498 dn->dn_struct_rwlock dmu_buf_hold_array_by_dnode+0x1e2 nsec ------ Time Distribution ------ count Stack 32768 | 131 dmu_read_uio_dnode+0x40 65536 | 136 dmu_read_uio_dbuf+0x3b 131072 | 47 zfs_freebsd_read+0x5fd 262144 |@@ 662 VOP_READ_APV+0xa0 524288 |@@@@@@@@@@@@@@@@ 4143 vn_read+0x15b 1048576 |@@@@@@@@@ 2475 vn_io_fault+0x10b 2097152 | 14 dofileread+0x98 4194304 | 3 kern_readv+0x68 8388608 | 7 sys_read+0x60 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 149331 16% 41% 0.00 14371 dn->dn_struct_rwlock dnode_hold_impl+0x17f nsec ------ Time Distribution ------ count Stack 8192 | 1 dmu_bonus_hold+0x1d 16384 |@@@@@@@@@@@@@@@@@@@@@@@@ 121918 zfs_zget+0xdf 32768 |@@@@ 22782 zfs_fhtovp+0x30a 65536 | 1042 nfsvno_fhtovp+0x72 131072 | 1345 nfsd_fhtovp+0xc1 262144 | 1008 nfsrvd_dorpc+0xe34 524288 | 48 nfssvc_program+0x58e 1048576 | 2 svc_run_internal+0xf69 2097152 | 1 svc_thread_start+0xb 4194304 | 0 fork_exit+0x9c 8388608 | 2 0xffffffff80f098fe 16777216 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 147947 11% 52% 0.00 10039 dn->dn_struct_rwlock dbuf_read+0x416 nsec ------ Time Distribution ------ count Stack 4096 | 1 dnode_hold_impl+0x199 8192 |@@@@@@@@@@@@@@ 70713 dmu_bonus_hold+0x1d 16384 |@@@@@@@@@@@@@@ 68856 zfs_zget+0xdf 32768 |@ 5373 zfs_fhtovp+0x30a 65536 | 804 nfsvno_fhtovp+0x72 131072 | 1026 nfsd_fhtovp+0xc1 262144 | 684 nfsrvd_dorpc+0xe34 524288 | 25 nfssvc_program+0x58e 1048576 | 2 svc_run_internal+0xf69 2097152 | 4 svc_thread_start+0xb 4194304 | 0 fork_exit+0x9c 8388608 | 2 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 144372 8% 59% 0.00 7292 dn->dn_struct_rwlock dbuf_read+0x416 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@ 118120 dmu_bonus_hold+0x15a 16384 |@@@@ 23172 zfs_zget+0xdf 32768 | 748 zfs_fhtovp+0x30a 65536 | 431 nfsvno_fhtovp+0x72 131072 | 631 nfsd_fhtovp+0xc1 262144 | 444 nfsrvd_dorpc+0xe34 524288 | 16 nfssvc_program+0x58e 1048576 | 1 svc_run_internal+0xf69 2097152 | 0 svc_thread_start+0xb 4194304 | 0 fork_exit+0x9c 8388608 | 1 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 143081 7% 67% 0.00 7142 dn->dn_struct_rwlock dmu_object_info_from_db+0x2d nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@ 116990 zfs_zget+0xf8 16384 |@@@@ 22652 zfs_fhtovp+0x30a 32768 | 732 nfsvno_fhtovp+0x72 65536 | 406 nfsd_fhtovp+0xc1 131072 | 633 nfsrvd_dorpc+0xe34 262144 | 389 nfssvc_program+0x58e 524288 | 8 svc_run_internal+0xf69 1048576 | 0 svc_thread_start+0xb 2097152 | 0 fork_exit+0x9c 4194304 | 1 0xffffffff80f098fe 8388608 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2075 5% 72% 0.00 339110 dn->dn_struct_rwlock dmu_buf_hold_array_by_dnode+0x1e2 nsec ------ Time Distribution ------ count Stack 16384 |@ 84 dmu_read_uio_dnode+0x40 32768 |@@@@@@@@@ 645 dmu_read_uio_dbuf+0x3b 65536 |@@@@@@@@ 575 zfs_freebsd_read+0x5fd 131072 |@@ 182 VOP_READ_APV+0xa0 262144 |@ 133 vn_read+0x15b 524288 | 50 vn_io_fault+0x10b 1048576 | 29 dofileread+0x98 2097152 |@@@@@ 347 kern_preadv+0x92 4194304 | 12 sys_pread+0x66 8388608 | 7 amd64_syscall+0x508 16777216 | 2 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8935 5% 77% 0.00 73321 vm map (user) unlock_and_deallocate+0x27e nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@ 6084 vm_fault_hold+0x1ea8 131072 |@@@@@@ 1906 vm_fault+0x78 262144 |@@ 834 trap_pfault+0xfa 524288 | 107 trap+0x518 1048576 | 1 0xffffffff80f093c7 2097152 | 0 4194304 | 1 8388608 | 0 16777216 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 82861 5% 81% 0.00 7603 zfsvfs->z_teardown_inactive_lock zfs_freebsd_inactive+0xd nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@ 66574 VOP_INACTIVE_APV+0xa6 16384 |@@@@ 13552 vinactive+0x189 32768 | 553 vputx+0x2ef 65536 | 305 nfsrvd_dorpc+0x1760 131072 | 410 nfssvc_program+0x58e 262144 | 290 svc_run_internal+0xf69 524288 | 6 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 19048 2% 83% 0.00 14257 dn->dn_struct_rwlock dnode_hold_impl+0x17f nsec ------ Time Distribution ------ count Stack 8192 | 11 dmu_bonus_hold+0x1d 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@ 16026 zfs_zget+0xdf 32768 |@@@ 2255 zfs_root+0x61 65536 | 129 lookup+0x82d 131072 | 173 nfsvno_namei+0x243 262144 | 129 nfsrvd_lookup+0x13b 524288 | 4 nfsrvd_dorpc+0xbe5 1048576 | 0 nfssvc_program+0x58e 2097152 | 0 svc_run_internal+0xf69 4194304 | 0 svc_thread_start+0xb 8388608 | 2 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 147119 2% 85% 0.00 1725 dn->dn_struct_rwlock dmu_bonus_hold+0x139 nsec ------ Time Distribution ------ count Stack 1024 | 7 zfs_zget+0xdf 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@ 124530 zfs_fhtovp+0x30a 4096 |@@@@ 20727 nfsvno_fhtovp+0x72 8192 | 32 nfsd_fhtovp+0xc1 16384 | 128 nfsrvd_dorpc+0xe34 32768 | 51 nfssvc_program+0x58e 65536 | 58 svc_run_internal+0xf69 131072 | 108 svc_thread_start+0xb 262144 | 51 fork_exit+0x9c 524288 | 1 0xffffffff80f098fe -------------------------------------------------------------------------------