Adaptive mutex hold: 55816587 events in 70.288 seconds (794116 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 243181 7% 7% 0.00 98232 vm page vm_fault+0x78 nsec ------ Time Distribution ------ count Stack 2048 | 53 trap_pfault+0xfa 4096 | 0 trap+0x518 8192 |@@@@@@@@@@@ 91683 0xffffffff80f093c7 16384 |@@@@@@@@@@@@@@@@@ 140951 32768 | 2891 65536 | 1242 131072 | 2426 262144 | 1656 524288 | 648 1048576 | 508 2097152 | 329 4194304 | 173 8388608 | 117 16777216 | 147 33554432 | 176 67108864 | 114 134217728 | 54 268435456 | 12 536870912 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 479675 6% 13% 0.00 37267 vnode interlock vputx+0x322 nsec ------ Time Distribution ------ count Stack 1024 | 130 nfsrvd_dorpc+0x1760 2048 |@@@@@ 84195 nfssvc_program+0x58e 4096 | 5242 svc_run_internal+0xf69 8192 |@@@@ 65123 svc_thread_start+0xb 16384 |@@@@@@@@@@@@@@@@@ 272191 fork_exit+0x9c 32768 |@ 16204 0xffffffff80f098fe 65536 | 5530 131072 | 8536 262144 | 8721 524288 | 6156 1048576 | 4815 2097152 | 2054 4194304 | 608 8388608 | 150 16777216 | 19 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 5% 18% 0.00 5068618859 vm object_list sysctl_root_handler_locked+0xb1 nsec ------ Time Distribution ------ count Stack 8589934592 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 3 sysctl_root+0x1e1 userland_sysctl+0x1b6 sys___sysctl+0x74 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 378137 4% 22% 0.00 32947 so_rcv tcp_do_segment+0x2385 nsec ------ Time Distribution ------ count Stack 1024 | 1 tcp_input+0x121b 2048 |@@@@ 51026 ip_input+0x15b 4096 |@@@@@@ 83434 netisr_dispatch_src+0x61 8192 |@ 20208 ether_demux+0x13e 16384 |@@@@@@@ 93763 ether_nh_input+0x32e 32768 |@@ 36239 netisr_dispatch_src+0x61 65536 |@@@ 46092 ether_input+0x26 131072 |@ 20698 vlan_input+0x18a 262144 |@ 20449 ether_demux+0x91 524288 | 5559 ether_nh_input+0x32e 1048576 | 621 netisr_dispatch_src+0x61 2097152 | 13 ether_input+0x26 4194304 | 6 igb_rxeof+0x755 8388608 | 25 igb_msix_que+0x167 16777216 | 3 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 492760 4% 25% 0.00 24327 sellck _cv_timedwait_sig_sbt+0x1bf nsec ------ Time Distribution ------ count Stack 16384 |@ 17333 seltdwait+0xba 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 458810 kern_select+0x96f 65536 | 15438 sys_select+0x54 131072 | 667 amd64_syscall+0x508 262144 | 384 0xffffffff80f096ab 524288 | 73 1048576 | 41 2097152 | 10 4194304 | 3 8388608 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 267876 3% 29% 0.00 38825 vnode interlock vref+0x15 nsec ------ Time Distribution ------ count Stack 2048 | 20 zfs_zget+0x174 4096 | 9 zfs_fhtovp+0x30a 8192 |@@@ 34670 nfsvno_fhtovp+0x72 16384 |@@@@@@@@@@@@@@@@@@@@@@ 199732 nfsd_fhtovp+0xc1 32768 |@ 11239 nfsrvd_dorpc+0xe34 65536 | 3410 nfssvc_program+0x58e 131072 | 5464 svc_run_internal+0xf69 262144 | 5403 svc_thread_start+0xb 524288 | 3737 fork_exit+0x9c 1048576 | 2740 0xffffffff80f098fe 2097152 | 1068 4194304 | 312 8388608 | 61 16777216 | 11 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1114236 3% 32% 0.00 8764 sg_lock svc_thread_start+0xb nsec ------ Time Distribution ------ count Stack 1024 | 2101 fork_exit+0x9c 2048 |@@@@@@@@@@@@@@@@@@@@@@ 831670 0xffffffff80f098fe 4096 |@ 41011 8192 | 212 16384 | 6981 32768 |@@@@ 165287 65536 |@ 48717 131072 | 10114 262144 | 6839 524288 | 1201 1048576 | 80 2097152 | 7 4194304 | 0 8388608 | 16 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 153587 3% 35% 0.00 61540 MPT2SAS lock intr_event_execute_handlers+0x1b6 nsec ------ Time Distribution ------ count Stack 1024 | 144 ithread_loop+0xa6 2048 |@@@@ 23536 fork_exit+0x9c 4096 | 182 0xffffffff80f098fe 8192 | 1 16384 | 4826 32768 |@@@@ 24088 65536 |@@@@@@@@@@@@ 64817 131072 |@@@@ 21855 262144 |@ 9280 524288 | 3821 1048576 | 863 2097152 | 143 4194304 | 20 8388608 | 7 16777216 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 564931 2% 37% 0.00 12632 pipe mutex kern_select+0x7df nsec ------ Time Distribution ------ count Stack 1024 | 11 sys_select+0x54 2048 | 5027 amd64_syscall+0x508 4096 | 216 0xffffffff80f096ab 8192 |@@@@@@@@@@@@@@@@ 303819 16384 |@@@@@@@@@@@@ 232832 32768 | 6098 65536 | 5618 131072 | 5379 262144 | 4098 524288 | 1100 1048576 | 507 2097152 | 156 4194304 | 47 8388608 | 21 16777216 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 300028 2% 39% 0.00 22521 sg_lock _cv_timedwait_sig_sbt+0x1bf nsec ------ Time Distribution ------ count Stack 16384 |@@@ 30447 svc_run_internal+0x910 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 264363 svc_thread_start+0xb 65536 | 4360 fork_exit+0x9c 131072 | 502 0xffffffff80f098fe 262144 | 315 524288 | 39 1048576 | 1 2097152 | 1 ------------------------------------------------------------------------------- Spin lock hold: 45627755 events in 70.288 seconds (649157 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10815 2% 2% 0.00 478920662 sched lock 0 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 4 taskqueue_thread_loop+0x131 32768 | 291 fork_exit+0x9c 65536 | 165 0xffffffff80f098fe 131072 | 93 262144 | 138 524288 | 122 1048576 | 230 2097152 |@ 371 4194304 |@ 590 8388608 |@@ 919 16777216 |@@ 1048 33554432 |@@@ 1087 67108864 |@@ 967 134217728 |@@ 1077 268435456 |@@@ 1197 536870912 |@@@ 1133 1073741824 |@ 713 2147483648 | 277 4294967296 | 114 8589934592 | 165 17179869184 | 70 34359738368 | 31 68719476736 | 13 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10832 2% 3% 0.00 476381679 sched lock 13 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 6 taskqueue_thread_loop+0x131 32768 | 325 fork_exit+0x9c 65536 | 149 0xffffffff80f098fe 131072 | 100 262144 | 145 524288 | 186 1048576 | 225 2097152 | 309 4194304 |@ 532 8388608 |@@ 899 16777216 |@@@ 1104 33554432 |@@ 1018 67108864 |@@ 1033 134217728 |@@@ 1094 268435456 |@@@ 1187 536870912 |@@ 1072 1073741824 |@@ 756 2147483648 | 282 4294967296 | 106 8589934592 | 171 17179869184 | 88 34359738368 | 40 68719476736 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9729 2% 5% 0.00 529961518 sched lock 16 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 2 taskqueue_thread_loop+0x131 32768 | 230 fork_exit+0x9c 65536 | 129 0xffffffff80f098fe 131072 | 89 262144 | 146 524288 | 164 1048576 | 187 2097152 | 304 4194304 |@ 462 8388608 |@@ 804 16777216 |@@ 902 33554432 |@@ 849 67108864 |@@ 888 134217728 |@@@ 999 268435456 |@@@ 1085 536870912 |@@@ 1049 1073741824 |@@ 745 2147483648 | 305 4294967296 | 125 8589934592 | 124 17179869184 | 93 34359738368 | 38 68719476736 | 10 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10359 2% 6% 0.00 497056919 sched lock 17 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 3 taskqueue_thread_loop+0x131 32768 | 264 fork_exit+0x9c 65536 | 149 0xffffffff80f098fe 131072 | 122 262144 | 146 524288 | 180 1048576 | 209 2097152 | 331 4194304 |@ 488 8388608 |@@ 851 16777216 |@@@ 1052 33554432 |@@ 1004 67108864 |@@ 896 134217728 |@@ 1030 268435456 |@@@ 1082 536870912 |@@@ 1100 1073741824 |@@ 735 2147483648 | 310 4294967296 | 106 8589934592 | 164 17179869184 | 101 34359738368 | 31 68719476736 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 13279 2% 8% 0.00 382084549 sched lock 31 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 7 taskqueue_thread_loop+0x131 32768 | 353 fork_exit+0x9c 65536 | 150 0xffffffff80f098fe 131072 | 118 262144 | 207 524288 | 217 1048576 | 271 2097152 |@ 464 4194304 |@ 707 8388608 |@@ 1224 16777216 |@@@ 1456 33554432 |@@@ 1350 67108864 |@@ 1260 134217728 |@@@ 1410 268435456 |@@@ 1532 536870912 |@@ 1247 1073741824 |@ 667 2147483648 | 250 4294967296 | 100 8589934592 | 176 17179869184 | 79 34359738368 | 25 68719476736 | 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9540 2% 9% 0.00 529093698 sched lock 15 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 4 taskqueue_thread_loop+0x131 32768 | 262 fork_exit+0x9c 65536 | 100 0xffffffff80f098fe 131072 | 78 262144 | 121 524288 | 137 1048576 | 177 2097152 | 269 4194304 |@ 446 8388608 |@@ 775 16777216 |@@ 904 33554432 |@@ 846 67108864 |@@ 909 134217728 |@@ 952 268435456 |@@@ 1087 536870912 |@@@ 1088 1073741824 |@@ 719 2147483648 | 265 4294967296 | 116 8589934592 | 151 17179869184 | 93 34359738368 | 32 68719476736 | 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9473 2% 11% 0.00 531402111 sched lock 9 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 3 taskqueue_thread_loop+0x131 32768 | 235 fork_exit+0x9c 65536 | 140 0xffffffff80f098fe 131072 | 98 262144 | 130 524288 | 132 1048576 | 171 2097152 | 270 4194304 |@ 465 8388608 |@@ 723 16777216 |@@ 845 33554432 |@@ 877 67108864 |@@ 883 134217728 |@@@ 1006 268435456 |@@@ 1095 536870912 |@@@ 1084 1073741824 |@@ 634 2147483648 | 276 4294967296 | 111 8589934592 | 169 17179869184 | 85 34359738368 | 34 68719476736 | 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9338 2% 13% 0.00 538898472 sched lock 12 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 3 taskqueue_thread_loop+0x131 32768 | 239 fork_exit+0x9c 65536 | 122 0xffffffff80f098fe 131072 | 87 262144 | 128 524288 | 126 1048576 | 157 2097152 | 262 4194304 |@ 409 8388608 |@@ 712 16777216 |@@ 833 33554432 |@@ 815 67108864 |@@ 889 134217728 |@@@ 974 268435456 |@@@ 1071 536870912 |@@@ 1109 1073741824 |@@ 757 2147483648 | 249 4294967296 | 113 8589934592 | 153 17179869184 | 91 34359738368 | 30 68719476736 | 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11706 2% 14% 0.00 428935739 sched lock 30 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 10 taskqueue_thread_loop+0x131 32768 | 315 fork_exit+0x9c 65536 | 155 0xffffffff80f098fe 131072 | 79 262144 | 169 524288 | 212 1048576 | 278 2097152 | 385 4194304 |@ 624 8388608 |@@ 1062 16777216 |@@@ 1237 33554432 |@@@ 1180 67108864 |@@ 1013 134217728 |@@@ 1172 268435456 |@@@ 1289 536870912 |@@@ 1202 1073741824 |@ 686 2147483648 | 260 4294967296 | 99 8589934592 | 157 17179869184 | 88 34359738368 | 24 68719476736 | 10 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9767 2% 16% 0.00 512019205 sched lock 5 _sleep+0x2df nsec ------ Time Distribution ------ count Stack 16384 | 2 taskqueue_thread_loop+0x131 32768 | 252 fork_exit+0x9c 65536 | 134 0xffffffff80f098fe 131072 | 97 262144 | 169 524288 | 140 1048576 | 212 2097152 | 305 4194304 |@ 505 8388608 |@@ 802 16777216 |@@ 939 33554432 |@@ 914 67108864 |@@ 807 134217728 |@@ 965 268435456 |@@@ 1058 536870912 |@@@ 1086 1073741824 |@@ 691 2147483648 | 317 4294967296 | 107 8589934592 | 143 17179869184 | 78 34359738368 | 34 68719476736 | 10 ------------------------------------------------------------------------------- R/W writer hold: 10064812 events in 70.288 seconds (143195 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1356934 45% 45% 0.00 97669 tcpinp tcp_input+0x121b nsec ------ Time Distribution ------ count Stack 8192 | 48 ip_input+0x15b 16384 | 27817 netisr_dispatch_src+0x61 32768 |@@@ 136577 ether_demux+0x13e 65536 |@@@@@@@ 326456 ether_nh_input+0x32e 131072 |@@@@@@@@@@@@@ 631674 netisr_dispatch_src+0x61 262144 |@@@@ 188170 ether_input+0x26 524288 | 42938 vlan_input+0x18a 1048576 | 2872 ether_demux+0x91 2097152 | 147 ether_nh_input+0x32e 4194304 | 16 netisr_dispatch_src+0x61 8388608 | 207 ether_input+0x26 16777216 | 12 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 556871 22% 66% 0.00 114831 tcpinp sosend_generic+0x570 nsec ------ Time Distribution ------ count Stack 16384 | 12 svc_vc_reply+0x16a 32768 | 20 svc_sendreply_common+0x9a 65536 |@@@@@@@@@ 175361 svc_sendreply_mbuf+0x5a 131072 |@@@@@@@@@@@@@@ 265777 nfssvc_program+0x879 262144 |@@@@ 78820 svc_run_internal+0xf69 524288 |@ 29389 svc_thread_start+0xb 1048576 | 6152 fork_exit+0x9c 2097152 | 1025 0xffffffff80f098fe 4194304 | 192 8388608 | 106 16777216 | 16 33554432 | 0 67108864 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 226463 8% 74% 0.00 109838 vm object vm_fault_hold+0x1ea8 nsec ------ Time Distribution ------ count Stack 8192 | 4 vm_fault+0x78 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 198046 trap_pfault+0xfa 32768 |@@ 16644 trap+0x518 65536 | 2154 0xffffffff80f093c7 131072 | 4269 262144 | 2777 524288 | 818 1048576 | 615 2097152 | 361 4194304 | 180 8388608 | 115 16777216 | 141 33554432 | 165 67108864 | 108 134217728 | 54 268435456 | 11 536870912 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 181963 6% 81% 0.00 105881 tcpinp tcp_input+0x121b nsec ------ Time Distribution ------ count Stack 8192 | 18 ip_input+0x15b 16384 | 3809 netisr_dispatch_src+0x61 32768 |@@ 13309 ether_demux+0x13e 65536 |@@@@ 29008 ether_nh_input+0x32e 131072 |@@@@@@@@@@@@@@@@ 99574 netisr_dispatch_src+0x61 262144 |@@@@ 30060 ether_input+0x26 524288 | 5991 vlan_input+0x18a 1048576 | 122 ether_demux+0x91 2097152 | 21 ether_nh_input+0x32e 4194304 | 0 netisr_dispatch_src+0x61 8388608 | 50 ether_input+0x26 16777216 | 1 igb_rxeof+0x755 igb_handle_que+0x39 taskqueue_run_locked+0x13b taskqueue_thread_loop+0xd8 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5766310 4% 85% 0.00 2162 vm object sysctl_root_handler_locked+0xb1 nsec ------ Time Distribution ------ count Stack 1024 | 11048 sysctl_root+0x1e1 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 5112711 userland_sysctl+0x1b6 4096 |@@@ 617613 sys___sysctl+0x74 8192 | 218 amd64_syscall+0x508 16384 | 3851 0xffffffff80f096ab 32768 | 5358 65536 | 4638 131072 | 7110 262144 | 3011 524288 | 475 1048576 | 194 2097152 | 55 4194304 | 10 8388608 | 16 16777216 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 53898 3% 88% 0.00 150182 tcpinp sosend_generic+0x570 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@ 12431 kern_sendit+0x247 32768 |@ 2263 sendit+0x126 65536 | 878 sys_sendto+0x4d 131072 |@@@@@@@@@@ 19528 amd64_syscall+0x508 262144 |@@@@@ 10573 0xffffffff80f096ab 524288 |@@@ 6195 1048576 | 1741 2097152 | 242 4194304 | 33 8388608 | 14 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 208635 2% 90% 0.00 25833 tcpinp sosend_generic+0x570 nsec ------ Time Distribution ------ count Stack 4096 | 1 soo_write+0x49 8192 | 3 dofilewrite+0x89 16384 |@@@@@@@@@@@@@@@@@@@@@@ 158832 kern_writev+0x68 32768 |@@@@ 27819 sys_write+0x60 65536 |@ 8895 amd64_syscall+0x508 131072 |@ 8027 0xffffffff80f096ab 262144 | 3849 524288 | 825 1048576 | 287 2097152 | 76 4194304 | 12 8388608 | 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11813 1% 91% 0.00 351611 kmem vm object kmem_free+0x44 nsec ------ Time Distribution ------ count Stack 16384 | 26 zone_drain_wait+0x424 32768 |@@ 929 uma_reclaim_locked+0x7d 65536 |@ 760 uma_reclaim+0x69 131072 | 136 vm_pageout+0x49e 262144 |@@@@@@@ 2935 fork_exit+0x9c 524288 |@@@@@@@@@@@@@ 5244 0xffffffff80f098fe 1048576 |@@@ 1494 2097152 | 235 4194304 | 42 8388608 | 12 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 24021 1% 92% 0.00 117245 tcpinp sosend_generic+0x570 nsec ------ Time Distribution ------ count Stack 16384 | 2 svc_vc_reply+0x16a 32768 | 1 svc_sendreply_common+0x9a 65536 |@@@@@@@ 5767 svc_sendreply_mbuf+0x5a 131072 |@@@@@@@@@@@@@@@@ 13134 nfssvc_program+0x879 262144 |@@@@ 3517 svc_run_internal+0xf69 524288 |@ 1283 svc_run+0x16d 1048576 | 254 nfsrvd_nfsd+0x31b 2097152 | 49 nfssvc_nfsd+0x1c9 4194304 | 10 sys_nfssvc+0xba 8388608 | 4 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 19311 1% 93% 0.00 142933 tcpinp tcp_input+0x121b nsec ------ Time Distribution ------ count Stack 32768 | 1 ip_input+0x15b 65536 | 397 swi_net+0x1b3 131072 |@@@@@@@@@@@@@@@@@@@@ 12922 intr_event_execute_handlers+0x1b6 262144 |@@@@@@@ 4507 ithread_loop+0xa6 524288 |@ 1222 fork_exit+0x9c 1048576 | 230 0xffffffff80f098fe 2097152 | 25 4194304 | 4 8388608 | 3 ------------------------------------------------------------------------------- R/W reader hold: 7476812 events in 70.288 seconds (106374 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 557710 13% 13% 0.00 12663 rib head lock rtalloc1_fib+0x12a nsec ------ Time Distribution ------ count Stack 2048 | 412 rtalloc_ign_fib+0x117 4096 |@@@@@@@@@@@@@@@@@@@@@@ 427481 ip_output+0x447 8192 |@@ 44739 tcp_output+0x1a21 16384 |@@ 50320 tcp_usr_send+0x417 32768 | 11482 sosend_generic+0x570 65536 | 3655 svc_vc_reply+0x16a 131072 | 7542 svc_sendreply_common+0x9a 262144 | 8210 svc_sendreply_mbuf+0x5a 524288 | 2665 nfssvc_program+0x879 1048576 | 837 svc_run_internal+0xf69 2097152 | 284 svc_thread_start+0xb 4194304 | 62 fork_exit+0x9c 8388608 | 19 0xffffffff80f098fe 16777216 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 226539 13% 26% 0.00 29715 vm object vm_fault+0x78 nsec ------ Time Distribution ------ count Stack 8192 | 6041 trap_pfault+0xfa 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@ 196248 trap+0x518 32768 |@ 11068 0xffffffff80f093c7 65536 | 2166 131072 | 4577 262144 | 3240 524288 | 1273 1048576 | 1028 2097152 | 606 4194304 | 216 8388608 | 70 16777216 | 4 33554432 | 1 67108864 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 72595 8% 34% 0.00 59516 Name Cache cache_lookup+0x648 nsec ------ Time Distribution ------ count Stack 2048 | 3 vfs_cache_lookup+0xac 4096 | 404 VOP_LOOKUP_APV+0xa0 8192 | 1 lookup+0x59e 16384 |@@@@@ 12378 nfsvno_namei+0x243 32768 |@@@@@@@@@@@@@@@@@@@@ 49802 nfsrvd_open+0xa63 65536 | 2308 nfsrvd_dorpc+0xbe5 131072 | 2333 nfssvc_program+0x58e 262144 | 2320 svc_run_internal+0xf69 524288 | 1418 svc_thread_start+0xb 1048576 | 1051 fork_exit+0x9c 2097152 | 428 0xffffffff80f098fe 4194304 | 120 8388608 | 24 16777216 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 607607 7% 41% 0.00 6027 rib head lock rtalloc1_fib+0x12a nsec ------ Time Distribution ------ count Stack 2048 | 5958 rtalloc_ign_fib+0x117 4096 |@@@@@@@@@@@@@@@@@@@@@@@@ 490394 ip_output+0x447 8192 |@@ 44957 tcp_output+0x1a21 16384 |@@ 45658 tcp_do_segment+0x27dc 32768 | 9755 tcp_input+0x121b 65536 | 3890 ip_input+0x15b 131072 | 3751 netisr_dispatch_src+0x61 262144 | 2865 ether_demux+0x13e 524288 | 342 ether_nh_input+0x32e 1048576 | 11 netisr_dispatch_src+0x61 2097152 | 13 ether_input+0x26 4194304 | 2 vlan_input+0x18a 8388608 | 11 ether_demux+0x91 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 158883 7% 48% 0.00 22729 Name Cache cache_lookup+0x648 nsec ------ Time Distribution ------ count Stack 2048 | 5022 vfs_cache_lookup+0xac 4096 |@@@@@@@@@@@@@@@@@@@ 105878 VOP_LOOKUP_APV+0xa0 8192 | 122 lookup+0x59e 16384 |@ 6322 nfsvno_namei+0x243 32768 |@@@@@@ 34133 nfsrvd_lookup+0x13b 65536 | 1645 nfsrvd_dorpc+0xbe5 131072 | 1769 nfssvc_program+0x58e 262144 | 1694 svc_run_internal+0xf69 524288 | 986 svc_thread_start+0xb 1048576 | 786 fork_exit+0x9c 2097152 | 370 0xffffffff80f098fe 4194304 | 126 8388608 | 21 16777216 | 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11806 7% 55% 0.00 300778 pmap pv global pmap_remove+0x5f1 nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@ 7398 kmem_unback+0x2b 524288 |@@@@@@@@ 3471 kmem_free+0x44 1048576 |@@ 797 zone_drain_wait+0x424 2097152 | 94 uma_reclaim_locked+0x7d 4194304 | 26 uma_reclaim+0x69 8388608 | 14 vm_pageout+0x49e 16777216 | 4 fork_exit+0x9c 33554432 | 2 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 226545 6% 61% 0.00 15162 pmap pv global pmap_enter+0xb32 nsec ------ Time Distribution ------ count Stack 4096 | 22 vm_fault_hold+0x1c84 8192 |@ 11608 vm_fault+0x78 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@ 192302 trap_pfault+0xfa 32768 |@@ 16019 trap+0x518 65536 | 1632 0xffffffff80f093c7 131072 | 2736 262144 | 1526 524288 | 381 1048576 | 201 2097152 | 87 4194304 | 22 8388608 | 8 16777216 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1358121 6% 67% 0.00 2467 pcbinfohash in_pcblookup_hash+0x237 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@ 467273 tcp_input+0x764 4096 |@@@@@@@@@@@@@@@@@@@ 883665 ip_input+0x15b 8192 | 613 netisr_dispatch_src+0x61 16384 | 1571 ether_demux+0x13e 32768 | 2041 ether_nh_input+0x32e 65536 | 652 netisr_dispatch_src+0x61 131072 | 944 ether_input+0x26 262144 | 1354 vlan_input+0x18a 524288 | 5 ether_demux+0x91 1048576 | 0 ether_nh_input+0x32e 2097152 | 1 netisr_dispatch_src+0x61 4194304 | 0 ether_input+0x26 8388608 | 2 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 545093 3% 70% 0.00 2969 radix node head vfs_stdcheckexp+0xd1 nsec ------ Time Distribution ------ count Stack 2048 |@@@@ 87617 nfsvno_fhtovp+0xd5 4096 |@@@@@@@@@@@@@@@@@@@@@@@@ 453977 nfsd_fhtovp+0xc1 8192 | 194 nfsrvd_dorpc+0xe34 16384 | 587 nfssvc_program+0x58e 32768 | 431 svc_run_internal+0xf69 65536 | 641 svc_thread_start+0xb 131072 | 833 fork_exit+0x9c 262144 | 578 0xffffffff80f098fe 524288 | 143 1048576 | 70 2097152 | 16 4194304 | 3 8388608 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 556870 3% 73% 0.00 2890 radix node head vfs_stdcheckexp+0xd1 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@ 113145 nfsvno_v4rootexport+0x2c 4096 |@@@@@@@@@@@@@@@@@@@@@@@ 440422 nfssvc_program+0x397 8192 | 171 svc_run_internal+0xf69 16384 | 575 svc_thread_start+0xb 32768 | 369 fork_exit+0x9c 65536 | 609 0xffffffff80f098fe 131072 | 766 262144 | 531 524288 | 182 1048576 | 77 2097152 | 18 4194304 | 4 8388608 | 1 ------------------------------------------------------------------------------- SX shared hold: 62143954 events in 70.288 seconds (884137 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6715 15% 15% 0.00 21154907 ds->ds_opening_lock dmu_objset_from_ds+0xc7 nsec ------ Time Distribution ------ count Stack 2048 | 36 zfs_ioc_snapshot_list_next+0x178 4096 | 29 zfsdev_ioctl+0x565 8192 | 0 devfs_ioctl_f+0x13b 16384 | 0 kern_ioctl+0x40c 32768 | 0 sys_ioctl+0x17e 65536 | 0 amd64_syscall+0x508 131072 | 0 0xffffffff80f096ab 262144 | 0 524288 | 0 1048576 | 0 2097152 | 0 4194304 | 0 8388608 | 0 16777216 |@@ 643 33554432 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 5887 67108864 | 118 134217728 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 545095 13% 28% 0.00 219769 zfsvfs->z_hold_mtx[i] zfs_zget+0x1aa nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@ 157233 zfs_fhtovp+0x30a 262144 |@@@@@@@@@@@@@@@@ 294477 nfsvno_fhtovp+0x72 524288 |@@@ 64186 nfsd_fhtovp+0xc1 1048576 |@ 21996 nfsrvd_dorpc+0xe34 2097152 | 5603 nfssvc_program+0x58e 4194304 | 1195 svc_run_internal+0xf69 8388608 | 349 svc_thread_start+0xb 16777216 | 53 fork_exit+0x9c 33554432 | 2 0xffffffff80f098fe 67108864 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 556869 11% 39% 0.00 186934 so_snd_sx sosend_generic+0x99c nsec ------ Time Distribution ------ count Stack 32768 | 18 svc_vc_reply+0x16a 65536 | 9146 svc_sendreply_common+0x9a 131072 |@@@@@@@@@@@@@@@@ 309759 svc_sendreply_mbuf+0x5a 262144 |@@@@@@@ 137292 nfssvc_program+0x879 524288 |@@@ 74235 svc_run_internal+0xf69 1048576 |@ 22336 svc_thread_start+0xb 2097152 | 3358 fork_exit+0x9c 4194304 | 485 0xffffffff80f098fe 8388608 | 207 16777216 | 32 33554432 | 0 67108864 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 557114 3% 42% 0.00 57537 xprt->xp_lock svc_run_internal+0x4e6 nsec ------ Time Distribution ------ count Stack 2048 | 7558 svc_thread_start+0xb 4096 |@@@@@@@@@@@@@ 254313 fork_exit+0x9c 8192 |@ 21369 0xffffffff80f098fe 16384 |@@@ 59520 32768 | 6954 65536 |@@@@@ 110084 131072 |@@ 40851 262144 |@ 30097 524288 | 15919 1048576 | 7819 2097152 | 2107 4194304 | 414 8388608 | 92 16777216 | 16 33554432 | 0 67108864 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 201924 3% 45% 0.00 127129 so_rcv_sx soreceive_generic+0x1358 nsec ------ Time Distribution ------ count Stack 8192 | 159 svc_vc_recv+0x14f 16384 | 74 svc_run_internal+0x4e6 32768 |@@@@@@@@@@@@@ 92670 svc_thread_start+0xb 65536 |@@@@ 29825 fork_exit+0x9c 131072 |@@@@ 33420 0xffffffff80f098fe 262144 |@@@ 23084 524288 |@@ 13521 1048576 |@ 6781 2097152 | 1911 4194304 | 384 8388608 | 79 16777216 | 15 33554432 | 0 67108864 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 478004 3% 48% 0.00 52959 db->db_mtx dbuf_hold_impl+0x4b8 nsec ------ Time Distribution ------ count Stack 8192 | 791 dbuf_hold+0x25 16384 | 2684 dmu_buf_hold_array_by_dnode+0x130 32768 |@@@@@@@@@@@@@@@ 244687 dmu_read_uio_dnode+0x40 65536 |@@@@@@@@@@@ 185179 dmu_read_uio_dbuf+0x3b 131072 |@ 22105 zfs_freebsd_read+0x5fd 262144 | 15267 VOP_READ_APV+0xa0 524288 | 3577 vn_read+0x15b 1048576 | 2250 vn_io_fault+0x10b 2097152 | 1058 dofileread+0x98 4194304 | 293 kern_readv+0x68 8388608 | 81 sys_read+0x60 16777216 | 19 amd64_syscall+0x508 33554432 | 10 0xffffffff80f096ab 67108864 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 477636 3% 50% 0.00 50568 db->db_mtx dmu_read_uio_dnode+0xe4 nsec ------ Time Distribution ------ count Stack 2048 | 2783 dmu_read_uio_dbuf+0x3b 4096 | 650 zfs_freebsd_read+0x5fd 8192 | 4 VOP_READ_APV+0xa0 16384 | 3 vn_read+0x15b 32768 |@@@@@@@@@@@@@@@@@@@ 304659 vn_io_fault+0x10b 65536 |@@@@@@@ 125764 dofileread+0x98 131072 |@ 21878 kern_readv+0x68 262144 | 15003 sys_read+0x60 524288 | 3592 amd64_syscall+0x508 1048576 | 2022 0xffffffff80f096ab 2097152 | 899 4194304 | 279 8388608 | 86 16777216 | 12 33554432 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 37870 2% 52% 0.00 583038 so_rcv_sx soreceive_generic+0x1358 nsec ------ Time Distribution ------ count Stack 8192 | 172 dofileread+0x98 16384 | 392 kern_readv+0x68 32768 |@ 1831 sys_read+0x60 65536 |@@@@@@@@@@@@@@ 18930 amd64_syscall+0x508 131072 |@@@@@@@ 9262 0xffffffff80f096ab 262144 |@@ 3480 524288 | 995 1048576 | 438 2097152 | 748 4194304 | 1000 8388608 | 312 16777216 | 147 33554432 | 118 67108864 | 26 134217728 | 3 268435456 | 2 536870912 | 10 1073741824 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 545094 2% 54% 0.00 34446 zp->z_lock zfs_zget+0x188 nsec ------ Time Distribution ------ count Stack 1024 | 1 zfs_fhtovp+0x30a 2048 |@@@@ 82228 nfsvno_fhtovp+0x72 4096 |@ 25038 nfsd_fhtovp+0xc1 8192 |@ 26908 nfsrvd_dorpc+0xe34 16384 |@@@@@@@ 136314 nfssvc_program+0x58e 32768 |@@@@@@@@@@@@ 222726 svc_run_internal+0xf69 65536 |@ 18287 svc_thread_start+0xb 131072 | 11888 fork_exit+0x9c 262144 | 11041 0xffffffff80f098fe 524288 | 5358 1048576 | 3512 2097152 | 1325 4194304 | 371 8388608 | 82 16777216 | 15 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 298889 2% 56% 0.00 50296 so_snd_sx sosend_generic+0x99c nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@ 62965 soo_write+0x49 16384 |@@ 26204 dofilewrite+0x89 32768 |@@ 24846 kern_writev+0x68 65536 |@@@@@@@@@@@@@ 138392 sys_write+0x60 131072 |@@ 26977 amd64_syscall+0x508 262144 |@ 14363 0xffffffff80f096ab 524288 | 3651 1048576 | 1141 2097152 | 229 4194304 | 54 8388608 | 54 16777216 | 11 33554432 | 1 67108864 | 0 134217728 | 1 ------------------------------------------------------------------------------- SX exclusive hold: 10201278 events in 70.288 seconds (145136 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 278174 20% 20% 0.00 291325 dn->dn_struct_rwlock dmu_buf_hold_array_by_dnode+0x1e2 nsec ------ Time Distribution ------ count Stack 16384 | 26 dmu_read_uio_dnode+0x40 32768 | 1635 dmu_read_uio_dbuf+0x3b 65536 |@@@@@@@@@@@@@@@@@@@@ 193389 zfs_freebsd_read+0x5fd 131072 |@@@@ 41427 VOP_READ_APV+0xa0 262144 |@ 18449 vn_read+0x15b 524288 | 5259 vn_io_fault+0x10b 1048576 | 2480 dofileread+0x98 2097152 | 4342 kern_preadv+0x92 4194304 | 9148 sys_pread+0x66 8388608 | 1317 amd64_syscall+0x508 16777216 | 263 0xffffffff80f096ab 33554432 | 242 67108864 | 155 134217728 | 39 268435456 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 61693 14% 33% 0.00 901531 dn->dn_struct_rwlock dmu_buf_hold_array_by_dnode+0x1e2 nsec ------ Time Distribution ------ count Stack 32768 | 86 dmu_read_uio_dnode+0x40 65536 | 1499 dmu_read_uio_dbuf+0x3b 131072 | 260 zfs_freebsd_read+0x5fd 262144 | 342 VOP_READ_APV+0xa0 524288 |@@ 5004 vn_read+0x15b 1048576 |@@@@@@@@@@@@@@@@@@@@@ 44221 vn_io_fault+0x10b 2097152 |@@@ 7344 dofileread+0x98 4194304 |@ 2303 kern_readv+0x68 8388608 | 506 sys_read+0x60 16777216 | 103 amd64_syscall+0x508 33554432 | 20 0xffffffff80f096ab 67108864 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 226451 12% 45% 0.00 217603 vm map (user) unlock_and_deallocate+0x27e nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@ 68836 vm_fault_hold+0x1ea8 131072 |@@@@@@@@@@@@@@ 111196 vm_fault+0x78 262144 |@@@@ 31607 trap_pfault+0xfa 524288 |@ 8055 trap+0x518 1048576 | 3529 0xffffffff80f093c7 2097152 | 1771 4194304 | 668 8388608 | 266 16777216 | 160 33554432 | 177 67108864 | 119 134217728 | 51 268435456 | 15 536870912 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 545259 4% 49% 0.00 27320 dn->dn_struct_rwlock dnode_hold_impl+0x17f nsec ------ Time Distribution ------ count Stack 16384 |@@@@ 86073 dmu_bonus_hold+0x1d 32768 |@@@@@@@@@@@@@@@@@@@@@@@ 421799 zfs_zget+0xdf 65536 | 13022 zfs_fhtovp+0x30a 131072 | 10864 nfsvno_fhtovp+0x72 262144 | 9078 nfsd_fhtovp+0xc1 524288 | 2702 nfsrvd_dorpc+0xe34 1048576 | 1284 nfssvc_program+0x58e 2097152 | 328 svc_run_internal+0xf69 4194304 | 68 svc_thread_start+0xb 8388608 | 36 fork_exit+0x9c 16777216 | 4 0xffffffff80f098fe 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 545259 3% 53% 0.00 26219 dn->dn_struct_rwlock dbuf_read+0x416 nsec ------ Time Distribution ------ count Stack 8192 | 17509 dnode_hold_impl+0x199 16384 |@@@@@@@@@@@ 204488 dmu_bonus_hold+0x1d 32768 |@@@@@@@@@@@@@@@ 288941 zfs_zget+0xdf 65536 | 10534 zfs_fhtovp+0x30a 131072 | 10470 nfsvno_fhtovp+0x72 262144 | 8366 nfsd_fhtovp+0xc1 524288 | 2719 nfsrvd_dorpc+0xe34 1048576 | 1549 nfssvc_program+0x58e 2097152 | 493 svc_run_internal+0xf69 4194304 | 139 svc_thread_start+0xb 8388608 | 40 fork_exit+0x9c 16777216 | 10 0xffffffff80f098fe 33554432 | 0 67108864 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 545109 2% 54% 0.00 13394 dn->dn_struct_rwlock dmu_object_info_from_db+0x2d nsec ------ Time Distribution ------ count Stack 8192 |@@@@ 74658 zfs_zget+0xf8 16384 |@@@@@@@@@@@@@@@@@@@@@@@@ 448224 zfs_fhtovp+0x30a 32768 | 6424 nfsvno_fhtovp+0x72 65536 | 4596 nfsd_fhtovp+0xc1 131072 | 5663 nfsrvd_dorpc+0xe34 262144 | 3928 nfssvc_program+0x58e 524288 | 1010 svc_run_internal+0xf69 1048576 | 451 svc_thread_start+0xb 2097152 | 121 fork_exit+0x9c 4194304 | 20 0xffffffff80f098fe 8388608 | 13 16777216 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 545110 2% 56% 0.00 13347 dn->dn_struct_rwlock dbuf_read+0x416 nsec ------ Time Distribution ------ count Stack 8192 |@@@@ 73989 dmu_bonus_hold+0x15a 16384 |@@@@@@@@@@@@@@@@@@@@@@@@ 449065 zfs_zget+0xdf 32768 | 6358 zfs_fhtovp+0x30a 65536 | 4423 nfsvno_fhtovp+0x72 131072 | 5744 nfsd_fhtovp+0xc1 262144 | 3893 nfsrvd_dorpc+0xe34 524288 | 1058 nfssvc_program+0x58e 1048576 | 456 svc_run_internal+0xf69 2097152 | 88 svc_thread_start+0xb 4194304 | 21 fork_exit+0x9c 8388608 | 13 0xffffffff80f098fe 16777216 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 479622 2% 58% 0.00 13353 zfsvfs->z_teardown_inactive_lock zfs_freebsd_inactive+0xd nsec ------ Time Distribution ------ count Stack 8192 |@@ 46945 VOP_INACTIVE_APV+0xa6 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@ 413911 vinactive+0x189 32768 | 5760 vputx+0x2ef 65536 | 3642 nfsrvd_dorpc+0x1760 131072 | 4649 nfssvc_program+0x58e 262144 | 3400 svc_run_internal+0xf69 524288 | 876 svc_thread_start+0xb 1048576 | 328 fork_exit+0x9c 2097152 | 82 0xffffffff80f098fe 4194304 | 14 8388608 | 13 16777216 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 35122 1% 59% 0.00 111593 zap->zap_rwlock zap_lookup_norm+0x278 nsec ------ Time Distribution ------ count Stack 2048 | 596 zap_lookup+0x2e 4096 |@@@@@@@@@@@@@@@@ 19634 dsl_prop_get_dd+0xfc 8192 | 141 dsl_prop_get_ds+0x14d 16384 | 39 dsl_prop_register+0x37 32768 | 15 dmu_objset_open_impl+0x2f8 65536 | 23 dmu_objset_from_ds+0x65 131072 | 388 zfs_ioc_snapshot_list_next+0x178 262144 |@@@@@@@@@ 10594 zfsdev_ioctl+0x565 524288 |@@ 2950 devfs_ioctl_f+0x13b 1048576 | 557 kern_ioctl+0x40c 2097152 | 136 sys_ioctl+0x17e 4194304 | 37 amd64_syscall+0x508 8388608 | 11 0xffffffff80f096ab 16777216 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 17163 1% 60% 0.00 226376 dn->dn_struct_rwlock dbuf_read+0x71e nsec ------ Time Distribution ------ count Stack 131072 | 459 dmu_buf_hold+0x3d 262144 |@@@@@@@@@@@@@@@@@@@@@@@ 13503 zap_lockdir+0x42 524288 |@@@@ 2675 zap_lookup_norm+0x3a 1048576 | 431 zap_contains+0x39 2097152 | 75 dsl_prop_get_dd+0x122 4194304 | 12 dsl_prop_get_ds+0x14d 8388608 | 5 dsl_prop_register+0x37 16777216 | 3 dmu_objset_open_impl+0x2f8 dmu_objset_from_ds+0x65 zfs_ioc_snapshot_list_next+0x178 zfsdev_ioctl+0x565 devfs_ioctl_f+0x13b kern_ioctl+0x40c sys_ioctl+0x17e amd64_syscall+0x508 0xffffffff80f096ab -------------------------------------------------------------------------------