Adaptive mutex spin: 1873576 events in 60.426 seconds (31006 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 241994 12% 12% 0.00 5381 sg_lock _cv_wait_sig+0x255 nsec ------ Time Distribution ------ count Stack 128 | 1 svc_run_internal+0xb51 256 | 742 svc_thread_start+0xb 512 | 7132 fork_exit+0x9c 1024 |@ 15339 0xffffffff80f098fe 2048 |@@@ 25786 4096 |@@@@@@@@@@@ 96331 8192 |@@@@@@@ 59644 16384 |@@@ 25430 32768 |@ 9267 65536 | 2116 131072 | 187 262144 | 10 524288 | 8 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 163799 8% 20% 0.00 5312 sg_lock svc_run_internal+0xd72 nsec ------ Time Distribution ------ count Stack 128 | 5 svc_thread_start+0xb 256 | 768 fork_exit+0x9c 512 |@ 7058 0xffffffff80f098fe 1024 |@@@ 16976 2048 |@@@@@ 29490 4096 |@@@@@@@@ 48296 8192 |@@@@@@ 35962 16384 |@@ 16107 32768 |@ 6845 65536 | 1910 131072 | 321 262144 | 26 524288 | 32 1048576 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 154895 8% 27% 0.00 5533 sg_lock svc_run_internal+0x99e nsec ------ Time Distribution ------ count Stack 256 | 443 svc_thread_start+0xb 512 | 2789 fork_exit+0x9c 1024 |@ 7354 0xffffffff80f098fe 2048 |@@@@ 21968 4096 |@@@@@@@@@ 49126 8192 |@@@@@@@@@ 48392 16384 |@@@ 17167 32768 |@ 6599 65536 | 974 131072 | 76 262144 | 0 524288 | 6 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 158953 6% 33% 0.00 4050 sg_lock xprt_active+0x68 nsec ------ Time Distribution ------ count Stack 128 | 10 svc_vc_soupcall+0x35 256 | 1991 sowakeup+0x83 512 |@ 6847 tcp_do_segment+0x272c 1024 |@@@ 17525 tcp_input+0x121b 2048 |@@@@@@@ 38790 ip_input+0x15b 4096 |@@@@@@@ 40565 netisr_dispatch_src+0x61 8192 |@@@@@@ 36103 ether_demux+0x13e 16384 |@@ 13657 ether_nh_input+0x32e 32768 | 2938 netisr_dispatch_src+0x61 65536 | 448 ether_input+0x26 131072 | 47 vlan_input+0x18a 262144 | 11 ether_demux+0x91 524288 | 21 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 32154 5% 39% 0.00 18518 taskqueue _sleep+0x38d nsec ------ Time Distribution ------ count Stack 128 | 4 taskqueue_thread_loop+0x131 256 | 126 fork_exit+0x9c 512 | 873 0xffffffff80f098fe 1024 |@@ 2490 2048 |@ 2139 4096 |@@ 3054 8192 |@@@@ 5163 16384 |@@@@@@@ 7637 32768 |@@@@@ 5795 65536 |@@@ 3321 131072 |@ 1215 262144 | 274 524288 | 57 1048576 | 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 127503 5% 44% 0.00 4621 sg_lock xprt_active+0x68 nsec ------ Time Distribution ------ count Stack 128 | 10 svc_vc_soupcall+0x35 256 | 1211 sowakeup+0x83 512 |@ 6309 tcp_do_segment+0x2385 1024 |@@@ 15254 tcp_input+0x121b 2048 |@@@@@ 23292 ip_input+0x15b 4096 |@@@@@@@@ 36210 netisr_dispatch_src+0x61 8192 |@@@@@@ 28603 ether_demux+0x13e 16384 |@@ 12365 ether_nh_input+0x32e 32768 | 3403 netisr_dispatch_src+0x61 65536 | 711 ether_input+0x26 131072 | 72 vlan_input+0x18a 262144 | 26 ether_demux+0x91 524288 | 34 ether_nh_input+0x32e 1048576 | 1 netisr_dispatch_src+0x61 2097152 | 0 ether_input+0x26 4194304 | 2 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 40564 5% 49% 0.00 14052 taskqueue taskqueue_run_locked+0x1cf nsec ------ Time Distribution ------ count Stack 128 | 3 taskqueue_thread_loop+0xd8 256 | 57 fork_exit+0x9c 512 |@ 1524 0xffffffff80f098fe 1024 |@ 1683 2048 |@ 1370 4096 |@@@@@@@@@@@@ 17454 8192 |@@@@ 6029 16384 |@@@ 4180 32768 |@@ 3701 65536 |@@ 2751 131072 | 1344 262144 | 398 524288 | 68 1048576 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 81773 5% 54% 0.00 6539 vnode_free_list _vhold+0x15f nsec ------ Time Distribution ------ count Stack 128 | 2 vref+0x15 256 | 214 zfs_zget+0x174 512 | 1488 zfs_vget+0x88 1024 |@@ 5946 nfsrvd_readdirplus+0xb18 2048 |@@@@@@@@ 23225 nfsrvd_dorpc+0x12e8 4096 |@@@@ 13149 nfssvc_program+0x58e 8192 |@@@@@@ 17161 svc_run_internal+0xf69 16384 |@@@@ 13391 svc_thread_start+0xb 32768 |@@ 5467 fork_exit+0x9c 65536 | 1471 0xffffffff80f098fe 131072 | 237 262144 | 16 524288 | 4 1048576 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 23481 5% 59% 0.00 22644 taskqueue taskqueue_enqueue+0x9c nsec ------ Time Distribution ------ count Stack 128 | 5 zio_interrupt+0x85 256 | 66 g_io_deliver+0x294 512 | 592 g_io_deliver+0x294 1024 |@ 1374 g_io_deliver+0x294 2048 |@@ 2000 g_disk_done+0x14c 4096 |@@@ 2753 dadone+0xeee 8192 |@@@@ 3796 xpt_done_process+0x781 16384 |@@@@@ 4282 xpt_done_td+0x176 32768 |@@@@@ 3973 fork_exit+0x9c 65536 |@@@ 2810 0xffffffff80f098fe 131072 |@ 1299 262144 | 438 524288 | 93 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 80726 4% 63% 0.00 5977 vnode_free_list _vdrop+0x33e nsec ------ Time Distribution ------ count Stack 128 | 1 vputx+0x322 256 | 248 nfsrvd_readdirplus+0x1165 512 | 2103 nfsrvd_dorpc+0x12e8 1024 |@@@@@ 14955 nfssvc_program+0x58e 2048 |@@@@@@@ 20230 svc_run_internal+0xf69 4096 |@@@ 8104 svc_thread_start+0xb 8192 |@@@@@@ 16667 fork_exit+0x9c 16384 |@@@@ 11964 0xffffffff80f098fe 32768 |@ 4890 65536 | 1338 131072 | 200 262144 | 20 524288 | 6 ------------------------------------------------------------------------------- Adaptive mutex block: 15390 events in 60.426 seconds (255 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2194 14% 14% 0.00 40740 sg_lock _cv_wait_sig+0x255 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@ 630 svc_run_internal+0xb51 65536 |@@@@@@@@@@@@@@@@@@@ 1442 svc_thread_start+0xb 131072 |@ 119 fork_exit+0x9c 262144 | 3 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1806 12% 25% 0.00 42051 sg_lock svc_run_internal+0xd72 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@ 517 svc_thread_start+0xb 65536 |@@@@@@@@@@@@@@@@@@@ 1155 fork_exit+0x9c 131072 |@@ 134 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 866 5% 31% 0.00 39021 sg_lock xprt_active+0x68 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@ 278 svc_vc_soupcall+0x35 65536 |@@@@@@@@@@@@@@@@@@@ 564 sowakeup+0x83 131072 | 24 tcp_do_segment+0x2385 tcp_input+0x121b ip_input+0x15b netisr_dispatch_src+0x61 ether_demux+0x13e ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 vlan_input+0x18a ether_demux+0x91 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 778 5% 35% 0.00 38058 sg_lock xprt_active+0x68 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@ 236 svc_vc_soupcall+0x35 65536 |@@@@@@@@@@@@@@@@@@@@ 523 sowakeup+0x83 131072 | 18 tcp_do_segment+0x272c 262144 | 1 tcp_input+0x121b ip_input+0x15b netisr_dispatch_src+0x61 ether_demux+0x13e ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 vlan_input+0x18a ether_demux+0x91 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 782 4% 40% 0.00 36695 sellck _cv_timedwait_sig_sbt+0x288 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@ 361 seltdwait+0xba 65536 |@@@@@@@@@@@@@@@ 401 kern_poll+0x4fa 131072 | 19 sys_poll+0x61 262144 | 0 amd64_syscall+0x508 524288 | 1 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 574 4% 44% 0.00 49640 taskqueue _sleep+0x38d nsec ------ Time Distribution ------ count Stack 32768 |@@@ 68 taskqueue_thread_loop+0x131 65536 |@@@@@@@@@@@@@@@@@@@@@ 420 fork_exit+0x9c 131072 |@@@@ 81 0xffffffff80f098fe 262144 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 403 3% 47% 0.00 54602 taskqueue taskqueue_enqueue+0x9c nsec ------ Time Distribution ------ count Stack 32768 |@@ 38 zio_interrupt+0x85 65536 |@@@@@@@@@@@@@@@@@@@@ 280 g_io_deliver+0x294 131072 |@@@@@ 73 g_io_deliver+0x294 262144 | 11 g_io_deliver+0x294 524288 | 1 g_disk_done+0x14c dadone+0xeee xpt_done_process+0x781 xpt_done_td+0x176 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 423 3% 50% 0.00 40377 nfs_state_mutex nfsrv_checkgetattr+0xd3 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@ 135 nfsrvd_getattr+0x1e9 65536 |@@@@@@@@@@@@@@@@@@@ 268 nfsrvd_dorpc+0x12e8 131072 |@ 18 nfssvc_program+0x58e 262144 | 2 svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 423 2% 52% 0.00 38280 umtxql umtxq_sleep+0x26a nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 144 do_wait+0x403 65536 |@@@@@@@@@@@@@@@@@@ 266 __umtx_op_wait_uint_private+0x7d 131072 | 13 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 260 2% 55% 0.00 55562 taskqueue taskqueue_run_locked+0x1cf nsec ------ Time Distribution ------ count Stack 32768 |@@ 24 taskqueue_thread_loop+0xd8 65536 |@@@@@@@@@@@@@@@@@@@ 170 fork_exit+0x9c 131072 |@@@@@@@ 63 0xffffffff80f098fe 262144 | 2 524288 | 1 ------------------------------------------------------------------------------- Spin lock spin: 1940363 events in 60.426 seconds (32112 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 111748 5% 5% 0.00 4772 sleepq chain wakeup_one+0xf nsec ------ Time Distribution ------ count Stack 128 | 9 taskqueue_enqueue_locked+0x1e7 256 | 24 taskqueue_enqueue+0xb8 512 | 87 dbuf_rele_and_unlock+0x11e 1024 | 301 zap_lookup_norm+0x283 2048 | 1764 zap_lookup+0x2e 4096 |@@@@@@@@@@@ 42351 dsl_prop_get_dd+0x158 8192 |@@@@@@@@@@@@@@@@@ 65327 dsl_prop_get_ds+0x14d 16384 | 1673 dsl_prop_register+0x37 32768 | 188 dmu_objset_open_impl+0x2f8 65536 | 22 dmu_objset_from_ds+0x65 131072 | 0 zfs_ioc_snapshot_list_next+0x178 262144 | 0 zfsdev_ioctl+0x565 524288 | 1 devfs_ioctl_f+0x13b 1048576 | 1 kern_ioctl+0x40c sys_ioctl+0x17e amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 109066 5% 10% 0.00 4791 sleepq chain wakeup_one+0xf nsec ------ Time Distribution ------ count Stack 128 | 6 taskqueue_enqueue_locked+0x1e7 256 | 22 taskqueue_enqueue+0xb8 512 | 66 dbuf_rele_and_unlock+0x11e 1024 | 281 zap_lookup_norm+0x283 2048 | 1256 zap_lookup+0x2e 4096 |@@@@@@@@@@@ 42046 dsl_prop_get_dd+0x158 8192 |@@@@@@@@@@@@@@@@@ 63601 dsl_prop_get_ds+0x14d 16384 | 1593 dsl_prop_register+0x37 32768 | 172 dmu_objset_open_impl+0x31f 65536 | 19 dmu_objset_from_ds+0x65 131072 | 1 zfs_ioc_snapshot_list_next+0x178 262144 | 2 zfsdev_ioctl+0x565 524288 | 1 devfs_ioctl_f+0x13b kern_ioctl+0x40c sys_ioctl+0x17e amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 46093 4% 14% 0.00 9960 sleepq chain wakeup_one+0xf nsec ------ Time Distribution ------ count Stack 128 | 88 taskqueue_enqueue_locked+0x1e7 256 | 164 taskqueue_enqueue+0xb8 512 | 672 zio_interrupt+0x85 1024 |@ 1853 g_io_deliver+0x294 2048 |@@@ 5538 g_io_deliver+0x294 4096 |@@@@@@ 10753 g_io_deliver+0x294 8192 |@@@@@@@ 11822 g_disk_done+0x14c 16384 |@@@@ 7492 dadone+0xeee 32768 |@@@ 4980 xpt_done_process+0x781 65536 |@ 2202 xpt_done_td+0x176 131072 | 480 fork_exit+0x9c 262144 | 40 0xffffffff80f098fe 524288 | 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 91126 4% 18% 0.00 4783 sleepq chain wakeup_one+0xf nsec ------ Time Distribution ------ count Stack 128 | 11 taskqueue_enqueue_locked+0x1e7 256 | 24 taskqueue_enqueue+0xb8 512 | 73 dbuf_rele_and_unlock+0x11e 1024 | 240 zap_lookup_norm+0x283 2048 | 1176 zap_contains+0x39 4096 |@@@@@@@@@@@ 34988 dsl_prop_get_dd+0x122 8192 |@@@@@@@@@@@@@@@@@ 53439 dsl_prop_get_ds+0x14d 16384 | 1027 dsl_prop_register+0x37 32768 | 133 dmu_objset_open_impl+0x31f 65536 | 15 dmu_objset_from_ds+0x65 zfs_ioc_snapshot_list_next+0x178 zfsdev_ioctl+0x565 devfs_ioctl_f+0x13b kern_ioctl+0x40c sys_ioctl+0x17e amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 91850 4% 23% 0.00 4729 sleepq chain wakeup_one+0xf nsec ------ Time Distribution ------ count Stack 64 | 1 taskqueue_enqueue_locked+0x1e7 128 | 8 taskqueue_enqueue+0xb8 256 | 21 dbuf_rele_and_unlock+0x11e 512 | 73 zap_lookup_norm+0x283 1024 | 274 zap_contains+0x39 2048 | 1969 dsl_prop_get_dd+0x122 4096 |@@@@@@@@@@@ 34264 dsl_prop_get_ds+0x14d 8192 |@@@@@@@@@@@@@@@@@ 54056 dsl_prop_register+0x37 16384 | 1027 dmu_objset_open_impl+0x2f8 32768 | 144 dmu_objset_from_ds+0x65 65536 | 12 zfs_ioc_snapshot_list_next+0x178 131072 | 1 zfsdev_ioctl+0x565 devfs_ioctl_f+0x13b kern_ioctl+0x40c sys_ioctl+0x17e amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 47327 4% 27% 0.00 9141 sched lock 8 tdq_lock_pair+0x121 nsec ------ Time Distribution ------ count Stack 64 | 43 sched_idletd+0x2a2 128 | 96 fork_exit+0x9c 256 | 310 0xffffffff80f098fe 512 | 1048 1024 |@ 2733 2048 |@@@@ 6413 4096 |@@@@@@@ 12264 8192 |@@@@@ 7931 16384 |@@@@@ 9085 32768 |@@@ 5386 65536 |@ 1579 131072 | 418 262144 | 16 524288 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 57726 3% 30% 0.00 5866 sleepq chain _sleep+0x109 nsec ------ Time Distribution ------ count Stack 64 | 7 taskqueue_thread_loop+0x131 128 | 142 fork_exit+0x9c 256 | 318 0xffffffff80f098fe 512 | 905 1024 |@ 2331 2048 |@@@ 7587 4096 |@@@@@@@@@@ 19687 8192 |@@@@@@@@ 16378 16384 |@@@ 6837 32768 |@ 2937 65536 | 552 131072 | 36 262144 | 5 524288 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 108889 3% 33% 0.00 3066 callout callout_lock+0xd1 nsec ------ Time Distribution ------ count Stack 32 | 2 callout_reset_sbt_on+0x14a 64 | 1257 tcp_timer_activate+0x147 128 |@@ 7722 tcp_do_segment+0x87 256 |@@@ 12366 tcp_input+0x121b 512 |@@@@@ 18276 ip_input+0x15b 1024 |@@@ 13585 netisr_dispatch_src+0x61 2048 |@@ 9651 ether_demux+0x13e 4096 |@@@@@@ 23176 ether_nh_input+0x32e 8192 |@@@ 12908 netisr_dispatch_src+0x61 16384 |@ 7113 ether_input+0x26 32768 | 2389 vlan_input+0x18a 65536 | 399 ether_demux+0x91 131072 | 29 ether_nh_input+0x32e 262144 | 4 netisr_dispatch_src+0x61 524288 | 11 ether_input+0x26 1048576 | 1 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 50031 2% 35% 0.00 4818 sleepq chain wakeup_one+0xf nsec ------ Time Distribution ------ count Stack 128 | 47 taskqueue_enqueue_locked+0x1e7 256 | 71 taskqueue_enqueue+0xb8 512 | 289 dbuf_rele_and_unlock+0x11e 1024 | 900 zap_cursor_fini+0x6b 2048 |@ 1967 dsl_prop_get_all_impl+0x410 4096 |@@@@@@@@ 14727 dsl_prop_get_all_ds+0x13e 8192 |@@@@@@@@@@@@@@@@@@ 31257 zfs_ioc_objset_stats_impl+0x3c 16384 | 633 zfs_ioc_snapshot_list_next+0x18c 32768 | 125 zfsdev_ioctl+0x565 65536 | 13 devfs_ioctl_f+0x13b 131072 | 1 kern_ioctl+0x40c 262144 | 0 sys_ioctl+0x17e 524288 | 1 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 44259 2% 37% 0.00 4452 sleepq chain wakeup_one+0xf nsec ------ Time Distribution ------ count Stack 128 | 52 taskqueue_enqueue_locked+0x1e7 256 | 86 taskqueue_enqueue+0xb8 512 | 334 dbuf_rele_and_unlock+0x11e 1024 | 1160 zap_lookup_norm+0x283 2048 |@ 2273 zap_lookup+0x2e 4096 |@@@@@@@@@@@@ 17842 dsl_prop_get_dd+0xfc 8192 |@@@@@@@@@@@@@@ 21341 dsl_prop_get_ds+0x14d 16384 | 970 dsl_prop_register+0x37 32768 | 187 dmu_objset_open_impl+0x31f 65536 | 12 dmu_objset_from_ds+0x65 131072 | 1 zfs_ioc_snapshot_list_next+0x178 262144 | 0 zfsdev_ioctl+0x565 524288 | 1 devfs_ioctl_f+0x13b kern_ioctl+0x40c sys_ioctl+0x17e amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Thread lock spin: 29329097 events in 60.426 seconds (485376 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1551987 6% 6% 0.00 138 sched lock 8 ithread_loop+0x167 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1432272 fork_exit+0x9c 128 |@ 97981 0xffffffff80f098fe 256 | 2135 512 | 351 1024 | 698 2048 | 1068 4096 | 4699 8192 | 6507 16384 | 4614 32768 | 1494 65536 | 161 131072 | 5 262144 | 0 524288 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 174364 3% 9% 0.00 547 sched lock 22 ithread_loop+0x167 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@@@@@@@@@@@@@@@@@@@ 147137 fork_exit+0x9c 128 |@@ 17199 0xffffffff80f098fe 256 | 430 512 | 36 1024 | 15 2048 | 28 4096 | 1634 8192 | 3448 16384 | 3361 32768 | 1007 65536 | 66 131072 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 199146 2% 11% 0.00 444 sched lock 26 ithread_loop+0x167 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 175060 fork_exit+0x9c 128 |@@ 14636 0xffffffff80f098fe 256 | 549 512 | 87 1024 | 102 2048 | 153 4096 | 1770 8192 | 3024 16384 | 2684 32768 | 982 65536 | 96 131072 | 2 262144 | 0 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 118866 2% 13% 0.00 689 sched lock 11 ithread_loop+0x167 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 108221 fork_exit+0x9c 128 | 1815 0xffffffff80f098fe 256 | 149 512 | 21 1024 | 21 2048 | 73 4096 | 2078 8192 | 2906 16384 | 2644 32768 | 868 65536 | 65 131072 | 3 262144 | 0 524288 | 1 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1423055 2% 15% 0.00 46 sched lock 8 intr_event_schedule_thread+0x7f nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1319023 intr_event_handle+0xe2 128 |@@ 99975 intr_execute_handlers+0x48 256 | 2380 lapic_handle_intr+0x3f 512 | 308 0xffffffff80f09aa7 1024 | 289 sched_idletd+0x3e3 2048 | 295 fork_exit+0x9c 4096 | 351 0xffffffff80f098fe 8192 | 313 16384 | 96 32768 | 25 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 239630 2% 17% 0.00 272 sched lock 6 ithread_loop+0x167 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 209873 fork_exit+0x9c 128 |@@ 20124 0xffffffff80f098fe 256 | 497 512 | 247 1024 | 603 2048 | 1123 4096 | 2468 8192 | 2568 16384 | 1593 32768 | 488 65536 | 44 131072 | 1 262144 | 0 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1422938 2% 19% 0.00 43 sched lock 8 critical_exit+0x58 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1383306 intr_event_handle+0x102 128 | 30066 intr_execute_handlers+0x48 256 | 4921 lapic_handle_intr+0x3f 512 | 3655 0xffffffff80f09aa7 1024 | 792 sched_idletd+0x3e3 2048 | 19 fork_exit+0x9c 4096 | 9 0xffffffff80f098fe 8192 | 144 16384 | 23 32768 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 93665 2% 20% 0.00 615 sched lock 10 ithread_loop+0x167 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 83988 fork_exit+0x9c 128 |@ 3429 0xffffffff80f098fe 256 | 137 512 | 24 1024 | 41 2048 | 63 4096 | 1108 8192 | 2198 16384 | 2033 32768 | 606 65536 | 34 131072 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 94511 1% 22% 0.00 550 sched lock 0 ithread_loop+0x167 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 87244 fork_exit+0x9c 128 | 1588 0xffffffff80f098fe 256 | 73 512 | 25 1024 | 33 2048 | 99 4096 | 1131 8192 | 2140 16384 | 1606 32768 | 534 65536 | 33 131072 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 61603 1% 23% 0.00 715 sched lock 17 ithread_loop+0x167 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 55597 fork_exit+0x9c 128 | 1579 0xffffffff80f098fe 256 | 118 512 | 13 1024 | 11 2048 | 15 4096 | 781 8192 | 1390 16384 | 1493 32768 | 574 65536 | 31 131072 | 0 262144 | 0 524288 | 0 1048576 | 1 ------------------------------------------------------------------------------- R/W writer blocked by writer: 7447 events in 60.426 seconds (123 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4299 53% 53% 0.00 35731 tcpinp in_pcblookup_hash+0x301 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@ 2008 tcp_input+0x764 65536 |@@@@@@@@@@@@@@@ 2213 ip_input+0x15b 131072 | 75 netisr_dispatch_src+0x61 262144 | 2 ether_demux+0x13e 524288 | 0 ether_nh_input+0x32e 1048576 | 1 netisr_dispatch_src+0x61 ether_input+0x26 vlan_input+0x18a ether_demux+0x91 ether_nh_input+0x32e netisr_dispatch_src+0x61 ether_input+0x26 igb_rxeof+0x755 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1772 25% 78% 0.00 40467 tcpinp tcp_usr_rcvd+0x78 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@ 573 soreceive_generic+0xfe1 65536 |@@@@@@@@@@@@@@@@@@ 1106 svc_vc_recv+0x14f 131072 |@ 88 svc_run_internal+0x4e6 262144 | 5 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 756 13% 91% 0.00 50332 tcpinp tcp_usr_send+0xb3 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@ 164 sosend_generic+0x570 65536 |@@@@@@@@@@@@@@@@ 416 svc_vc_reply+0x16a 131072 |@@@@@@ 173 svc_sendreply_common+0x9a 262144 | 3 svc_sendreply_mbuf+0x5a nfssvc_program+0x879 svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 259 4% 95% 0.00 41186 tcpinp tcp_usr_rcvd+0x78 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 87 soreceive_generic+0xfe1 65536 |@@@@@@@@@@@@@@@@@ 152 svc_vc_recv+0x14f 131072 |@@ 19 svc_run_internal+0x4e6 262144 | 1 svc_run+0x16d nfsrvd_nfsd+0x31b nfssvc_nfsd+0x1c9 sys_nfssvc+0xba amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 119 2% 97% 0.00 57948 tcpinp tcp_usr_send+0xb3 nsec ------ Time Distribution ------ count Stack 32768 |@@@ 14 sosend_generic+0x570 65536 |@@@@@@@@@@@@@@@@ 64 svc_vc_reply+0x16a 131072 |@@@@@@@@@ 39 svc_sendreply_common+0x9a 262144 | 2 svc_sendreply_mbuf+0x5a nfssvc_program+0x879 svc_run_internal+0xf69 svc_run+0x16d nfsrvd_nfsd+0x31b nfssvc_nfsd+0x1c9 sys_nfssvc+0xba amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 150 2% 99% 0.00 34585 tcpinp tcp_usr_rcvd+0x78 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@ 91 soreceive_generic+0xfe1 65536 |@@@@@@@@@@@ 57 kern_recvit+0x212 131072 | 2 sys_recvfrom+0x86 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 39 1% 99% 0.00 39372 tcpinp tcp_usr_send+0xb3 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@ 10 sosend_generic+0x570 65536 |@@@@@@@@@@@@@@@@@@@@@@ 29 soo_write+0x49 dofilewrite+0x89 kern_writev+0x68 sys_write+0x60 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 27 0% 100% 0.00 32431 tcpinp tcp_usr_rcvd+0x78 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@ 17 soreceive_generic+0xfe1 65536 |@@@@@@@@@@@ 10 dofileread+0x98 kern_readv+0x68 sys_read+0x60 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 13 0% 100% 0.00 34204 tcpinp tcp_usr_send+0xb3 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@ 8 sosend_generic+0x570 65536 |@@@@@@@@@@@ 5 soo_write+0x49 dofilewrite+0x89 kern_writev+0x68 sys_writev+0x36 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 34742 pmap pv list pmap_remove_pages+0x4ca nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 1 vmspace_exit+0xa1 65536 |@@@@@@@@@@@@@@@@@@@@ 2 exit1+0x80b sys_sys_exit+0xd amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- R/W writer blocked by readers: 2 events in 60.426 seconds (0 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 60% 60% 0.00 34539 vm object vm_fault_hold+0xa70 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_fault+0x78 trap_pfault+0xfa trap+0x518 0xffffffff80f093c7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 40% 100% 0.00 22680 Name Cache cache_lookup+0x495 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vfs_cache_lookup+0xac VOP_LOOKUP_APV+0xa0 lookup+0x59e nfsvno_namei+0x243 nfsrvd_lookup+0x13b nfsrvd_dorpc+0xbe5 nfssvc_program+0x58e svc_run_internal+0xf69 svc_run+0x16d nfsrvd_nfsd+0x31b nfssvc_nfsd+0x1c9 sys_nfssvc+0xba amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- R/W reader blocked by writer: 3 events in 60.426 seconds (0 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 66% 66% 0.00 37944 Name Cache cache_lookup+0x1b7 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 vfs_cache_lookup+0xac VOP_LOOKUP_APV+0xa0 lookup+0x59e namei+0x5a1 kern_statat+0xab sys_stat+0x2d amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 34% 100% 0.00 39142 Name Cache cache_lookup+0x1b7 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vfs_cache_lookup+0xac VOP_LOOKUP_APV+0xa0 lookup+0x59e nfsvno_namei+0x243 nfsrvd_lookup+0x13b nfsrvd_dorpc+0xbe5 nfssvc_program+0x58e svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- R/W reader spin on writer: 1060350 events in 60.426 seconds (17548 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 357264 34% 34% 0.00 6244 tcpinp in_pcblookup_hash+0x301 nsec ------ Time Distribution ------ count Stack 128 | 1 tcp_input+0x764 256 | 678 ip_input+0x15b 512 | 6995 netisr_dispatch_src+0x61 1024 |@ 12795 ether_demux+0x13e 2048 |@@ 27100 ether_nh_input+0x32e 4096 |@@@@@@@@@ 114782 netisr_dispatch_src+0x61 8192 |@@@@@@@@ 98211 ether_input+0x26 16384 |@@@@@@ 82394 vlan_input+0x18a 32768 |@ 13629 ether_demux+0x91 65536 | 637 ether_nh_input+0x32e 131072 | 12 netisr_dispatch_src+0x61 262144 | 10 ether_input+0x26 524288 | 17 igb_rxeof+0x755 1048576 | 3 igb_msix_que+0x167 intr_event_execute_handlers+0x1b6 ithread_loop+0xa6 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 333440 30% 64% 0.00 5887 tcpinp tcp_usr_rcvd+0x78 nsec ------ Time Distribution ------ count Stack 128 | 1 soreceive_generic+0xfe1 256 | 1423 svc_vc_recv+0x14f 512 |@ 16737 svc_run_internal+0x4e6 1024 |@@@ 33797 svc_thread_start+0xb 2048 |@@@@ 45661 fork_exit+0x9c 4096 |@@@@@ 57936 0xffffffff80f098fe 8192 |@@@@@@@@@ 102285 16384 |@@@@@ 57317 32768 |@ 15999 65536 | 2148 131072 | 115 262144 | 9 524288 | 9 1048576 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 242650 23% 87% 0.00 6268 tcpinp tcp_usr_send+0xb3 nsec ------ Time Distribution ------ count Stack 256 | 711 sosend_generic+0x570 512 |@ 10992 svc_vc_reply+0x16a 1024 |@@ 21539 svc_sendreply_common+0x9a 2048 |@@@ 31499 svc_sendreply_mbuf+0x5a 4096 |@@@@@ 46998 nfssvc_program+0x879 8192 |@@@@@@@@ 65855 svc_run_internal+0xf69 16384 |@@@@@@ 51467 svc_thread_start+0xb 32768 |@ 11849 fork_exit+0x9c 65536 | 1578 0xffffffff80f098fe 131072 | 137 262144 | 8 524288 | 15 1048576 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 42863 4% 91% 0.00 6068 tcpinp tcp_usr_rcvd+0x78 nsec ------ Time Distribution ------ count Stack 256 | 176 soreceive_generic+0xfe1 512 |@ 1880 svc_vc_recv+0x14f 1024 |@@ 3639 svc_run_internal+0x4e6 2048 |@@@ 5711 svc_run+0x16d 4096 |@@@@@ 7798 nfsrvd_nfsd+0x31b 8192 |@@@@@@@@@ 13571 nfssvc_nfsd+0x1c9 16384 |@@@@@ 7652 sys_nfssvc+0xba 32768 |@ 2108 amd64_syscall+0x508 65536 | 312 0xffffffff80f096ab 131072 | 14 262144 | 1 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 32491 3% 94% 0.00 6338 tcpinp tcp_usr_send+0xb3 nsec ------ Time Distribution ------ count Stack 128 | 1 sosend_generic+0x570 256 | 77 svc_vc_reply+0x16a 512 |@ 1494 svc_sendreply_common+0x9a 1024 |@@ 2848 svc_sendreply_mbuf+0x5a 2048 |@@@ 4024 nfssvc_program+0x879 4096 |@@@@@ 6464 svc_run_internal+0xf69 8192 |@@@@@@@ 8645 svc_run+0x16d 16384 |@@@@@@ 7035 nfsrvd_nfsd+0x31b 32768 |@ 1650 nfssvc_nfsd+0x1c9 65536 | 229 sys_nfssvc+0xba 131072 | 23 amd64_syscall+0x508 262144 | 0 0xffffffff80f096ab 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 20230 2% 97% 0.00 6935 tcpinp tcp_usr_send+0xb3 nsec ------ Time Distribution ------ count Stack 256 | 65 sosend_generic+0x570 512 | 317 soo_write+0x49 1024 |@ 1143 dofilewrite+0x89 2048 |@@@ 2510 kern_writev+0x68 4096 |@@@@@ 3893 sys_writev+0x36 8192 |@@@@@@@@@@ 6982 amd64_syscall+0x508 16384 |@@@@@ 3953 0xffffffff80f096ab 32768 |@ 1099 65536 | 249 131072 | 16 262144 | 2 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 18270 2% 99% 0.00 7432 tcpinp tcp_usr_send+0xb3 nsec ------ Time Distribution ------ count Stack 256 | 35 sosend_generic+0x570 512 | 362 soo_write+0x49 1024 |@ 946 dofilewrite+0x89 2048 |@@@ 2088 kern_writev+0x68 4096 |@@@@@@ 4025 sys_write+0x60 8192 |@@@@@@@ 4559 amd64_syscall+0x508 16384 |@@@@@@@ 4738 0xffffffff80f096ab 32768 |@@ 1245 65536 | 243 131072 | 29 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8259 1% 99% 0.00 6743 tcpinp tcp_usr_rcvd+0x78 nsec ------ Time Distribution ------ count Stack 256 | 17 soreceive_generic+0xfe1 512 | 241 dofileread+0x98 1024 |@@@ 1033 kern_readv+0x68 2048 |@@@@@ 1397 sys_read+0x60 4096 |@@@@@@ 1708 amd64_syscall+0x508 8192 |@@@@@@ 1762 0xffffffff80f096ab 16384 |@@@@ 1296 32768 |@@ 638 65536 | 154 131072 | 12 262144 | 0 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2673 0% 100% 0.00 6992 tcpinp tcp_usr_rcvd+0x78 nsec ------ Time Distribution ------ count Stack 256 | 11 soreceive_generic+0xfe1 512 | 87 kern_recvit+0x212 1024 |@@ 227 sys_recvfrom+0x86 2048 |@@@@ 425 amd64_syscall+0x508 4096 |@@@@@@ 555 0xffffffff80f096ab 8192 |@@@@@@@ 673 16384 |@@@@ 437 32768 |@@ 215 65536 | 36 131072 | 6 262144 | 0 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 593 0% 100% 0.00 7319 tcpinp tcp_usr_send+0xb3 nsec ------ Time Distribution ------ count Stack 256 | 2 sosend_generic+0x570 512 |@ 21 kern_sendit+0x247 1024 |@@@@ 83 sendit+0x126 2048 |@@@ 75 sys_sendto+0x4d 4096 |@@@ 62 amd64_syscall+0x508 8192 |@@@@@@@@@ 195 0xffffffff80f096ab 16384 |@@@@ 85 32768 |@@ 57 65536 | 12 131072 | 1 ------------------------------------------------------------------------------- R/W reader spin on write wanted: 405 events in 60.426 seconds (7 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 164 29% 29% 0.00 3538 Name Cache cache_lookup+0x495 nsec ------ Time Distribution ------ count Stack 256 |@ 7 vfs_cache_lookup+0xac 512 |@@@@ 27 VOP_LOOKUP_APV+0xa0 1024 |@@@@@@ 35 lookup+0x59e 2048 |@@@@@@ 35 namei+0x5a1 4096 |@@@@ 24 kern_accessat+0x11e 8192 |@@@ 20 amd64_syscall+0x508 16384 |@ 10 0xffffffff80f096ab 32768 | 4 65536 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 35 14% 43% 0.00 8103 Name Cache cache_lookup+0x495 nsec ------ Time Distribution ------ count Stack 256 | 1 vfs_cache_lookup+0xac 512 |@@@ 4 VOP_LOOKUP_APV+0xa0 1024 |@@@ 4 lookup+0x59e 2048 |@@@@@@ 7 namei+0x5a1 4096 |@@ 3 kern_statat+0xab 8192 |@@@@@ 6 sys_stat+0x2d 16384 |@@@@ 5 amd64_syscall+0x508 32768 |@@@ 4 0xffffffff80f096ab 65536 | 0 131072 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 16 9% 52% 0.00 10844 Name Cache cache_lookup+0x1b7 nsec ------ Time Distribution ------ count Stack 512 |@ 1 vfs_cache_lookup+0xac 1024 | 0 VOP_LOOKUP_APV+0xa0 2048 |@@@@@@@ 4 lookup+0x59e 4096 |@@@ 2 nfsvno_namei+0x243 8192 |@@@@@ 3 nfsrvd_lookup+0x13b 16384 |@@@ 2 nfsrvd_dorpc+0xbe5 32768 |@@@ 2 nfssvc_program+0x58e 65536 |@@@ 2 svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 35 7% 59% 0.00 3893 Name Cache cache_lookup+0x495 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@ 12 vfs_cache_lookup+0xac 1024 |@@@@@@@@@@ 12 VOP_LOOKUP_APV+0xa0 2048 |@@@ 4 lookup+0x59e 4096 | 1 nfsvno_namei+0x243 8192 | 0 nfsrvd_lookup+0x13b 16384 |@ 2 nfsrvd_dorpc+0xbe5 32768 |@@@ 4 nfssvc_program+0x58e svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 22 5% 64% 0.00 4765 Name Cache cache_lookup+0x1b7 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@ 5 vfs_cache_lookup+0xac 1024 |@@@@ 3 VOP_LOOKUP_APV+0xa0 2048 |@ 1 lookup+0x59e 4096 |@@ 2 namei+0x5a1 8192 |@@@@@@@@@ 7 kern_statat+0xab 16384 |@@@@@ 4 sys_stat+0x2d amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 4% 68% 0.00 18694 vm object vm_fault_hold+0x398 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 2 vm_fault+0x78 2048 | 0 trap_pfault+0xfa 4096 | 0 trap+0x518 8192 | 0 0xffffffff80f093c7 16384 |@@@@@@@ 1 32768 | 0 65536 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 19 4% 72% 0.00 3751 vm object vm_fault_hold+0xa70 nsec ------ Time Distribution ------ count Stack 512 |@ 1 vm_fault+0x78 1024 |@@@ 2 trap_pfault+0xfa 2048 |@@@@@@@ 5 trap+0x518 4096 |@@@@@@@ 5 0xffffffff80f093c7 8192 |@@@@@@@ 5 16384 | 0 32768 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 24 3% 75% 0.00 2670 Name Cache cache_lookup+0x495 nsec ------ Time Distribution ------ count Stack 256 |@@ 2 vfs_cache_lookup+0xac 512 |@@@ 3 VOP_LOOKUP_APV+0xa0 1024 |@@@@@@@@@@ 8 lookup+0x59e 2048 |@@@ 3 namei+0x5a1 4096 |@@ 2 vn_open_cred+0x21c 8192 |@@@@@ 4 kern_openat+0x25f 16384 |@@ 2 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 3% 78% 0.00 58653 Name Cache cache_enter_time+0x484 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 zfs_lookup+0x54b zfs_freebsd_lookup+0x6d VOP_CACHEDLOOKUP_APV+0xa0 vfs_cache_lookup+0xd6 VOP_LOOKUP_APV+0xa0 lookup+0x59e nfsvno_namei+0x243 nfsrvd_lookup+0x13b nfsrvd_dorpc+0xbe5 nfssvc_program+0x58e svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 3% 81% 0.00 14377 Name Cache cache_lookup+0x1b7 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@ 2 vfs_cache_lookup+0xac 16384 |@@@@@@@ 1 VOP_LOOKUP_APV+0xa0 32768 | 0 lookup+0x59e 65536 |@@@@@@@ 1 namei+0x5a1 vn_open_cred+0x21c kern_openat+0x25f amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- SX exclusive block: 5161 events in 60.426 seconds (85 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3758 36% 36% 0.00 42244 so_snd_sx sosend_generic+0xe8 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 1291 svc_vc_reply+0x16a 65536 |@@@@@@@@@@@@@@@@@ 2133 svc_sendreply_common+0x9a 131072 |@@ 313 svc_sendreply_mbuf+0x5a 262144 | 20 nfssvc_program+0x879 524288 | 1 svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 19% 55% 0.00 14243108 proctree kern_wait6+0x2e6 nsec ------ Time Distribution ------ count Stack 4194304 |@@@@@@@@@@ 2 sys_wait4+0x73 8388608 |@@@@@ 1 amd64_syscall+0x508 16777216 | 0 0xffffffff80f096ab 33554432 |@@@@@@@@@@@@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 11% 66% 0.00 16176821 proctree killjobc+0x146 nsec ------ Time Distribution ------ count Stack 16777216 |@@@@@@@@@@@@@@@@@@@@ 2 exit1+0x810 33554432 |@@@@@@@@@@ 1 sys_sys_exit+0xd amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 71% 0.00 24569269 proctree sys_setsid+0xa6 nsec ------ Time Distribution ------ count Stack 33554432 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 468 4% 76% 0.00 41192 so_snd_sx sosend_generic+0xe8 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@ 177 svc_vc_reply+0x16a 65536 |@@@@@@@@@@@@@@@ 244 svc_sendreply_common+0x9a 131072 |@@ 45 svc_sendreply_mbuf+0x5a 262144 | 2 nfssvc_program+0x879 svc_run_internal+0xf69 svc_run+0x16d nfsrvd_nfsd+0x31b nfssvc_nfsd+0x1c9 sys_nfssvc+0xba amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 3% 79% 0.00 13280324 allproc exit1+0x8af nsec ------ Time Distribution ------ count Stack 16777216 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sys_sys_exit+0xd amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 65 2% 81% 0.00 170563 dp->dp_lock dsl_pool_undirty_space+0x3a nsec ------ Time Distribution ------ count Stack 32768 |@ 4 zio_vdev_io_assess+0x2e5 65536 |@@@@@@@@@@@@@ 29 zio_execute+0x236 131072 |@@@@@@ 13 zio_done+0x94f 262144 |@@@@ 10 zio_execute+0x236 524288 |@@ 5 taskqueue_run_locked+0x13b 1048576 | 1 taskqueue_thread_loop+0xd8 2097152 |@ 3 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 2% 84% 0.00 1021559 dp->dp_lock dsl_pool_undirty_space+0x3a nsec ------ Time Distribution ------ count Stack 131072 |@@@ 1 zio_vdev_io_assess+0x2e5 262144 |@@@@@@@@@ 3 zio_execute+0x236 524288 |@@@@@@ 2 vdev_queue_io_to_issue+0x8fd 1048576 |@@@@@@ 2 vdev_queue_io+0x10e 2097152 | 0 zio_vdev_io_start+0x30b 4194304 |@@@ 1 zio_execute+0x236 8388608 |@@@ 1 zio_nowait+0x49 vdev_raidz_io_start+0x51c zio_vdev_io_start+0x34d zio_execute+0x236 zio_nowait+0x49 vdev_mirror_io_start+0x385 zio_vdev_io_start+0x1cc zio_execute+0x236 taskqueue_run_locked+0x13b taskqueue_thread_loop+0xd8 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 2% 86% 0.00 9395429 allproc fork1+0x763 nsec ------ Time Distribution ------ count Stack 16777216 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sys_fork+0x39 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 22 2% 88% 0.00 412729 scl->scl_lock spa_config_enter+0x63 nsec ------ Time Distribution ------ count Stack 65536 |@@ 2 metaslab_alloc+0x59 131072 |@@@@ 3 zio_dva_allocate+0x75 262144 |@@@@@@ 5 zio_execute+0x236 524288 |@@@@@@@@@@@@ 9 taskqueue_run_locked+0x13b 1048576 |@@ 2 taskqueue_thread_loop+0xd8 2097152 | 0 fork_exit+0x9c 4194304 |@ 1 0xffffffff80f098fe ------------------------------------------------------------------------------- SX shared block: 14 events in 60.426 seconds (0 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 44% 44% 0.00 31900 zf->zf_rwlock dmu_zfetch+0x48 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@ 5 dbuf_read+0x286 65536 |@@@@@@@@ 2 dnode_hold_impl+0x199 dmu_bonus_hold+0x1d zfs_zget+0xdf zfs_vget+0x88 nfsrvd_readdirplus+0xb18 nfsrvd_dorpc+0x12e8 nfssvc_program+0x58e svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 30% 74% 0.00 38443 zf->zf_rwlock dmu_zfetch+0x48 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@ 3 dbuf_read+0x286 65536 | 0 dnode_hold_impl+0x199 131072 |@@@@@@@ 1 dmu_bonus_hold+0x1d zfs_zget+0xdf zfs_fhtovp+0x30a nfsvno_fhtovp+0x72 nfsd_fhtovp+0xc1 nfsrvd_dorpc+0xe34 nfssvc_program+0x58e svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 13% 87% 0.00 68678 vm map (user) vm_map_lookup+0x34 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_fault_hold+0xd1 vm_fault+0x78 trap_pfault+0xfa trap+0x518 0xffffffff80f093c7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 8% 95% 0.00 39639 zf->zf_rwlock dmu_zfetch+0x48 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 dbuf_read+0x286 dnode_hold_impl+0x199 dmu_bonus_hold+0x1d zfs_zget+0xdf zfs_fhtovp+0x30a nfsvno_fhtovp+0x72 nfsd_fhtovp+0xc1 nfsrvd_dorpc+0xe34 nfssvc_program+0x58e svc_run_internal+0xf69 svc_run+0x16d nfsrvd_nfsd+0x31b nfssvc_nfsd+0x1c9 sys_nfssvc+0xba amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 5% 100% 0.00 26988 zf->zf_rwlock dmu_zfetch+0x48 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 dbuf_read+0x286 dnode_hold_impl+0x199 dmu_tx_hold_object_impl+0x32 dmu_tx_hold_sa+0x3c zfs_freebsd_setattr+0xd85 VOP_SETATTR_APV+0xa0 setutimes+0x18b kern_utimensat+0xfd sys_utimensat+0x24 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- SX exclusive spin: 194752 events in 60.426 seconds (3223 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 126109 40% 40% 0.00 6794 so_snd_sx sosend_generic+0xe8 nsec ------ Time Distribution ------ count Stack 256 | 304 svc_vc_reply+0x16a 512 | 4114 svc_sendreply_common+0x9a 1024 |@@ 9139 svc_sendreply_mbuf+0x5a 2048 |@@@@ 18126 nfssvc_program+0x879 4096 |@@@@@@ 28862 svc_run_internal+0xf69 8192 |@@@@@@@ 33208 svc_thread_start+0xb 16384 |@@@@@ 21414 fork_exit+0x9c 32768 |@@ 8896 0xffffffff80f098fe 65536 | 1869 131072 | 143 262144 | 16 524288 | 17 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 19155 27% 67% 0.00 29380 dp->dp_lock dsl_pool_undirty_space+0x3a nsec ------ Time Distribution ------ count Stack 256 | 5 zio_vdev_io_assess+0x2e5 512 | 70 zio_execute+0x236 1024 | 200 zio_done+0x94f 2048 |@ 814 zio_execute+0x236 4096 |@@@@@@@@ 5111 taskqueue_run_locked+0x13b 8192 |@@@@ 2708 taskqueue_thread_loop+0xd8 16384 |@@@@ 3172 fork_exit+0x9c 32768 |@@@@ 2611 0xffffffff80f098fe 65536 |@@@ 2076 131072 |@@ 1533 262144 |@ 681 524288 | 156 1048576 | 12 2097152 | 4 4194304 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15589 5% 72% 0.00 7066 so_snd_sx sosend_generic+0xe8 nsec ------ Time Distribution ------ count Stack 256 | 42 svc_vc_reply+0x16a 512 | 493 svc_sendreply_common+0x9a 1024 |@@ 1149 svc_sendreply_mbuf+0x5a 2048 |@@@@ 2245 nfssvc_program+0x879 4096 |@@@@@@ 3522 svc_run_internal+0xf69 8192 |@@@@@@@ 4054 svc_run+0x16d 16384 |@@@@@ 2673 nfsrvd_nfsd+0x31b 32768 |@@ 1120 nfssvc_nfsd+0x1c9 65536 | 258 sys_nfssvc+0xba 131072 | 24 amd64_syscall+0x508 262144 | 2 0xffffffff80f096ab 524288 | 6 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 731 4% 76% 0.00 105663 scl->scl_lock spa_config_enter+0x63 nsec ------ Time Distribution ------ count Stack 256 | 1 metaslab_alloc+0x59 512 | 10 zio_dva_allocate+0x75 1024 | 15 zio_execute+0x236 2048 | 11 taskqueue_run_locked+0x13b 4096 |@ 33 taskqueue_thread_loop+0xd8 8192 |@ 45 fork_exit+0x9c 16384 |@@@ 78 0xffffffff80f098fe 32768 |@@@ 93 65536 |@@@@@ 128 131072 |@@@@@ 126 262144 |@@@@ 109 524288 |@@ 65 1048576 | 17 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 584 3% 79% 0.00 116376 dp->dp_lock dsl_pool_undirty_space+0x3a nsec ------ Time Distribution ------ count Stack 256 | 1 zio_vdev_io_assess+0x2e5 512 | 4 zio_execute+0x236 1024 | 11 vdev_queue_io_to_issue+0x8fd 2048 | 19 vdev_queue_io+0x10e 4096 |@@@ 65 zio_vdev_io_start+0x30b 8192 |@@ 48 zio_execute+0x236 16384 |@@ 46 zio_nowait+0x49 32768 |@@@@ 85 vdev_raidz_io_start+0x51c 65536 |@@@@@ 112 zio_vdev_io_start+0x34d 131072 |@@@ 60 zio_execute+0x236 262144 |@@ 56 zio_nowait+0x49 524288 |@@ 45 vdev_mirror_io_start+0x385 1048576 |@ 27 zio_vdev_io_start+0x1cc 2097152 | 5 zio_execute+0x236 taskqueue_run_locked+0x13b taskqueue_thread_loop+0xd8 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 755 3% 82% 0.00 82446 scl->scl_lock spa_config_exit+0x45 nsec ------ Time Distribution ------ count Stack 1024 | 6 metaslab_alloc+0xbbc 2048 | 19 zio_dva_allocate+0x75 4096 |@@@@@ 144 zio_execute+0x236 8192 |@@ 74 taskqueue_run_locked+0x13b 16384 |@@ 60 taskqueue_thread_loop+0xd8 32768 |@@@ 84 fork_exit+0x9c 65536 |@@@ 92 0xffffffff80f098fe 131072 |@@@@ 120 262144 |@@@ 93 524288 |@ 48 1048576 | 15 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1200 2% 84% 0.00 36891 dp->dp_lock dsl_pool_undirty_space+0x3a nsec ------ Time Distribution ------ count Stack 256 | 1 zio_vdev_io_assess+0x2e5 512 | 5 zio_execute+0x236 1024 | 32 vdev_queue_io_to_issue+0x8fd 2048 |@@ 112 vdev_queue_io_done+0xc6 4096 |@@@@@ 204 zio_vdev_io_done+0xd9 8192 |@@@ 154 zio_execute+0x236 16384 |@@@@ 178 taskqueue_run_locked+0x13b 32768 |@@@@ 195 taskqueue_thread_loop+0xd8 65536 |@@@ 150 fork_exit+0x9c 131072 |@@ 90 0xffffffff80f098fe 262144 |@ 57 524288 | 17 1048576 | 3 2097152 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 581 1% 85% 0.00 35587 dd->dd_lock dsl_dir_diduse_space+0x6f nsec ------ Time Distribution ------ count Stack 512 | 1 dsl_dir_diduse_space+0x123 1024 |@ 27 dsl_dataset_block_kill+0x1de 2048 |@@ 49 dbuf_write_done+0x4a 4096 |@@@@ 92 arc_write_done+0x2ef 8192 |@@@@ 88 zio_done+0x7ee 16384 |@@@@@ 108 zio_execute+0x236 32768 |@@@@ 87 zio_done+0x94f 65536 |@@ 52 zio_execute+0x236 131072 |@@ 42 zio_done+0x94f 262144 |@ 21 zio_execute+0x236 524288 | 12 zio_done+0x94f 1048576 | 1 zio_execute+0x236 2097152 | 1 taskqueue_run_locked+0x13b taskqueue_thread_loop+0xd8 fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 233 1% 86% 0.00 86942 dp->dp_lock dsl_pool_undirty_space+0x3a nsec ------ Time Distribution ------ count Stack 1024 | 4 zio_vdev_io_assess+0x2e5 2048 | 7 zio_execute+0x236 4096 |@@@ 25 vdev_queue_io_to_issue+0x8fd 8192 |@@@ 24 vdev_queue_io+0x10e 16384 |@@ 22 zio_vdev_io_start+0x30b 32768 |@@@@ 34 zio_execute+0x236 65536 |@@@@@ 45 zio_nowait+0x49 131072 |@@@@ 37 vdev_raidz_io_start+0x51c 262144 |@@ 21 zio_vdev_io_start+0x34d 524288 |@ 10 zio_execute+0x236 1048576 | 2 zio_nowait+0x49 2097152 | 1 vdev_mirror_io_start+0x385 4194304 | 1 zio_vdev_io_start+0x1cc zio_execute+0x236 zio_ready+0x1ee zio_execute+0x236 taskqueue_run_locked+0x13b taskqueue_thread_loop+0xd8 fork_exit+0x9c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 541 1% 87% 0.00 34222 dp->dp_lock dsl_pool_undirty_space+0x3a nsec ------ Time Distribution ------ count Stack 256 | 2 zio_vdev_io_assess+0x2e5 512 | 5 zio_execute+0x236 1024 | 15 taskqueue_run_locked+0x13b 2048 |@@ 38 taskqueue_thread_loop+0xd8 4096 |@@@ 60 fork_exit+0x9c 8192 |@@@@ 77 0xffffffff80f098fe 16384 |@@@ 72 32768 |@@@@@ 93 65536 |@@@@@ 101 131072 |@@ 53 262144 |@ 21 524288 | 4 ------------------------------------------------------------------------------- SX shared spin: 2205 events in 60.426 seconds (36 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 267 61% 61% 0.00 3378 vm map (user) vm_map_lookup+0x34 nsec ------ Time Distribution ------ count Stack 64 | 4 vm_fault_hold+0xd1 128 | 6 vm_fault+0x78 256 |@ 12 trap_pfault+0xfa 512 |@@ 23 trap+0x518 1024 |@@@@@@ 61 0xffffffff80f093c7 2048 |@@@ 32 4096 |@@@@@ 52 8192 |@@@@@ 46 16384 |@@@ 27 32768 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 382 12% 73% 0.00 467 zf->zf_rwlock dmu_zfetch+0x48 nsec ------ Time Distribution ------ count Stack 64 | 5 dbuf_read+0x286 128 |@ 21 dnode_hold_impl+0x199 256 |@@@@@@@@@ 119 dmu_bonus_hold+0x1d 512 |@@@@@@@@@@ 135 zfs_zget+0xdf 1024 |@@@@@@@ 93 zfs_vget+0x88 2048 | 2 nfsrvd_readdirplus+0xb18 4096 | 5 nfsrvd_dorpc+0x12e8 8192 | 0 nfssvc_program+0x58e 16384 | 1 svc_run_internal+0xf69 32768 | 1 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 3% 76% 0.00 14263 vm map (user) vm_map_lookup+0x2fb nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@ 1 vm_fault_hold+0xd1 16384 |@@@@@@@@@@ 1 vm_fault+0x78 32768 |@@@@@@@@@@ 1 trap_pfault+0xfa trap+0x518 0xffffffff80f093c7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 44 3% 78% 0.00 956 zf->zf_rwlock dmu_zfetch+0x48 nsec ------ Time Distribution ------ count Stack 64 | 1 dbuf_read+0x286 128 |@ 2 dnode_hold_impl+0x199 256 |@@@@@@@@@@@ 17 dmu_bonus_hold+0x1d 512 |@@@@@@@@@@@ 17 zfs_zget+0xdf 1024 |@@ 3 zfs_fhtovp+0x30a 2048 | 0 nfsvno_fhtovp+0x72 4096 | 1 nfsd_fhtovp+0xc1 8192 | 1 nfsrvd_dorpc+0xe34 16384 |@ 2 nfssvc_program+0x58e svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 75 2% 80% 0.00 406 zf->zf_rwlock dmu_zfetch+0x48 nsec ------ Time Distribution ------ count Stack 64 | 1 dbuf_read+0x286 128 |@ 4 dnode_hold_impl+0x199 256 |@@@@@ 14 dmu_buf_hold_noread+0x28 512 |@@@@@@@@@@@ 29 dmu_buf_hold+0x1c 1024 |@@@@@@@@@@ 27 zap_get_leaf_byblk+0x45 fzap_cursor_retrieve+0x15f zap_cursor_retrieve+0x20e zfs_freebsd_readdir+0x460 VOP_READDIR_APV+0xa6 nfsrvd_readdirplus+0x423 nfsrvd_dorpc+0x12e8 nfssvc_program+0x58e svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 178 2% 82% 0.00 148 dn->dn_struct_rwlock dbuf_read+0x8a nsec ------ Time Distribution ------ count Stack 64 |@@@@@ 35 dnode_hold_impl+0x199 128 |@@@@@@@@@ 58 dmu_bonus_hold+0x1d 256 |@@@@@@@@@ 59 zfs_zget+0xdf 512 |@@@@ 24 zfs_vget+0x88 1024 | 2 nfsrvd_readdirplus+0xb18 nfsrvd_dorpc+0x12e8 nfssvc_program+0x58e svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 227 2% 84% 0.00 103 dn->dn_struct_rwlock dnode_hold_impl+0x112 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@@ 54 dmu_bonus_hold+0x1d 128 |@@@@@@@@@@@@@@@ 119 zfs_zget+0xdf 256 |@@@@@ 43 zfs_vget+0x88 512 |@ 11 nfsrvd_readdirplus+0xb18 nfsrvd_dorpc+0x12e8 nfssvc_program+0x58e svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 177 2% 85% 0.00 126 zfsvfs->z_teardown_inactive_lock zfs_inactive+0x36 nsec ------ Time Distribution ------ count Stack 64 |@@@@@@ 41 zfs_freebsd_inactive+0xd 128 |@@@@@@@@@@ 63 VOP_INACTIVE_APV+0xa6 256 |@@@@@@@@@ 57 vinactive+0x189 512 |@@ 16 vputx+0x2ef nfsrvd_readdirplus+0x1165 nfsrvd_dorpc+0x12e8 nfssvc_program+0x58e svc_run_internal+0xf69 svc_thread_start+0xb fork_exit+0x9c 0xffffffff80f098fe ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 1% 87% 0.00 7313 proctree sysctl_kern_proc+0x194 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@ 1 sysctl_root_handler_locked+0xb1 8192 | 0 sysctl_root+0x1e1 16384 |@@@@@@@@@@@@@@@@@@@@ 2 userland_sysctl+0x1b6 sys___sysctl+0x74 amd64_syscall+0x508 0xffffffff80f096ab ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 1% 88% 0.00 21375 vm map (user) vm_map_madvise+0x83 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sys_madvise+0x85 amd64_syscall+0x508 0xffffffff80f096ab -------------------------------------------------------------------------------