16 #include "kmp_error.h" 20 #include "kmp_stats.h" 23 #include "ompt-specific.h" 26 #define MAX_MESSAGE 512 42 if ((env = getenv(
"KMP_INITIAL_THREAD_BIND")) != NULL &&
43 __kmp_str_match_true(env)) {
44 __kmp_middle_initialize();
45 KC_TRACE(10, (
"__kmpc_begin: middle initialization called\n"));
46 }
else if (__kmp_ignore_mppbeg() == FALSE) {
48 __kmp_internal_begin();
49 KC_TRACE(10, (
"__kmpc_begin: called\n"));
67 if (__kmp_ignore_mppend() == FALSE) {
68 KC_TRACE(10, (
"__kmpc_end: called\n"));
69 KA_TRACE(30, (
"__kmpc_end\n"));
71 __kmp_internal_end_thread(-1);
94 kmp_int32 gtid = __kmp_entry_gtid();
96 KC_TRACE(10, (
"__kmpc_global_thread_num: T#%d\n", gtid));
117 (
"__kmpc_global_num_threads: num_threads = %d\n", __kmp_all_nth));
119 return TCR_4(__kmp_all_nth);
129 KC_TRACE(10, (
"__kmpc_bound_thread_num: called\n"));
130 return __kmp_tid_from_gtid(__kmp_entry_gtid());
139 KC_TRACE(10, (
"__kmpc_bound_num_threads: called\n"));
141 return __kmp_entry_thread()->th.th_team->t.t_nproc;
161 if (__kmp_par_range == 0) {
168 semi2 = strchr(semi2,
';');
172 semi2 = strchr(semi2 + 1,
';');
176 if (__kmp_par_range_filename[0]) {
177 const char *name = semi2 - 1;
178 while ((name > loc->
psource) && (*name !=
'/') && (*name !=
';')) {
181 if ((*name ==
'/') || (*name ==
';')) {
184 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
185 return __kmp_par_range < 0;
188 semi3 = strchr(semi2 + 1,
';');
189 if (__kmp_par_range_routine[0]) {
190 if ((semi3 != NULL) && (semi3 > semi2) &&
191 (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
192 return __kmp_par_range < 0;
195 if (KMP_SSCANF(semi3 + 1,
"%d", &line_no) == 1) {
196 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
197 return __kmp_par_range > 0;
199 return __kmp_par_range < 0;
213 return __kmp_entry_thread()->th.th_root->r.r_active;
226 kmp_int32 num_threads) {
227 KA_TRACE(20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
228 global_tid, num_threads));
230 __kmp_push_num_threads(loc, global_tid, num_threads);
233 void __kmpc_pop_num_threads(
ident_t *loc, kmp_int32 global_tid) {
234 KA_TRACE(20, (
"__kmpc_pop_num_threads: enter\n"));
241 void __kmpc_push_proc_bind(
ident_t *loc, kmp_int32 global_tid,
242 kmp_int32 proc_bind) {
243 KA_TRACE(20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid,
246 __kmp_push_proc_bind(loc, global_tid, (kmp_proc_bind_t)proc_bind);
262 int gtid = __kmp_entry_gtid();
264 #if (KMP_STATS_ENABLED) 276 va_start(ap, microtask);
279 ompt_frame_t *ompt_frame;
280 if (ompt_enabled.enabled) {
281 kmp_info_t *master_th = __kmp_threads[gtid];
282 kmp_team_t *parent_team = master_th->th.th_team;
283 ompt_lw_taskteam_t *lwt = parent_team->t.ompt_serialized_team_info;
285 ompt_frame = &(lwt->ompt_task_info.frame);
287 int tid = __kmp_tid_from_gtid(gtid);
289 parent_team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame);
291 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
292 OMPT_STORE_RETURN_ADDRESS(gtid);
296 #if INCLUDE_SSC_MARKS 299 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
300 VOLATILE_CAST(microtask_t) microtask,
301 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
303 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
309 #if INCLUDE_SSC_MARKS 312 __kmp_join_call(loc, gtid
336 kmp_int32 num_teams, kmp_int32 num_threads) {
338 (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
339 global_tid, num_teams, num_threads));
341 __kmp_push_num_teams(loc, global_tid, num_teams, num_threads);
356 int gtid = __kmp_entry_gtid();
357 kmp_info_t *this_thr = __kmp_threads[gtid];
359 va_start(ap, microtask);
364 this_thr->th.th_teams_microtask = microtask;
365 this_thr->th.th_teams_level =
366 this_thr->th.th_team->t.t_level;
369 kmp_team_t *parent_team = this_thr->th.th_team;
370 int tid = __kmp_tid_from_gtid(gtid);
371 if (ompt_enabled.enabled) {
372 parent_team->t.t_implicit_task_taskdata[tid]
373 .ompt_task_info.frame.enter_frame = OMPT_GET_FRAME_ADDRESS(1);
375 OMPT_STORE_RETURN_ADDRESS(gtid);
380 if (this_thr->th.th_teams_size.nteams == 0) {
381 __kmp_push_num_teams(loc, gtid, 0, 0);
383 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
384 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
385 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
387 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
388 VOLATILE_CAST(microtask_t)
390 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
391 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
397 __kmp_join_call(loc, gtid
404 this_thr->th.th_teams_microtask = NULL;
405 this_thr->th.th_teams_level = 0;
406 *(kmp_int64 *)(&this_thr->th.th_teams_size) = 0L;
415 int __kmpc_invoke_task_func(
int gtid) {
return __kmp_invoke_task_func(gtid); }
434 OMPT_STORE_RETURN_ADDRESS(global_tid);
436 __kmp_serialized_parallel(loc, global_tid);
447 kmp_internal_control_t *top;
448 kmp_info_t *this_thr;
449 kmp_team_t *serial_team;
452 (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid));
460 if (!TCR_4(__kmp_init_parallel))
461 __kmp_parallel_initialize();
463 this_thr = __kmp_threads[global_tid];
464 serial_team = this_thr->th.th_serial_team;
467 kmp_task_team_t *task_team = this_thr->th.th_task_team;
470 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks)
471 __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL));
475 KMP_DEBUG_ASSERT(serial_team);
476 KMP_ASSERT(serial_team->t.t_serialized);
477 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
478 KMP_DEBUG_ASSERT(serial_team != this_thr->th.th_root->r.r_root_team);
479 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
480 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
483 if (ompt_enabled.enabled &&
484 this_thr->th.ompt_thread_info.state != omp_state_overhead) {
485 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = NULL;
486 if (ompt_enabled.ompt_callback_implicit_task) {
487 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
488 ompt_scope_end, NULL, OMPT_CUR_TASK_DATA(this_thr), 1,
489 __kmp_tid_from_gtid(global_tid));
493 ompt_data_t *parent_task_data;
494 __ompt_get_task_info_internal(1, NULL, &parent_task_data, NULL, NULL, NULL);
496 if (ompt_enabled.ompt_callback_parallel_end) {
497 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
498 &(serial_team->t.ompt_team_info.parallel_data), parent_task_data,
499 ompt_invoker_program, OMPT_LOAD_RETURN_ADDRESS(global_tid));
501 __ompt_lw_taskteam_unlink(this_thr);
502 this_thr->th.ompt_thread_info.state = omp_state_overhead;
508 top = serial_team->t.t_control_stack_top;
509 if (top && top->serial_nesting_level == serial_team->t.t_serialized) {
510 copy_icvs(&serial_team->t.t_threads[0]->th.th_current_task->td_icvs, top);
511 serial_team->t.t_control_stack_top = top->next;
516 serial_team->t.t_level--;
519 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
521 dispatch_private_info_t *disp_buffer =
522 serial_team->t.t_dispatch->th_disp_buffer;
523 serial_team->t.t_dispatch->th_disp_buffer =
524 serial_team->t.t_dispatch->th_disp_buffer->next;
525 __kmp_free(disp_buffer);
528 --serial_team->t.t_serialized;
529 if (serial_team->t.t_serialized == 0) {
533 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 534 if (__kmp_inherit_fp_control && serial_team->t.t_fp_control_saved) {
535 __kmp_clear_x87_fpu_status_word();
536 __kmp_load_x87_fpu_control_word(&serial_team->t.t_x87_fpu_control_word);
537 __kmp_load_mxcsr(&serial_team->t.t_mxcsr);
541 this_thr->th.th_team = serial_team->t.t_parent;
542 this_thr->th.th_info.ds.ds_tid = serial_team->t.t_master_tid;
545 this_thr->th.th_team_nproc = serial_team->t.t_parent->t.t_nproc;
546 this_thr->th.th_team_master =
547 serial_team->t.t_parent->t.t_threads[0];
548 this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized;
551 this_thr->th.th_dispatch =
552 &this_thr->th.th_team->t.t_dispatch[serial_team->t.t_master_tid];
554 __kmp_pop_current_task_from_thread(this_thr);
556 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0);
557 this_thr->th.th_current_task->td_flags.executing = 1;
559 if (__kmp_tasking_mode != tskm_immediate_exec) {
561 this_thr->th.th_task_team =
562 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
564 (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / " 566 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
569 if (__kmp_tasking_mode != tskm_immediate_exec) {
570 KA_TRACE(20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting " 571 "depth of serial team %p to %d\n",
572 global_tid, serial_team, serial_team->t.t_serialized));
576 if (__kmp_env_consistency_check)
577 __kmp_pop_parallel(global_tid, NULL);
579 if (ompt_enabled.enabled)
580 this_thr->th.ompt_thread_info.state =
581 ((this_thr->th.th_team_serialized) ? omp_state_work_serial
582 : omp_state_work_parallel);
595 KC_TRACE(10, (
"__kmpc_flush: called\n"));
600 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 614 if (!__kmp_cpuinfo.initialized) {
615 __kmp_query_cpuid(&__kmp_cpuinfo);
617 if (!__kmp_cpuinfo.sse2) {
622 #elif KMP_COMPILER_MSVC 625 __sync_synchronize();
626 #endif // KMP_COMPILER_ICC 629 #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64) 645 #error Unknown or unsupported architecture 648 #if OMPT_SUPPORT && OMPT_OPTIONAL 649 if (ompt_enabled.ompt_callback_flush) {
650 ompt_callbacks.ompt_callback(ompt_callback_flush)(
651 __ompt_get_thread_data_internal(), OMPT_GET_RETURN_ADDRESS(0));
666 KC_TRACE(10, (
"__kmpc_barrier: called T#%d\n", global_tid));
668 if (!TCR_4(__kmp_init_parallel))
669 __kmp_parallel_initialize();
671 if (__kmp_env_consistency_check) {
673 KMP_WARNING(ConstructIdentInvalid);
676 __kmp_check_barrier(global_tid, ct_barrier, loc);
680 ompt_frame_t *ompt_frame;
681 if (ompt_enabled.enabled) {
682 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
683 if (ompt_frame->enter_frame == NULL)
684 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
685 OMPT_STORE_RETURN_ADDRESS(global_tid);
688 __kmp_threads[global_tid]->th.th_ident = loc;
696 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
697 #if OMPT_SUPPORT && OMPT_OPTIONAL 698 if (ompt_enabled.enabled) {
699 ompt_frame->enter_frame = NULL;
714 KC_TRACE(10, (
"__kmpc_master: called T#%d\n", global_tid));
716 if (!TCR_4(__kmp_init_parallel))
717 __kmp_parallel_initialize();
719 if (KMP_MASTER_GTID(global_tid)) {
721 KMP_PUSH_PARTITIONED_TIMER(OMP_master);
725 #if OMPT_SUPPORT && OMPT_OPTIONAL 727 if (ompt_enabled.ompt_callback_master) {
728 kmp_info_t *this_thr = __kmp_threads[global_tid];
729 kmp_team_t *team = this_thr->th.th_team;
731 int tid = __kmp_tid_from_gtid(global_tid);
732 ompt_callbacks.ompt_callback(ompt_callback_master)(
733 ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
734 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
735 OMPT_GET_RETURN_ADDRESS(0));
740 if (__kmp_env_consistency_check) {
741 #if KMP_USE_DYNAMIC_LOCK 743 __kmp_push_sync(global_tid, ct_master, loc, NULL, 0);
745 __kmp_check_sync(global_tid, ct_master, loc, NULL, 0);
748 __kmp_push_sync(global_tid, ct_master, loc, NULL);
750 __kmp_check_sync(global_tid, ct_master, loc, NULL);
766 KC_TRACE(10, (
"__kmpc_end_master: called T#%d\n", global_tid));
768 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(global_tid));
769 KMP_POP_PARTITIONED_TIMER();
771 #if OMPT_SUPPORT && OMPT_OPTIONAL 772 kmp_info_t *this_thr = __kmp_threads[global_tid];
773 kmp_team_t *team = this_thr->th.th_team;
774 if (ompt_enabled.ompt_callback_master) {
775 int tid = __kmp_tid_from_gtid(global_tid);
776 ompt_callbacks.ompt_callback(ompt_callback_master)(
777 ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
778 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
779 OMPT_GET_RETURN_ADDRESS(0));
783 if (__kmp_env_consistency_check) {
785 KMP_WARNING(ThreadIdentInvalid);
787 if (KMP_MASTER_GTID(global_tid))
788 __kmp_pop_sync(global_tid, ct_master, loc);
802 KMP_DEBUG_ASSERT(__kmp_init_serial);
804 KC_TRACE(10, (
"__kmpc_ordered: called T#%d\n", gtid));
806 if (!TCR_4(__kmp_init_parallel))
807 __kmp_parallel_initialize();
810 __kmp_itt_ordered_prep(gtid);
814 th = __kmp_threads[gtid];
816 #if OMPT_SUPPORT && OMPT_OPTIONAL 820 if (ompt_enabled.enabled) {
821 OMPT_STORE_RETURN_ADDRESS(gtid);
822 team = __kmp_team_from_gtid(gtid);
823 lck = (ompt_wait_id_t)&team->t.t_ordered.dt.t_value;
825 th->th.ompt_thread_info.wait_id = lck;
826 th->th.ompt_thread_info.state = omp_state_wait_ordered;
829 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
830 if (ompt_enabled.ompt_callback_mutex_acquire) {
831 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
832 ompt_mutex_ordered, omp_lock_hint_none, ompt_mutex_impl_spin,
833 (ompt_wait_id_t)lck, codeptr_ra);
838 if (th->th.th_dispatch->th_deo_fcn != 0)
839 (*th->th.th_dispatch->th_deo_fcn)(>id, &cid, loc);
841 __kmp_parallel_deo(>id, &cid, loc);
843 #if OMPT_SUPPORT && OMPT_OPTIONAL 844 if (ompt_enabled.enabled) {
846 th->th.ompt_thread_info.state = omp_state_work_parallel;
847 th->th.ompt_thread_info.wait_id = 0;
850 if (ompt_enabled.ompt_callback_mutex_acquired) {
851 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
852 ompt_mutex_ordered, (ompt_wait_id_t)lck, codeptr_ra);
858 __kmp_itt_ordered_start(gtid);
873 KC_TRACE(10, (
"__kmpc_end_ordered: called T#%d\n", gtid));
876 __kmp_itt_ordered_end(gtid);
880 th = __kmp_threads[gtid];
882 if (th->th.th_dispatch->th_dxo_fcn != 0)
883 (*th->th.th_dispatch->th_dxo_fcn)(>id, &cid, loc);
885 __kmp_parallel_dxo(>id, &cid, loc);
887 #if OMPT_SUPPORT && OMPT_OPTIONAL 888 OMPT_STORE_RETURN_ADDRESS(gtid);
889 if (ompt_enabled.ompt_callback_mutex_released) {
890 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
892 (ompt_wait_id_t)&__kmp_team_from_gtid(gtid)->t.t_ordered.dt.t_value,
893 OMPT_LOAD_RETURN_ADDRESS(gtid));
898 #if KMP_USE_DYNAMIC_LOCK 900 static __forceinline
void 901 __kmp_init_indirect_csptr(kmp_critical_name *crit,
ident_t const *loc,
902 kmp_int32 gtid, kmp_indirect_locktag_t tag) {
906 kmp_indirect_lock_t **lck;
907 lck = (kmp_indirect_lock_t **)crit;
908 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
909 KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
910 KMP_SET_I_LOCK_LOCATION(ilk, loc);
911 KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
913 (
"__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
915 __kmp_itt_critical_creating(ilk->lock, loc);
917 int status = KMP_COMPARE_AND_STORE_PTR(lck,
nullptr, ilk);
920 __kmp_itt_critical_destroyed(ilk->lock);
926 KMP_DEBUG_ASSERT(*lck != NULL);
930 #define KMP_ACQUIRE_TAS_LOCK(lock, gtid) \ 932 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 933 if (l->lk.poll != KMP_LOCK_FREE(tas) || \ 934 !KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), \ 935 KMP_LOCK_BUSY(gtid + 1, tas))) { \ 937 KMP_FSYNC_PREPARE(l); \ 938 KMP_INIT_YIELD(spins); \ 939 if (TCR_4(__kmp_nth) > \ 940 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 943 KMP_YIELD_SPIN(spins); \ 945 kmp_backoff_t backoff = __kmp_spin_backoff_params; \ 946 while (l->lk.poll != KMP_LOCK_FREE(tas) || \ 947 !KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), \ 948 KMP_LOCK_BUSY(gtid + 1, tas))) { \ 949 __kmp_spin_backoff(&backoff); \ 950 if (TCR_4(__kmp_nth) > \ 951 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 954 KMP_YIELD_SPIN(spins); \ 958 KMP_FSYNC_ACQUIRED(l); \ 962 #define KMP_TEST_TAS_LOCK(lock, gtid, rc) \ 964 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 965 rc = l->lk.poll == KMP_LOCK_FREE(tas) && \ 966 KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), \ 967 KMP_LOCK_BUSY(gtid + 1, tas)); \ 971 #define KMP_RELEASE_TAS_LOCK(lock, gtid) \ 973 TCW_4(((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); \ 979 #include <sys/syscall.h> 989 #define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) \ 991 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 992 kmp_int32 gtid_code = (gtid + 1) << 1; \ 994 KMP_FSYNC_PREPARE(ftx); \ 995 kmp_int32 poll_val; \ 996 while ((poll_val = KMP_COMPARE_AND_STORE_RET32( \ 997 &(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 998 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \ 999 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \ 1001 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, \ 1003 KMP_LOCK_BUSY(1, futex))) { \ 1006 poll_val |= KMP_LOCK_BUSY(1, futex); \ 1009 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, \ 1010 NULL, NULL, 0)) != 0) { \ 1015 KMP_FSYNC_ACQUIRED(ftx); \ 1019 #define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) \ 1021 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1022 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 1023 KMP_LOCK_BUSY(gtid + 1 << 1, futex))) { \ 1024 KMP_FSYNC_ACQUIRED(ftx); \ 1032 #define KMP_RELEASE_FUTEX_LOCK(lock, gtid) \ 1034 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1036 KMP_FSYNC_RELEASING(ftx); \ 1037 kmp_int32 poll_val = \ 1038 KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \ 1039 if (KMP_LOCK_STRIP(poll_val) & 1) { \ 1040 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, \ 1041 KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \ 1044 KMP_YIELD(TCR_4(__kmp_nth) > \ 1045 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \ 1048 #endif // KMP_USE_FUTEX 1050 #else // KMP_USE_DYNAMIC_LOCK 1052 static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit,
1055 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1058 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1065 lck = __kmp_user_lock_allocate(&idx, gtid, kmp_lf_critical_section);
1066 __kmp_init_user_lock_with_checks(lck);
1067 __kmp_set_user_lock_location(lck, loc);
1069 __kmp_itt_critical_creating(lck);
1080 int status = KMP_COMPARE_AND_STORE_PTR(lck_pp, 0, lck);
1085 __kmp_itt_critical_destroyed(lck);
1089 __kmp_destroy_user_lock_with_checks(lck);
1090 __kmp_user_lock_free(&idx, gtid, lck);
1091 lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1092 KMP_DEBUG_ASSERT(lck != NULL);
1098 #endif // KMP_USE_DYNAMIC_LOCK 1111 kmp_critical_name *crit) {
1112 #if KMP_USE_DYNAMIC_LOCK 1113 #if OMPT_SUPPORT && OMPT_OPTIONAL 1114 OMPT_STORE_RETURN_ADDRESS(global_tid);
1115 #endif // OMPT_SUPPORT 1116 __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
1119 KMP_TIME_PARTITIONED_BLOCK(
1121 #if OMPT_SUPPORT && OMPT_OPTIONAL 1122 omp_state_t prev_state = omp_state_undefined;
1123 ompt_thread_info_t ti;
1125 kmp_user_lock_p lck;
1127 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1131 KMP_CHECK_USER_LOCK_INIT();
1133 if ((__kmp_user_lock_kind == lk_tas) &&
1134 (
sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1135 lck = (kmp_user_lock_p)crit;
1138 else if ((__kmp_user_lock_kind == lk_futex) &&
1139 (
sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1140 lck = (kmp_user_lock_p)crit;
1144 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
1147 if (__kmp_env_consistency_check)
1148 __kmp_push_sync(global_tid, ct_critical, loc, lck);
1156 __kmp_itt_critical_acquiring(lck);
1158 #if OMPT_SUPPORT && OMPT_OPTIONAL 1159 OMPT_STORE_RETURN_ADDRESS(gtid);
1160 void *codeptr_ra = NULL;
1161 if (ompt_enabled.enabled) {
1162 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1164 prev_state = ti.state;
1165 ti.wait_id = (ompt_wait_id_t)lck;
1166 ti.state = omp_state_wait_critical;
1169 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1170 if (ompt_enabled.ompt_callback_mutex_acquire) {
1171 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1172 ompt_mutex_critical, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
1173 (ompt_wait_id_t)crit, codeptr_ra);
1179 __kmp_acquire_user_lock_with_checks(lck, global_tid);
1182 __kmp_itt_critical_acquired(lck);
1184 #if OMPT_SUPPORT && OMPT_OPTIONAL 1185 if (ompt_enabled.enabled) {
1187 ti.state = prev_state;
1191 if (ompt_enabled.ompt_callback_mutex_acquired) {
1192 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1193 ompt_mutex_critical, (ompt_wait_id_t)crit, codeptr_ra);
1199 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1200 #endif // KMP_USE_DYNAMIC_LOCK 1203 #if KMP_USE_DYNAMIC_LOCK 1206 static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
1208 #define KMP_TSX_LOCK(seq) lockseq_##seq 1210 #define KMP_TSX_LOCK(seq) __kmp_user_lock_seq 1213 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1214 #define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm) 1216 #define KMP_CPUINFO_RTM 0 1220 if (hint & kmp_lock_hint_hle)
1221 return KMP_TSX_LOCK(hle);
1222 if (hint & kmp_lock_hint_rtm)
1223 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm) : __kmp_user_lock_seq;
1224 if (hint & kmp_lock_hint_adaptive)
1225 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive) : __kmp_user_lock_seq;
1228 if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
1229 return __kmp_user_lock_seq;
1230 if ((hint & omp_lock_hint_speculative) &&
1231 (hint & omp_lock_hint_nonspeculative))
1232 return __kmp_user_lock_seq;
1235 if (hint & omp_lock_hint_contended)
1236 return lockseq_queuing;
1239 if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
1243 if (hint & omp_lock_hint_speculative)
1244 return KMP_TSX_LOCK(hle);
1246 return __kmp_user_lock_seq;
1249 #if OMPT_SUPPORT && OMPT_OPTIONAL 1250 static ompt_mutex_impl_t
1251 __ompt_get_mutex_impl_type(
void *user_lock, kmp_indirect_lock_t *ilock = 0) {
1253 switch (KMP_EXTRACT_D_TAG(user_lock)) {
1258 return ompt_mutex_impl_queuing;
1261 return ompt_mutex_impl_spin;
1264 return ompt_mutex_impl_speculative;
1267 return ompt_mutex_impl_unknown;
1269 ilock = KMP_LOOKUP_I_LOCK(user_lock);
1272 switch (ilock->type) {
1274 case locktag_adaptive:
1276 return ompt_mutex_impl_speculative;
1278 case locktag_nested_tas:
1279 return ompt_mutex_impl_spin;
1281 case locktag_nested_futex:
1283 case locktag_ticket:
1284 case locktag_queuing:
1286 case locktag_nested_ticket:
1287 case locktag_nested_queuing:
1288 case locktag_nested_drdpa:
1289 return ompt_mutex_impl_queuing;
1291 return ompt_mutex_impl_unknown;
1296 static ompt_mutex_impl_t __ompt_get_mutex_impl_type() {
1297 switch (__kmp_user_lock_kind) {
1299 return ompt_mutex_impl_spin;
1306 return ompt_mutex_impl_queuing;
1311 return ompt_mutex_impl_speculative;
1314 return ompt_mutex_impl_unknown;
1332 void __kmpc_critical_with_hint(
ident_t *loc, kmp_int32 global_tid,
1333 kmp_critical_name *crit, uintptr_t hint) {
1335 kmp_user_lock_p lck;
1336 #if OMPT_SUPPORT && OMPT_OPTIONAL 1337 omp_state_t prev_state = omp_state_undefined;
1338 ompt_thread_info_t ti;
1340 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1342 codeptr = OMPT_GET_RETURN_ADDRESS(0);
1345 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1347 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
1350 kmp_dyna_lockseq_t lckseq = __kmp_map_hint_to_lock(hint);
1351 if (KMP_IS_D_LOCK(lckseq)) {
1352 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0,
1353 KMP_GET_D_TAG(lckseq));
1355 __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lckseq));
1361 if (KMP_EXTRACT_D_TAG(lk) != 0) {
1362 lck = (kmp_user_lock_p)lk;
1363 if (__kmp_env_consistency_check) {
1364 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1365 __kmp_map_hint_to_lock(hint));
1368 __kmp_itt_critical_acquiring(lck);
1370 #if OMPT_SUPPORT && OMPT_OPTIONAL 1371 if (ompt_enabled.enabled) {
1372 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1374 prev_state = ti.state;
1375 ti.wait_id = (ompt_wait_id_t)lck;
1376 ti.state = omp_state_wait_critical;
1379 if (ompt_enabled.ompt_callback_mutex_acquire) {
1380 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1381 ompt_mutex_critical, (
unsigned int)hint,
1382 __ompt_get_mutex_impl_type(crit), (ompt_wait_id_t)crit, codeptr);
1386 #if KMP_USE_INLINED_TAS 1387 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1388 KMP_ACQUIRE_TAS_LOCK(lck, global_tid);
1390 #elif KMP_USE_INLINED_FUTEX 1391 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1392 KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid);
1396 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
1399 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
1401 if (__kmp_env_consistency_check) {
1402 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1403 __kmp_map_hint_to_lock(hint));
1406 __kmp_itt_critical_acquiring(lck);
1408 #if OMPT_SUPPORT && OMPT_OPTIONAL 1409 if (ompt_enabled.enabled) {
1410 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1412 prev_state = ti.state;
1413 ti.wait_id = (ompt_wait_id_t)lck;
1414 ti.state = omp_state_wait_critical;
1417 if (ompt_enabled.ompt_callback_mutex_acquire) {
1418 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1419 ompt_mutex_critical, (
unsigned int)hint,
1420 __ompt_get_mutex_impl_type(0, ilk), (ompt_wait_id_t)crit, codeptr);
1424 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
1428 __kmp_itt_critical_acquired(lck);
1430 #if OMPT_SUPPORT && OMPT_OPTIONAL 1431 if (ompt_enabled.enabled) {
1433 ti.state = prev_state;
1437 if (ompt_enabled.ompt_callback_mutex_acquired) {
1438 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1439 ompt_mutex_critical, (ompt_wait_id_t)crit, codeptr);
1444 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1445 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1448 #endif // KMP_USE_DYNAMIC_LOCK 1460 kmp_critical_name *crit) {
1461 kmp_user_lock_p lck;
1463 KC_TRACE(10, (
"__kmpc_end_critical: called T#%d\n", global_tid));
1465 #if KMP_USE_DYNAMIC_LOCK 1466 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
1467 lck = (kmp_user_lock_p)crit;
1468 KMP_ASSERT(lck != NULL);
1469 if (__kmp_env_consistency_check) {
1470 __kmp_pop_sync(global_tid, ct_critical, loc);
1473 __kmp_itt_critical_releasing(lck);
1475 #if KMP_USE_INLINED_TAS 1476 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1477 KMP_RELEASE_TAS_LOCK(lck, global_tid);
1479 #elif KMP_USE_INLINED_FUTEX 1480 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1481 KMP_RELEASE_FUTEX_LOCK(lck, global_tid);
1485 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
1488 kmp_indirect_lock_t *ilk =
1489 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
1490 KMP_ASSERT(ilk != NULL);
1492 if (__kmp_env_consistency_check) {
1493 __kmp_pop_sync(global_tid, ct_critical, loc);
1496 __kmp_itt_critical_releasing(lck);
1498 KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
1501 #else // KMP_USE_DYNAMIC_LOCK 1503 if ((__kmp_user_lock_kind == lk_tas) &&
1504 (
sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1505 lck = (kmp_user_lock_p)crit;
1508 else if ((__kmp_user_lock_kind == lk_futex) &&
1509 (
sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1510 lck = (kmp_user_lock_p)crit;
1514 lck = (kmp_user_lock_p)TCR_PTR(*((kmp_user_lock_p *)crit));
1517 KMP_ASSERT(lck != NULL);
1519 if (__kmp_env_consistency_check)
1520 __kmp_pop_sync(global_tid, ct_critical, loc);
1523 __kmp_itt_critical_releasing(lck);
1527 __kmp_release_user_lock_with_checks(lck, global_tid);
1529 #endif // KMP_USE_DYNAMIC_LOCK 1531 #if OMPT_SUPPORT && OMPT_OPTIONAL 1534 OMPT_STORE_RETURN_ADDRESS(global_tid);
1535 if (ompt_enabled.ompt_callback_mutex_released) {
1536 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
1537 ompt_mutex_critical, (ompt_wait_id_t)crit, OMPT_LOAD_RETURN_ADDRESS(0));
1541 KMP_POP_PARTITIONED_TIMER();
1542 KA_TRACE(15, (
"__kmpc_end_critical: done T#%d\n", global_tid));
1557 KC_TRACE(10, (
"__kmpc_barrier_master: called T#%d\n", global_tid));
1559 if (!TCR_4(__kmp_init_parallel))
1560 __kmp_parallel_initialize();
1562 if (__kmp_env_consistency_check)
1563 __kmp_check_barrier(global_tid, ct_barrier, loc);
1566 ompt_frame_t *ompt_frame;
1567 if (ompt_enabled.enabled) {
1568 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1569 if (ompt_frame->enter_frame == NULL)
1570 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
1571 OMPT_STORE_RETURN_ADDRESS(global_tid);
1575 __kmp_threads[global_tid]->th.th_ident = loc;
1577 status = __kmp_barrier(bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL);
1578 #if OMPT_SUPPORT && OMPT_OPTIONAL 1579 if (ompt_enabled.enabled) {
1580 ompt_frame->enter_frame = NULL;
1584 return (status != 0) ? 0 : 1;
1597 KC_TRACE(10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid));
1599 __kmp_end_split_barrier(bs_plain_barrier, global_tid);
1615 KC_TRACE(10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
1617 if (!TCR_4(__kmp_init_parallel))
1618 __kmp_parallel_initialize();
1620 if (__kmp_env_consistency_check) {
1622 KMP_WARNING(ConstructIdentInvalid);
1624 __kmp_check_barrier(global_tid, ct_barrier, loc);
1628 ompt_frame_t *ompt_frame;
1629 if (ompt_enabled.enabled) {
1630 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1631 if (ompt_frame->enter_frame == NULL)
1632 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
1633 OMPT_STORE_RETURN_ADDRESS(global_tid);
1637 __kmp_threads[global_tid]->th.th_ident = loc;
1639 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
1640 #if OMPT_SUPPORT && OMPT_OPTIONAL 1641 if (ompt_enabled.enabled) {
1642 ompt_frame->enter_frame = NULL;
1648 if (__kmp_env_consistency_check) {
1652 if (global_tid < 0) {
1653 KMP_WARNING(ThreadIdentInvalid);
1659 __kmp_pop_sync(global_tid, ct_master, loc);
1679 kmp_int32 rc = __kmp_enter_single(global_tid, loc, TRUE);
1684 KMP_PUSH_PARTITIONED_TIMER(OMP_single);
1687 #if OMPT_SUPPORT && OMPT_OPTIONAL 1688 kmp_info_t *this_thr = __kmp_threads[global_tid];
1689 kmp_team_t *team = this_thr->th.th_team;
1690 int tid = __kmp_tid_from_gtid(global_tid);
1692 if (ompt_enabled.enabled) {
1694 if (ompt_enabled.ompt_callback_work) {
1695 ompt_callbacks.ompt_callback(ompt_callback_work)(
1696 ompt_work_single_executor, ompt_scope_begin,
1697 &(team->t.ompt_team_info.parallel_data),
1698 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1699 1, OMPT_GET_RETURN_ADDRESS(0));
1702 if (ompt_enabled.ompt_callback_work) {
1703 ompt_callbacks.ompt_callback(ompt_callback_work)(
1704 ompt_work_single_other, ompt_scope_begin,
1705 &(team->t.ompt_team_info.parallel_data),
1706 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1707 1, OMPT_GET_RETURN_ADDRESS(0));
1708 ompt_callbacks.ompt_callback(ompt_callback_work)(
1709 ompt_work_single_other, ompt_scope_end,
1710 &(team->t.ompt_team_info.parallel_data),
1711 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1712 1, OMPT_GET_RETURN_ADDRESS(0));
1731 __kmp_exit_single(global_tid);
1732 KMP_POP_PARTITIONED_TIMER();
1734 #if OMPT_SUPPORT && OMPT_OPTIONAL 1735 kmp_info_t *this_thr = __kmp_threads[global_tid];
1736 kmp_team_t *team = this_thr->th.th_team;
1737 int tid = __kmp_tid_from_gtid(global_tid);
1739 if (ompt_enabled.ompt_callback_work) {
1740 ompt_callbacks.ompt_callback(ompt_callback_work)(
1741 ompt_work_single_executor, ompt_scope_end,
1742 &(team->t.ompt_team_info.parallel_data),
1743 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
1744 OMPT_GET_RETURN_ADDRESS(0));
1757 KE_TRACE(10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1759 #if OMPT_SUPPORT && OMPT_OPTIONAL 1760 if (ompt_enabled.ompt_callback_work) {
1761 ompt_work_type_t ompt_work_type = ompt_work_loop;
1762 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
1763 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
1766 if ((loc->
flags & KMP_IDENT_WORK_LOOP) != 0) {
1767 ompt_work_type = ompt_work_loop;
1768 }
else if ((loc->
flags & KMP_IDENT_WORK_SECTIONS) != 0) {
1769 ompt_work_type = ompt_work_sections;
1770 }
else if ((loc->
flags & KMP_IDENT_WORK_DISTRIBUTE) != 0) {
1771 ompt_work_type = ompt_work_distribute;
1776 KMP_DEBUG_ASSERT(ompt_work_type);
1778 ompt_callbacks.ompt_callback(ompt_callback_work)(
1779 ompt_work_type, ompt_scope_end, &(team_info->parallel_data),
1780 &(task_info->task_data), 0, OMPT_GET_RETURN_ADDRESS(0));
1784 if (__kmp_env_consistency_check)
1785 __kmp_pop_workshare(global_tid, ct_pdo, loc);
1791 void ompc_set_num_threads(
int arg) {
1793 __kmp_set_num_threads(arg, __kmp_entry_gtid());
1796 void ompc_set_dynamic(
int flag) {
1800 thread = __kmp_entry_thread();
1802 __kmp_save_internal_controls(thread);
1804 set__dynamic(thread, flag ? TRUE : FALSE);
1807 void ompc_set_nested(
int flag) {
1811 thread = __kmp_entry_thread();
1813 __kmp_save_internal_controls(thread);
1815 set__nested(thread, flag ? TRUE : FALSE);
1818 void ompc_set_max_active_levels(
int max_active_levels) {
1823 __kmp_set_max_active_levels(__kmp_entry_gtid(), max_active_levels);
1826 void ompc_set_schedule(omp_sched_t kind,
int modifier) {
1828 __kmp_set_schedule(__kmp_entry_gtid(), (kmp_sched_t)kind, modifier);
1831 int ompc_get_ancestor_thread_num(
int level) {
1832 return __kmp_get_ancestor_thread_num(__kmp_entry_gtid(), level);
1835 int ompc_get_team_size(
int level) {
1836 return __kmp_get_team_size(__kmp_entry_gtid(), level);
1839 void kmpc_set_stacksize(
int arg) {
1841 __kmp_aux_set_stacksize(arg);
1844 void kmpc_set_stacksize_s(
size_t arg) {
1846 __kmp_aux_set_stacksize(arg);
1849 void kmpc_set_blocktime(
int arg) {
1853 gtid = __kmp_entry_gtid();
1854 tid = __kmp_tid_from_gtid(gtid);
1855 thread = __kmp_thread_from_gtid(gtid);
1857 __kmp_aux_set_blocktime(arg, thread, tid);
1860 void kmpc_set_library(
int arg) {
1862 __kmp_user_set_library((
enum library_type)arg);
1865 void kmpc_set_defaults(
char const *str) {
1867 __kmp_aux_set_defaults(str, KMP_STRLEN(str));
1870 void kmpc_set_disp_num_buffers(
int arg) {
1873 if (__kmp_init_serial == 0 && arg > 0)
1874 __kmp_dispatch_num_buffers = arg;
1877 int kmpc_set_affinity_mask_proc(
int proc,
void **mask) {
1878 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1881 if (!TCR_4(__kmp_init_middle)) {
1882 __kmp_middle_initialize();
1884 return __kmp_aux_set_affinity_mask_proc(proc, mask);
1888 int kmpc_unset_affinity_mask_proc(
int proc,
void **mask) {
1889 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1892 if (!TCR_4(__kmp_init_middle)) {
1893 __kmp_middle_initialize();
1895 return __kmp_aux_unset_affinity_mask_proc(proc, mask);
1899 int kmpc_get_affinity_mask_proc(
int proc,
void **mask) {
1900 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1903 if (!TCR_4(__kmp_init_middle)) {
1904 __kmp_middle_initialize();
1906 return __kmp_aux_get_affinity_mask_proc(proc, mask);
1956 void *cpy_data,
void (*cpy_func)(
void *,
void *),
1960 KC_TRACE(10, (
"__kmpc_copyprivate: called T#%d\n", gtid));
1964 data_ptr = &__kmp_team_from_gtid(gtid)->t.t_copypriv_data;
1966 if (__kmp_env_consistency_check) {
1968 KMP_WARNING(ConstructIdentInvalid);
1975 *data_ptr = cpy_data;
1978 ompt_frame_t *ompt_frame;
1979 if (ompt_enabled.enabled) {
1980 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1981 if (ompt_frame->enter_frame == NULL)
1982 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
1983 OMPT_STORE_RETURN_ADDRESS(gtid);
1988 __kmp_threads[gtid]->th.th_ident = loc;
1990 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
1993 (*cpy_func)(cpy_data, *data_ptr);
1999 if (ompt_enabled.enabled) {
2000 OMPT_STORE_RETURN_ADDRESS(gtid);
2004 __kmp_threads[gtid]->th.th_ident = loc;
2007 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2008 #if OMPT_SUPPORT && OMPT_OPTIONAL 2009 if (ompt_enabled.enabled) {
2010 ompt_frame->enter_frame = NULL;
2017 #define INIT_LOCK __kmp_init_user_lock_with_checks 2018 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks 2019 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks 2020 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed 2021 #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks 2022 #define ACQUIRE_NESTED_LOCK_TIMED \ 2023 __kmp_acquire_nested_user_lock_with_checks_timed 2024 #define RELEASE_LOCK __kmp_release_user_lock_with_checks 2025 #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks 2026 #define TEST_LOCK __kmp_test_user_lock_with_checks 2027 #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks 2028 #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks 2029 #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks 2034 #if KMP_USE_DYNAMIC_LOCK 2037 static __forceinline
void __kmp_init_lock_with_hint(
ident_t *loc,
void **lock,
2038 kmp_dyna_lockseq_t seq) {
2039 if (KMP_IS_D_LOCK(seq)) {
2040 KMP_INIT_D_LOCK(lock, seq);
2042 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
2045 KMP_INIT_I_LOCK(lock, seq);
2047 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2048 __kmp_itt_lock_creating(ilk->lock, loc);
2054 static __forceinline
void 2055 __kmp_init_nest_lock_with_hint(
ident_t *loc,
void **lock,
2056 kmp_dyna_lockseq_t seq) {
2059 if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive)
2060 seq = __kmp_user_lock_seq;
2064 seq = lockseq_nested_tas;
2068 seq = lockseq_nested_futex;
2071 case lockseq_ticket:
2072 seq = lockseq_nested_ticket;
2074 case lockseq_queuing:
2075 seq = lockseq_nested_queuing;
2078 seq = lockseq_nested_drdpa;
2081 seq = lockseq_nested_queuing;
2083 KMP_INIT_I_LOCK(lock, seq);
2085 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2086 __kmp_itt_lock_creating(ilk->lock, loc);
2091 void __kmpc_init_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
void **user_lock,
2093 KMP_DEBUG_ASSERT(__kmp_init_serial);
2094 if (__kmp_env_consistency_check && user_lock == NULL) {
2095 KMP_FATAL(LockIsUninitialized,
"omp_init_lock_with_hint");
2098 __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2100 #if OMPT_SUPPORT && OMPT_OPTIONAL 2102 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2104 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2105 if (ompt_enabled.ompt_callback_lock_init) {
2106 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2107 ompt_mutex_lock, (omp_lock_hint_t)hint,
2108 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2115 void __kmpc_init_nest_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
2116 void **user_lock, uintptr_t hint) {
2117 KMP_DEBUG_ASSERT(__kmp_init_serial);
2118 if (__kmp_env_consistency_check && user_lock == NULL) {
2119 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock_with_hint");
2122 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2124 #if OMPT_SUPPORT && OMPT_OPTIONAL 2126 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2128 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2129 if (ompt_enabled.ompt_callback_lock_init) {
2130 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2131 ompt_mutex_nest_lock, (omp_lock_hint_t)hint,
2132 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2138 #endif // KMP_USE_DYNAMIC_LOCK 2141 void __kmpc_init_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2142 #if KMP_USE_DYNAMIC_LOCK 2144 KMP_DEBUG_ASSERT(__kmp_init_serial);
2145 if (__kmp_env_consistency_check && user_lock == NULL) {
2146 KMP_FATAL(LockIsUninitialized,
"omp_init_lock");
2148 __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2150 #if OMPT_SUPPORT && OMPT_OPTIONAL 2152 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2154 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2155 if (ompt_enabled.ompt_callback_lock_init) {
2156 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2157 ompt_mutex_lock, omp_lock_hint_none,
2158 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2163 #else // KMP_USE_DYNAMIC_LOCK 2165 static char const *
const func =
"omp_init_lock";
2166 kmp_user_lock_p lck;
2167 KMP_DEBUG_ASSERT(__kmp_init_serial);
2169 if (__kmp_env_consistency_check) {
2170 if (user_lock == NULL) {
2171 KMP_FATAL(LockIsUninitialized, func);
2175 KMP_CHECK_USER_LOCK_INIT();
2177 if ((__kmp_user_lock_kind == lk_tas) &&
2178 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2179 lck = (kmp_user_lock_p)user_lock;
2182 else if ((__kmp_user_lock_kind == lk_futex) &&
2183 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2184 lck = (kmp_user_lock_p)user_lock;
2188 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2191 __kmp_set_user_lock_location(lck, loc);
2193 #if OMPT_SUPPORT && OMPT_OPTIONAL 2195 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2197 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2198 if (ompt_enabled.ompt_callback_lock_init) {
2199 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2200 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2201 (ompt_wait_id_t)user_lock, codeptr);
2206 __kmp_itt_lock_creating(lck);
2209 #endif // KMP_USE_DYNAMIC_LOCK 2213 void __kmpc_init_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2214 #if KMP_USE_DYNAMIC_LOCK 2216 KMP_DEBUG_ASSERT(__kmp_init_serial);
2217 if (__kmp_env_consistency_check && user_lock == NULL) {
2218 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock");
2220 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2222 #if OMPT_SUPPORT && OMPT_OPTIONAL 2224 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2226 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2227 if (ompt_enabled.ompt_callback_lock_init) {
2228 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2229 ompt_mutex_nest_lock, omp_lock_hint_none,
2230 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2235 #else // KMP_USE_DYNAMIC_LOCK 2237 static char const *
const func =
"omp_init_nest_lock";
2238 kmp_user_lock_p lck;
2239 KMP_DEBUG_ASSERT(__kmp_init_serial);
2241 if (__kmp_env_consistency_check) {
2242 if (user_lock == NULL) {
2243 KMP_FATAL(LockIsUninitialized, func);
2247 KMP_CHECK_USER_LOCK_INIT();
2249 if ((__kmp_user_lock_kind == lk_tas) &&
2250 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2251 OMP_NEST_LOCK_T_SIZE)) {
2252 lck = (kmp_user_lock_p)user_lock;
2255 else if ((__kmp_user_lock_kind == lk_futex) &&
2256 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2257 OMP_NEST_LOCK_T_SIZE)) {
2258 lck = (kmp_user_lock_p)user_lock;
2262 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2265 INIT_NESTED_LOCK(lck);
2266 __kmp_set_user_lock_location(lck, loc);
2268 #if OMPT_SUPPORT && OMPT_OPTIONAL 2270 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2272 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2273 if (ompt_enabled.ompt_callback_lock_init) {
2274 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2275 ompt_mutex_nest_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2276 (ompt_wait_id_t)user_lock, codeptr);
2281 __kmp_itt_lock_creating(lck);
2284 #endif // KMP_USE_DYNAMIC_LOCK 2287 void __kmpc_destroy_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2288 #if KMP_USE_DYNAMIC_LOCK 2291 kmp_user_lock_p lck;
2292 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2293 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2295 lck = (kmp_user_lock_p)user_lock;
2297 __kmp_itt_lock_destroyed(lck);
2299 #if OMPT_SUPPORT && OMPT_OPTIONAL 2301 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2303 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2304 if (ompt_enabled.ompt_callback_lock_destroy) {
2305 kmp_user_lock_p lck;
2306 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2307 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2309 lck = (kmp_user_lock_p)user_lock;
2311 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2312 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2315 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2317 kmp_user_lock_p lck;
2319 if ((__kmp_user_lock_kind == lk_tas) &&
2320 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2321 lck = (kmp_user_lock_p)user_lock;
2324 else if ((__kmp_user_lock_kind == lk_futex) &&
2325 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2326 lck = (kmp_user_lock_p)user_lock;
2330 lck = __kmp_lookup_user_lock(user_lock,
"omp_destroy_lock");
2333 #if OMPT_SUPPORT && OMPT_OPTIONAL 2335 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2337 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2338 if (ompt_enabled.ompt_callback_lock_destroy) {
2339 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2340 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2345 __kmp_itt_lock_destroyed(lck);
2349 if ((__kmp_user_lock_kind == lk_tas) &&
2350 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2354 else if ((__kmp_user_lock_kind == lk_futex) &&
2355 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2360 __kmp_user_lock_free(user_lock, gtid, lck);
2362 #endif // KMP_USE_DYNAMIC_LOCK 2366 void __kmpc_destroy_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2367 #if KMP_USE_DYNAMIC_LOCK 2370 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
2371 __kmp_itt_lock_destroyed(ilk->lock);
2373 #if OMPT_SUPPORT && OMPT_OPTIONAL 2375 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2377 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2378 if (ompt_enabled.ompt_callback_lock_destroy) {
2379 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2380 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2383 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2385 #else // KMP_USE_DYNAMIC_LOCK 2387 kmp_user_lock_p lck;
2389 if ((__kmp_user_lock_kind == lk_tas) &&
2390 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2391 OMP_NEST_LOCK_T_SIZE)) {
2392 lck = (kmp_user_lock_p)user_lock;
2395 else if ((__kmp_user_lock_kind == lk_futex) &&
2396 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2397 OMP_NEST_LOCK_T_SIZE)) {
2398 lck = (kmp_user_lock_p)user_lock;
2402 lck = __kmp_lookup_user_lock(user_lock,
"omp_destroy_nest_lock");
2405 #if OMPT_SUPPORT && OMPT_OPTIONAL 2407 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2409 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2410 if (ompt_enabled.ompt_callback_lock_destroy) {
2411 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2412 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2417 __kmp_itt_lock_destroyed(lck);
2420 DESTROY_NESTED_LOCK(lck);
2422 if ((__kmp_user_lock_kind == lk_tas) &&
2423 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2424 OMP_NEST_LOCK_T_SIZE)) {
2428 else if ((__kmp_user_lock_kind == lk_futex) &&
2429 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2430 OMP_NEST_LOCK_T_SIZE)) {
2435 __kmp_user_lock_free(user_lock, gtid, lck);
2437 #endif // KMP_USE_DYNAMIC_LOCK 2440 void __kmpc_set_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2442 #if KMP_USE_DYNAMIC_LOCK 2443 int tag = KMP_EXTRACT_D_TAG(user_lock);
2445 __kmp_itt_lock_acquiring(
2449 #if OMPT_SUPPORT && OMPT_OPTIONAL 2451 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2453 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2454 if (ompt_enabled.ompt_callback_mutex_acquire) {
2455 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2456 ompt_mutex_lock, omp_lock_hint_none,
2457 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2461 #if KMP_USE_INLINED_TAS 2462 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2463 KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
2465 #elif KMP_USE_INLINED_FUTEX 2466 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2467 KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
2471 __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2474 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2476 #if OMPT_SUPPORT && OMPT_OPTIONAL 2477 if (ompt_enabled.ompt_callback_mutex_acquired) {
2478 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2479 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2483 #else // KMP_USE_DYNAMIC_LOCK 2485 kmp_user_lock_p lck;
2487 if ((__kmp_user_lock_kind == lk_tas) &&
2488 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2489 lck = (kmp_user_lock_p)user_lock;
2492 else if ((__kmp_user_lock_kind == lk_futex) &&
2493 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2494 lck = (kmp_user_lock_p)user_lock;
2498 lck = __kmp_lookup_user_lock(user_lock,
"omp_set_lock");
2502 __kmp_itt_lock_acquiring(lck);
2504 #if OMPT_SUPPORT && OMPT_OPTIONAL 2506 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2508 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2509 if (ompt_enabled.ompt_callback_mutex_acquire) {
2510 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2511 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2512 (ompt_wait_id_t)lck, codeptr);
2516 ACQUIRE_LOCK(lck, gtid);
2519 __kmp_itt_lock_acquired(lck);
2522 #if OMPT_SUPPORT && OMPT_OPTIONAL 2523 if (ompt_enabled.ompt_callback_mutex_acquired) {
2524 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2525 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
2529 #endif // KMP_USE_DYNAMIC_LOCK 2532 void __kmpc_set_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2533 #if KMP_USE_DYNAMIC_LOCK 2536 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2538 #if OMPT_SUPPORT && OMPT_OPTIONAL 2540 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2542 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2543 if (ompt_enabled.enabled) {
2544 if (ompt_enabled.ompt_callback_mutex_acquire) {
2545 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2546 ompt_mutex_nest_lock, omp_lock_hint_none,
2547 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2552 int acquire_status =
2553 KMP_D_LOCK_FUNC(user_lock,
set)((kmp_dyna_lock_t *)user_lock, gtid);
2555 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2558 #if OMPT_SUPPORT && OMPT_OPTIONAL 2559 if (ompt_enabled.enabled) {
2560 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2561 if (ompt_enabled.ompt_callback_mutex_acquired) {
2563 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2564 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2567 if (ompt_enabled.ompt_callback_nest_lock) {
2569 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2570 ompt_scope_begin, (ompt_wait_id_t)user_lock, codeptr);
2576 #else // KMP_USE_DYNAMIC_LOCK 2578 kmp_user_lock_p lck;
2580 if ((__kmp_user_lock_kind == lk_tas) &&
2581 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2582 OMP_NEST_LOCK_T_SIZE)) {
2583 lck = (kmp_user_lock_p)user_lock;
2586 else if ((__kmp_user_lock_kind == lk_futex) &&
2587 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2588 OMP_NEST_LOCK_T_SIZE)) {
2589 lck = (kmp_user_lock_p)user_lock;
2593 lck = __kmp_lookup_user_lock(user_lock,
"omp_set_nest_lock");
2597 __kmp_itt_lock_acquiring(lck);
2599 #if OMPT_SUPPORT && OMPT_OPTIONAL 2601 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2603 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2604 if (ompt_enabled.enabled) {
2605 if (ompt_enabled.ompt_callback_mutex_acquire) {
2606 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2607 ompt_mutex_nest_lock, omp_lock_hint_none,
2608 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)lck, codeptr);
2613 ACQUIRE_NESTED_LOCK(lck, gtid, &acquire_status);
2616 __kmp_itt_lock_acquired(lck);
2619 #if OMPT_SUPPORT && OMPT_OPTIONAL 2620 if (ompt_enabled.enabled) {
2621 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2622 if (ompt_enabled.ompt_callback_mutex_acquired) {
2624 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2625 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
2628 if (ompt_enabled.ompt_callback_nest_lock) {
2630 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2631 ompt_scope_begin, (ompt_wait_id_t)lck, codeptr);
2637 #endif // KMP_USE_DYNAMIC_LOCK 2640 void __kmpc_unset_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2641 #if KMP_USE_DYNAMIC_LOCK 2643 int tag = KMP_EXTRACT_D_TAG(user_lock);
2645 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2647 #if KMP_USE_INLINED_TAS 2648 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2649 KMP_RELEASE_TAS_LOCK(user_lock, gtid);
2651 #elif KMP_USE_INLINED_FUTEX 2652 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2653 KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
2657 __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2660 #if OMPT_SUPPORT && OMPT_OPTIONAL 2662 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2664 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2665 if (ompt_enabled.ompt_callback_mutex_released) {
2666 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2667 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2671 #else // KMP_USE_DYNAMIC_LOCK 2673 kmp_user_lock_p lck;
2678 if ((__kmp_user_lock_kind == lk_tas) &&
2679 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2680 #if KMP_OS_LINUX && \ 2681 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2684 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2686 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
2689 #if OMPT_SUPPORT && OMPT_OPTIONAL 2691 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2693 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2694 if (ompt_enabled.ompt_callback_mutex_released) {
2695 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2696 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
2702 lck = (kmp_user_lock_p)user_lock;
2706 else if ((__kmp_user_lock_kind == lk_futex) &&
2707 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2708 lck = (kmp_user_lock_p)user_lock;
2712 lck = __kmp_lookup_user_lock(user_lock,
"omp_unset_lock");
2716 __kmp_itt_lock_releasing(lck);
2719 RELEASE_LOCK(lck, gtid);
2721 #if OMPT_SUPPORT && OMPT_OPTIONAL 2723 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2725 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2726 if (ompt_enabled.ompt_callback_mutex_released) {
2727 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2728 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
2732 #endif // KMP_USE_DYNAMIC_LOCK 2736 void __kmpc_unset_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2737 #if KMP_USE_DYNAMIC_LOCK 2740 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2742 int release_status =
2743 KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
2745 #if OMPT_SUPPORT && OMPT_OPTIONAL 2747 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2749 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2750 if (ompt_enabled.enabled) {
2751 if (release_status == KMP_LOCK_RELEASED) {
2752 if (ompt_enabled.ompt_callback_mutex_released) {
2754 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2755 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2757 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2759 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2760 ompt_scope_end, (ompt_wait_id_t)user_lock, codeptr);
2765 #else // KMP_USE_DYNAMIC_LOCK 2767 kmp_user_lock_p lck;
2771 if ((__kmp_user_lock_kind == lk_tas) &&
2772 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2773 OMP_NEST_LOCK_T_SIZE)) {
2774 #if KMP_OS_LINUX && \ 2775 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2777 kmp_tas_lock_t *tl = (kmp_tas_lock_t *)user_lock;
2779 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2782 #if OMPT_SUPPORT && OMPT_OPTIONAL 2783 int release_status = KMP_LOCK_STILL_HELD;
2786 if (--(tl->lk.depth_locked) == 0) {
2787 TCW_4(tl->lk.poll, 0);
2788 #if OMPT_SUPPORT && OMPT_OPTIONAL 2789 release_status = KMP_LOCK_RELEASED;
2794 #if OMPT_SUPPORT && OMPT_OPTIONAL 2796 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2798 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2799 if (ompt_enabled.enabled) {
2800 if (release_status == KMP_LOCK_RELEASED) {
2801 if (ompt_enabled.ompt_callback_mutex_released) {
2803 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2804 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
2806 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2808 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2809 ompt_mutex_scope_end, (ompt_wait_id_t)lck, codeptr);
2816 lck = (kmp_user_lock_p)user_lock;
2820 else if ((__kmp_user_lock_kind == lk_futex) &&
2821 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2822 OMP_NEST_LOCK_T_SIZE)) {
2823 lck = (kmp_user_lock_p)user_lock;
2827 lck = __kmp_lookup_user_lock(user_lock,
"omp_unset_nest_lock");
2831 __kmp_itt_lock_releasing(lck);
2835 release_status = RELEASE_NESTED_LOCK(lck, gtid);
2836 #if OMPT_SUPPORT && OMPT_OPTIONAL 2838 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2840 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2841 if (ompt_enabled.enabled) {
2842 if (release_status == KMP_LOCK_RELEASED) {
2843 if (ompt_enabled.ompt_callback_mutex_released) {
2845 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2846 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
2848 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2850 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2851 ompt_mutex_scope_end, (ompt_wait_id_t)lck, codeptr);
2856 #endif // KMP_USE_DYNAMIC_LOCK 2860 int __kmpc_test_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2863 #if KMP_USE_DYNAMIC_LOCK 2865 int tag = KMP_EXTRACT_D_TAG(user_lock);
2867 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2869 #if OMPT_SUPPORT && OMPT_OPTIONAL 2871 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2873 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2874 if (ompt_enabled.ompt_callback_mutex_acquire) {
2875 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2876 ompt_mutex_lock, omp_lock_hint_none,
2877 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2881 #if KMP_USE_INLINED_TAS 2882 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2883 KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
2885 #elif KMP_USE_INLINED_FUTEX 2886 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2887 KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
2891 rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2895 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2897 #if OMPT_SUPPORT && OMPT_OPTIONAL 2898 if (ompt_enabled.ompt_callback_mutex_acquired) {
2899 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2900 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2906 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2911 #else // KMP_USE_DYNAMIC_LOCK 2913 kmp_user_lock_p lck;
2916 if ((__kmp_user_lock_kind == lk_tas) &&
2917 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2918 lck = (kmp_user_lock_p)user_lock;
2921 else if ((__kmp_user_lock_kind == lk_futex) &&
2922 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2923 lck = (kmp_user_lock_p)user_lock;
2927 lck = __kmp_lookup_user_lock(user_lock,
"omp_test_lock");
2931 __kmp_itt_lock_acquiring(lck);
2933 #if OMPT_SUPPORT && OMPT_OPTIONAL 2935 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2937 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2938 if (ompt_enabled.ompt_callback_mutex_acquire) {
2939 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2940 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2941 (ompt_wait_id_t)lck, codeptr);
2945 rc = TEST_LOCK(lck, gtid);
2948 __kmp_itt_lock_acquired(lck);
2950 __kmp_itt_lock_cancelled(lck);
2953 #if OMPT_SUPPORT && OMPT_OPTIONAL 2954 if (rc && ompt_enabled.ompt_callback_mutex_acquired) {
2955 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2956 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
2960 return (rc ? FTN_TRUE : FTN_FALSE);
2964 #endif // KMP_USE_DYNAMIC_LOCK 2968 int __kmpc_test_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2969 #if KMP_USE_DYNAMIC_LOCK 2972 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2974 #if OMPT_SUPPORT && OMPT_OPTIONAL 2976 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2978 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2979 if (ompt_enabled.ompt_callback_mutex_acquire) {
2980 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2981 ompt_mutex_nest_lock, omp_lock_hint_none,
2982 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2986 rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
2989 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2991 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2994 #if OMPT_SUPPORT && OMPT_OPTIONAL 2995 if (ompt_enabled.enabled && rc) {
2997 if (ompt_enabled.ompt_callback_mutex_acquired) {
2999 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3000 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
3003 if (ompt_enabled.ompt_callback_nest_lock) {
3005 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3006 ompt_scope_begin, (ompt_wait_id_t)user_lock, codeptr);
3013 #else // KMP_USE_DYNAMIC_LOCK 3015 kmp_user_lock_p lck;
3018 if ((__kmp_user_lock_kind == lk_tas) &&
3019 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
3020 OMP_NEST_LOCK_T_SIZE)) {
3021 lck = (kmp_user_lock_p)user_lock;
3024 else if ((__kmp_user_lock_kind == lk_futex) &&
3025 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
3026 OMP_NEST_LOCK_T_SIZE)) {
3027 lck = (kmp_user_lock_p)user_lock;
3031 lck = __kmp_lookup_user_lock(user_lock,
"omp_test_nest_lock");
3035 __kmp_itt_lock_acquiring(lck);
3038 #if OMPT_SUPPORT && OMPT_OPTIONAL 3040 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3042 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3043 if (ompt_enabled.enabled) &&
3044 ompt_enabled.ompt_callback_mutex_acquire) {
3045 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3046 ompt_mutex_nest_lock, omp_lock_hint_none,
3047 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)lck, codeptr);
3051 rc = TEST_NESTED_LOCK(lck, gtid);
3054 __kmp_itt_lock_acquired(lck);
3056 __kmp_itt_lock_cancelled(lck);
3059 #if OMPT_SUPPORT && OMPT_OPTIONAL 3060 if (ompt_enabled.enabled && rc) {
3062 if (ompt_enabled.ompt_callback_mutex_acquired) {
3064 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3065 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
3068 if (ompt_enabled.ompt_callback_nest_lock) {
3070 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3071 ompt_mutex_scope_begin, (ompt_wait_id_t)lck, codeptr);
3080 #endif // KMP_USE_DYNAMIC_LOCK 3090 #define __KMP_SET_REDUCTION_METHOD(gtid, rmethod) \ 3091 ((__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) = (rmethod)) 3093 #define __KMP_GET_REDUCTION_METHOD(gtid) \ 3094 (__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) 3100 static __forceinline
void 3101 __kmp_enter_critical_section_reduce_block(
ident_t *loc, kmp_int32 global_tid,
3102 kmp_critical_name *crit) {
3108 kmp_user_lock_p lck;
3110 #if KMP_USE_DYNAMIC_LOCK 3112 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
3115 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3116 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0,
3117 KMP_GET_D_TAG(__kmp_user_lock_seq));
3119 __kmp_init_indirect_csptr(crit, loc, global_tid,
3120 KMP_GET_I_TAG(__kmp_user_lock_seq));
3126 if (KMP_EXTRACT_D_TAG(lk) != 0) {
3127 lck = (kmp_user_lock_p)lk;
3128 KMP_DEBUG_ASSERT(lck != NULL);
3129 if (__kmp_env_consistency_check) {
3130 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3132 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
3134 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
3136 KMP_DEBUG_ASSERT(lck != NULL);
3137 if (__kmp_env_consistency_check) {
3138 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3140 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
3143 #else // KMP_USE_DYNAMIC_LOCK 3148 if (__kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE) {
3149 lck = (kmp_user_lock_p)crit;
3151 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
3153 KMP_DEBUG_ASSERT(lck != NULL);
3155 if (__kmp_env_consistency_check)
3156 __kmp_push_sync(global_tid, ct_critical, loc, lck);
3158 __kmp_acquire_user_lock_with_checks(lck, global_tid);
3160 #endif // KMP_USE_DYNAMIC_LOCK 3164 static __forceinline
void 3165 __kmp_end_critical_section_reduce_block(
ident_t *loc, kmp_int32 global_tid,
3166 kmp_critical_name *crit) {
3168 kmp_user_lock_p lck;
3170 #if KMP_USE_DYNAMIC_LOCK 3172 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3173 lck = (kmp_user_lock_p)crit;
3174 if (__kmp_env_consistency_check)
3175 __kmp_pop_sync(global_tid, ct_critical, loc);
3176 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
3178 kmp_indirect_lock_t *ilk =
3179 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
3180 if (__kmp_env_consistency_check)
3181 __kmp_pop_sync(global_tid, ct_critical, loc);
3182 KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
3185 #else // KMP_USE_DYNAMIC_LOCK 3190 if (__kmp_base_user_lock_size > 32) {
3191 lck = *((kmp_user_lock_p *)crit);
3192 KMP_ASSERT(lck != NULL);
3194 lck = (kmp_user_lock_p)crit;
3197 if (__kmp_env_consistency_check)
3198 __kmp_pop_sync(global_tid, ct_critical, loc);
3200 __kmp_release_user_lock_with_checks(lck, global_tid);
3202 #endif // KMP_USE_DYNAMIC_LOCK 3206 static __forceinline
int 3207 __kmp_swap_teams_for_teams_reduction(kmp_info_t *th, kmp_team_t **team_p,
3212 if (th->th.th_teams_microtask) {
3213 *team_p = team = th->th.th_team;
3214 if (team->t.t_level == th->th.th_teams_level) {
3216 KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid);
3218 th->th.th_info.ds.ds_tid = team->t.t_master_tid;
3219 th->th.th_team = team->t.t_parent;
3220 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
3221 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
3222 *task_state = th->th.th_task_state;
3223 th->th.th_task_state = 0;
3231 static __forceinline
void 3232 __kmp_restore_swapped_teams(kmp_info_t *th, kmp_team_t *team,
int task_state) {
3234 th->th.th_info.ds.ds_tid = 0;
3235 th->th.th_team = team;
3236 th->th.th_team_nproc = team->t.t_nproc;
3237 th->th.th_task_team = team->t.t_task_team[task_state];
3238 th->th.th_task_state = task_state;
3260 size_t reduce_size,
void *reduce_data,
3261 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3262 kmp_critical_name *lck) {
3266 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3270 int teams_swapped = 0, task_state;
3272 KA_TRACE(10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid));
3280 if (!TCR_4(__kmp_init_parallel))
3281 __kmp_parallel_initialize();
3284 #if KMP_USE_DYNAMIC_LOCK 3285 if (__kmp_env_consistency_check)
3286 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3288 if (__kmp_env_consistency_check)
3289 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3293 th = __kmp_thread_from_gtid(global_tid);
3294 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3295 #endif // OMP_40_ENABLED 3313 packed_reduction_method = __kmp_determine_reduction_method(
3314 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3315 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3317 if (packed_reduction_method == critical_reduce_block) {
3319 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3322 }
else if (packed_reduction_method == empty_reduce_block) {
3328 }
else if (packed_reduction_method == atomic_reduce_block) {
3338 if (__kmp_env_consistency_check)
3339 __kmp_pop_sync(global_tid, ct_reduce, loc);
3341 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3342 tree_reduce_block)) {
3362 ompt_frame_t *ompt_frame;
3363 if (ompt_enabled.enabled) {
3364 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3365 if (ompt_frame->enter_frame == NULL)
3366 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3367 OMPT_STORE_RETURN_ADDRESS(global_tid);
3371 __kmp_threads[global_tid]->th.th_ident = loc;
3374 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3375 global_tid, FALSE, reduce_size, reduce_data, reduce_func);
3376 retval = (retval != 0) ? (0) : (1);
3377 #if OMPT_SUPPORT && OMPT_OPTIONAL 3378 if (ompt_enabled.enabled) {
3379 ompt_frame->enter_frame = NULL;
3385 if (__kmp_env_consistency_check) {
3387 __kmp_pop_sync(global_tid, ct_reduce, loc);
3397 if (teams_swapped) {
3398 __kmp_restore_swapped_teams(th, team, task_state);
3403 (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n",
3404 global_tid, packed_reduction_method, retval));
3418 kmp_critical_name *lck) {
3420 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3422 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid));
3424 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3426 if (packed_reduction_method == critical_reduce_block) {
3428 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3430 }
else if (packed_reduction_method == empty_reduce_block) {
3435 }
else if (packed_reduction_method == atomic_reduce_block) {
3442 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3443 tree_reduce_block)) {
3453 if (__kmp_env_consistency_check)
3454 __kmp_pop_sync(global_tid, ct_reduce, loc);
3456 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n",
3457 global_tid, packed_reduction_method));
3480 size_t reduce_size,
void *reduce_data,
3481 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3482 kmp_critical_name *lck) {
3485 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3489 int teams_swapped = 0, task_state;
3492 KA_TRACE(10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid));
3500 if (!TCR_4(__kmp_init_parallel))
3501 __kmp_parallel_initialize();
3504 #if KMP_USE_DYNAMIC_LOCK 3505 if (__kmp_env_consistency_check)
3506 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3508 if (__kmp_env_consistency_check)
3509 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3513 th = __kmp_thread_from_gtid(global_tid);
3514 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3515 #endif // OMP_40_ENABLED 3517 packed_reduction_method = __kmp_determine_reduction_method(
3518 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3519 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3521 if (packed_reduction_method == critical_reduce_block) {
3523 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3526 }
else if (packed_reduction_method == empty_reduce_block) {
3532 }
else if (packed_reduction_method == atomic_reduce_block) {
3536 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3537 tree_reduce_block)) {
3543 ompt_frame_t *ompt_frame;
3544 if (ompt_enabled.enabled) {
3545 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3546 if (ompt_frame->enter_frame == NULL)
3547 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3548 OMPT_STORE_RETURN_ADDRESS(global_tid);
3552 __kmp_threads[global_tid]->th.th_ident =
3556 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3557 global_tid, TRUE, reduce_size, reduce_data, reduce_func);
3558 retval = (retval != 0) ? (0) : (1);
3559 #if OMPT_SUPPORT && OMPT_OPTIONAL 3560 if (ompt_enabled.enabled) {
3561 ompt_frame->enter_frame = NULL;
3567 if (__kmp_env_consistency_check) {
3569 __kmp_pop_sync(global_tid, ct_reduce, loc);
3579 if (teams_swapped) {
3580 __kmp_restore_swapped_teams(th, team, task_state);
3585 (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n",
3586 global_tid, packed_reduction_method, retval));
3602 kmp_critical_name *lck) {
3604 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3608 int teams_swapped = 0, task_state;
3611 KA_TRACE(10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid));
3614 th = __kmp_thread_from_gtid(global_tid);
3615 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3616 #endif // OMP_40_ENABLED 3618 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3623 if (packed_reduction_method == critical_reduce_block) {
3625 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3629 ompt_frame_t *ompt_frame;
3630 if (ompt_enabled.enabled) {
3631 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3632 if (ompt_frame->enter_frame == NULL)
3633 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3634 OMPT_STORE_RETURN_ADDRESS(global_tid);
3638 __kmp_threads[global_tid]->th.th_ident = loc;
3640 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3641 #if OMPT_SUPPORT && OMPT_OPTIONAL 3642 if (ompt_enabled.enabled) {
3643 ompt_frame->enter_frame = NULL;
3647 }
else if (packed_reduction_method == empty_reduce_block) {
3653 ompt_frame_t *ompt_frame;
3654 if (ompt_enabled.enabled) {
3655 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3656 if (ompt_frame->enter_frame == NULL)
3657 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3658 OMPT_STORE_RETURN_ADDRESS(global_tid);
3662 __kmp_threads[global_tid]->th.th_ident = loc;
3664 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3665 #if OMPT_SUPPORT && OMPT_OPTIONAL 3666 if (ompt_enabled.enabled) {
3667 ompt_frame->enter_frame = NULL;
3671 }
else if (packed_reduction_method == atomic_reduce_block) {
3674 ompt_frame_t *ompt_frame;
3675 if (ompt_enabled.enabled) {
3676 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3677 if (ompt_frame->enter_frame == NULL)
3678 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3679 OMPT_STORE_RETURN_ADDRESS(global_tid);
3684 __kmp_threads[global_tid]->th.th_ident = loc;
3686 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3687 #if OMPT_SUPPORT && OMPT_OPTIONAL 3688 if (ompt_enabled.enabled) {
3689 ompt_frame->enter_frame = NULL;
3693 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3694 tree_reduce_block)) {
3697 __kmp_end_split_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3706 if (teams_swapped) {
3707 __kmp_restore_swapped_teams(th, team, task_state);
3711 if (__kmp_env_consistency_check)
3712 __kmp_pop_sync(global_tid, ct_reduce, loc);
3714 KA_TRACE(10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n",
3715 global_tid, packed_reduction_method));
3720 #undef __KMP_GET_REDUCTION_METHOD 3721 #undef __KMP_SET_REDUCTION_METHOD 3725 kmp_uint64 __kmpc_get_taskid() {
3730 gtid = __kmp_get_gtid();
3734 thread = __kmp_thread_from_gtid(gtid);
3735 return thread->th.th_current_task->td_task_id;
3739 kmp_uint64 __kmpc_get_parent_taskid() {
3743 kmp_taskdata_t *parent_task;
3745 gtid = __kmp_get_gtid();
3749 thread = __kmp_thread_from_gtid(gtid);
3750 parent_task = thread->th.th_current_task->td_parent;
3751 return (parent_task == NULL ? 0 : parent_task->td_task_id);
3767 void __kmpc_doacross_init(
ident_t *loc,
int gtid,
int num_dims,
3768 struct kmp_dim *dims) {
3770 kmp_int64 last, trace_count;
3771 kmp_info_t *th = __kmp_threads[gtid];
3772 kmp_team_t *team = th->th.th_team;
3774 kmp_disp_t *pr_buf = th->th.th_dispatch;
3775 dispatch_shared_info_t *sh_buf;
3779 (
"__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
3780 gtid, num_dims, !team->t.t_serialized));
3781 KMP_DEBUG_ASSERT(dims != NULL);
3782 KMP_DEBUG_ASSERT(num_dims > 0);
3784 if (team->t.t_serialized) {
3785 KA_TRACE(20, (
"__kmpc_doacross_init() exit: serialized team\n"));
3788 KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
3789 idx = pr_buf->th_doacross_buf_idx++;
3791 sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
3794 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
3795 pr_buf->th_doacross_info = (kmp_int64 *)__kmp_thread_malloc(
3796 th,
sizeof(kmp_int64) * (4 * num_dims + 1));
3797 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
3798 pr_buf->th_doacross_info[0] =
3799 (kmp_int64)num_dims;
3802 pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done;
3803 pr_buf->th_doacross_info[2] = dims[0].lo;
3804 pr_buf->th_doacross_info[3] = dims[0].up;
3805 pr_buf->th_doacross_info[4] = dims[0].st;
3807 for (j = 1; j < num_dims; ++j) {
3810 if (dims[j].st == 1) {
3812 range_length = dims[j].up - dims[j].lo + 1;
3814 if (dims[j].st > 0) {
3815 KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo);
3816 range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1;
3818 KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up);
3820 (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1;
3823 pr_buf->th_doacross_info[last++] = range_length;
3824 pr_buf->th_doacross_info[last++] = dims[j].lo;
3825 pr_buf->th_doacross_info[last++] = dims[j].up;
3826 pr_buf->th_doacross_info[last++] = dims[j].st;
3831 if (dims[0].st == 1) {
3832 trace_count = dims[0].up - dims[0].lo + 1;
3833 }
else if (dims[0].st > 0) {
3834 KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo);
3835 trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1;
3837 KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
3838 trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
3840 for (j = 1; j < num_dims; ++j) {
3841 trace_count *= pr_buf->th_doacross_info[4 * j + 1];
3843 KMP_DEBUG_ASSERT(trace_count > 0);
3847 if (idx != sh_buf->doacross_buf_idx) {
3849 __kmp_wait_yield_4((
volatile kmp_uint32 *)&sh_buf->doacross_buf_idx, idx,
3856 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET32(
3857 (
volatile kmp_int32 *)&sh_buf->doacross_flags, NULL, 1);
3859 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET64(
3860 (
volatile kmp_int64 *)&sh_buf->doacross_flags, NULL, 1LL);
3862 if (flags == NULL) {
3864 size_t size = trace_count / 8 + 8;
3865 flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
3867 sh_buf->doacross_flags = flags;
3868 }
else if (flags == (kmp_uint32 *)1) {
3871 while (*(
volatile kmp_int32 *)&sh_buf->doacross_flags == 1)
3873 while (*(
volatile kmp_int64 *)&sh_buf->doacross_flags == 1LL)
3880 KMP_DEBUG_ASSERT(sh_buf->doacross_flags > (kmp_uint32 *)1);
3881 pr_buf->th_doacross_flags =
3882 sh_buf->doacross_flags;
3884 KA_TRACE(20, (
"__kmpc_doacross_init() exit: T#%d\n", gtid));
3887 void __kmpc_doacross_wait(
ident_t *loc,
int gtid,
long long *vec) {
3888 kmp_int32 shft, num_dims, i;
3890 kmp_int64 iter_number;
3891 kmp_info_t *th = __kmp_threads[gtid];
3892 kmp_team_t *team = th->th.th_team;
3894 kmp_int64 lo, up, st;
3896 KA_TRACE(20, (
"__kmpc_doacross_wait() enter: called T#%d\n", gtid));
3897 if (team->t.t_serialized) {
3898 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: serialized team\n"));
3903 pr_buf = th->th.th_dispatch;
3904 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
3905 num_dims = pr_buf->th_doacross_info[0];
3906 lo = pr_buf->th_doacross_info[2];
3907 up = pr_buf->th_doacross_info[3];
3908 st = pr_buf->th_doacross_info[4];
3910 if (vec[0] < lo || vec[0] > up) {
3911 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3912 "bounds [%lld,%lld]\n",
3913 gtid, vec[0], lo, up));
3916 iter_number = vec[0] - lo;
3917 }
else if (st > 0) {
3918 if (vec[0] < lo || vec[0] > up) {
3919 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3920 "bounds [%lld,%lld]\n",
3921 gtid, vec[0], lo, up));
3924 iter_number = (kmp_uint64)(vec[0] - lo) / st;
3926 if (vec[0] > lo || vec[0] < up) {
3927 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3928 "bounds [%lld,%lld]\n",
3929 gtid, vec[0], lo, up));
3932 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
3934 for (i = 1; i < num_dims; ++i) {
3936 kmp_int32 j = i * 4;
3937 ln = pr_buf->th_doacross_info[j + 1];
3938 lo = pr_buf->th_doacross_info[j + 2];
3939 up = pr_buf->th_doacross_info[j + 3];
3940 st = pr_buf->th_doacross_info[j + 4];
3942 if (vec[i] < lo || vec[i] > up) {
3943 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3944 "bounds [%lld,%lld]\n",
3945 gtid, vec[i], lo, up));
3949 }
else if (st > 0) {
3950 if (vec[i] < lo || vec[i] > up) {
3951 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3952 "bounds [%lld,%lld]\n",
3953 gtid, vec[i], lo, up));
3956 iter = (kmp_uint64)(vec[i] - lo) / st;
3958 if (vec[i] > lo || vec[i] < up) {
3959 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3960 "bounds [%lld,%lld]\n",
3961 gtid, vec[i], lo, up));
3964 iter = (kmp_uint64)(lo - vec[i]) / (-st);
3966 iter_number = iter + ln * iter_number;
3968 shft = iter_number % 32;
3971 while ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) {
3976 (
"__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
3977 gtid, (iter_number << 5) + shft));
3980 void __kmpc_doacross_post(
ident_t *loc,
int gtid,
long long *vec) {
3981 kmp_int32 shft, num_dims, i;
3983 kmp_int64 iter_number;
3984 kmp_info_t *th = __kmp_threads[gtid];
3985 kmp_team_t *team = th->th.th_team;
3989 KA_TRACE(20, (
"__kmpc_doacross_post() enter: called T#%d\n", gtid));
3990 if (team->t.t_serialized) {
3991 KA_TRACE(20, (
"__kmpc_doacross_post() exit: serialized team\n"));
3997 pr_buf = th->th.th_dispatch;
3998 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
3999 num_dims = pr_buf->th_doacross_info[0];
4000 lo = pr_buf->th_doacross_info[2];
4001 st = pr_buf->th_doacross_info[4];
4003 iter_number = vec[0] - lo;
4004 }
else if (st > 0) {
4005 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4007 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4009 for (i = 1; i < num_dims; ++i) {
4011 kmp_int32 j = i * 4;
4012 ln = pr_buf->th_doacross_info[j + 1];
4013 lo = pr_buf->th_doacross_info[j + 2];
4014 st = pr_buf->th_doacross_info[j + 4];
4017 }
else if (st > 0) {
4018 iter = (kmp_uint64)(vec[i] - lo) / st;
4020 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4022 iter_number = iter + ln * iter_number;
4024 shft = iter_number % 32;
4028 if ((flag & pr_buf->th_doacross_flags[iter_number]) == 0)
4029 KMP_TEST_THEN_OR32(&pr_buf->th_doacross_flags[iter_number], flag);
4030 KA_TRACE(20, (
"__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid,
4031 (iter_number << 5) + shft));
4034 void __kmpc_doacross_fini(
ident_t *loc,
int gtid) {
4036 kmp_info_t *th = __kmp_threads[gtid];
4037 kmp_team_t *team = th->th.th_team;
4038 kmp_disp_t *pr_buf = th->th.th_dispatch;
4040 KA_TRACE(20, (
"__kmpc_doacross_fini() enter: called T#%d\n", gtid));
4041 if (team->t.t_serialized) {
4042 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: serialized team %p\n", team));
4045 num_done = KMP_TEST_THEN_INC32((kmp_int32 *)pr_buf->th_doacross_info[1]) + 1;
4046 if (num_done == th->th.th_team_nproc) {
4048 int idx = pr_buf->th_doacross_buf_idx - 1;
4049 dispatch_shared_info_t *sh_buf =
4050 &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
4051 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] ==
4052 (kmp_int64)&sh_buf->doacross_num_done);
4053 KMP_DEBUG_ASSERT(num_done == sh_buf->doacross_num_done);
4054 KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
4055 __kmp_thread_free(th, CCAST(kmp_uint32 *, sh_buf->doacross_flags));
4056 sh_buf->doacross_flags = NULL;
4057 sh_buf->doacross_num_done = 0;
4058 sh_buf->doacross_buf_idx +=
4059 __kmp_dispatch_num_buffers;
4062 __kmp_thread_free(th, (
void *)pr_buf->th_doacross_info);
4063 pr_buf->th_doacross_info = NULL;
4064 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: T#%d\n", gtid));
kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
void __kmpc_flush(ident_t *loc)
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end(ident_t *loc)
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
#define KMP_IDENT_AUTOPAR
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
kmp_int32 __kmpc_in_parallel(ident_t *loc)
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)