23 #include "tsan_annotations.h" 26 #include <sys/syscall.h> 44 void __kmp_validate_locks(
void) {
49 x = ~((kmp_uint32)0) - 2;
52 for (i = 0; i < 8; ++i, ++x, ++y) {
53 kmp_uint32 z = (x - y);
57 KMP_ASSERT(offsetof(kmp_base_queuing_lock, tail_id) % 8 == 0);
71 static kmp_int32 __kmp_get_tas_lock_owner(kmp_tas_lock_t *lck) {
72 return KMP_LOCK_STRIP(TCR_4(lck->lk.poll)) - 1;
75 static inline bool __kmp_is_tas_lock_nestable(kmp_tas_lock_t *lck) {
76 return lck->lk.depth_locked != -1;
79 __forceinline
static int 80 __kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) {
83 #ifdef USE_LOCK_PROFILE 84 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll));
85 if ((curr != 0) && (curr != gtid + 1))
86 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
90 if ((lck->lk.poll == KMP_LOCK_FREE(tas)) &&
91 KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(tas),
92 KMP_LOCK_BUSY(gtid + 1, tas))) {
93 KMP_FSYNC_ACQUIRED(lck);
94 return KMP_LOCK_ACQUIRED_FIRST;
98 KMP_FSYNC_PREPARE(lck);
99 KMP_INIT_YIELD(spins);
100 if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
103 KMP_YIELD_SPIN(spins);
106 kmp_backoff_t backoff = __kmp_spin_backoff_params;
107 while ((lck->lk.poll != KMP_LOCK_FREE(tas)) ||
108 (!KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(tas),
109 KMP_LOCK_BUSY(gtid + 1, tas)))) {
111 __kmp_spin_backoff(&backoff);
112 if (TCR_4(__kmp_nth) >
113 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
116 KMP_YIELD_SPIN(spins);
119 KMP_FSYNC_ACQUIRED(lck);
120 return KMP_LOCK_ACQUIRED_FIRST;
123 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
124 int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid);
125 ANNOTATE_TAS_ACQUIRED(lck);
129 static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck,
131 char const *
const func =
"omp_set_lock";
132 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
133 __kmp_is_tas_lock_nestable(lck)) {
134 KMP_FATAL(LockNestableUsedAsSimple, func);
136 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) {
137 KMP_FATAL(LockIsAlreadyOwned, func);
139 return __kmp_acquire_tas_lock(lck, gtid);
142 int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
143 if ((lck->lk.poll == KMP_LOCK_FREE(tas)) &&
144 KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(tas),
145 KMP_LOCK_BUSY(gtid + 1, tas))) {
146 KMP_FSYNC_ACQUIRED(lck);
152 static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck,
154 char const *
const func =
"omp_test_lock";
155 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
156 __kmp_is_tas_lock_nestable(lck)) {
157 KMP_FATAL(LockNestableUsedAsSimple, func);
159 return __kmp_test_tas_lock(lck, gtid);
162 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
165 KMP_FSYNC_RELEASING(lck);
166 ANNOTATE_TAS_RELEASED(lck);
167 KMP_ST_REL32(&(lck->lk.poll), KMP_LOCK_FREE(tas));
170 KMP_YIELD(TCR_4(__kmp_nth) >
171 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
172 return KMP_LOCK_RELEASED;
175 static int __kmp_release_tas_lock_with_checks(kmp_tas_lock_t *lck,
177 char const *
const func =
"omp_unset_lock";
179 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
180 __kmp_is_tas_lock_nestable(lck)) {
181 KMP_FATAL(LockNestableUsedAsSimple, func);
183 if (__kmp_get_tas_lock_owner(lck) == -1) {
184 KMP_FATAL(LockUnsettingFree, func);
186 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) >= 0) &&
187 (__kmp_get_tas_lock_owner(lck) != gtid)) {
188 KMP_FATAL(LockUnsettingSetByAnother, func);
190 return __kmp_release_tas_lock(lck, gtid);
193 void __kmp_init_tas_lock(kmp_tas_lock_t *lck) {
194 TCW_4(lck->lk.poll, KMP_LOCK_FREE(tas));
197 static void __kmp_init_tas_lock_with_checks(kmp_tas_lock_t *lck) {
198 __kmp_init_tas_lock(lck);
201 void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck) { lck->lk.poll = 0; }
203 static void __kmp_destroy_tas_lock_with_checks(kmp_tas_lock_t *lck) {
204 char const *
const func =
"omp_destroy_lock";
205 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
206 __kmp_is_tas_lock_nestable(lck)) {
207 KMP_FATAL(LockNestableUsedAsSimple, func);
209 if (__kmp_get_tas_lock_owner(lck) != -1) {
210 KMP_FATAL(LockStillOwned, func);
212 __kmp_destroy_tas_lock(lck);
217 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
218 KMP_DEBUG_ASSERT(gtid >= 0);
220 if (__kmp_get_tas_lock_owner(lck) == gtid) {
221 lck->lk.depth_locked += 1;
222 return KMP_LOCK_ACQUIRED_NEXT;
224 __kmp_acquire_tas_lock_timed_template(lck, gtid);
225 ANNOTATE_TAS_ACQUIRED(lck);
226 lck->lk.depth_locked = 1;
227 return KMP_LOCK_ACQUIRED_FIRST;
231 static int __kmp_acquire_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
233 char const *
const func =
"omp_set_nest_lock";
234 if (!__kmp_is_tas_lock_nestable(lck)) {
235 KMP_FATAL(LockSimpleUsedAsNestable, func);
237 return __kmp_acquire_nested_tas_lock(lck, gtid);
240 int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
243 KMP_DEBUG_ASSERT(gtid >= 0);
245 if (__kmp_get_tas_lock_owner(lck) == gtid) {
246 retval = ++lck->lk.depth_locked;
247 }
else if (!__kmp_test_tas_lock(lck, gtid)) {
251 retval = lck->lk.depth_locked = 1;
256 static int __kmp_test_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
258 char const *
const func =
"omp_test_nest_lock";
259 if (!__kmp_is_tas_lock_nestable(lck)) {
260 KMP_FATAL(LockSimpleUsedAsNestable, func);
262 return __kmp_test_nested_tas_lock(lck, gtid);
265 int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
266 KMP_DEBUG_ASSERT(gtid >= 0);
269 if (--(lck->lk.depth_locked) == 0) {
270 __kmp_release_tas_lock(lck, gtid);
271 return KMP_LOCK_RELEASED;
273 return KMP_LOCK_STILL_HELD;
276 static int __kmp_release_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
278 char const *
const func =
"omp_unset_nest_lock";
280 if (!__kmp_is_tas_lock_nestable(lck)) {
281 KMP_FATAL(LockSimpleUsedAsNestable, func);
283 if (__kmp_get_tas_lock_owner(lck) == -1) {
284 KMP_FATAL(LockUnsettingFree, func);
286 if (__kmp_get_tas_lock_owner(lck) != gtid) {
287 KMP_FATAL(LockUnsettingSetByAnother, func);
289 return __kmp_release_nested_tas_lock(lck, gtid);
292 void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck) {
293 __kmp_init_tas_lock(lck);
294 lck->lk.depth_locked = 0;
297 static void __kmp_init_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
298 __kmp_init_nested_tas_lock(lck);
301 void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck) {
302 __kmp_destroy_tas_lock(lck);
303 lck->lk.depth_locked = 0;
306 static void __kmp_destroy_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
307 char const *
const func =
"omp_destroy_nest_lock";
308 if (!__kmp_is_tas_lock_nestable(lck)) {
309 KMP_FATAL(LockSimpleUsedAsNestable, func);
311 if (__kmp_get_tas_lock_owner(lck) != -1) {
312 KMP_FATAL(LockStillOwned, func);
314 __kmp_destroy_nested_tas_lock(lck);
327 static kmp_int32 __kmp_get_futex_lock_owner(kmp_futex_lock_t *lck) {
328 return KMP_LOCK_STRIP((TCR_4(lck->lk.poll) >> 1)) - 1;
331 static inline bool __kmp_is_futex_lock_nestable(kmp_futex_lock_t *lck) {
332 return lck->lk.depth_locked != -1;
335 __forceinline
static int 336 __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) {
337 kmp_int32 gtid_code = (gtid + 1) << 1;
341 #ifdef USE_LOCK_PROFILE 342 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll));
343 if ((curr != 0) && (curr != gtid_code))
344 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
348 KMP_FSYNC_PREPARE(lck);
349 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
350 lck, lck->lk.poll, gtid));
354 while ((poll_val = KMP_COMPARE_AND_STORE_RET32(
355 &(lck->lk.poll), KMP_LOCK_FREE(futex),
356 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) {
358 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1;
361 (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
362 lck, gtid, poll_val, cond));
373 if (!KMP_COMPARE_AND_STORE_REL32(&(lck->lk.poll), poll_val,
374 poll_val | KMP_LOCK_BUSY(1, futex))) {
377 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
378 lck, lck->lk.poll, gtid));
381 poll_val |= KMP_LOCK_BUSY(1, futex);
384 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n", lck,
385 lck->lk.poll, gtid));
390 (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
391 lck, gtid, poll_val));
394 if ((rc = syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAIT, poll_val, NULL,
396 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) " 397 "failed (rc=%d errno=%d)\n",
398 lck, gtid, poll_val, rc, errno));
403 (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
404 lck, gtid, poll_val));
411 KMP_FSYNC_ACQUIRED(lck);
412 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
413 lck->lk.poll, gtid));
414 return KMP_LOCK_ACQUIRED_FIRST;
417 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
418 int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid);
419 ANNOTATE_FUTEX_ACQUIRED(lck);
423 static int __kmp_acquire_futex_lock_with_checks(kmp_futex_lock_t *lck,
425 char const *
const func =
"omp_set_lock";
426 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
427 __kmp_is_futex_lock_nestable(lck)) {
428 KMP_FATAL(LockNestableUsedAsSimple, func);
430 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) == gtid)) {
431 KMP_FATAL(LockIsAlreadyOwned, func);
433 return __kmp_acquire_futex_lock(lck, gtid);
436 int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
437 if (KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(futex),
438 KMP_LOCK_BUSY((gtid + 1) << 1, futex))) {
439 KMP_FSYNC_ACQUIRED(lck);
445 static int __kmp_test_futex_lock_with_checks(kmp_futex_lock_t *lck,
447 char const *
const func =
"omp_test_lock";
448 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
449 __kmp_is_futex_lock_nestable(lck)) {
450 KMP_FATAL(LockNestableUsedAsSimple, func);
452 return __kmp_test_futex_lock(lck, gtid);
455 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
458 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
459 lck, lck->lk.poll, gtid));
461 KMP_FSYNC_RELEASING(lck);
462 ANNOTATE_FUTEX_RELEASED(lck);
464 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(lck->lk.poll), KMP_LOCK_FREE(futex));
467 (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
468 lck, gtid, poll_val));
470 if (KMP_LOCK_STRIP(poll_val) & 1) {
472 (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
474 syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex),
480 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
481 lck->lk.poll, gtid));
483 KMP_YIELD(TCR_4(__kmp_nth) >
484 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
485 return KMP_LOCK_RELEASED;
488 static int __kmp_release_futex_lock_with_checks(kmp_futex_lock_t *lck,
490 char const *
const func =
"omp_unset_lock";
492 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
493 __kmp_is_futex_lock_nestable(lck)) {
494 KMP_FATAL(LockNestableUsedAsSimple, func);
496 if (__kmp_get_futex_lock_owner(lck) == -1) {
497 KMP_FATAL(LockUnsettingFree, func);
499 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) >= 0) &&
500 (__kmp_get_futex_lock_owner(lck) != gtid)) {
501 KMP_FATAL(LockUnsettingSetByAnother, func);
503 return __kmp_release_futex_lock(lck, gtid);
506 void __kmp_init_futex_lock(kmp_futex_lock_t *lck) {
507 TCW_4(lck->lk.poll, KMP_LOCK_FREE(futex));
510 static void __kmp_init_futex_lock_with_checks(kmp_futex_lock_t *lck) {
511 __kmp_init_futex_lock(lck);
514 void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck) { lck->lk.poll = 0; }
516 static void __kmp_destroy_futex_lock_with_checks(kmp_futex_lock_t *lck) {
517 char const *
const func =
"omp_destroy_lock";
518 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
519 __kmp_is_futex_lock_nestable(lck)) {
520 KMP_FATAL(LockNestableUsedAsSimple, func);
522 if (__kmp_get_futex_lock_owner(lck) != -1) {
523 KMP_FATAL(LockStillOwned, func);
525 __kmp_destroy_futex_lock(lck);
530 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
531 KMP_DEBUG_ASSERT(gtid >= 0);
533 if (__kmp_get_futex_lock_owner(lck) == gtid) {
534 lck->lk.depth_locked += 1;
535 return KMP_LOCK_ACQUIRED_NEXT;
537 __kmp_acquire_futex_lock_timed_template(lck, gtid);
538 ANNOTATE_FUTEX_ACQUIRED(lck);
539 lck->lk.depth_locked = 1;
540 return KMP_LOCK_ACQUIRED_FIRST;
544 static int __kmp_acquire_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
546 char const *
const func =
"omp_set_nest_lock";
547 if (!__kmp_is_futex_lock_nestable(lck)) {
548 KMP_FATAL(LockSimpleUsedAsNestable, func);
550 return __kmp_acquire_nested_futex_lock(lck, gtid);
553 int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
556 KMP_DEBUG_ASSERT(gtid >= 0);
558 if (__kmp_get_futex_lock_owner(lck) == gtid) {
559 retval = ++lck->lk.depth_locked;
560 }
else if (!__kmp_test_futex_lock(lck, gtid)) {
564 retval = lck->lk.depth_locked = 1;
569 static int __kmp_test_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
571 char const *
const func =
"omp_test_nest_lock";
572 if (!__kmp_is_futex_lock_nestable(lck)) {
573 KMP_FATAL(LockSimpleUsedAsNestable, func);
575 return __kmp_test_nested_futex_lock(lck, gtid);
578 int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
579 KMP_DEBUG_ASSERT(gtid >= 0);
582 if (--(lck->lk.depth_locked) == 0) {
583 __kmp_release_futex_lock(lck, gtid);
584 return KMP_LOCK_RELEASED;
586 return KMP_LOCK_STILL_HELD;
589 static int __kmp_release_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
591 char const *
const func =
"omp_unset_nest_lock";
593 if (!__kmp_is_futex_lock_nestable(lck)) {
594 KMP_FATAL(LockSimpleUsedAsNestable, func);
596 if (__kmp_get_futex_lock_owner(lck) == -1) {
597 KMP_FATAL(LockUnsettingFree, func);
599 if (__kmp_get_futex_lock_owner(lck) != gtid) {
600 KMP_FATAL(LockUnsettingSetByAnother, func);
602 return __kmp_release_nested_futex_lock(lck, gtid);
605 void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck) {
606 __kmp_init_futex_lock(lck);
607 lck->lk.depth_locked = 0;
610 static void __kmp_init_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
611 __kmp_init_nested_futex_lock(lck);
614 void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck) {
615 __kmp_destroy_futex_lock(lck);
616 lck->lk.depth_locked = 0;
619 static void __kmp_destroy_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
620 char const *
const func =
"omp_destroy_nest_lock";
621 if (!__kmp_is_futex_lock_nestable(lck)) {
622 KMP_FATAL(LockSimpleUsedAsNestable, func);
624 if (__kmp_get_futex_lock_owner(lck) != -1) {
625 KMP_FATAL(LockStillOwned, func);
627 __kmp_destroy_nested_futex_lock(lck);
630 #endif // KMP_USE_FUTEX 635 static kmp_int32 __kmp_get_ticket_lock_owner(kmp_ticket_lock_t *lck) {
636 return std::atomic_load_explicit(&lck->lk.owner_id,
637 std::memory_order_relaxed) -
641 static inline bool __kmp_is_ticket_lock_nestable(kmp_ticket_lock_t *lck) {
642 return std::atomic_load_explicit(&lck->lk.depth_locked,
643 std::memory_order_relaxed) != -1;
646 static kmp_uint32 __kmp_bakery_check(
void *now_serving, kmp_uint32 my_ticket) {
647 return std::atomic_load_explicit((std::atomic<unsigned> *)now_serving,
648 std::memory_order_acquire) == my_ticket;
651 __forceinline
static int 652 __kmp_acquire_ticket_lock_timed_template(kmp_ticket_lock_t *lck,
654 kmp_uint32 my_ticket = std::atomic_fetch_add_explicit(
655 &lck->lk.next_ticket, 1U, std::memory_order_relaxed);
657 #ifdef USE_LOCK_PROFILE 658 if (std::atomic_load_explicit(&lck->lk.now_serving,
659 std::memory_order_relaxed) != my_ticket)
660 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
664 if (std::atomic_load_explicit(&lck->lk.now_serving,
665 std::memory_order_acquire) == my_ticket) {
666 return KMP_LOCK_ACQUIRED_FIRST;
668 KMP_WAIT_YIELD_PTR(&lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck);
669 return KMP_LOCK_ACQUIRED_FIRST;
672 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
673 int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid);
674 ANNOTATE_TICKET_ACQUIRED(lck);
678 static int __kmp_acquire_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
680 char const *
const func =
"omp_set_lock";
682 if (!std::atomic_load_explicit(&lck->lk.initialized,
683 std::memory_order_relaxed)) {
684 KMP_FATAL(LockIsUninitialized, func);
686 if (lck->lk.self != lck) {
687 KMP_FATAL(LockIsUninitialized, func);
689 if (__kmp_is_ticket_lock_nestable(lck)) {
690 KMP_FATAL(LockNestableUsedAsSimple, func);
692 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) == gtid)) {
693 KMP_FATAL(LockIsAlreadyOwned, func);
696 __kmp_acquire_ticket_lock(lck, gtid);
698 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
699 std::memory_order_relaxed);
700 return KMP_LOCK_ACQUIRED_FIRST;
703 int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
704 kmp_uint32 my_ticket = std::atomic_load_explicit(&lck->lk.next_ticket,
705 std::memory_order_relaxed);
707 if (std::atomic_load_explicit(&lck->lk.now_serving,
708 std::memory_order_relaxed) == my_ticket) {
709 kmp_uint32 next_ticket = my_ticket + 1;
710 if (std::atomic_compare_exchange_strong_explicit(
711 &lck->lk.next_ticket, &my_ticket, next_ticket,
712 std::memory_order_acquire, std::memory_order_acquire)) {
719 static int __kmp_test_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
721 char const *
const func =
"omp_test_lock";
723 if (!std::atomic_load_explicit(&lck->lk.initialized,
724 std::memory_order_relaxed)) {
725 KMP_FATAL(LockIsUninitialized, func);
727 if (lck->lk.self != lck) {
728 KMP_FATAL(LockIsUninitialized, func);
730 if (__kmp_is_ticket_lock_nestable(lck)) {
731 KMP_FATAL(LockNestableUsedAsSimple, func);
734 int retval = __kmp_test_ticket_lock(lck, gtid);
737 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
738 std::memory_order_relaxed);
743 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
744 kmp_uint32 distance = std::atomic_load_explicit(&lck->lk.next_ticket,
745 std::memory_order_relaxed) -
746 std::atomic_load_explicit(&lck->lk.now_serving,
747 std::memory_order_relaxed);
749 ANNOTATE_TICKET_RELEASED(lck);
750 std::atomic_fetch_add_explicit(&lck->lk.now_serving, 1U,
751 std::memory_order_release);
754 (kmp_uint32)(__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
755 return KMP_LOCK_RELEASED;
758 static int __kmp_release_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
760 char const *
const func =
"omp_unset_lock";
762 if (!std::atomic_load_explicit(&lck->lk.initialized,
763 std::memory_order_relaxed)) {
764 KMP_FATAL(LockIsUninitialized, func);
766 if (lck->lk.self != lck) {
767 KMP_FATAL(LockIsUninitialized, func);
769 if (__kmp_is_ticket_lock_nestable(lck)) {
770 KMP_FATAL(LockNestableUsedAsSimple, func);
772 if (__kmp_get_ticket_lock_owner(lck) == -1) {
773 KMP_FATAL(LockUnsettingFree, func);
775 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) >= 0) &&
776 (__kmp_get_ticket_lock_owner(lck) != gtid)) {
777 KMP_FATAL(LockUnsettingSetByAnother, func);
779 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
780 return __kmp_release_ticket_lock(lck, gtid);
783 void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck) {
784 lck->lk.location = NULL;
786 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
787 std::memory_order_relaxed);
788 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
789 std::memory_order_relaxed);
790 std::atomic_store_explicit(
791 &lck->lk.owner_id, 0,
792 std::memory_order_relaxed);
793 std::atomic_store_explicit(
794 &lck->lk.depth_locked, -1,
795 std::memory_order_relaxed);
796 std::atomic_store_explicit(&lck->lk.initialized,
true,
797 std::memory_order_release);
800 static void __kmp_init_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
801 __kmp_init_ticket_lock(lck);
804 void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck) {
805 std::atomic_store_explicit(&lck->lk.initialized,
false,
806 std::memory_order_release);
808 lck->lk.location = NULL;
809 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
810 std::memory_order_relaxed);
811 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
812 std::memory_order_relaxed);
813 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
814 std::atomic_store_explicit(&lck->lk.depth_locked, -1,
815 std::memory_order_relaxed);
818 static void __kmp_destroy_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
819 char const *
const func =
"omp_destroy_lock";
821 if (!std::atomic_load_explicit(&lck->lk.initialized,
822 std::memory_order_relaxed)) {
823 KMP_FATAL(LockIsUninitialized, func);
825 if (lck->lk.self != lck) {
826 KMP_FATAL(LockIsUninitialized, func);
828 if (__kmp_is_ticket_lock_nestable(lck)) {
829 KMP_FATAL(LockNestableUsedAsSimple, func);
831 if (__kmp_get_ticket_lock_owner(lck) != -1) {
832 KMP_FATAL(LockStillOwned, func);
834 __kmp_destroy_ticket_lock(lck);
839 int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
840 KMP_DEBUG_ASSERT(gtid >= 0);
842 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
843 std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
844 std::memory_order_relaxed);
845 return KMP_LOCK_ACQUIRED_NEXT;
847 __kmp_acquire_ticket_lock_timed_template(lck, gtid);
848 ANNOTATE_TICKET_ACQUIRED(lck);
849 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
850 std::memory_order_relaxed);
851 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
852 std::memory_order_relaxed);
853 return KMP_LOCK_ACQUIRED_FIRST;
857 static int __kmp_acquire_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
859 char const *
const func =
"omp_set_nest_lock";
861 if (!std::atomic_load_explicit(&lck->lk.initialized,
862 std::memory_order_relaxed)) {
863 KMP_FATAL(LockIsUninitialized, func);
865 if (lck->lk.self != lck) {
866 KMP_FATAL(LockIsUninitialized, func);
868 if (!__kmp_is_ticket_lock_nestable(lck)) {
869 KMP_FATAL(LockSimpleUsedAsNestable, func);
871 return __kmp_acquire_nested_ticket_lock(lck, gtid);
874 int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
877 KMP_DEBUG_ASSERT(gtid >= 0);
879 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
880 retval = std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
881 std::memory_order_relaxed) +
883 }
else if (!__kmp_test_ticket_lock(lck, gtid)) {
886 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
887 std::memory_order_relaxed);
888 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
889 std::memory_order_relaxed);
895 static int __kmp_test_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
897 char const *
const func =
"omp_test_nest_lock";
899 if (!std::atomic_load_explicit(&lck->lk.initialized,
900 std::memory_order_relaxed)) {
901 KMP_FATAL(LockIsUninitialized, func);
903 if (lck->lk.self != lck) {
904 KMP_FATAL(LockIsUninitialized, func);
906 if (!__kmp_is_ticket_lock_nestable(lck)) {
907 KMP_FATAL(LockSimpleUsedAsNestable, func);
909 return __kmp_test_nested_ticket_lock(lck, gtid);
912 int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
913 KMP_DEBUG_ASSERT(gtid >= 0);
915 if ((std::atomic_fetch_add_explicit(&lck->lk.depth_locked, -1,
916 std::memory_order_relaxed) -
918 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
919 __kmp_release_ticket_lock(lck, gtid);
920 return KMP_LOCK_RELEASED;
922 return KMP_LOCK_STILL_HELD;
925 static int __kmp_release_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
927 char const *
const func =
"omp_unset_nest_lock";
929 if (!std::atomic_load_explicit(&lck->lk.initialized,
930 std::memory_order_relaxed)) {
931 KMP_FATAL(LockIsUninitialized, func);
933 if (lck->lk.self != lck) {
934 KMP_FATAL(LockIsUninitialized, func);
936 if (!__kmp_is_ticket_lock_nestable(lck)) {
937 KMP_FATAL(LockSimpleUsedAsNestable, func);
939 if (__kmp_get_ticket_lock_owner(lck) == -1) {
940 KMP_FATAL(LockUnsettingFree, func);
942 if (__kmp_get_ticket_lock_owner(lck) != gtid) {
943 KMP_FATAL(LockUnsettingSetByAnother, func);
945 return __kmp_release_nested_ticket_lock(lck, gtid);
948 void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck) {
949 __kmp_init_ticket_lock(lck);
950 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
951 std::memory_order_relaxed);
955 static void __kmp_init_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
956 __kmp_init_nested_ticket_lock(lck);
959 void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck) {
960 __kmp_destroy_ticket_lock(lck);
961 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
962 std::memory_order_relaxed);
966 __kmp_destroy_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
967 char const *
const func =
"omp_destroy_nest_lock";
969 if (!std::atomic_load_explicit(&lck->lk.initialized,
970 std::memory_order_relaxed)) {
971 KMP_FATAL(LockIsUninitialized, func);
973 if (lck->lk.self != lck) {
974 KMP_FATAL(LockIsUninitialized, func);
976 if (!__kmp_is_ticket_lock_nestable(lck)) {
977 KMP_FATAL(LockSimpleUsedAsNestable, func);
979 if (__kmp_get_ticket_lock_owner(lck) != -1) {
980 KMP_FATAL(LockStillOwned, func);
982 __kmp_destroy_nested_ticket_lock(lck);
987 static int __kmp_is_ticket_lock_initialized(kmp_ticket_lock_t *lck) {
988 return std::atomic_load_explicit(&lck->lk.initialized,
989 std::memory_order_relaxed) &&
990 (lck->lk.self == lck);
993 static const ident_t *__kmp_get_ticket_lock_location(kmp_ticket_lock_t *lck) {
994 return lck->lk.location;
997 static void __kmp_set_ticket_lock_location(kmp_ticket_lock_t *lck,
999 lck->lk.location = loc;
1002 static kmp_lock_flags_t __kmp_get_ticket_lock_flags(kmp_ticket_lock_t *lck) {
1003 return lck->lk.flags;
1006 static void __kmp_set_ticket_lock_flags(kmp_ticket_lock_t *lck,
1007 kmp_lock_flags_t flags) {
1008 lck->lk.flags = flags;
1066 #ifdef DEBUG_QUEUING_LOCKS 1069 #define TRACE_BUF_ELE 1024 1070 static char traces[TRACE_BUF_ELE][128] = {0};
1072 #define TRACE_LOCK(X, Y) \ 1073 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y); 1074 #define TRACE_LOCK_T(X, Y, Z) \ 1075 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X, Y, Z); 1076 #define TRACE_LOCK_HT(X, Y, Z, Q) \ 1077 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, \ 1080 static void __kmp_dump_queuing_lock(kmp_info_t *this_thr, kmp_int32 gtid,
1081 kmp_queuing_lock_t *lck, kmp_int32 head_id,
1082 kmp_int32 tail_id) {
1085 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n");
1087 i = tc % TRACE_BUF_ELE;
1088 __kmp_printf_no_lock(
"%s\n", traces[i]);
1089 i = (i + 1) % TRACE_BUF_ELE;
1090 while (i != (tc % TRACE_BUF_ELE)) {
1091 __kmp_printf_no_lock(
"%s", traces[i]);
1092 i = (i + 1) % TRACE_BUF_ELE;
1094 __kmp_printf_no_lock(
"\n");
1096 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, " 1097 "next_wait:%d, head_id:%d, tail_id:%d\n",
1098 gtid + 1, this_thr->th.th_spin_here,
1099 this_thr->th.th_next_waiting, head_id, tail_id);
1101 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id);
1103 if (lck->lk.head_id >= 1) {
1104 t = __kmp_threads[lck->lk.head_id - 1]->th.th_next_waiting;
1106 __kmp_printf_no_lock(
"-> %d ", t);
1107 t = __kmp_threads[t - 1]->th.th_next_waiting;
1110 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id);
1111 __kmp_printf_no_lock(
"\n\n");
1116 static kmp_int32 __kmp_get_queuing_lock_owner(kmp_queuing_lock_t *lck) {
1117 return TCR_4(lck->lk.owner_id) - 1;
1120 static inline bool __kmp_is_queuing_lock_nestable(kmp_queuing_lock_t *lck) {
1121 return lck->lk.depth_locked != -1;
1125 template <
bool takeTime>
1128 __forceinline
static int 1129 __kmp_acquire_queuing_lock_timed_template(kmp_queuing_lock_t *lck,
1131 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid);
1132 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1133 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1134 volatile kmp_uint32 *spin_here_p;
1135 kmp_int32 need_mf = 1;
1138 omp_state_t prev_state = omp_state_undefined;
1142 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1144 KMP_FSYNC_PREPARE(lck);
1145 KMP_DEBUG_ASSERT(this_thr != NULL);
1146 spin_here_p = &this_thr->th.th_spin_here;
1148 #ifdef DEBUG_QUEUING_LOCKS 1149 TRACE_LOCK(gtid + 1,
"acq ent");
1151 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1152 if (this_thr->th.th_next_waiting != 0)
1153 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1155 KMP_DEBUG_ASSERT(!*spin_here_p);
1156 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1165 *spin_here_p = TRUE;
1177 #ifdef DEBUG_QUEUING_LOCKS 1179 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1188 enqueued = KMP_COMPARE_AND_STORE_ACQ64((
volatile kmp_int64 *)tail_id_p,
1190 KMP_PACK_64(gtid + 1, gtid + 1));
1191 #ifdef DEBUG_QUEUING_LOCKS 1193 TRACE_LOCK(gtid + 1,
"acq enq: (-1,0)->(tid,tid)");
1199 KMP_DEBUG_ASSERT(tail != gtid + 1);
1201 #ifdef DEBUG_QUEUING_LOCKS 1202 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1210 enqueued = KMP_COMPARE_AND_STORE_ACQ32(tail_id_p, tail, gtid + 1);
1212 #ifdef DEBUG_QUEUING_LOCKS 1214 TRACE_LOCK(gtid + 1,
"acq enq: (h,t)->(h,tid)");
1221 kmp_int32 grabbed_lock;
1223 #ifdef DEBUG_QUEUING_LOCKS 1225 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1231 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1);
1235 *spin_here_p = FALSE;
1239 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1241 #ifdef DEBUG_QUEUING_LOCKS 1242 TRACE_LOCK_HT(gtid + 1,
"acq exit: ", head, 0);
1246 if (ompt_enabled.enabled && prev_state != omp_state_undefined) {
1248 this_thr->th.ompt_thread_info.state = prev_state;
1249 this_thr->th.ompt_thread_info.wait_id = 0;
1253 KMP_FSYNC_ACQUIRED(lck);
1254 return KMP_LOCK_ACQUIRED_FIRST;
1261 if (ompt_enabled.enabled && prev_state == omp_state_undefined) {
1263 prev_state = this_thr->th.ompt_thread_info.state;
1264 this_thr->th.ompt_thread_info.wait_id = (uint64_t)lck;
1265 this_thr->th.ompt_thread_info.state = omp_state_wait_lock;
1271 kmp_info_t *tail_thr = __kmp_thread_from_gtid(tail - 1);
1272 KMP_ASSERT(tail_thr != NULL);
1273 tail_thr->th.th_next_waiting = gtid + 1;
1277 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n",
1283 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1285 #ifdef DEBUG_QUEUING_LOCKS 1286 TRACE_LOCK(gtid + 1,
"acq spin");
1288 if (this_thr->th.th_next_waiting != 0)
1289 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1291 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1292 KA_TRACE(1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after " 1293 "waiting on queue\n",
1296 #ifdef DEBUG_QUEUING_LOCKS 1297 TRACE_LOCK(gtid + 1,
"acq exit 2");
1302 this_thr->th.ompt_thread_info.state = prev_state;
1303 this_thr->th.ompt_thread_info.wait_id = 0;
1307 return KMP_LOCK_ACQUIRED_FIRST;
1313 KMP_YIELD(TCR_4(__kmp_nth) >
1314 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
1315 #ifdef DEBUG_QUEUING_LOCKS 1316 TRACE_LOCK(gtid + 1,
"acq retry");
1319 KMP_ASSERT2(0,
"should not get here");
1320 return KMP_LOCK_ACQUIRED_FIRST;
1323 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1324 KMP_DEBUG_ASSERT(gtid >= 0);
1326 int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1327 ANNOTATE_QUEUING_ACQUIRED(lck);
1331 static int __kmp_acquire_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1333 char const *
const func =
"omp_set_lock";
1334 if (lck->lk.initialized != lck) {
1335 KMP_FATAL(LockIsUninitialized, func);
1337 if (__kmp_is_queuing_lock_nestable(lck)) {
1338 KMP_FATAL(LockNestableUsedAsSimple, func);
1340 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1341 KMP_FATAL(LockIsAlreadyOwned, func);
1344 __kmp_acquire_queuing_lock(lck, gtid);
1346 lck->lk.owner_id = gtid + 1;
1347 return KMP_LOCK_ACQUIRED_FIRST;
1350 int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1351 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1354 kmp_info_t *this_thr;
1357 KA_TRACE(1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid));
1358 KMP_DEBUG_ASSERT(gtid >= 0);
1360 this_thr = __kmp_thread_from_gtid(gtid);
1361 KMP_DEBUG_ASSERT(this_thr != NULL);
1362 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1369 if (KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1)) {
1371 (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid));
1372 KMP_FSYNC_ACQUIRED(lck);
1373 ANNOTATE_QUEUING_ACQUIRED(lck);
1379 (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid));
1383 static int __kmp_test_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1385 char const *
const func =
"omp_test_lock";
1386 if (lck->lk.initialized != lck) {
1387 KMP_FATAL(LockIsUninitialized, func);
1389 if (__kmp_is_queuing_lock_nestable(lck)) {
1390 KMP_FATAL(LockNestableUsedAsSimple, func);
1393 int retval = __kmp_test_queuing_lock(lck, gtid);
1396 lck->lk.owner_id = gtid + 1;
1401 int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1402 kmp_info_t *this_thr;
1403 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1404 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1407 (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1408 KMP_DEBUG_ASSERT(gtid >= 0);
1409 this_thr = __kmp_thread_from_gtid(gtid);
1410 KMP_DEBUG_ASSERT(this_thr != NULL);
1411 #ifdef DEBUG_QUEUING_LOCKS 1412 TRACE_LOCK(gtid + 1,
"rel ent");
1414 if (this_thr->th.th_spin_here)
1415 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1416 if (this_thr->th.th_next_waiting != 0)
1417 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1419 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1420 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1422 KMP_FSYNC_RELEASING(lck);
1423 ANNOTATE_QUEUING_RELEASED(lck);
1432 #ifdef DEBUG_QUEUING_LOCKS 1434 TRACE_LOCK_HT(gtid + 1,
"rel read: ", head, tail);
1436 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1438 KMP_DEBUG_ASSERT(head !=
1443 if (KMP_COMPARE_AND_STORE_REL32(head_id_p, -1, 0)) {
1446 (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1448 #ifdef DEBUG_QUEUING_LOCKS 1449 TRACE_LOCK_HT(gtid + 1,
"rel exit: ", 0, 0);
1455 return KMP_LOCK_RELEASED;
1462 #ifdef DEBUG_QUEUING_LOCKS 1464 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1466 KMP_DEBUG_ASSERT(head > 0);
1469 dequeued = KMP_COMPARE_AND_STORE_REL64(
1470 RCAST(
volatile kmp_int64 *, tail_id_p), KMP_PACK_64(head, head),
1471 KMP_PACK_64(-1, 0));
1472 #ifdef DEBUG_QUEUING_LOCKS 1473 TRACE_LOCK(gtid + 1,
"rel deq: (h,h)->(-1,0)");
1477 volatile kmp_int32 *waiting_id_p;
1478 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1479 KMP_DEBUG_ASSERT(head_thr != NULL);
1480 waiting_id_p = &head_thr->th.th_next_waiting;
1483 #ifdef DEBUG_QUEUING_LOCKS 1484 if (head <= 0 || tail <= 0)
1485 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1487 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1493 *head_id_p = KMP_WAIT_YIELD((
volatile kmp_uint32 *)waiting_id_p, 0,
1495 #ifdef DEBUG_QUEUING_LOCKS 1496 TRACE_LOCK(gtid + 1,
"rel deq: (h,t)->(h',t)");
1503 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1504 KMP_DEBUG_ASSERT(head_thr != NULL);
1507 #ifdef DEBUG_QUEUING_LOCKS 1508 if (head <= 0 || tail <= 0)
1509 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1511 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1515 head_thr->th.th_next_waiting = 0;
1516 #ifdef DEBUG_QUEUING_LOCKS 1517 TRACE_LOCK_T(gtid + 1,
"rel nw=0 for t=", head);
1522 head_thr->th.th_spin_here = FALSE;
1524 KA_TRACE(1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after " 1527 #ifdef DEBUG_QUEUING_LOCKS 1528 TRACE_LOCK(gtid + 1,
"rel exit 2");
1530 return KMP_LOCK_RELEASED;
1535 #ifdef DEBUG_QUEUING_LOCKS 1536 TRACE_LOCK(gtid + 1,
"rel retry");
1540 KMP_ASSERT2(0,
"should not get here");
1541 return KMP_LOCK_RELEASED;
1544 static int __kmp_release_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1546 char const *
const func =
"omp_unset_lock";
1548 if (lck->lk.initialized != lck) {
1549 KMP_FATAL(LockIsUninitialized, func);
1551 if (__kmp_is_queuing_lock_nestable(lck)) {
1552 KMP_FATAL(LockNestableUsedAsSimple, func);
1554 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1555 KMP_FATAL(LockUnsettingFree, func);
1557 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1558 KMP_FATAL(LockUnsettingSetByAnother, func);
1560 lck->lk.owner_id = 0;
1561 return __kmp_release_queuing_lock(lck, gtid);
1564 void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck) {
1565 lck->lk.location = NULL;
1566 lck->lk.head_id = 0;
1567 lck->lk.tail_id = 0;
1568 lck->lk.next_ticket = 0;
1569 lck->lk.now_serving = 0;
1570 lck->lk.owner_id = 0;
1571 lck->lk.depth_locked = -1;
1572 lck->lk.initialized = lck;
1574 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1577 static void __kmp_init_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1578 __kmp_init_queuing_lock(lck);
1581 void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck) {
1582 lck->lk.initialized = NULL;
1583 lck->lk.location = NULL;
1584 lck->lk.head_id = 0;
1585 lck->lk.tail_id = 0;
1586 lck->lk.next_ticket = 0;
1587 lck->lk.now_serving = 0;
1588 lck->lk.owner_id = 0;
1589 lck->lk.depth_locked = -1;
1592 static void __kmp_destroy_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1593 char const *
const func =
"omp_destroy_lock";
1594 if (lck->lk.initialized != lck) {
1595 KMP_FATAL(LockIsUninitialized, func);
1597 if (__kmp_is_queuing_lock_nestable(lck)) {
1598 KMP_FATAL(LockNestableUsedAsSimple, func);
1600 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1601 KMP_FATAL(LockStillOwned, func);
1603 __kmp_destroy_queuing_lock(lck);
1608 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1609 KMP_DEBUG_ASSERT(gtid >= 0);
1611 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1612 lck->lk.depth_locked += 1;
1613 return KMP_LOCK_ACQUIRED_NEXT;
1615 __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1616 ANNOTATE_QUEUING_ACQUIRED(lck);
1618 lck->lk.depth_locked = 1;
1620 lck->lk.owner_id = gtid + 1;
1621 return KMP_LOCK_ACQUIRED_FIRST;
1626 __kmp_acquire_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1628 char const *
const func =
"omp_set_nest_lock";
1629 if (lck->lk.initialized != lck) {
1630 KMP_FATAL(LockIsUninitialized, func);
1632 if (!__kmp_is_queuing_lock_nestable(lck)) {
1633 KMP_FATAL(LockSimpleUsedAsNestable, func);
1635 return __kmp_acquire_nested_queuing_lock(lck, gtid);
1638 int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1641 KMP_DEBUG_ASSERT(gtid >= 0);
1643 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1644 retval = ++lck->lk.depth_locked;
1645 }
else if (!__kmp_test_queuing_lock(lck, gtid)) {
1649 retval = lck->lk.depth_locked = 1;
1651 lck->lk.owner_id = gtid + 1;
1656 static int __kmp_test_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1658 char const *
const func =
"omp_test_nest_lock";
1659 if (lck->lk.initialized != lck) {
1660 KMP_FATAL(LockIsUninitialized, func);
1662 if (!__kmp_is_queuing_lock_nestable(lck)) {
1663 KMP_FATAL(LockSimpleUsedAsNestable, func);
1665 return __kmp_test_nested_queuing_lock(lck, gtid);
1668 int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1669 KMP_DEBUG_ASSERT(gtid >= 0);
1672 if (--(lck->lk.depth_locked) == 0) {
1674 lck->lk.owner_id = 0;
1675 __kmp_release_queuing_lock(lck, gtid);
1676 return KMP_LOCK_RELEASED;
1678 return KMP_LOCK_STILL_HELD;
1682 __kmp_release_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1684 char const *
const func =
"omp_unset_nest_lock";
1686 if (lck->lk.initialized != lck) {
1687 KMP_FATAL(LockIsUninitialized, func);
1689 if (!__kmp_is_queuing_lock_nestable(lck)) {
1690 KMP_FATAL(LockSimpleUsedAsNestable, func);
1692 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1693 KMP_FATAL(LockUnsettingFree, func);
1695 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1696 KMP_FATAL(LockUnsettingSetByAnother, func);
1698 return __kmp_release_nested_queuing_lock(lck, gtid);
1701 void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1702 __kmp_init_queuing_lock(lck);
1703 lck->lk.depth_locked = 0;
1707 __kmp_init_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1708 __kmp_init_nested_queuing_lock(lck);
1711 void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1712 __kmp_destroy_queuing_lock(lck);
1713 lck->lk.depth_locked = 0;
1717 __kmp_destroy_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1718 char const *
const func =
"omp_destroy_nest_lock";
1719 if (lck->lk.initialized != lck) {
1720 KMP_FATAL(LockIsUninitialized, func);
1722 if (!__kmp_is_queuing_lock_nestable(lck)) {
1723 KMP_FATAL(LockSimpleUsedAsNestable, func);
1725 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1726 KMP_FATAL(LockStillOwned, func);
1728 __kmp_destroy_nested_queuing_lock(lck);
1733 static int __kmp_is_queuing_lock_initialized(kmp_queuing_lock_t *lck) {
1734 return lck == lck->lk.initialized;
1737 static const ident_t *__kmp_get_queuing_lock_location(kmp_queuing_lock_t *lck) {
1738 return lck->lk.location;
1741 static void __kmp_set_queuing_lock_location(kmp_queuing_lock_t *lck,
1743 lck->lk.location = loc;
1746 static kmp_lock_flags_t __kmp_get_queuing_lock_flags(kmp_queuing_lock_t *lck) {
1747 return lck->lk.flags;
1750 static void __kmp_set_queuing_lock_flags(kmp_queuing_lock_t *lck,
1751 kmp_lock_flags_t flags) {
1752 lck->lk.flags = flags;
1755 #if KMP_USE_ADAPTIVE_LOCKS 1759 #if KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 1761 #include <immintrin.h> 1762 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1767 #define _XBEGIN_STARTED (~0u) 1768 #define _XABORT_EXPLICIT (1 << 0) 1769 #define _XABORT_RETRY (1 << 1) 1770 #define _XABORT_CONFLICT (1 << 2) 1771 #define _XABORT_CAPACITY (1 << 3) 1772 #define _XABORT_DEBUG (1 << 4) 1773 #define _XABORT_NESTED (1 << 5) 1774 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF)) 1777 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1779 #define STRINGIZE_INTERNAL(arg) #arg 1780 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg) 1786 static __inline
int _xbegin() {
1814 #endif // KMP_ARCH_X86_64 1823 __asm__
volatile(
"1: .byte 0xC7; .byte 0xF8;\n" 1826 "1: movl %%eax,%0\n" 1828 :
"+r"(res)::
"memory",
"%eax");
1829 #endif // KMP_OS_WINDOWS 1834 static __inline
void _xend() {
1842 __asm__
volatile(
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1851 #define _xabort(ARG) _asm _emit 0xc6 _asm _emit 0xf8 _asm _emit ARG 1853 #define _xabort(ARG) \ 1854 __asm__ volatile(".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG):::"memory"); 1857 #endif // KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 1860 #if KMP_DEBUG_ADAPTIVE_LOCKS 1865 static kmp_adaptive_lock_statistics_t destroyedStats;
1868 static kmp_adaptive_lock_info_t liveLocks;
1871 static kmp_bootstrap_lock_t chain_lock;
1874 void __kmp_init_speculative_stats() {
1875 kmp_adaptive_lock_info_t *lck = &liveLocks;
1877 memset((
void *)&(lck->stats), 0,
sizeof(lck->stats));
1878 lck->stats.next = lck;
1879 lck->stats.prev = lck;
1881 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1882 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1884 __kmp_init_bootstrap_lock(&chain_lock);
1888 static void __kmp_remember_lock(kmp_adaptive_lock_info_t *lck) {
1889 __kmp_acquire_bootstrap_lock(&chain_lock);
1891 lck->stats.next = liveLocks.stats.next;
1892 lck->stats.prev = &liveLocks;
1894 liveLocks.stats.next = lck;
1895 lck->stats.next->stats.prev = lck;
1897 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1898 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1900 __kmp_release_bootstrap_lock(&chain_lock);
1903 static void __kmp_forget_lock(kmp_adaptive_lock_info_t *lck) {
1904 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1905 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1907 kmp_adaptive_lock_info_t *n = lck->stats.next;
1908 kmp_adaptive_lock_info_t *p = lck->stats.prev;
1914 static void __kmp_zero_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1915 memset((
void *)&lck->stats, 0,
sizeof(lck->stats));
1916 __kmp_remember_lock(lck);
1919 static void __kmp_add_stats(kmp_adaptive_lock_statistics_t *t,
1920 kmp_adaptive_lock_info_t *lck) {
1921 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
1923 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
1924 t->successfulSpeculations += s->successfulSpeculations;
1925 t->hardFailedSpeculations += s->hardFailedSpeculations;
1926 t->softFailedSpeculations += s->softFailedSpeculations;
1927 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
1928 t->lemmingYields += s->lemmingYields;
1931 static void __kmp_accumulate_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1932 kmp_adaptive_lock_statistics_t *t = &destroyedStats;
1934 __kmp_acquire_bootstrap_lock(&chain_lock);
1936 __kmp_add_stats(&destroyedStats, lck);
1937 __kmp_forget_lock(lck);
1939 __kmp_release_bootstrap_lock(&chain_lock);
1942 static float percent(kmp_uint32 count, kmp_uint32 total) {
1943 return (total == 0) ? 0.0 : (100.0 * count) / total;
1946 static FILE *__kmp_open_stats_file() {
1947 if (strcmp(__kmp_speculative_statsfile,
"-") == 0)
1950 size_t buffLen = KMP_STRLEN(__kmp_speculative_statsfile) + 20;
1951 char buffer[buffLen];
1952 KMP_SNPRINTF(&buffer[0], buffLen, __kmp_speculative_statsfile,
1953 (kmp_int32)getpid());
1954 FILE *result = fopen(&buffer[0],
"w");
1957 return result ? result : stdout;
1960 void __kmp_print_speculative_stats() {
1961 if (__kmp_user_lock_kind != lk_adaptive)
1964 FILE *statsFile = __kmp_open_stats_file();
1966 kmp_adaptive_lock_statistics_t total = destroyedStats;
1967 kmp_adaptive_lock_info_t *lck;
1969 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
1970 __kmp_add_stats(&total, lck);
1972 kmp_adaptive_lock_statistics_t *t = &total;
1973 kmp_uint32 totalSections =
1974 t->nonSpeculativeAcquires + t->successfulSpeculations;
1975 kmp_uint32 totalSpeculations = t->successfulSpeculations +
1976 t->hardFailedSpeculations +
1977 t->softFailedSpeculations;
1979 fprintf(statsFile,
"Speculative lock statistics (all approximate!)\n");
1980 fprintf(statsFile,
" Lock parameters: \n" 1981 " max_soft_retries : %10d\n" 1982 " max_badness : %10d\n",
1983 __kmp_adaptive_backoff_params.max_soft_retries,
1984 __kmp_adaptive_backoff_params.max_badness);
1985 fprintf(statsFile,
" Non-speculative acquire attempts : %10d\n",
1986 t->nonSpeculativeAcquireAttempts);
1987 fprintf(statsFile,
" Total critical sections : %10d\n",
1989 fprintf(statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
1990 t->successfulSpeculations,
1991 percent(t->successfulSpeculations, totalSections));
1992 fprintf(statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
1993 t->nonSpeculativeAcquires,
1994 percent(t->nonSpeculativeAcquires, totalSections));
1995 fprintf(statsFile,
" Lemming yields : %10d\n\n",
1998 fprintf(statsFile,
" Speculative acquire attempts : %10d\n",
2000 fprintf(statsFile,
" Successes : %10d (%5.1f%%)\n",
2001 t->successfulSpeculations,
2002 percent(t->successfulSpeculations, totalSpeculations));
2003 fprintf(statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2004 t->softFailedSpeculations,
2005 percent(t->softFailedSpeculations, totalSpeculations));
2006 fprintf(statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2007 t->hardFailedSpeculations,
2008 percent(t->hardFailedSpeculations, totalSpeculations));
2010 if (statsFile != stdout)
2014 #define KMP_INC_STAT(lck, stat) (lck->lk.adaptive.stats.stat++) 2016 #define KMP_INC_STAT(lck, stat) 2018 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 2020 static inline bool __kmp_is_unlocked_queuing_lock(kmp_queuing_lock_t *lck) {
2023 bool res = lck->lk.head_id == 0;
2027 #if KMP_COMPILER_ICC 2030 __sync_synchronize();
2037 static __inline
void 2038 __kmp_update_badness_after_success(kmp_adaptive_lock_t *lck) {
2040 lck->lk.adaptive.badness = 0;
2041 KMP_INC_STAT(lck, successfulSpeculations);
2045 static __inline
void __kmp_step_badness(kmp_adaptive_lock_t *lck) {
2046 kmp_uint32 newBadness = (lck->lk.adaptive.badness << 1) | 1;
2047 if (newBadness > lck->lk.adaptive.max_badness) {
2050 lck->lk.adaptive.badness = newBadness;
2055 static __inline
int __kmp_should_speculate(kmp_adaptive_lock_t *lck,
2057 kmp_uint32 badness = lck->lk.adaptive.badness;
2058 kmp_uint32 attempts = lck->lk.adaptive.acquire_attempts;
2059 int res = (attempts & badness) == 0;
2065 static int __kmp_test_adaptive_lock_only(kmp_adaptive_lock_t *lck,
2067 int retries = lck->lk.adaptive.max_soft_retries;
2074 kmp_uint32 status = _xbegin();
2079 if (status == _XBEGIN_STARTED) {
2084 if (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2088 KMP_ASSERT2(0,
"should not get here");
2093 if (status & SOFT_ABORT_MASK) {
2094 KMP_INC_STAT(lck, softFailedSpeculations);
2097 KMP_INC_STAT(lck, hardFailedSpeculations);
2102 }
while (retries--);
2106 __kmp_step_badness(lck);
2113 static int __kmp_test_adaptive_lock(kmp_adaptive_lock_t *lck, kmp_int32 gtid) {
2115 if (__kmp_should_speculate(lck, gtid) &&
2116 __kmp_test_adaptive_lock_only(lck, gtid))
2121 lck->lk.adaptive.acquire_attempts++;
2124 if (__kmp_test_queuing_lock(GET_QLK_PTR(lck), gtid)) {
2125 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2132 static int __kmp_test_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2134 char const *
const func =
"omp_test_lock";
2135 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2136 KMP_FATAL(LockIsUninitialized, func);
2139 int retval = __kmp_test_adaptive_lock(lck, gtid);
2142 lck->lk.qlk.owner_id = gtid + 1;
2158 static void __kmp_acquire_adaptive_lock(kmp_adaptive_lock_t *lck,
2160 if (__kmp_should_speculate(lck, gtid)) {
2161 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2162 if (__kmp_test_adaptive_lock_only(lck, gtid))
2171 while (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2172 KMP_INC_STAT(lck, lemmingYields);
2176 if (__kmp_test_adaptive_lock_only(lck, gtid))
2183 lck->lk.adaptive.acquire_attempts++;
2185 __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid);
2187 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2188 ANNOTATE_QUEUING_ACQUIRED(lck);
2191 static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2193 char const *
const func =
"omp_set_lock";
2194 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2195 KMP_FATAL(LockIsUninitialized, func);
2197 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == gtid) {
2198 KMP_FATAL(LockIsAlreadyOwned, func);
2201 __kmp_acquire_adaptive_lock(lck, gtid);
2203 lck->lk.qlk.owner_id = gtid + 1;
2206 static int __kmp_release_adaptive_lock(kmp_adaptive_lock_t *lck,
2208 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(
2213 __kmp_update_badness_after_success(lck);
2216 __kmp_release_queuing_lock(GET_QLK_PTR(lck), gtid);
2218 return KMP_LOCK_RELEASED;
2221 static int __kmp_release_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2223 char const *
const func =
"omp_unset_lock";
2225 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2226 KMP_FATAL(LockIsUninitialized, func);
2228 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == -1) {
2229 KMP_FATAL(LockUnsettingFree, func);
2231 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != gtid) {
2232 KMP_FATAL(LockUnsettingSetByAnother, func);
2234 lck->lk.qlk.owner_id = 0;
2235 __kmp_release_adaptive_lock(lck, gtid);
2236 return KMP_LOCK_RELEASED;
2239 static void __kmp_init_adaptive_lock(kmp_adaptive_lock_t *lck) {
2240 __kmp_init_queuing_lock(GET_QLK_PTR(lck));
2241 lck->lk.adaptive.badness = 0;
2242 lck->lk.adaptive.acquire_attempts = 0;
2243 lck->lk.adaptive.max_soft_retries =
2244 __kmp_adaptive_backoff_params.max_soft_retries;
2245 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2246 #if KMP_DEBUG_ADAPTIVE_LOCKS 2247 __kmp_zero_speculative_stats(&lck->lk.adaptive);
2249 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2252 static void __kmp_init_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2253 __kmp_init_adaptive_lock(lck);
2256 static void __kmp_destroy_adaptive_lock(kmp_adaptive_lock_t *lck) {
2257 #if KMP_DEBUG_ADAPTIVE_LOCKS 2258 __kmp_accumulate_speculative_stats(&lck->lk.adaptive);
2260 __kmp_destroy_queuing_lock(GET_QLK_PTR(lck));
2264 static void __kmp_destroy_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2265 char const *
const func =
"omp_destroy_lock";
2266 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2267 KMP_FATAL(LockIsUninitialized, func);
2269 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != -1) {
2270 KMP_FATAL(LockStillOwned, func);
2272 __kmp_destroy_adaptive_lock(lck);
2275 #endif // KMP_USE_ADAPTIVE_LOCKS 2281 static kmp_int32 __kmp_get_drdpa_lock_owner(kmp_drdpa_lock_t *lck) {
2282 return TCR_4(lck->lk.owner_id) - 1;
2285 static inline bool __kmp_is_drdpa_lock_nestable(kmp_drdpa_lock_t *lck) {
2286 return lck->lk.depth_locked != -1;
2289 __forceinline
static int 2290 __kmp_acquire_drdpa_lock_timed_template(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2292 KMP_TEST_THEN_INC64(RCAST(
volatile kmp_int64 *, &lck->lk.next_ticket));
2293 kmp_uint64 mask = TCR_8(lck->lk.mask);
2294 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls = lck->lk.polls;
2296 #ifdef USE_LOCK_PROFILE 2297 if (TCR_8(polls[ticket & mask].poll) != ticket)
2298 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2312 KMP_FSYNC_PREPARE(lck);
2313 KMP_INIT_YIELD(spins);
2314 while (TCR_8(polls[ticket & mask].poll) < ticket) {
2319 KMP_YIELD(TCR_4(__kmp_nth) >
2320 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2321 KMP_YIELD_SPIN(spins);
2330 mask = TCR_8(lck->lk.mask);
2331 polls = lck->lk.polls;
2335 KMP_FSYNC_ACQUIRED(lck);
2336 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2338 lck->lk.now_serving = ticket;
2345 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2346 __kmp_free(CCAST(kmp_base_drdpa_lock::kmp_lock_poll *, lck->lk.old_polls));
2347 lck->lk.old_polls = NULL;
2348 lck->lk.cleanup_ticket = 0;
2354 if (lck->lk.old_polls == NULL) {
2355 bool reconfigure =
false;
2356 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *old_polls = polls;
2357 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2359 if (TCR_4(__kmp_nth) >
2360 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2363 if (num_polls > 1) {
2365 num_polls = TCR_4(lck->lk.num_polls);
2368 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2369 __kmp_allocate(num_polls *
sizeof(*polls));
2370 polls[0].poll = ticket;
2376 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2377 if (num_waiting > num_polls) {
2378 kmp_uint32 old_num_polls = num_polls;
2381 mask = (mask << 1) | 1;
2383 }
while (num_polls <= num_waiting);
2389 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2390 __kmp_allocate(num_polls *
sizeof(*polls));
2392 for (i = 0; i < old_num_polls; i++) {
2393 polls[i].poll = old_polls[i].poll;
2408 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring " 2409 "lock %p to %d polls\n",
2410 ticket, lck, num_polls));
2412 lck->lk.old_polls = old_polls;
2413 lck->lk.polls = polls;
2417 lck->lk.num_polls = num_polls;
2418 lck->lk.mask = mask;
2426 lck->lk.cleanup_ticket = TCR_8(lck->lk.next_ticket);
2429 return KMP_LOCK_ACQUIRED_FIRST;
2432 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2433 int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2434 ANNOTATE_DRDPA_ACQUIRED(lck);
2438 static int __kmp_acquire_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2440 char const *
const func =
"omp_set_lock";
2441 if (lck->lk.initialized != lck) {
2442 KMP_FATAL(LockIsUninitialized, func);
2444 if (__kmp_is_drdpa_lock_nestable(lck)) {
2445 KMP_FATAL(LockNestableUsedAsSimple, func);
2447 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) == gtid)) {
2448 KMP_FATAL(LockIsAlreadyOwned, func);
2451 __kmp_acquire_drdpa_lock(lck, gtid);
2453 lck->lk.owner_id = gtid + 1;
2454 return KMP_LOCK_ACQUIRED_FIRST;
2457 int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2460 kmp_uint64 ticket = TCR_8(lck->lk.next_ticket);
2461 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls = lck->lk.polls;
2462 kmp_uint64 mask = TCR_8(lck->lk.mask);
2463 if (TCR_8(polls[ticket & mask].poll) == ticket) {
2464 kmp_uint64 next_ticket = ticket + 1;
2465 if (KMP_COMPARE_AND_STORE_ACQ64(&lck->lk.next_ticket, ticket,
2467 KMP_FSYNC_ACQUIRED(lck);
2468 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2470 lck->lk.now_serving = ticket;
2484 static int __kmp_test_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2486 char const *
const func =
"omp_test_lock";
2487 if (lck->lk.initialized != lck) {
2488 KMP_FATAL(LockIsUninitialized, func);
2490 if (__kmp_is_drdpa_lock_nestable(lck)) {
2491 KMP_FATAL(LockNestableUsedAsSimple, func);
2494 int retval = __kmp_test_drdpa_lock(lck, gtid);
2497 lck->lk.owner_id = gtid + 1;
2502 int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2505 kmp_uint64 ticket = lck->lk.now_serving + 1;
2506 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls = lck->lk.polls;
2507 kmp_uint64 mask = TCR_8(lck->lk.mask);
2508 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2510 KMP_FSYNC_RELEASING(lck);
2511 ANNOTATE_DRDPA_RELEASED(lck);
2512 KMP_ST_REL64(&(polls[ticket & mask].poll), ticket);
2513 return KMP_LOCK_RELEASED;
2516 static int __kmp_release_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2518 char const *
const func =
"omp_unset_lock";
2520 if (lck->lk.initialized != lck) {
2521 KMP_FATAL(LockIsUninitialized, func);
2523 if (__kmp_is_drdpa_lock_nestable(lck)) {
2524 KMP_FATAL(LockNestableUsedAsSimple, func);
2526 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2527 KMP_FATAL(LockUnsettingFree, func);
2529 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) >= 0) &&
2530 (__kmp_get_drdpa_lock_owner(lck) != gtid)) {
2531 KMP_FATAL(LockUnsettingSetByAnother, func);
2533 lck->lk.owner_id = 0;
2534 return __kmp_release_drdpa_lock(lck, gtid);
2537 void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck) {
2538 lck->lk.location = NULL;
2540 lck->lk.num_polls = 1;
2542 (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)__kmp_allocate(
2543 lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2544 lck->lk.cleanup_ticket = 0;
2545 lck->lk.old_polls = NULL;
2546 lck->lk.next_ticket = 0;
2547 lck->lk.now_serving = 0;
2548 lck->lk.owner_id = 0;
2549 lck->lk.depth_locked = -1;
2550 lck->lk.initialized = lck;
2552 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2555 static void __kmp_init_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2556 __kmp_init_drdpa_lock(lck);
2559 void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck) {
2560 lck->lk.initialized = NULL;
2561 lck->lk.location = NULL;
2562 if (lck->lk.polls != NULL) {
2563 __kmp_free(CCAST(kmp_base_drdpa_lock::kmp_lock_poll *, lck->lk.polls));
2564 lck->lk.polls = NULL;
2566 if (lck->lk.old_polls != NULL) {
2567 __kmp_free(CCAST(kmp_base_drdpa_lock::kmp_lock_poll *, lck->lk.old_polls));
2568 lck->lk.old_polls = NULL;
2571 lck->lk.num_polls = 0;
2572 lck->lk.cleanup_ticket = 0;
2573 lck->lk.next_ticket = 0;
2574 lck->lk.now_serving = 0;
2575 lck->lk.owner_id = 0;
2576 lck->lk.depth_locked = -1;
2579 static void __kmp_destroy_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2580 char const *
const func =
"omp_destroy_lock";
2581 if (lck->lk.initialized != lck) {
2582 KMP_FATAL(LockIsUninitialized, func);
2584 if (__kmp_is_drdpa_lock_nestable(lck)) {
2585 KMP_FATAL(LockNestableUsedAsSimple, func);
2587 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2588 KMP_FATAL(LockStillOwned, func);
2590 __kmp_destroy_drdpa_lock(lck);
2595 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2596 KMP_DEBUG_ASSERT(gtid >= 0);
2598 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2599 lck->lk.depth_locked += 1;
2600 return KMP_LOCK_ACQUIRED_NEXT;
2602 __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2603 ANNOTATE_DRDPA_ACQUIRED(lck);
2605 lck->lk.depth_locked = 1;
2607 lck->lk.owner_id = gtid + 1;
2608 return KMP_LOCK_ACQUIRED_FIRST;
2612 static void __kmp_acquire_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2614 char const *
const func =
"omp_set_nest_lock";
2615 if (lck->lk.initialized != lck) {
2616 KMP_FATAL(LockIsUninitialized, func);
2618 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2619 KMP_FATAL(LockSimpleUsedAsNestable, func);
2621 __kmp_acquire_nested_drdpa_lock(lck, gtid);
2624 int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2627 KMP_DEBUG_ASSERT(gtid >= 0);
2629 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2630 retval = ++lck->lk.depth_locked;
2631 }
else if (!__kmp_test_drdpa_lock(lck, gtid)) {
2635 retval = lck->lk.depth_locked = 1;
2637 lck->lk.owner_id = gtid + 1;
2642 static int __kmp_test_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2644 char const *
const func =
"omp_test_nest_lock";
2645 if (lck->lk.initialized != lck) {
2646 KMP_FATAL(LockIsUninitialized, func);
2648 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2649 KMP_FATAL(LockSimpleUsedAsNestable, func);
2651 return __kmp_test_nested_drdpa_lock(lck, gtid);
2654 int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2655 KMP_DEBUG_ASSERT(gtid >= 0);
2658 if (--(lck->lk.depth_locked) == 0) {
2660 lck->lk.owner_id = 0;
2661 __kmp_release_drdpa_lock(lck, gtid);
2662 return KMP_LOCK_RELEASED;
2664 return KMP_LOCK_STILL_HELD;
2667 static int __kmp_release_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2669 char const *
const func =
"omp_unset_nest_lock";
2671 if (lck->lk.initialized != lck) {
2672 KMP_FATAL(LockIsUninitialized, func);
2674 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2675 KMP_FATAL(LockSimpleUsedAsNestable, func);
2677 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2678 KMP_FATAL(LockUnsettingFree, func);
2680 if (__kmp_get_drdpa_lock_owner(lck) != gtid) {
2681 KMP_FATAL(LockUnsettingSetByAnother, func);
2683 return __kmp_release_nested_drdpa_lock(lck, gtid);
2686 void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2687 __kmp_init_drdpa_lock(lck);
2688 lck->lk.depth_locked = 0;
2691 static void __kmp_init_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2692 __kmp_init_nested_drdpa_lock(lck);
2695 void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2696 __kmp_destroy_drdpa_lock(lck);
2697 lck->lk.depth_locked = 0;
2700 static void __kmp_destroy_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2701 char const *
const func =
"omp_destroy_nest_lock";
2702 if (lck->lk.initialized != lck) {
2703 KMP_FATAL(LockIsUninitialized, func);
2705 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2706 KMP_FATAL(LockSimpleUsedAsNestable, func);
2708 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2709 KMP_FATAL(LockStillOwned, func);
2711 __kmp_destroy_nested_drdpa_lock(lck);
2716 static int __kmp_is_drdpa_lock_initialized(kmp_drdpa_lock_t *lck) {
2717 return lck == lck->lk.initialized;
2720 static const ident_t *__kmp_get_drdpa_lock_location(kmp_drdpa_lock_t *lck) {
2721 return lck->lk.location;
2724 static void __kmp_set_drdpa_lock_location(kmp_drdpa_lock_t *lck,
2726 lck->lk.location = loc;
2729 static kmp_lock_flags_t __kmp_get_drdpa_lock_flags(kmp_drdpa_lock_t *lck) {
2730 return lck->lk.flags;
2733 static void __kmp_set_drdpa_lock_flags(kmp_drdpa_lock_t *lck,
2734 kmp_lock_flags_t flags) {
2735 lck->lk.flags = flags;
2739 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 2740 #define __kmp_tsc() __kmp_hardware_timestamp() 2742 kmp_backoff_t __kmp_spin_backoff_params = {1, 4096, 100};
2745 extern kmp_uint64 __kmp_now_nsec();
2746 kmp_backoff_t __kmp_spin_backoff_params = {1, 256, 100};
2747 #define __kmp_tsc() __kmp_now_nsec() 2757 static inline bool before(kmp_uint64 a, kmp_uint64 b) {
2758 return ((kmp_int64)b - (kmp_int64)a) > 0;
2762 void __kmp_spin_backoff(kmp_backoff_t *boff) {
2765 for (i = boff->step; i > 0; i--) {
2766 kmp_uint64 goal = __kmp_tsc() + boff->min_tick;
2769 }
while (before(__kmp_tsc(), goal));
2771 boff->step = (boff->step << 1 | 1) & (boff->max_backoff - 1);
2774 #if KMP_USE_DYNAMIC_LOCK 2778 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck,
2779 kmp_dyna_lockseq_t seq) {
2780 TCW_4(*lck, KMP_GET_D_TAG(seq));
2783 (
"__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
2789 #define HLE_ACQUIRE ".byte 0xf2;" 2790 #define HLE_RELEASE ".byte 0xf3;" 2792 static inline kmp_uint32 swap4(kmp_uint32
volatile *p, kmp_uint32 v) {
2793 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" :
"+r"(v),
"+m"(*p) : :
"memory");
2797 static void __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck) { TCW_4(*lck, 0); }
2799 static void __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2801 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
2804 while (*(kmp_uint32
volatile *)lck != KMP_LOCK_FREE(hle)) {
2805 for (
int i = delay; i != 0; --i)
2807 delay = ((delay << 1) | 1) & 7;
2809 }
while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
2813 static void __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2815 __kmp_acquire_hle_lock(lck, gtid);
2818 static int __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2819 __asm__
volatile(HLE_RELEASE
"movl %1,%0" 2821 :
"r"(KMP_LOCK_FREE(hle))
2823 return KMP_LOCK_RELEASED;
2826 static int __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2828 return __kmp_release_hle_lock(lck, gtid);
2831 static int __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2832 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
2835 static int __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2837 return __kmp_test_hle_lock(lck, gtid);
2840 static void __kmp_init_rtm_lock(kmp_queuing_lock_t *lck) {
2841 __kmp_init_queuing_lock(lck);
2844 static void __kmp_destroy_rtm_lock(kmp_queuing_lock_t *lck) {
2845 __kmp_destroy_queuing_lock(lck);
2848 static void __kmp_acquire_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2849 unsigned retries = 3, status;
2852 if (status == _XBEGIN_STARTED) {
2853 if (__kmp_is_unlocked_queuing_lock(lck))
2857 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
2859 while (!__kmp_is_unlocked_queuing_lock(lck))
2861 }
else if (!(status & _XABORT_RETRY))
2863 }
while (retries--);
2866 __kmp_acquire_queuing_lock(lck, gtid);
2869 static void __kmp_acquire_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2871 __kmp_acquire_rtm_lock(lck, gtid);
2874 static int __kmp_release_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2875 if (__kmp_is_unlocked_queuing_lock(lck)) {
2880 __kmp_release_queuing_lock(lck, gtid);
2882 return KMP_LOCK_RELEASED;
2885 static int __kmp_release_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2887 return __kmp_release_rtm_lock(lck, gtid);
2890 static int __kmp_test_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2891 unsigned retries = 3, status;
2894 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
2897 if (!(status & _XABORT_RETRY))
2899 }
while (retries--);
2901 return (__kmp_is_unlocked_queuing_lock(lck)) ? 1 : 0;
2904 static int __kmp_test_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2906 return __kmp_test_rtm_lock(lck, gtid);
2909 #endif // KMP_USE_TSX 2912 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *l,
2913 kmp_dyna_lockseq_t tag);
2914 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock);
2915 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2916 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2917 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2918 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2920 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2922 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2929 #define expand(l, op) 0, __kmp_init_direct_lock, 2930 void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t) = {
2931 __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init)};
2935 #define expand(l, op) 0, (void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock, 2936 void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *) = {
2937 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
2941 #define expand(l, op) \ 2942 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 2943 static int (*direct_set[])(kmp_dyna_lock_t *, kmp_int32) = {
2944 __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire)};
2946 #define expand(l, op) \ 2947 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 2948 static int (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2949 __kmp_set_indirect_lock_with_checks, 0,
2950 KMP_FOREACH_D_LOCK(expand, acquire)};
2954 #define expand(l, op) \ 2955 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 2956 static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32) = {
2957 __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release)};
2958 static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32) = {
2959 __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test)};
2961 #define expand(l, op) \ 2962 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 2963 static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2964 __kmp_unset_indirect_lock_with_checks, 0,
2965 KMP_FOREACH_D_LOCK(expand, release)};
2966 static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2967 __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test)};
2971 int (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32) = 0;
2972 int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32) = 0;
2973 int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32) = 0;
2976 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock, 2977 void (*__kmp_indirect_init[])(kmp_user_lock_p) = {
2978 KMP_FOREACH_I_LOCK(expand, init)};
2979 void (*__kmp_indirect_destroy[])(kmp_user_lock_p) = {
2980 KMP_FOREACH_I_LOCK(expand, destroy)};
2984 #define expand(l, op) \ 2985 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock, 2986 static int (*indirect_set[])(kmp_user_lock_p,
2987 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, acquire)};
2989 #define expand(l, op) \ 2990 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks, 2991 static int (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = {
2992 KMP_FOREACH_I_LOCK(expand, acquire)};
2996 #define expand(l, op) \ 2997 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock, 2998 static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = {
2999 KMP_FOREACH_I_LOCK(expand, release)};
3000 static int (*indirect_test[])(kmp_user_lock_p,
3001 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, test)};
3003 #define expand(l, op) \ 3004 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks, 3005 static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = {
3006 KMP_FOREACH_I_LOCK(expand, release)};
3007 static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = {
3008 KMP_FOREACH_I_LOCK(expand, test)};
3012 int (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32) = 0;
3013 int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32) = 0;
3014 int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32) = 0;
3017 kmp_indirect_lock_table_t __kmp_i_lock_table;
3020 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = {0};
3023 void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3025 void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3026 kmp_lock_flags_t) = {0};
3027 const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
3028 kmp_user_lock_p) = {0};
3029 kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
3030 kmp_user_lock_p) = {0};
3033 static kmp_indirect_lock_t *__kmp_indirect_lock_pool[KMP_NUM_I_LOCKS] = {0};
3040 kmp_indirect_lock_t *__kmp_allocate_indirect_lock(
void **user_lock,
3042 kmp_indirect_locktag_t tag) {
3043 kmp_indirect_lock_t *lck;
3044 kmp_lock_index_t idx;
3046 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3048 if (__kmp_indirect_lock_pool[tag] != NULL) {
3050 lck = __kmp_indirect_lock_pool[tag];
3051 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3052 idx = lck->lock->pool.index;
3053 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3054 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: reusing an existing lock %p\n",
3057 idx = __kmp_i_lock_table.next;
3059 if (idx == __kmp_i_lock_table.size) {
3061 int row = __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK;
3062 kmp_indirect_lock_t **new_table = (kmp_indirect_lock_t **)__kmp_allocate(
3063 2 * row *
sizeof(kmp_indirect_lock_t *));
3064 KMP_MEMCPY(new_table, __kmp_i_lock_table.table,
3065 row *
sizeof(kmp_indirect_lock_t *));
3066 kmp_indirect_lock_t **old_table = __kmp_i_lock_table.table;
3067 __kmp_i_lock_table.table = new_table;
3068 __kmp_free(old_table);
3070 for (
int i = row; i < 2 * row; ++i)
3071 *(__kmp_i_lock_table.table + i) = (kmp_indirect_lock_t *)__kmp_allocate(
3072 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3073 __kmp_i_lock_table.size = 2 * idx;
3075 __kmp_i_lock_table.next++;
3076 lck = KMP_GET_I_LOCK(idx);
3078 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3080 (
"__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3083 __kmp_release_lock(&__kmp_global_lock, gtid);
3087 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3088 *((kmp_lock_index_t *)user_lock) = idx
3091 *((kmp_indirect_lock_t **)user_lock) = lck;
3098 static __forceinline kmp_indirect_lock_t *
3099 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func) {
3100 if (__kmp_env_consistency_check) {
3101 kmp_indirect_lock_t *lck = NULL;
3102 if (user_lock == NULL) {
3103 KMP_FATAL(LockIsUninitialized, func);
3105 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3106 kmp_lock_index_t idx = KMP_EXTRACT_I_INDEX(user_lock);
3107 if (idx >= __kmp_i_lock_table.size) {
3108 KMP_FATAL(LockIsUninitialized, func);
3110 lck = KMP_GET_I_LOCK(idx);
3112 lck = *((kmp_indirect_lock_t **)user_lock);
3115 KMP_FATAL(LockIsUninitialized, func);
3119 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3120 return KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(user_lock));
3122 return *((kmp_indirect_lock_t **)user_lock);
3127 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *lock,
3128 kmp_dyna_lockseq_t seq) {
3129 #if KMP_USE_ADAPTIVE_LOCKS 3130 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3131 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3132 seq = lockseq_queuing;
3136 if (seq == lockseq_rtm && !__kmp_cpuinfo.rtm) {
3137 seq = lockseq_queuing;
3140 kmp_indirect_locktag_t tag = KMP_GET_I_TAG(seq);
3141 kmp_indirect_lock_t *l =
3142 __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3143 KMP_I_LOCK_FUNC(l, init)(l->lock);
3145 20, (
"__kmp_init_indirect_lock: initialized indirect lock with type#%d\n",
3149 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock) {
3150 kmp_uint32 gtid = __kmp_entry_gtid();
3151 kmp_indirect_lock_t *l =
3152 __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3153 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3154 kmp_indirect_locktag_t tag = l->type;
3156 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3159 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3160 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3161 l->lock->pool.index = KMP_EXTRACT_I_INDEX(lock);
3163 __kmp_indirect_lock_pool[tag] = l;
3165 __kmp_release_lock(&__kmp_global_lock, gtid);
3168 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3169 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3170 return KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3173 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3174 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3175 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3178 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3179 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3180 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3183 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3185 kmp_indirect_lock_t *l =
3186 __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3187 return KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3190 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3192 kmp_indirect_lock_t *l =
3193 __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3194 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3197 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3199 kmp_indirect_lock_t *l =
3200 __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3201 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3204 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3207 kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq) {
3210 case lockseq_nested_tas:
3211 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3214 case lockseq_nested_futex:
3215 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3217 case lockseq_ticket:
3218 case lockseq_nested_ticket:
3219 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3220 case lockseq_queuing:
3221 case lockseq_nested_queuing:
3222 #if KMP_USE_ADAPTIVE_LOCKS 3223 case lockseq_adaptive:
3225 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3227 case lockseq_nested_drdpa:
3228 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3235 void __kmp_init_dynamic_user_locks() {
3237 if (__kmp_env_consistency_check) {
3238 __kmp_direct_set = direct_set_check;
3239 __kmp_direct_unset = direct_unset_check;
3240 __kmp_direct_test = direct_test_check;
3241 __kmp_indirect_set = indirect_set_check;
3242 __kmp_indirect_unset = indirect_unset_check;
3243 __kmp_indirect_test = indirect_test_check;
3245 __kmp_direct_set = direct_set;
3246 __kmp_direct_unset = direct_unset;
3247 __kmp_direct_test = direct_test;
3248 __kmp_indirect_set = indirect_set;
3249 __kmp_indirect_unset = indirect_unset;
3250 __kmp_indirect_test = indirect_test;
3255 if (__kmp_init_user_locks)
3259 __kmp_i_lock_table.size = KMP_I_LOCK_CHUNK;
3260 __kmp_i_lock_table.table =
3261 (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *));
3262 *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)__kmp_allocate(
3263 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3264 __kmp_i_lock_table.next = 0;
3267 __kmp_indirect_lock_size[locktag_ticket] =
sizeof(kmp_ticket_lock_t);
3268 __kmp_indirect_lock_size[locktag_queuing] =
sizeof(kmp_queuing_lock_t);
3269 #if KMP_USE_ADAPTIVE_LOCKS 3270 __kmp_indirect_lock_size[locktag_adaptive] =
sizeof(kmp_adaptive_lock_t);
3272 __kmp_indirect_lock_size[locktag_drdpa] =
sizeof(kmp_drdpa_lock_t);
3274 __kmp_indirect_lock_size[locktag_rtm] =
sizeof(kmp_queuing_lock_t);
3276 __kmp_indirect_lock_size[locktag_nested_tas] =
sizeof(kmp_tas_lock_t);
3278 __kmp_indirect_lock_size[locktag_nested_futex] =
sizeof(kmp_futex_lock_t);
3280 __kmp_indirect_lock_size[locktag_nested_ticket] =
sizeof(kmp_ticket_lock_t);
3281 __kmp_indirect_lock_size[locktag_nested_queuing] =
sizeof(kmp_queuing_lock_t);
3282 __kmp_indirect_lock_size[locktag_nested_drdpa] =
sizeof(kmp_drdpa_lock_t);
3285 #define fill_jumps(table, expand, sep) \ 3287 table[locktag##sep##ticket] = expand(ticket); \ 3288 table[locktag##sep##queuing] = expand(queuing); \ 3289 table[locktag##sep##drdpa] = expand(drdpa); \ 3292 #if KMP_USE_ADAPTIVE_LOCKS 3293 #define fill_table(table, expand) \ 3295 fill_jumps(table, expand, _); \ 3296 table[locktag_adaptive] = expand(queuing); \ 3297 fill_jumps(table, expand, _nested_); \ 3300 #define fill_table(table, expand) \ 3302 fill_jumps(table, expand, _); \ 3303 fill_jumps(table, expand, _nested_); \ 3305 #endif // KMP_USE_ADAPTIVE_LOCKS 3308 (void (*)(kmp_user_lock_p, const ident_t *)) __kmp_set_##l##_lock_location 3309 fill_table(__kmp_indirect_set_location, expand);
3312 (void (*)(kmp_user_lock_p, kmp_lock_flags_t)) __kmp_set_##l##_lock_flags 3313 fill_table(__kmp_indirect_set_flags, expand);
3316 (const ident_t *(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_location 3317 fill_table(__kmp_indirect_get_location, expand);
3320 (kmp_lock_flags_t(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_flags 3321 fill_table(__kmp_indirect_get_flags, expand);
3324 __kmp_init_user_locks = TRUE;
3328 void __kmp_cleanup_indirect_user_locks() {
3334 for (k = 0; k < KMP_NUM_I_LOCKS; ++k) {
3335 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3337 kmp_indirect_lock_t *ll = l;
3338 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3339 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: freeing %p from pool\n",
3341 __kmp_free(ll->lock);
3344 __kmp_indirect_lock_pool[k] = NULL;
3347 for (i = 0; i < __kmp_i_lock_table.next; i++) {
3348 kmp_indirect_lock_t *l = KMP_GET_I_LOCK(i);
3349 if (l->lock != NULL) {
3351 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3354 (
"__kmp_cleanup_indirect_user_locks: destroy/freeing %p from table\n",
3356 __kmp_free(l->lock);
3360 for (i = 0; i < __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK; i++)
3361 __kmp_free(__kmp_i_lock_table.table[i]);
3362 __kmp_free(__kmp_i_lock_table.table);
3364 __kmp_init_user_locks = FALSE;
3367 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3368 int __kmp_num_locks_in_block = 1;
3370 #else // KMP_USE_DYNAMIC_LOCK 3376 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3378 size_t __kmp_base_user_lock_size = 0;
3379 size_t __kmp_user_lock_size = 0;
3381 kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck) = NULL;
3382 int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
3383 kmp_int32 gtid) = NULL;
3385 int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
3386 kmp_int32 gtid) = NULL;
3387 int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
3388 kmp_int32 gtid) = NULL;
3389 void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3390 void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck) = NULL;
3391 void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3392 int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3393 kmp_int32 gtid) = NULL;
3395 int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3396 kmp_int32 gtid) = NULL;
3397 int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3398 kmp_int32 gtid) = NULL;
3399 void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3400 void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3402 int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck) = NULL;
3403 const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck) = NULL;
3404 void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
3406 kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck) = NULL;
3407 void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
3408 kmp_lock_flags_t flags) = NULL;
3410 void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind) {
3411 switch (user_lock_kind) {
3417 __kmp_base_user_lock_size =
sizeof(kmp_base_tas_lock_t);
3418 __kmp_user_lock_size =
sizeof(kmp_tas_lock_t);
3420 __kmp_get_user_lock_owner_ =
3421 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_tas_lock_owner);
3423 if (__kmp_env_consistency_check) {
3424 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3425 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3427 KMP_BIND_USER_LOCK(tas);
3428 KMP_BIND_NESTED_USER_LOCK(tas);
3431 __kmp_destroy_user_lock_ =
3432 (void (*)(kmp_user_lock_p))(&__kmp_destroy_tas_lock);
3434 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3436 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3438 __kmp_set_user_lock_location_ =
3439 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3441 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3443 __kmp_set_user_lock_flags_ =
3444 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3450 __kmp_base_user_lock_size =
sizeof(kmp_base_futex_lock_t);
3451 __kmp_user_lock_size =
sizeof(kmp_futex_lock_t);
3453 __kmp_get_user_lock_owner_ =
3454 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_futex_lock_owner);
3456 if (__kmp_env_consistency_check) {
3457 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3458 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3460 KMP_BIND_USER_LOCK(futex);
3461 KMP_BIND_NESTED_USER_LOCK(futex);
3464 __kmp_destroy_user_lock_ =
3465 (void (*)(kmp_user_lock_p))(&__kmp_destroy_futex_lock);
3467 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3469 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3471 __kmp_set_user_lock_location_ =
3472 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3474 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3476 __kmp_set_user_lock_flags_ =
3477 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3480 #endif // KMP_USE_FUTEX 3483 __kmp_base_user_lock_size =
sizeof(kmp_base_ticket_lock_t);
3484 __kmp_user_lock_size =
sizeof(kmp_ticket_lock_t);
3486 __kmp_get_user_lock_owner_ =
3487 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_owner);
3489 if (__kmp_env_consistency_check) {
3490 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3491 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3493 KMP_BIND_USER_LOCK(ticket);
3494 KMP_BIND_NESTED_USER_LOCK(ticket);
3497 __kmp_destroy_user_lock_ =
3498 (void (*)(kmp_user_lock_p))(&__kmp_destroy_ticket_lock);
3500 __kmp_is_user_lock_initialized_ =
3501 (int (*)(kmp_user_lock_p))(&__kmp_is_ticket_lock_initialized);
3503 __kmp_get_user_lock_location_ =
3504 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_location);
3506 __kmp_set_user_lock_location_ = (void (*)(
3507 kmp_user_lock_p,
const ident_t *))(&__kmp_set_ticket_lock_location);
3509 __kmp_get_user_lock_flags_ =
3510 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_flags);
3512 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3513 &__kmp_set_ticket_lock_flags);
3517 __kmp_base_user_lock_size =
sizeof(kmp_base_queuing_lock_t);
3518 __kmp_user_lock_size =
sizeof(kmp_queuing_lock_t);
3520 __kmp_get_user_lock_owner_ =
3521 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3523 if (__kmp_env_consistency_check) {
3524 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3525 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3527 KMP_BIND_USER_LOCK(queuing);
3528 KMP_BIND_NESTED_USER_LOCK(queuing);
3531 __kmp_destroy_user_lock_ =
3532 (void (*)(kmp_user_lock_p))(&__kmp_destroy_queuing_lock);
3534 __kmp_is_user_lock_initialized_ =
3535 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3537 __kmp_get_user_lock_location_ =
3538 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3540 __kmp_set_user_lock_location_ = (void (*)(
3541 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3543 __kmp_get_user_lock_flags_ =
3544 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3546 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3547 &__kmp_set_queuing_lock_flags);
3550 #if KMP_USE_ADAPTIVE_LOCKS 3552 __kmp_base_user_lock_size =
sizeof(kmp_base_adaptive_lock_t);
3553 __kmp_user_lock_size =
sizeof(kmp_adaptive_lock_t);
3555 __kmp_get_user_lock_owner_ =
3556 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3558 if (__kmp_env_consistency_check) {
3559 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3561 KMP_BIND_USER_LOCK(adaptive);
3564 __kmp_destroy_user_lock_ =
3565 (void (*)(kmp_user_lock_p))(&__kmp_destroy_adaptive_lock);
3567 __kmp_is_user_lock_initialized_ =
3568 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3570 __kmp_get_user_lock_location_ =
3571 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3573 __kmp_set_user_lock_location_ = (void (*)(
3574 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3576 __kmp_get_user_lock_flags_ =
3577 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3579 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3580 &__kmp_set_queuing_lock_flags);
3583 #endif // KMP_USE_ADAPTIVE_LOCKS 3586 __kmp_base_user_lock_size =
sizeof(kmp_base_drdpa_lock_t);
3587 __kmp_user_lock_size =
sizeof(kmp_drdpa_lock_t);
3589 __kmp_get_user_lock_owner_ =
3590 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_owner);
3592 if (__kmp_env_consistency_check) {
3593 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3594 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3596 KMP_BIND_USER_LOCK(drdpa);
3597 KMP_BIND_NESTED_USER_LOCK(drdpa);
3600 __kmp_destroy_user_lock_ =
3601 (void (*)(kmp_user_lock_p))(&__kmp_destroy_drdpa_lock);
3603 __kmp_is_user_lock_initialized_ =
3604 (int (*)(kmp_user_lock_p))(&__kmp_is_drdpa_lock_initialized);
3606 __kmp_get_user_lock_location_ =
3607 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_location);
3609 __kmp_set_user_lock_location_ = (void (*)(
3610 kmp_user_lock_p,
const ident_t *))(&__kmp_set_drdpa_lock_location);
3612 __kmp_get_user_lock_flags_ =
3613 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_flags);
3615 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3616 &__kmp_set_drdpa_lock_flags);
3624 kmp_lock_table_t __kmp_user_lock_table = {1, 0, NULL};
3625 kmp_user_lock_p __kmp_lock_pool = NULL;
3628 kmp_block_of_locks *__kmp_lock_blocks = NULL;
3629 int __kmp_num_locks_in_block = 1;
3631 static kmp_lock_index_t __kmp_lock_table_insert(kmp_user_lock_p lck) {
3633 kmp_lock_index_t index;
3634 if (__kmp_user_lock_table.used >= __kmp_user_lock_table.allocated) {
3635 kmp_lock_index_t size;
3636 kmp_user_lock_p *table;
3638 if (__kmp_user_lock_table.allocated == 0) {
3641 size = __kmp_user_lock_table.allocated * 2;
3643 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof(kmp_user_lock_p) * size);
3644 KMP_MEMCPY(table + 1, __kmp_user_lock_table.table + 1,
3645 sizeof(kmp_user_lock_p) * (__kmp_user_lock_table.used - 1));
3646 table[0] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3651 __kmp_user_lock_table.table = table;
3652 __kmp_user_lock_table.allocated = size;
3654 KMP_DEBUG_ASSERT(__kmp_user_lock_table.used <
3655 __kmp_user_lock_table.allocated);
3656 index = __kmp_user_lock_table.used;
3657 __kmp_user_lock_table.table[index] = lck;
3658 ++__kmp_user_lock_table.used;
3662 static kmp_user_lock_p __kmp_lock_block_allocate() {
3664 static int last_index = 0;
3665 if ((last_index >= __kmp_num_locks_in_block) || (__kmp_lock_blocks == NULL)) {
3669 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3670 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3672 (
char *)__kmp_allocate(space_for_locks +
sizeof(kmp_block_of_locks));
3674 kmp_block_of_locks *new_block =
3675 (kmp_block_of_locks *)(&buffer[space_for_locks]);
3676 new_block->next_block = __kmp_lock_blocks;
3677 new_block->locks = (
void *)buffer;
3680 __kmp_lock_blocks = new_block;
3682 kmp_user_lock_p ret = (kmp_user_lock_p)(&(
3683 ((
char *)(__kmp_lock_blocks->locks))[last_index * __kmp_user_lock_size]));
3690 kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3691 kmp_lock_flags_t flags) {
3692 kmp_user_lock_p lck;
3693 kmp_lock_index_t index;
3694 KMP_DEBUG_ASSERT(user_lock);
3696 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3698 if (__kmp_lock_pool == NULL) {
3703 ANNOTATE_IGNORE_WRITES_BEGIN();
3704 if (__kmp_num_locks_in_block <= 1) {
3705 lck = (kmp_user_lock_p)__kmp_allocate(__kmp_user_lock_size);
3707 lck = __kmp_lock_block_allocate();
3709 ANNOTATE_IGNORE_WRITES_END();
3713 index = __kmp_lock_table_insert(lck);
3716 lck = __kmp_lock_pool;
3717 index = __kmp_lock_pool->pool.index;
3718 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3723 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3724 *((kmp_lock_index_t *)user_lock) = index;
3726 *((kmp_user_lock_p *)user_lock) = lck;
3730 __kmp_set_user_lock_flags(lck, flags);
3732 __kmp_release_lock(&__kmp_global_lock, gtid);
3738 void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid,
3739 kmp_user_lock_p lck) {
3740 KMP_DEBUG_ASSERT(user_lock != NULL);
3741 KMP_DEBUG_ASSERT(lck != NULL);
3743 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3745 lck->pool.next = __kmp_lock_pool;
3746 __kmp_lock_pool = lck;
3747 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3748 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3749 KMP_DEBUG_ASSERT(0 < index && index <= __kmp_user_lock_table.used);
3750 lck->pool.index = index;
3753 __kmp_release_lock(&__kmp_global_lock, gtid);
3756 kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func) {
3757 kmp_user_lock_p lck = NULL;
3759 if (__kmp_env_consistency_check) {
3760 if (user_lock == NULL) {
3761 KMP_FATAL(LockIsUninitialized, func);
3765 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3766 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3767 if (__kmp_env_consistency_check) {
3768 if (!(0 < index && index < __kmp_user_lock_table.used)) {
3769 KMP_FATAL(LockIsUninitialized, func);
3772 KMP_DEBUG_ASSERT(0 < index && index < __kmp_user_lock_table.used);
3773 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3774 lck = __kmp_user_lock_table.table[index];
3776 lck = *((kmp_user_lock_p *)user_lock);
3779 if (__kmp_env_consistency_check) {
3781 KMP_FATAL(LockIsUninitialized, func);
3788 void __kmp_cleanup_user_locks(
void) {
3791 __kmp_lock_pool = NULL;
3793 #define IS_CRITICAL(lck) \ 3794 ((__kmp_get_user_lock_flags_ != NULL) && \ 3795 ((*__kmp_get_user_lock_flags_)(lck)&kmp_lf_critical_section)) 3820 while (__kmp_user_lock_table.used > 1) {
3825 kmp_user_lock_p lck =
3826 __kmp_user_lock_table.table[--__kmp_user_lock_table.used];
3828 if ((__kmp_is_user_lock_initialized_ != NULL) &&
3829 (*__kmp_is_user_lock_initialized_)(lck)) {
3833 if (__kmp_env_consistency_check && (!IS_CRITICAL(lck)) &&
3834 ((loc = __kmp_get_user_lock_location(lck)) != NULL) &&
3836 kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->
psource, 0);
3837 KMP_WARNING(CnsLockNotDestroyed, str_loc.file, str_loc.line);
3838 __kmp_str_loc_free(&str_loc);
3842 if (IS_CRITICAL(lck)) {
3845 (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n",
3846 lck, *(
void **)lck));
3848 KA_TRACE(20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck,
3854 __kmp_destroy_user_lock(lck);
3858 if (__kmp_lock_blocks == NULL) {
3866 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
3867 __kmp_user_lock_table.table = NULL;
3868 __kmp_user_lock_table.allocated = 0;
3870 while (table_ptr != NULL) {
3873 kmp_user_lock_p *next = (kmp_user_lock_p *)(table_ptr[0]);
3874 __kmp_free(table_ptr);
3879 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
3880 __kmp_lock_blocks = NULL;
3882 while (block_ptr != NULL) {
3883 kmp_block_of_locks_t *next = block_ptr->next_block;
3884 __kmp_free(block_ptr->locks);
3889 TCW_4(__kmp_init_user_locks, FALSE);
3892 #endif // KMP_USE_DYNAMIC_LOCK