compiler-rt/lib/tsan/rtl/tsan_vector_clock.cpp

   1 //===-- tsan_vector_clock.cpp ---------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of ThreadSanitizer (TSan), a race detector.
  10 //
  11 //===----------------------------------------------------------------------===//
  12 #include "tsan_vector_clock.h"
  13
  14 #include "sanitizer_common/sanitizer_placement_new.h"
  15 #include "tsan_mman.h"
  16
  17 namespace __tsan {
  18
  19 #if TSAN_VECTORIZE
  20 const uptr kVectorClockSize = kThreadSlotCount * sizeof(Epoch) / sizeof(m128);
  21 #endif
  22
  23 VectorClock::VectorClock() { Reset(); }
  24
  25 void VectorClock::Reset() {
  26 #if !TSAN_VECTORIZE
  27   for (uptr i = 0; i < kThreadSlotCount; i++)
  28     clk_[i] = kEpochZero;
  29 #else
  30   m128 z = _mm_setzero_si128();
  31   m128* vclk = reinterpret_cast<m128*>(clk_);
  32   for (uptr i = 0; i < kVectorClockSize; i++) _mm_store_si128(&vclk[i], z);
  33 #endif
  34 }
  35
  36 void VectorClock::Acquire(const VectorClock* src) {
  37   if (!src)
  38     return;
  39 #if !TSAN_VECTORIZE
  40   for (uptr i = 0; i < kThreadSlotCount; i++)
  41     clk_[i] = max(clk_[i], src->clk_[i]);
  42 #else
  43   m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
  44   m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(src->clk_);
  45   for (uptr i = 0; i < kVectorClockSize; i++) {
  46     m128 s = _mm_load_si128(&vsrc[i]);
  47     m128 d = _mm_load_si128(&vdst[i]);
  48     m128 m = _mm_max_epu16(s, d);
  49     _mm_store_si128(&vdst[i], m);
  50   }
  51 #endif
  52 }
  53
  54 static VectorClock* AllocClock(VectorClock** dstp) {
  55   if (UNLIKELY(!*dstp))
  56     *dstp = New<VectorClock>();
  57   return *dstp;
  58 }
  59
  60 void VectorClock::Release(VectorClock** dstp) const {
  61   VectorClock* dst = AllocClock(dstp);
  62   dst->Acquire(this);
  63 }
  64
  65 void VectorClock::ReleaseStore(VectorClock** dstp) const {
  66   VectorClock* dst = AllocClock(dstp);
  67   *dst = *this;
  68 }
  69
  70 VectorClock& VectorClock::operator=(const VectorClock& other) {
  71 #if !TSAN_VECTORIZE
  72   for (uptr i = 0; i < kThreadSlotCount; i++)
  73     clk_[i] = other.clk_[i];
  74 #else
  75   m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
  76   m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(other.clk_);
  77   for (uptr i = 0; i < kVectorClockSize; i++) {
  78     m128 s = _mm_load_si128(&vsrc[i]);
  79     _mm_store_si128(&vdst[i], s);
  80   }
  81 #endif
  82   return *this;
  83 }
  84
  85 void VectorClock::ReleaseStoreAcquire(VectorClock** dstp) {
  86   VectorClock* dst = AllocClock(dstp);
  87 #if !TSAN_VECTORIZE
  88   for (uptr i = 0; i < kThreadSlotCount; i++) {
  89     Epoch tmp = dst->clk_[i];
  90     dst->clk_[i] = clk_[i];
  91     clk_[i] = max(clk_[i], tmp);
  92   }
  93 #else
  94   m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
  95   m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
  96   for (uptr i = 0; i < kVectorClockSize; i++) {
  97     m128 t = _mm_load_si128(&vdst[i]);
  98     m128 c = _mm_load_si128(&vclk[i]);
  99     m128 m = _mm_max_epu16(c, t);
 100     _mm_store_si128(&vdst[i], c);
 101     _mm_store_si128(&vclk[i], m);
 102   }
 103 #endif
 104 }
 105
 106 void VectorClock::ReleaseAcquire(VectorClock** dstp) {
 107   VectorClock* dst = AllocClock(dstp);
 108 #if !TSAN_VECTORIZE
 109   for (uptr i = 0; i < kThreadSlotCount; i++) {
 110     dst->clk_[i] = max(dst->clk_[i], clk_[i]);
 111     clk_[i] = dst->clk_[i];
 112   }
 113 #else
 114   m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
 115   m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
 116   for (uptr i = 0; i < kVectorClockSize; i++) {
 117     m128 c = _mm_load_si128(&vclk[i]);
 118     m128 d = _mm_load_si128(&vdst[i]);
 119     m128 m = _mm_max_epu16(c, d);
 120     _mm_store_si128(&vdst[i], m);
 121     _mm_store_si128(&vclk[i], m);
 122   }
 123 #endif
 124 }
 125
 126 }  // namespace __tsan