src/hotspot/share/prims/unsafe.cpp

*** 50,10 ***
--- 50,13 ---
  #include "services/threadService.hpp"
  #include "utilities/align.hpp"
  #include "utilities/copy.hpp"
  #include "utilities/dtrace.hpp"
  #include "utilities/macros.hpp"
+ #if INCLUDE_TSAN
+ #include "tsan/tsanExternalDecls.hpp"
+ #endif
  
  /**
   * Implementation of the jdk.internal.misc.Unsafe class
   */
  

*** 261,69 ***
  // That is, it should be in the range [0, MAX_OBJECT_SIZE].
  UNSAFE_ENTRY(jobject, Unsafe_GetReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {
    oop p = JNIHandles::resolve(obj);
    assert_field_offset_sane(p, offset);
    oop v = HeapAccess<ON_UNKNOWN_OOP_REF>::oop_load_at(p, offset);
    return JNIHandles::make_local(env, v);
  } UNSAFE_END
  
  UNSAFE_ENTRY(void, Unsafe_PutReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) {
    oop x = JNIHandles::resolve(x_h);
    oop p = JNIHandles::resolve(obj);
    assert_field_offset_sane(p, offset);
    HeapAccess<ON_UNKNOWN_OOP_REF>::oop_store_at(p, offset, x);
  } UNSAFE_END
  
  UNSAFE_ENTRY(jobject, Unsafe_GetReferenceVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {
    oop p = JNIHandles::resolve(obj);
    assert_field_offset_sane(p, offset);
    oop v = HeapAccess<MO_SEQ_CST | ON_UNKNOWN_OOP_REF>::oop_load_at(p, offset);
    return JNIHandles::make_local(env, v);
  } UNSAFE_END
  
  UNSAFE_ENTRY(void, Unsafe_PutReferenceVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) {
    oop x = JNIHandles::resolve(x_h);
    oop p = JNIHandles::resolve(obj);
    assert_field_offset_sane(p, offset);
    HeapAccess<MO_SEQ_CST | ON_UNKNOWN_OOP_REF>::oop_store_at(p, offset, x);
  } UNSAFE_END
  
  UNSAFE_ENTRY(jobject, Unsafe_GetUncompressedObject(JNIEnv *env, jobject unsafe, jlong addr)) {
    oop v = *(oop*) (address) addr;
    return JNIHandles::make_local(env, v);
  } UNSAFE_END
  
! #define DEFINE_GETSETOOP(java_type, Type) \
   \
  UNSAFE_ENTRY(java_type, Unsafe_Get##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \
!   return MemoryAccess<java_type>(thread, obj, offset).get(); \
  } UNSAFE_END \
   \
  UNSAFE_ENTRY(void, Unsafe_Put##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \
    MemoryAccess<java_type>(thread, obj, offset).put(x); \
  } UNSAFE_END \
   \
  // END DEFINE_GETSETOOP.
  
! DEFINE_GETSETOOP(jboolean, Boolean)
! DEFINE_GETSETOOP(jbyte, Byte)
! DEFINE_GETSETOOP(jshort, Short);
! DEFINE_GETSETOOP(jchar, Char);
! DEFINE_GETSETOOP(jint, Int);
! DEFINE_GETSETOOP(jlong, Long);
! DEFINE_GETSETOOP(jfloat, Float);
! DEFINE_GETSETOOP(jdouble, Double);
  
  #undef DEFINE_GETSETOOP
  
  #define DEFINE_GETSETOOP_VOLATILE(java_type, Type) \
   \
  UNSAFE_ENTRY(java_type, Unsafe_Get##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \
!   return MemoryAccess<java_type>(thread, obj, offset).get_volatile(); \
  } UNSAFE_END \
   \
  UNSAFE_ENTRY(void, Unsafe_Put##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \
    MemoryAccess<java_type>(thread, obj, offset).put_volatile(x); \
  } UNSAFE_END \
   \
  // END DEFINE_GETSETOOP_VOLATILE.
  
--- 264,111 ---
  // That is, it should be in the range [0, MAX_OBJECT_SIZE].
  UNSAFE_ENTRY(jobject, Unsafe_GetReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {
    oop p = JNIHandles::resolve(obj);
    assert_field_offset_sane(p, offset);
    oop v = HeapAccess<ON_UNKNOWN_OOP_REF>::oop_load_at(p, offset);
+   TSAN_RUNTIME_ONLY(
+     void* addr = index_oop_from_field_offset_long(p, offset);
+     if (UseCompressedOops) {
+       __tsan_read4_pc(addr, SharedRuntime::tsan_code_location(0, 0));
+     } else {
+       __tsan_read8_pc(addr, SharedRuntime::tsan_code_location(0, 0));
+     }
+   );
    return JNIHandles::make_local(env, v);
  } UNSAFE_END
  
  UNSAFE_ENTRY(void, Unsafe_PutReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) {
    oop x = JNIHandles::resolve(x_h);
    oop p = JNIHandles::resolve(obj);
    assert_field_offset_sane(p, offset);
+   TSAN_RUNTIME_ONLY(
+     void* addr = index_oop_from_field_offset_long(p, offset);
+     if (UseCompressedOops) {
+       __tsan_write4_pc(addr, SharedRuntime::tsan_code_location(0, 0));
+     } else {
+       __tsan_write8_pc(addr, SharedRuntime::tsan_code_location(0, 0));
+     }
+   );
    HeapAccess<ON_UNKNOWN_OOP_REF>::oop_store_at(p, offset, x);
  } UNSAFE_END
  
  UNSAFE_ENTRY(jobject, Unsafe_GetReferenceVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {
    oop p = JNIHandles::resolve(obj);
    assert_field_offset_sane(p, offset);
    oop v = HeapAccess<MO_SEQ_CST | ON_UNKNOWN_OOP_REF>::oop_load_at(p, offset);
+   TSAN_RUNTIME_ONLY(
+     void* addr = index_oop_from_field_offset_long(p, offset);
+     __tsan_java_acquire(addr);
+   );
    return JNIHandles::make_local(env, v);
  } UNSAFE_END
  
  UNSAFE_ENTRY(void, Unsafe_PutReferenceVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) {
    oop x = JNIHandles::resolve(x_h);
    oop p = JNIHandles::resolve(obj);
    assert_field_offset_sane(p, offset);
+   TSAN_RUNTIME_ONLY(
+     void* addr = index_oop_from_field_offset_long(p, offset);
+     __tsan_java_release(addr);
+   );
    HeapAccess<MO_SEQ_CST | ON_UNKNOWN_OOP_REF>::oop_store_at(p, offset, x);
  } UNSAFE_END
  
  UNSAFE_ENTRY(jobject, Unsafe_GetUncompressedObject(JNIEnv *env, jobject unsafe, jlong addr)) {
    oop v = *(oop*) (address) addr;
    return JNIHandles::make_local(env, v);
  } UNSAFE_END
  
! #define DEFINE_GETSETOOP(java_type, Type, size) \
   \
  UNSAFE_ENTRY(java_type, Unsafe_Get##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \
!   java_type ret = MemoryAccess<java_type>(thread, obj, offset).get(); \
+   TSAN_RUNTIME_ONLY( \
+     void* addr = index_oop_from_field_offset_long(JNIHandles::resolve(obj), offset); \
+     __tsan_read##size##_pc(addr, SharedRuntime::tsan_code_location(0, 0)); \
+   ); \
+   return ret; \
  } UNSAFE_END \
   \
  UNSAFE_ENTRY(void, Unsafe_Put##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \
+   TSAN_RUNTIME_ONLY( \
+     void* addr = index_oop_from_field_offset_long(JNIHandles::resolve(obj), offset); \
+     __tsan_write##size##_pc(addr, SharedRuntime::tsan_code_location(0, 0)); \
+   ); \
    MemoryAccess<java_type>(thread, obj, offset).put(x); \
  } UNSAFE_END \
   \
  // END DEFINE_GETSETOOP.
  
! DEFINE_GETSETOOP(jboolean, Boolean, 1)
! DEFINE_GETSETOOP(jbyte, Byte, 1)
! DEFINE_GETSETOOP(jshort, Short, 2);
! DEFINE_GETSETOOP(jchar, Char, 2);
! DEFINE_GETSETOOP(jint, Int, 4);
! DEFINE_GETSETOOP(jlong, Long, 8);
! DEFINE_GETSETOOP(jfloat, Float, 4);
! DEFINE_GETSETOOP(jdouble, Double, 8);
  
  #undef DEFINE_GETSETOOP
  
  #define DEFINE_GETSETOOP_VOLATILE(java_type, Type) \
   \
  UNSAFE_ENTRY(java_type, Unsafe_Get##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \
!   java_type ret = MemoryAccess<java_type>(thread, obj, offset).get_volatile(); \
+   TSAN_RUNTIME_ONLY( \
+     void* addr = index_oop_from_field_offset_long(JNIHandles::resolve(obj), offset); \
+     __tsan_java_acquire(addr); \
+   ); \
+   return ret; \
  } UNSAFE_END \
   \
  UNSAFE_ENTRY(void, Unsafe_Put##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \
+   TSAN_RUNTIME_ONLY( \
+     void* addr = index_oop_from_field_offset_long(JNIHandles::resolve(obj), offset); \
+     __tsan_java_release(addr); \
+   ); \
    MemoryAccess<java_type>(thread, obj, offset).put_volatile(x); \
  } UNSAFE_END \
   \
  // END DEFINE_GETSETOOP_VOLATILE.
  

*** 897,68 ***
--- 942,102 ---
    env->Throw(thr);
  } UNSAFE_END
  
  // JSR166 ------------------------------------------------------------------
  
+ // Calls __tsan_java_release() on construct and __tsan_java_acquire() on destruct.
+ class ScopedReleaseAcquire: public StackObj {
+ private:
+   void* _addr;
+ public:
+   ScopedReleaseAcquire(volatile void* addr) {
+     TSAN_RUNTIME_ONLY(
+       _addr = const_cast<void*>(addr);
+       __tsan_java_release(_addr);
+     );
+   }
+ 
+   ScopedReleaseAcquire(oop obj, jlong offset) {
+     TSAN_RUNTIME_ONLY(
+       _addr = index_oop_from_field_offset_long(obj, offset);
+       __tsan_java_release(_addr);
+     );
+   }
+ 
+   ~ScopedReleaseAcquire() {
+     TSAN_RUNTIME_ONLY(__tsan_java_acquire(_addr));
+   }
+ };
+ 
  UNSAFE_ENTRY(jobject, Unsafe_CompareAndExchangeReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject e_h, jobject x_h)) {
    oop x = JNIHandles::resolve(x_h);
    oop e = JNIHandles::resolve(e_h);
    oop p = JNIHandles::resolve(obj);
    assert_field_offset_sane(p, offset);
+   ScopedReleaseAcquire releaseAcquire(p, offset);
    oop res = HeapAccess<ON_UNKNOWN_OOP_REF>::oop_atomic_cmpxchg_at(p, (ptrdiff_t)offset, e, x);
    return JNIHandles::make_local(env, res);
  } UNSAFE_END
  
  UNSAFE_ENTRY(jint, Unsafe_CompareAndExchangeInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint e, jint x)) {
    oop p = JNIHandles::resolve(obj);
    if (p == NULL) {
      volatile jint* addr = (volatile jint*)index_oop_from_field_offset_long(p, offset);
+     ScopedReleaseAcquire releaseAcquire(addr);
      return RawAccess<>::atomic_cmpxchg(addr, e, x);
    } else {
      assert_field_offset_sane(p, offset);
+     ScopedReleaseAcquire releaseAcquire(p, offset);
      return HeapAccess<>::atomic_cmpxchg_at(p, (ptrdiff_t)offset, e, x);
    }
  } UNSAFE_END
  
  UNSAFE_ENTRY(jlong, Unsafe_CompareAndExchangeLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong e, jlong x)) {
    oop p = JNIHandles::resolve(obj);
    if (p == NULL) {
      volatile jlong* addr = (volatile jlong*)index_oop_from_field_offset_long(p, offset);
+     ScopedReleaseAcquire releaseAcquire(addr);
      return RawAccess<>::atomic_cmpxchg(addr, e, x);
    } else {
      assert_field_offset_sane(p, offset);
+     ScopedReleaseAcquire releaseAcquire(p, offset);
      return HeapAccess<>::atomic_cmpxchg_at(p, (ptrdiff_t)offset, e, x);
    }
  } UNSAFE_END
  
  UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSetReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject e_h, jobject x_h)) {
    oop x = JNIHandles::resolve(x_h);
    oop e = JNIHandles::resolve(e_h);
    oop p = JNIHandles::resolve(obj);
    assert_field_offset_sane(p, offset);
+   ScopedReleaseAcquire releaseAcquire(p, offset);
    oop ret = HeapAccess<ON_UNKNOWN_OOP_REF>::oop_atomic_cmpxchg_at(p, (ptrdiff_t)offset, e, x);
    return ret == e;
  } UNSAFE_END
  
  UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSetInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint e, jint x)) {
    oop p = JNIHandles::resolve(obj);
    if (p == NULL) {
      volatile jint* addr = (volatile jint*)index_oop_from_field_offset_long(p, offset);
+     ScopedReleaseAcquire releaseAcquire(addr);
      return RawAccess<>::atomic_cmpxchg(addr, e, x) == e;
    } else {
      assert_field_offset_sane(p, offset);
+     ScopedReleaseAcquire releaseAcquire(p, offset);
      return HeapAccess<>::atomic_cmpxchg_at(p, (ptrdiff_t)offset, e, x) == e;
    }
  } UNSAFE_END
  
  UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSetLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong e, jlong x)) {
    oop p = JNIHandles::resolve(obj);
    if (p == NULL) {
      volatile jlong* addr = (volatile jlong*)index_oop_from_field_offset_long(p, offset);
+     ScopedReleaseAcquire releaseAcquire(addr);
      return RawAccess<>::atomic_cmpxchg(addr, e, x) == e;
    } else {
      assert_field_offset_sane(p, offset);
+     ScopedReleaseAcquire releaseAcquire(p, offset);
      return HeapAccess<>::atomic_cmpxchg_at(p, (ptrdiff_t)offset, e, x) == e;
    }
  } UNSAFE_END
  
  static void post_thread_park_event(EventThreadPark* event, const oop obj, jlong timeout_nanos, jlong until_epoch_millis) {