atomic_pointer.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"runtime/internal/atomic"
     9  	"unsafe"
    10  )
    11  
    12  // These functions cannot have go:noescape annotations,
    13  // because while ptr does not escape, new does.
    14  // If new is marked as not escaping, the compiler will make incorrect
    15  // escape analysis decisions about the pointer value being stored.
    16  
    17  // atomicwb performs a write barrier before an atomic pointer write.
    18  // The caller should guard the call with "if writeBarrier.enabled".
    19  //
    20  //go:nosplit
    21  func atomicwb(ptr *unsafe.Pointer, new unsafe.Pointer) {
    22  	slot := (*uintptr)(unsafe.Pointer(ptr))
    23  	if !getg().m.p.ptr().wbBuf.putFast(*slot, uintptr(new)) {
    24  		wbBufFlush(slot, uintptr(new))
    25  	}
    26  }
    27  
    28  // atomicstorep performs *ptr = new atomically and invokes a write barrier.
    29  //
    30  //go:nosplit
    31  func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) {
    32  	if writeBarrier.enabled {
    33  		atomicwb((*unsafe.Pointer)(ptr), new)
    34  	}
    35  	atomic.StorepNoWB(noescape(ptr), new)
    36  }
    37  
    38  // Like above, but implement in terms of sync/atomic's uintptr operations.
    39  // We cannot just call the runtime routines, because the race detector expects
    40  // to be able to intercept the sync/atomic forms but not the runtime forms.
    41  
    42  //go:linkname sync_atomic_StoreUintptr sync/atomic.StoreUintptr
    43  func sync_atomic_StoreUintptr(ptr *uintptr, new uintptr)
    44  
    45  //go:linkname sync_atomic_StorePointer sync/atomic.StorePointer
    46  //go:nosplit
    47  func sync_atomic_StorePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
    48  	if writeBarrier.enabled {
    49  		atomicwb(ptr, new)
    50  	}
    51  	sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
    52  }
    53  
    54  //go:linkname sync_atomic_SwapUintptr sync/atomic.SwapUintptr
    55  func sync_atomic_SwapUintptr(ptr *uintptr, new uintptr) uintptr
    56  
    57  //go:linkname sync_atomic_SwapPointer sync/atomic.SwapPointer
    58  //go:nosplit
    59  func sync_atomic_SwapPointer(ptr *unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer {
    60  	if writeBarrier.enabled {
    61  		atomicwb(ptr, new)
    62  	}
    63  	old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(new)))
    64  	return old
    65  }
    66  
    67  //go:linkname sync_atomic_CompareAndSwapUintptr sync/atomic.CompareAndSwapUintptr
    68  func sync_atomic_CompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool
    69  
    70  //go:linkname sync_atomic_CompareAndSwapPointer sync/atomic.CompareAndSwapPointer
    71  //go:nosplit
    72  func sync_atomic_CompareAndSwapPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
    73  	if writeBarrier.enabled {
    74  		atomicwb(ptr, new)
    75  	}
    76  	return sync_atomic_CompareAndSwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(old), uintptr(new))
    77  }
    78
View as plain text