Source file src/runtime/os_linux.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/goarch"
    10  	"runtime/internal/atomic"
    11  	"runtime/internal/syscall"
    12  	"unsafe"
    13  )
    14  
    15  // sigPerThreadSyscall is the same signal (SIGSETXID) used by glibc for
    16  // per-thread syscalls on Linux. We use it for the same purpose in non-cgo
    17  // binaries.
    18  const sigPerThreadSyscall = _SIGRTMIN + 1
    19  
    20  type mOS struct {
    21  	// profileTimer holds the ID of the POSIX interval timer for profiling CPU
    22  	// usage on this thread.
    23  	//
    24  	// It is valid when the profileTimerValid field is non-zero. A thread
    25  	// creates and manages its own timer, and these fields are read and written
    26  	// only by this thread. But because some of the reads on profileTimerValid
    27  	// are in signal handling code, access to that field uses atomic operations.
    28  	profileTimer      int32
    29  	profileTimerValid uint32
    30  
    31  	// needPerThreadSyscall indicates that a per-thread syscall is required
    32  	// for doAllThreadsSyscall.
    33  	needPerThreadSyscall atomic.Uint8
    34  }
    35  
    36  //go:noescape
    37  func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32
    38  
    39  // Linux futex.
    40  //
    41  //	futexsleep(uint32 *addr, uint32 val)
    42  //	futexwakeup(uint32 *addr)
    43  //
    44  // Futexsleep atomically checks if *addr == val and if so, sleeps on addr.
    45  // Futexwakeup wakes up threads sleeping on addr.
    46  // Futexsleep is allowed to wake up spuriously.
    47  
    48  const (
    49  	_FUTEX_PRIVATE_FLAG = 128
    50  	_FUTEX_WAIT_PRIVATE = 0 | _FUTEX_PRIVATE_FLAG
    51  	_FUTEX_WAKE_PRIVATE = 1 | _FUTEX_PRIVATE_FLAG
    52  )
    53  
    54  // Atomically,
    55  //	if(*addr == val) sleep
    56  // Might be woken up spuriously; that's allowed.
    57  // Don't sleep longer than ns; ns < 0 means forever.
    58  //go:nosplit
    59  func futexsleep(addr *uint32, val uint32, ns int64) {
    60  	// Some Linux kernels have a bug where futex of
    61  	// FUTEX_WAIT returns an internal error code
    62  	// as an errno. Libpthread ignores the return value
    63  	// here, and so can we: as it says a few lines up,
    64  	// spurious wakeups are allowed.
    65  	if ns < 0 {
    66  		futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, nil, nil, 0)
    67  		return
    68  	}
    69  
    70  	var ts timespec
    71  	ts.setNsec(ns)
    72  	futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, unsafe.Pointer(&ts), nil, 0)
    73  }
    74  
    75  // If any procs are sleeping on addr, wake up at most cnt.
    76  //go:nosplit
    77  func futexwakeup(addr *uint32, cnt uint32) {
    78  	ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE_PRIVATE, cnt, nil, nil, 0)
    79  	if ret >= 0 {
    80  		return
    81  	}
    82  
    83  	// I don't know that futex wakeup can return
    84  	// EAGAIN or EINTR, but if it does, it would be
    85  	// safe to loop and call futex again.
    86  	systemstack(func() {
    87  		print("futexwakeup addr=", addr, " returned ", ret, "\n")
    88  	})
    89  
    90  	*(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006
    91  }
    92  
    93  func getproccount() int32 {
    94  	// This buffer is huge (8 kB) but we are on the system stack
    95  	// and there should be plenty of space (64 kB).
    96  	// Also this is a leaf, so we're not holding up the memory for long.
    97  	// See golang.org/issue/11823.
    98  	// The suggested behavior here is to keep trying with ever-larger
    99  	// buffers, but we don't have a dynamic memory allocator at the
   100  	// moment, so that's a bit tricky and seems like overkill.
   101  	const maxCPUs = 64 * 1024
   102  	var buf [maxCPUs / 8]byte
   103  	r := sched_getaffinity(0, unsafe.Sizeof(buf), &buf[0])
   104  	if r < 0 {
   105  		return 1
   106  	}
   107  	n := int32(0)
   108  	for _, v := range buf[:r] {
   109  		for v != 0 {
   110  			n += int32(v & 1)
   111  			v >>= 1
   112  		}
   113  	}
   114  	if n == 0 {
   115  		n = 1
   116  	}
   117  	return n
   118  }
   119  
   120  // Clone, the Linux rfork.
   121  const (
   122  	_CLONE_VM             = 0x100
   123  	_CLONE_FS             = 0x200
   124  	_CLONE_FILES          = 0x400
   125  	_CLONE_SIGHAND        = 0x800
   126  	_CLONE_PTRACE         = 0x2000
   127  	_CLONE_VFORK          = 0x4000
   128  	_CLONE_PARENT         = 0x8000
   129  	_CLONE_THREAD         = 0x10000
   130  	_CLONE_NEWNS          = 0x20000
   131  	_CLONE_SYSVSEM        = 0x40000
   132  	_CLONE_SETTLS         = 0x80000
   133  	_CLONE_PARENT_SETTID  = 0x100000
   134  	_CLONE_CHILD_CLEARTID = 0x200000
   135  	_CLONE_UNTRACED       = 0x800000
   136  	_CLONE_CHILD_SETTID   = 0x1000000
   137  	_CLONE_STOPPED        = 0x2000000
   138  	_CLONE_NEWUTS         = 0x4000000
   139  	_CLONE_NEWIPC         = 0x8000000
   140  
   141  	// As of QEMU 2.8.0 (5ea2fc84d), user emulation requires all six of these
   142  	// flags to be set when creating a thread; attempts to share the other
   143  	// five but leave SYSVSEM unshared will fail with -EINVAL.
   144  	//
   145  	// In non-QEMU environments CLONE_SYSVSEM is inconsequential as we do not
   146  	// use System V semaphores.
   147  
   148  	cloneFlags = _CLONE_VM | /* share memory */
   149  		_CLONE_FS | /* share cwd, etc */
   150  		_CLONE_FILES | /* share fd table */
   151  		_CLONE_SIGHAND | /* share sig handler table */
   152  		_CLONE_SYSVSEM | /* share SysV semaphore undo lists (see issue #20763) */
   153  		_CLONE_THREAD /* revisit - okay for now */
   154  )
   155  
   156  //go:noescape
   157  func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32
   158  
   159  // May run with m.p==nil, so write barriers are not allowed.
   160  //go:nowritebarrier
   161  func newosproc(mp *m) {
   162  	stk := unsafe.Pointer(mp.g0.stack.hi)
   163  	/*
   164  	 * note: strace gets confused if we use CLONE_PTRACE here.
   165  	 */
   166  	if false {
   167  		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " clone=", abi.FuncPCABI0(clone), " id=", mp.id, " ostk=", &mp, "\n")
   168  	}
   169  
   170  	// Disable signals during clone, so that the new thread starts
   171  	// with signals disabled. It will enable them in minit.
   172  	var oset sigset
   173  	sigprocmask(_SIG_SETMASK, &sigset_all, &oset)
   174  	ret := clone(cloneFlags, stk, unsafe.Pointer(mp), unsafe.Pointer(mp.g0), unsafe.Pointer(abi.FuncPCABI0(mstart)))
   175  	sigprocmask(_SIG_SETMASK, &oset, nil)
   176  
   177  	if ret < 0 {
   178  		print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -ret, ")\n")
   179  		if ret == -_EAGAIN {
   180  			println("runtime: may need to increase max user processes (ulimit -u)")
   181  		}
   182  		throw("newosproc")
   183  	}
   184  }
   185  
   186  // Version of newosproc that doesn't require a valid G.
   187  //go:nosplit
   188  func newosproc0(stacksize uintptr, fn unsafe.Pointer) {
   189  	stack := sysAlloc(stacksize, &memstats.stacks_sys)
   190  	if stack == nil {
   191  		write(2, unsafe.Pointer(&failallocatestack[0]), int32(len(failallocatestack)))
   192  		exit(1)
   193  	}
   194  	ret := clone(cloneFlags, unsafe.Pointer(uintptr(stack)+stacksize), nil, nil, fn)
   195  	if ret < 0 {
   196  		write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate)))
   197  		exit(1)
   198  	}
   199  }
   200  
   201  var failallocatestack = []byte("runtime: failed to allocate stack for the new OS thread\n")
   202  var failthreadcreate = []byte("runtime: failed to create new OS thread\n")
   203  
   204  const (
   205  	_AT_NULL   = 0  // End of vector
   206  	_AT_PAGESZ = 6  // System physical page size
   207  	_AT_HWCAP  = 16 // hardware capability bit vector
   208  	_AT_RANDOM = 25 // introduced in 2.6.29
   209  	_AT_HWCAP2 = 26 // hardware capability bit vector 2
   210  )
   211  
   212  var procAuxv = []byte("/proc/self/auxv\x00")
   213  
   214  var addrspace_vec [1]byte
   215  
   216  func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32
   217  
   218  func sysargs(argc int32, argv **byte) {
   219  	n := argc + 1
   220  
   221  	// skip over argv, envp to get to auxv
   222  	for argv_index(argv, n) != nil {
   223  		n++
   224  	}
   225  
   226  	// skip NULL separator
   227  	n++
   228  
   229  	// now argv+n is auxv
   230  	auxv := (*[1 << 28]uintptr)(add(unsafe.Pointer(argv), uintptr(n)*goarch.PtrSize))
   231  	if sysauxv(auxv[:]) != 0 {
   232  		return
   233  	}
   234  	// In some situations we don't get a loader-provided
   235  	// auxv, such as when loaded as a library on Android.
   236  	// Fall back to /proc/self/auxv.
   237  	fd := open(&procAuxv[0], 0 /* O_RDONLY */, 0)
   238  	if fd < 0 {
   239  		// On Android, /proc/self/auxv might be unreadable (issue 9229), so we fallback to
   240  		// try using mincore to detect the physical page size.
   241  		// mincore should return EINVAL when address is not a multiple of system page size.
   242  		const size = 256 << 10 // size of memory region to allocate
   243  		p, err := mmap(nil, size, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
   244  		if err != 0 {
   245  			return
   246  		}
   247  		var n uintptr
   248  		for n = 4 << 10; n < size; n <<= 1 {
   249  			err := mincore(unsafe.Pointer(uintptr(p)+n), 1, &addrspace_vec[0])
   250  			if err == 0 {
   251  				physPageSize = n
   252  				break
   253  			}
   254  		}
   255  		if physPageSize == 0 {
   256  			physPageSize = size
   257  		}
   258  		munmap(p, size)
   259  		return
   260  	}
   261  	var buf [128]uintptr
   262  	n = read(fd, noescape(unsafe.Pointer(&buf[0])), int32(unsafe.Sizeof(buf)))
   263  	closefd(fd)
   264  	if n < 0 {
   265  		return
   266  	}
   267  	// Make sure buf is terminated, even if we didn't read
   268  	// the whole file.
   269  	buf[len(buf)-2] = _AT_NULL
   270  	sysauxv(buf[:])
   271  }
   272  
   273  // startupRandomData holds random bytes initialized at startup. These come from
   274  // the ELF AT_RANDOM auxiliary vector.
   275  var startupRandomData []byte
   276  
   277  func sysauxv(auxv []uintptr) int {
   278  	var i int
   279  	for ; auxv[i] != _AT_NULL; i += 2 {
   280  		tag, val := auxv[i], auxv[i+1]
   281  		switch tag {
   282  		case _AT_RANDOM:
   283  			// The kernel provides a pointer to 16-bytes
   284  			// worth of random data.
   285  			startupRandomData = (*[16]byte)(unsafe.Pointer(val))[:]
   286  
   287  		case _AT_PAGESZ:
   288  			physPageSize = val
   289  		}
   290  
   291  		archauxv(tag, val)
   292  		vdsoauxv(tag, val)
   293  	}
   294  	return i / 2
   295  }
   296  
   297  var sysTHPSizePath = []byte("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size\x00")
   298  
   299  func getHugePageSize() uintptr {
   300  	var numbuf [20]byte
   301  	fd := open(&sysTHPSizePath[0], 0 /* O_RDONLY */, 0)
   302  	if fd < 0 {
   303  		return 0
   304  	}
   305  	ptr := noescape(unsafe.Pointer(&numbuf[0]))
   306  	n := read(fd, ptr, int32(len(numbuf)))
   307  	closefd(fd)
   308  	if n <= 0 {
   309  		return 0
   310  	}
   311  	n-- // remove trailing newline
   312  	v, ok := atoi(slicebytetostringtmp((*byte)(ptr), int(n)))
   313  	if !ok || v < 0 {
   314  		v = 0
   315  	}
   316  	if v&(v-1) != 0 {
   317  		// v is not a power of 2
   318  		return 0
   319  	}
   320  	return uintptr(v)
   321  }
   322  
   323  func osinit() {
   324  	ncpu = getproccount()
   325  	physHugePageSize = getHugePageSize()
   326  	if iscgo {
   327  		// #42494 glibc and musl reserve some signals for
   328  		// internal use and require they not be blocked by
   329  		// the rest of a normal C runtime. When the go runtime
   330  		// blocks...unblocks signals, temporarily, the blocked
   331  		// interval of time is generally very short. As such,
   332  		// these expectations of *libc code are mostly met by
   333  		// the combined go+cgo system of threads. However,
   334  		// when go causes a thread to exit, via a return from
   335  		// mstart(), the combined runtime can deadlock if
   336  		// these signals are blocked. Thus, don't block these
   337  		// signals when exiting threads.
   338  		// - glibc: SIGCANCEL (32), SIGSETXID (33)
   339  		// - musl: SIGTIMER (32), SIGCANCEL (33), SIGSYNCCALL (34)
   340  		sigdelset(&sigsetAllExiting, 32)
   341  		sigdelset(&sigsetAllExiting, 33)
   342  		sigdelset(&sigsetAllExiting, 34)
   343  	}
   344  	osArchInit()
   345  }
   346  
   347  var urandom_dev = []byte("/dev/urandom\x00")
   348  
   349  func getRandomData(r []byte) {
   350  	if startupRandomData != nil {
   351  		n := copy(r, startupRandomData)
   352  		extendRandom(r, n)
   353  		return
   354  	}
   355  	fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0)
   356  	n := read(fd, unsafe.Pointer(&r[0]), int32(len(r)))
   357  	closefd(fd)
   358  	extendRandom(r, int(n))
   359  }
   360  
   361  func goenvs() {
   362  	goenvs_unix()
   363  }
   364  
   365  // Called to do synchronous initialization of Go code built with
   366  // -buildmode=c-archive or -buildmode=c-shared.
   367  // None of the Go runtime is initialized.
   368  //go:nosplit
   369  //go:nowritebarrierrec
   370  func libpreinit() {
   371  	initsig(true)
   372  }
   373  
   374  // Called to initialize a new m (including the bootstrap m).
   375  // Called on the parent thread (main thread in case of bootstrap), can allocate memory.
   376  func mpreinit(mp *m) {
   377  	mp.gsignal = malg(32 * 1024) // Linux wants >= 2K
   378  	mp.gsignal.m = mp
   379  }
   380  
   381  func gettid() uint32
   382  
   383  // Called to initialize a new m (including the bootstrap m).
   384  // Called on the new thread, cannot allocate memory.
   385  func minit() {
   386  	minitSignals()
   387  
   388  	// Cgo-created threads and the bootstrap m are missing a
   389  	// procid. We need this for asynchronous preemption and it's
   390  	// useful in debuggers.
   391  	getg().m.procid = uint64(gettid())
   392  }
   393  
   394  // Called from dropm to undo the effect of an minit.
   395  //go:nosplit
   396  func unminit() {
   397  	unminitSignals()
   398  }
   399  
   400  // Called from exitm, but not from drop, to undo the effect of thread-owned
   401  // resources in minit, semacreate, or elsewhere. Do not take locks after calling this.
   402  func mdestroy(mp *m) {
   403  }
   404  
   405  //#ifdef GOARCH_386
   406  //#define sa_handler k_sa_handler
   407  //#endif
   408  
   409  func sigreturn()
   410  func sigtramp() // Called via C ABI
   411  func cgoSigtramp()
   412  
   413  //go:noescape
   414  func sigaltstack(new, old *stackt)
   415  
   416  //go:noescape
   417  func setitimer(mode int32, new, old *itimerval)
   418  
   419  //go:noescape
   420  func timer_create(clockid int32, sevp *sigevent, timerid *int32) int32
   421  
   422  //go:noescape
   423  func timer_settime(timerid int32, flags int32, new, old *itimerspec) int32
   424  
   425  //go:noescape
   426  func timer_delete(timerid int32) int32
   427  
   428  //go:noescape
   429  func rtsigprocmask(how int32, new, old *sigset, size int32)
   430  
   431  //go:nosplit
   432  //go:nowritebarrierrec
   433  func sigprocmask(how int32, new, old *sigset) {
   434  	rtsigprocmask(how, new, old, int32(unsafe.Sizeof(*new)))
   435  }
   436  
   437  func raise(sig uint32)
   438  func raiseproc(sig uint32)
   439  
   440  //go:noescape
   441  func sched_getaffinity(pid, len uintptr, buf *byte) int32
   442  func osyield()
   443  
   444  //go:nosplit
   445  func osyield_no_g() {
   446  	osyield()
   447  }
   448  
   449  func pipe() (r, w int32, errno int32)
   450  func pipe2(flags int32) (r, w int32, errno int32)
   451  func setNonblock(fd int32)
   452  
   453  const (
   454  	_si_max_size    = 128
   455  	_sigev_max_size = 64
   456  )
   457  
   458  //go:nosplit
   459  //go:nowritebarrierrec
   460  func setsig(i uint32, fn uintptr) {
   461  	var sa sigactiont
   462  	sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTORER | _SA_RESTART
   463  	sigfillset(&sa.sa_mask)
   464  	// Although Linux manpage says "sa_restorer element is obsolete and
   465  	// should not be used". x86_64 kernel requires it. Only use it on
   466  	// x86.
   467  	if GOARCH == "386" || GOARCH == "amd64" {
   468  		sa.sa_restorer = abi.FuncPCABI0(sigreturn)
   469  	}
   470  	if fn == abi.FuncPCABIInternal(sighandler) { // abi.FuncPCABIInternal(sighandler) matches the callers in signal_unix.go
   471  		if iscgo {
   472  			fn = abi.FuncPCABI0(cgoSigtramp)
   473  		} else {
   474  			fn = abi.FuncPCABI0(sigtramp)
   475  		}
   476  	}
   477  	sa.sa_handler = fn
   478  	sigaction(i, &sa, nil)
   479  }
   480  
   481  //go:nosplit
   482  //go:nowritebarrierrec
   483  func setsigstack(i uint32) {
   484  	var sa sigactiont
   485  	sigaction(i, nil, &sa)
   486  	if sa.sa_flags&_SA_ONSTACK != 0 {
   487  		return
   488  	}
   489  	sa.sa_flags |= _SA_ONSTACK
   490  	sigaction(i, &sa, nil)
   491  }
   492  
   493  //go:nosplit
   494  //go:nowritebarrierrec
   495  func getsig(i uint32) uintptr {
   496  	var sa sigactiont
   497  	sigaction(i, nil, &sa)
   498  	return sa.sa_handler
   499  }
   500  
   501  // setSignaltstackSP sets the ss_sp field of a stackt.
   502  //go:nosplit
   503  func setSignalstackSP(s *stackt, sp uintptr) {
   504  	*(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp
   505  }
   506  
   507  //go:nosplit
   508  func (c *sigctxt) fixsigcode(sig uint32) {
   509  }
   510  
   511  // sysSigaction calls the rt_sigaction system call.
   512  //go:nosplit
   513  func sysSigaction(sig uint32, new, old *sigactiont) {
   514  	if rt_sigaction(uintptr(sig), new, old, unsafe.Sizeof(sigactiont{}.sa_mask)) != 0 {
   515  		// Workaround for bugs in QEMU user mode emulation.
   516  		//
   517  		// QEMU turns calls to the sigaction system call into
   518  		// calls to the C library sigaction call; the C
   519  		// library call rejects attempts to call sigaction for
   520  		// SIGCANCEL (32) or SIGSETXID (33).
   521  		//
   522  		// QEMU rejects calling sigaction on SIGRTMAX (64).
   523  		//
   524  		// Just ignore the error in these case. There isn't
   525  		// anything we can do about it anyhow.
   526  		if sig != 32 && sig != 33 && sig != 64 {
   527  			// Use system stack to avoid split stack overflow on ppc64/ppc64le.
   528  			systemstack(func() {
   529  				throw("sigaction failed")
   530  			})
   531  		}
   532  	}
   533  }
   534  
   535  // rt_sigaction is implemented in assembly.
   536  //go:noescape
   537  func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32
   538  
   539  func getpid() int
   540  func tgkill(tgid, tid, sig int)
   541  
   542  // signalM sends a signal to mp.
   543  func signalM(mp *m, sig int) {
   544  	tgkill(getpid(), int(mp.procid), sig)
   545  }
   546  
   547  // go118UseTimerCreateProfiler enables the per-thread CPU profiler.
   548  const go118UseTimerCreateProfiler = true
   549  
   550  // validSIGPROF compares this signal delivery's code against the signal sources
   551  // that the profiler uses, returning whether the delivery should be processed.
   552  // To be processed, a signal delivery from a known profiling mechanism should
   553  // correspond to the best profiling mechanism available to this thread. Signals
   554  // from other sources are always considered valid.
   555  //
   556  //go:nosplit
   557  func validSIGPROF(mp *m, c *sigctxt) bool {
   558  	code := int32(c.sigcode())
   559  	setitimer := code == _SI_KERNEL
   560  	timer_create := code == _SI_TIMER
   561  
   562  	if !(setitimer || timer_create) {
   563  		// The signal doesn't correspond to a profiling mechanism that the
   564  		// runtime enables itself. There's no reason to process it, but there's
   565  		// no reason to ignore it either.
   566  		return true
   567  	}
   568  
   569  	if mp == nil {
   570  		// Since we don't have an M, we can't check if there's an active
   571  		// per-thread timer for this thread. We don't know how long this thread
   572  		// has been around, and if it happened to interact with the Go scheduler
   573  		// at a time when profiling was active (causing it to have a per-thread
   574  		// timer). But it may have never interacted with the Go scheduler, or
   575  		// never while profiling was active. To avoid double-counting, process
   576  		// only signals from setitimer.
   577  		//
   578  		// When a custom cgo traceback function has been registered (on
   579  		// platforms that support runtime.SetCgoTraceback), SIGPROF signals
   580  		// delivered to a thread that cannot find a matching M do this check in
   581  		// the assembly implementations of runtime.cgoSigtramp.
   582  		return setitimer
   583  	}
   584  
   585  	// Having an M means the thread interacts with the Go scheduler, and we can
   586  	// check whether there's an active per-thread timer for this thread.
   587  	if atomic.Load(&mp.profileTimerValid) != 0 {
   588  		// If this M has its own per-thread CPU profiling interval timer, we
   589  		// should track the SIGPROF signals that come from that timer (for
   590  		// accurate reporting of its CPU usage; see issue 35057) and ignore any
   591  		// that it gets from the process-wide setitimer (to not over-count its
   592  		// CPU consumption).
   593  		return timer_create
   594  	}
   595  
   596  	// No active per-thread timer means the only valid profiler is setitimer.
   597  	return setitimer
   598  }
   599  
   600  func setProcessCPUProfiler(hz int32) {
   601  	setProcessCPUProfilerTimer(hz)
   602  }
   603  
   604  func setThreadCPUProfiler(hz int32) {
   605  	mp := getg().m
   606  	mp.profilehz = hz
   607  
   608  	if !go118UseTimerCreateProfiler {
   609  		return
   610  	}
   611  
   612  	// destroy any active timer
   613  	if atomic.Load(&mp.profileTimerValid) != 0 {
   614  		timerid := mp.profileTimer
   615  		atomic.Store(&mp.profileTimerValid, 0)
   616  		mp.profileTimer = 0
   617  
   618  		ret := timer_delete(timerid)
   619  		if ret != 0 {
   620  			print("runtime: failed to disable profiling timer; timer_delete(", timerid, ") errno=", -ret, "\n")
   621  			throw("timer_delete")
   622  		}
   623  	}
   624  
   625  	if hz == 0 {
   626  		// If the goal was to disable profiling for this thread, then the job's done.
   627  		return
   628  	}
   629  
   630  	// The period of the timer should be 1/Hz. For every "1/Hz" of additional
   631  	// work, the user should expect one additional sample in the profile.
   632  	//
   633  	// But to scale down to very small amounts of application work, to observe
   634  	// even CPU usage of "one tenth" of the requested period, set the initial
   635  	// timing delay in a different way: So that "one tenth" of a period of CPU
   636  	// spend shows up as a 10% chance of one sample (for an expected value of
   637  	// 0.1 samples), and so that "two and six tenths" periods of CPU spend show
   638  	// up as a 60% chance of 3 samples and a 40% chance of 2 samples (for an
   639  	// expected value of 2.6). Set the initial delay to a value in the unifom
   640  	// random distribution between 0 and the desired period. And because "0"
   641  	// means "disable timer", add 1 so the half-open interval [0,period) turns
   642  	// into (0,period].
   643  	//
   644  	// Otherwise, this would show up as a bias away from short-lived threads and
   645  	// from threads that are only occasionally active: for example, when the
   646  	// garbage collector runs on a mostly-idle system, the additional threads it
   647  	// activates may do a couple milliseconds of GC-related work and nothing
   648  	// else in the few seconds that the profiler observes.
   649  	spec := new(itimerspec)
   650  	spec.it_value.setNsec(1 + int64(fastrandn(uint32(1e9/hz))))
   651  	spec.it_interval.setNsec(1e9 / int64(hz))
   652  
   653  	var timerid int32
   654  	var sevp sigevent
   655  	sevp.notify = _SIGEV_THREAD_ID
   656  	sevp.signo = _SIGPROF
   657  	sevp.sigev_notify_thread_id = int32(mp.procid)
   658  	ret := timer_create(_CLOCK_THREAD_CPUTIME_ID, &sevp, &timerid)
   659  	if ret != 0 {
   660  		// If we cannot create a timer for this M, leave profileTimerValid false
   661  		// to fall back to the process-wide setitimer profiler.
   662  		return
   663  	}
   664  
   665  	ret = timer_settime(timerid, 0, spec, nil)
   666  	if ret != 0 {
   667  		print("runtime: failed to configure profiling timer; timer_settime(", timerid,
   668  			", 0, {interval: {",
   669  			spec.it_interval.tv_sec, "s + ", spec.it_interval.tv_nsec, "ns} value: {",
   670  			spec.it_value.tv_sec, "s + ", spec.it_value.tv_nsec, "ns}}, nil) errno=", -ret, "\n")
   671  		throw("timer_settime")
   672  	}
   673  
   674  	mp.profileTimer = timerid
   675  	atomic.Store(&mp.profileTimerValid, 1)
   676  }
   677  
   678  // perThreadSyscallArgs contains the system call number, arguments, and
   679  // expected return values for a system call to be executed on all threads.
   680  type perThreadSyscallArgs struct {
   681  	trap uintptr
   682  	a1   uintptr
   683  	a2   uintptr
   684  	a3   uintptr
   685  	a4   uintptr
   686  	a5   uintptr
   687  	a6   uintptr
   688  	r1   uintptr
   689  	r2   uintptr
   690  }
   691  
   692  // perThreadSyscall is the system call to execute for the ongoing
   693  // doAllThreadsSyscall.
   694  //
   695  // perThreadSyscall may only be written while mp.needPerThreadSyscall == 0 on
   696  // all Ms.
   697  var perThreadSyscall perThreadSyscallArgs
   698  
   699  // syscall_runtime_doAllThreadsSyscall and executes a specified system call on
   700  // all Ms.
   701  //
   702  // The system call is expected to succeed and return the same value on every
   703  // thread. If any threads do not match, the runtime throws.
   704  //
   705  //go:linkname syscall_runtime_doAllThreadsSyscall syscall.runtime_doAllThreadsSyscall
   706  //go:uintptrescapes
   707  func syscall_runtime_doAllThreadsSyscall(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) {
   708  	if iscgo {
   709  		// In cgo, we are not aware of threads created in C, so this approach will not work.
   710  		panic("doAllThreadsSyscall not supported with cgo enabled")
   711  	}
   712  
   713  	// STW to guarantee that user goroutines see an atomic change to thread
   714  	// state. Without STW, goroutines could migrate Ms while change is in
   715  	// progress and e.g., see state old -> new -> old -> new.
   716  	//
   717  	// N.B. Internally, this function does not depend on STW to
   718  	// successfully change every thread. It is only needed for user
   719  	// expectations, per above.
   720  	stopTheWorld("doAllThreadsSyscall")
   721  
   722  	// This function depends on several properties:
   723  	//
   724  	// 1. All OS threads that already exist are associated with an M in
   725  	//    allm. i.e., we won't miss any pre-existing threads.
   726  	// 2. All Ms listed in allm will eventually have an OS thread exist.
   727  	//    i.e., they will set procid and be able to receive signals.
   728  	// 3. OS threads created after we read allm will clone from a thread
   729  	//    that has executed the system call. i.e., they inherit the
   730  	//    modified state.
   731  	//
   732  	// We achieve these through different mechanisms:
   733  	//
   734  	// 1. Addition of new Ms to allm in allocm happens before clone of its
   735  	//    OS thread later in newm.
   736  	// 2. newm does acquirem to avoid being preempted, ensuring that new Ms
   737  	//    created in allocm will eventually reach OS thread clone later in
   738  	//    newm.
   739  	// 3. We take allocmLock for write here to prevent allocation of new Ms
   740  	//    while this function runs. Per (1), this prevents clone of OS
   741  	//    threads that are not yet in allm.
   742  	allocmLock.lock()
   743  
   744  	// Disable preemption, preventing us from changing Ms, as we handle
   745  	// this M specially.
   746  	//
   747  	// N.B. STW and lock() above do this as well, this is added for extra
   748  	// clarity.
   749  	acquirem()
   750  
   751  	// N.B. allocmLock also prevents concurrent execution of this function,
   752  	// serializing use of perThreadSyscall, mp.needPerThreadSyscall, and
   753  	// ensuring all threads execute system calls from multiple calls in the
   754  	// same order.
   755  
   756  	r1, r2, errno := syscall.Syscall6(trap, a1, a2, a3, a4, a5, a6)
   757  	if GOARCH == "ppc64" || GOARCH == "ppc64le" {
   758  		// TODO(https://go.dev/issue/51192 ): ppc64 doesn't use r2.
   759  		r2 = 0
   760  	}
   761  	if errno != 0 {
   762  		releasem(getg().m)
   763  		allocmLock.unlock()
   764  		startTheWorld()
   765  		return r1, r2, errno
   766  	}
   767  
   768  	perThreadSyscall = perThreadSyscallArgs{
   769  		trap: trap,
   770  		a1:   a1,
   771  		a2:   a2,
   772  		a3:   a3,
   773  		a4:   a4,
   774  		a5:   a5,
   775  		a6:   a6,
   776  		r1:   r1,
   777  		r2:   r2,
   778  	}
   779  
   780  	// Wait for all threads to start.
   781  	//
   782  	// As described above, some Ms have been added to allm prior to
   783  	// allocmLock, but not yet completed OS clone and set procid.
   784  	//
   785  	// At minimum we must wait for a thread to set procid before we can
   786  	// send it a signal.
   787  	//
   788  	// We take this one step further and wait for all threads to start
   789  	// before sending any signals. This prevents system calls from getting
   790  	// applied twice: once in the parent and once in the child, like so:
   791  	//
   792  	//          A                     B                  C
   793  	//                         add C to allm
   794  	// doAllThreadsSyscall
   795  	//   allocmLock.lock()
   796  	//   signal B
   797  	//                         <receive signal>
   798  	//                         execute syscall
   799  	//                         <signal return>
   800  	//                         clone C
   801  	//                                             <thread start>
   802  	//                                             set procid
   803  	//   signal C
   804  	//                                             <receive signal>
   805  	//                                             execute syscall
   806  	//                                             <signal return>
   807  	//
   808  	// In this case, thread C inherited the syscall-modified state from
   809  	// thread B and did not need to execute the syscall, but did anyway
   810  	// because doAllThreadsSyscall could not be sure whether it was
   811  	// required.
   812  	//
   813  	// Some system calls may not be idempotent, so we ensure each thread
   814  	// executes the system call exactly once.
   815  	for mp := allm; mp != nil; mp = mp.alllink {
   816  		for atomic.Load64(&mp.procid) == 0 {
   817  			// Thread is starting.
   818  			osyield()
   819  		}
   820  	}
   821  
   822  	// Signal every other thread, where they will execute perThreadSyscall
   823  	// from the signal handler.
   824  	gp := getg()
   825  	tid := gp.m.procid
   826  	for mp := allm; mp != nil; mp = mp.alllink {
   827  		if atomic.Load64(&mp.procid) == tid {
   828  			// Our thread already performed the syscall.
   829  			continue
   830  		}
   831  		mp.needPerThreadSyscall.Store(1)
   832  		signalM(mp, sigPerThreadSyscall)
   833  	}
   834  
   835  	// Wait for all threads to complete.
   836  	for mp := allm; mp != nil; mp = mp.alllink {
   837  		if mp.procid == tid {
   838  			continue
   839  		}
   840  		for mp.needPerThreadSyscall.Load() != 0 {
   841  			osyield()
   842  		}
   843  	}
   844  
   845  	perThreadSyscall = perThreadSyscallArgs{}
   846  
   847  	releasem(getg().m)
   848  	allocmLock.unlock()
   849  	startTheWorld()
   850  
   851  	return r1, r2, errno
   852  }
   853  
   854  // runPerThreadSyscall runs perThreadSyscall for this M if required.
   855  //
   856  // This function throws if the system call returns with anything other than the
   857  // expected values.
   858  //go:nosplit
   859  func runPerThreadSyscall() {
   860  	gp := getg()
   861  	if gp.m.needPerThreadSyscall.Load() == 0 {
   862  		return
   863  	}
   864  
   865  	args := perThreadSyscall
   866  	r1, r2, errno := syscall.Syscall6(args.trap, args.a1, args.a2, args.a3, args.a4, args.a5, args.a6)
   867  	if GOARCH == "ppc64" || GOARCH == "ppc64le" {
   868  		// TODO(https://go.dev/issue/51192 ): ppc64 doesn't use r2.
   869  		r2 = 0
   870  	}
   871  	if errno != 0 || r1 != args.r1 || r2 != args.r2 {
   872  		print("trap:", args.trap, ", a123456=[", args.a1, ",", args.a2, ",", args.a3, ",", args.a4, ",", args.a5, ",", args.a6, "]\n")
   873  		print("results: got {r1=", r1, ",r2=", r2, ",errno=", errno, "}, want {r1=", args.r1, ",r2=", args.r2, ",errno=0\n")
   874  		throw("AllThreadsSyscall6 results differ between threads; runtime corrupted")
   875  	}
   876  
   877  	gp.m.needPerThreadSyscall.Store(0)
   878  }
   879  

View as plain text