Source file src/runtime/mkpreempt.go

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build ignore
     6  
     7  // mkpreempt generates the asyncPreempt functions for each
     8  // architecture.
     9  package main
    10  
    11  import (
    12  	"flag"
    13  	"fmt"
    14  	"io"
    15  	"log"
    16  	"os"
    17  	"strings"
    18  )
    19  
    20  // Copied from cmd/compile/internal/ssa/gen/*Ops.go
    21  
    22  var regNames386 = []string{
    23  	"AX",
    24  	"CX",
    25  	"DX",
    26  	"BX",
    27  	"SP",
    28  	"BP",
    29  	"SI",
    30  	"DI",
    31  	"X0",
    32  	"X1",
    33  	"X2",
    34  	"X3",
    35  	"X4",
    36  	"X5",
    37  	"X6",
    38  	"X7",
    39  }
    40  
    41  var regNamesAMD64 = []string{
    42  	"AX",
    43  	"CX",
    44  	"DX",
    45  	"BX",
    46  	"SP",
    47  	"BP",
    48  	"SI",
    49  	"DI",
    50  	"R8",
    51  	"R9",
    52  	"R10",
    53  	"R11",
    54  	"R12",
    55  	"R13",
    56  	"R14",
    57  	"R15",
    58  	"X0",
    59  	"X1",
    60  	"X2",
    61  	"X3",
    62  	"X4",
    63  	"X5",
    64  	"X6",
    65  	"X7",
    66  	"X8",
    67  	"X9",
    68  	"X10",
    69  	"X11",
    70  	"X12",
    71  	"X13",
    72  	"X14",
    73  	"X15",
    74  }
    75  
    76  var out io.Writer
    77  
    78  var arches = map[string]func(){
    79  	"386":     gen386,
    80  	"amd64":   genAMD64,
    81  	"arm":     genARM,
    82  	"arm64":   genARM64,
    83  	"mips64x": func() { genMIPS(true) },
    84  	"mipsx":   func() { genMIPS(false) },
    85  	"ppc64x":  genPPC64,
    86  	"riscv64": genRISCV64,
    87  	"s390x":   genS390X,
    88  	"wasm":    genWasm,
    89  }
    90  var beLe = map[string]bool{"mips64x": true, "mipsx": true, "ppc64x": true}
    91  
    92  func main() {
    93  	flag.Parse()
    94  	if flag.NArg() > 0 {
    95  		out = os.Stdout
    96  		for _, arch := range flag.Args() {
    97  			gen, ok := arches[arch]
    98  			if !ok {
    99  				log.Fatalf("unknown arch %s", arch)
   100  			}
   101  			header(arch)
   102  			gen()
   103  		}
   104  		return
   105  	}
   106  
   107  	for arch, gen := range arches {
   108  		f, err := os.Create(fmt.Sprintf("preempt_%s.s", arch))
   109  		if err != nil {
   110  			log.Fatal(err)
   111  		}
   112  		out = f
   113  		header(arch)
   114  		gen()
   115  		if err := f.Close(); err != nil {
   116  			log.Fatal(err)
   117  		}
   118  	}
   119  }
   120  
   121  func header(arch string) {
   122  	fmt.Fprintf(out, "// Code generated by mkpreempt.go; DO NOT EDIT.\n\n")
   123  	if beLe[arch] {
   124  		base := arch[:len(arch)-1]
   125  		fmt.Fprintf(out, "//go:build %s || %sle\n", base, base)
   126  	}
   127  	fmt.Fprintf(out, "#include \"go_asm.h\"\n")
   128  	fmt.Fprintf(out, "#include \"textflag.h\"\n\n")
   129  	fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n")
   130  }
   131  
   132  func p(f string, args ...any) {
   133  	fmted := fmt.Sprintf(f, args...)
   134  	fmt.Fprintf(out, "\t%s\n", strings.ReplaceAll(fmted, "\n", "\n\t"))
   135  }
   136  
   137  func label(l string) {
   138  	fmt.Fprintf(out, "%s\n", l)
   139  }
   140  
   141  type layout struct {
   142  	stack int
   143  	regs  []regPos
   144  	sp    string // stack pointer register
   145  }
   146  
   147  type regPos struct {
   148  	pos int
   149  
   150  	op  string
   151  	reg string
   152  
   153  	// If this register requires special save and restore, these
   154  	// give those operations with a %d placeholder for the stack
   155  	// offset.
   156  	save, restore string
   157  }
   158  
   159  func (l *layout) add(op, reg string, size int) {
   160  	l.regs = append(l.regs, regPos{op: op, reg: reg, pos: l.stack})
   161  	l.stack += size
   162  }
   163  
   164  func (l *layout) addSpecial(save, restore string, size int) {
   165  	l.regs = append(l.regs, regPos{save: save, restore: restore, pos: l.stack})
   166  	l.stack += size
   167  }
   168  
   169  func (l *layout) save() {
   170  	for _, reg := range l.regs {
   171  		if reg.save != "" {
   172  			p(reg.save, reg.pos)
   173  		} else {
   174  			p("%s %s, %d(%s)", reg.op, reg.reg, reg.pos, l.sp)
   175  		}
   176  	}
   177  }
   178  
   179  func (l *layout) restore() {
   180  	for i := len(l.regs) - 1; i >= 0; i-- {
   181  		reg := l.regs[i]
   182  		if reg.restore != "" {
   183  			p(reg.restore, reg.pos)
   184  		} else {
   185  			p("%s %d(%s), %s", reg.op, reg.pos, l.sp, reg.reg)
   186  		}
   187  	}
   188  }
   189  
   190  func gen386() {
   191  	p("PUSHFL")
   192  	// Save general purpose registers.
   193  	var l = layout{sp: "SP"}
   194  	for _, reg := range regNames386 {
   195  		if reg == "SP" || strings.HasPrefix(reg, "X") {
   196  			continue
   197  		}
   198  		l.add("MOVL", reg, 4)
   199  	}
   200  
   201  	softfloat := "GO386_softfloat"
   202  
   203  	// Save SSE state only if supported.
   204  	lSSE := layout{stack: l.stack, sp: "SP"}
   205  	for i := 0; i < 8; i++ {
   206  		lSSE.add("MOVUPS", fmt.Sprintf("X%d", i), 16)
   207  	}
   208  
   209  	p("ADJSP $%d", lSSE.stack)
   210  	p("NOP SP")
   211  	l.save()
   212  	p("#ifndef %s", softfloat)
   213  	lSSE.save()
   214  	p("#endif")
   215  	p("CALL ·asyncPreempt2(SB)")
   216  	p("#ifndef %s", softfloat)
   217  	lSSE.restore()
   218  	p("#endif")
   219  	l.restore()
   220  	p("ADJSP $%d", -lSSE.stack)
   221  
   222  	p("POPFL")
   223  	p("RET")
   224  }
   225  
   226  func genAMD64() {
   227  	// Assign stack offsets.
   228  	var l = layout{sp: "SP"}
   229  	for _, reg := range regNamesAMD64 {
   230  		if reg == "SP" || reg == "BP" {
   231  			continue
   232  		}
   233  		if !strings.HasPrefix(reg, "X") {
   234  			l.add("MOVQ", reg, 8)
   235  		}
   236  	}
   237  	lSSE := layout{stack: l.stack, sp: "SP"}
   238  	for _, reg := range regNamesAMD64 {
   239  		if strings.HasPrefix(reg, "X") {
   240  			lSSE.add("MOVUPS", reg, 16)
   241  		}
   242  	}
   243  
   244  	// TODO: MXCSR register?
   245  
   246  	p("PUSHQ BP")
   247  	p("MOVQ SP, BP")
   248  	p("// Save flags before clobbering them")
   249  	p("PUSHFQ")
   250  	p("// obj doesn't understand ADD/SUB on SP, but does understand ADJSP")
   251  	p("ADJSP $%d", lSSE.stack)
   252  	p("// But vet doesn't know ADJSP, so suppress vet stack checking")
   253  	p("NOP SP")
   254  
   255  	l.save()
   256  
   257  	// Apparently, the signal handling code path in darwin kernel leaves
   258  	// the upper bits of Y registers in a dirty state, which causes
   259  	// many SSE operations (128-bit and narrower) become much slower.
   260  	// Clear the upper bits to get to a clean state. See issue #37174.
   261  	// It is safe here as Go code don't use the upper bits of Y registers.
   262  	p("#ifdef GOOS_darwin")
   263  	p("CMPB internal∕cpu·X86+const_offsetX86HasAVX(SB), $0")
   264  	p("JE 2(PC)")
   265  	p("VZEROUPPER")
   266  	p("#endif")
   267  
   268  	lSSE.save()
   269  	p("CALL ·asyncPreempt2(SB)")
   270  	lSSE.restore()
   271  	l.restore()
   272  	p("ADJSP $%d", -lSSE.stack)
   273  	p("POPFQ")
   274  	p("POPQ BP")
   275  	p("RET")
   276  }
   277  
   278  func genARM() {
   279  	// Add integer registers R0-R12.
   280  	// R13 (SP), R14 (LR), R15 (PC) are special and not saved here.
   281  	var l = layout{sp: "R13", stack: 4} // add LR slot
   282  	for i := 0; i <= 12; i++ {
   283  		reg := fmt.Sprintf("R%d", i)
   284  		if i == 10 {
   285  			continue // R10 is g register, no need to save/restore
   286  		}
   287  		l.add("MOVW", reg, 4)
   288  	}
   289  	// Add flag register.
   290  	l.addSpecial(
   291  		"MOVW CPSR, R0\nMOVW R0, %d(R13)",
   292  		"MOVW %d(R13), R0\nMOVW R0, CPSR",
   293  		4)
   294  
   295  	// Add floating point registers F0-F15 and flag register.
   296  	var lfp = layout{stack: l.stack, sp: "R13"}
   297  	lfp.addSpecial(
   298  		"MOVW FPCR, R0\nMOVW R0, %d(R13)",
   299  		"MOVW %d(R13), R0\nMOVW R0, FPCR",
   300  		4)
   301  	for i := 0; i <= 15; i++ {
   302  		reg := fmt.Sprintf("F%d", i)
   303  		lfp.add("MOVD", reg, 8)
   304  	}
   305  
   306  	p("MOVW.W R14, -%d(R13)", lfp.stack) // allocate frame, save LR
   307  	l.save()
   308  	p("MOVB ·goarm(SB), R0\nCMP $6, R0\nBLT nofp") // test goarm, and skip FP registers if goarm=5.
   309  	lfp.save()
   310  	label("nofp:")
   311  	p("CALL ·asyncPreempt2(SB)")
   312  	p("MOVB ·goarm(SB), R0\nCMP $6, R0\nBLT nofp2") // test goarm, and skip FP registers if goarm=5.
   313  	lfp.restore()
   314  	label("nofp2:")
   315  	l.restore()
   316  
   317  	p("MOVW %d(R13), R14", lfp.stack)     // sigctxt.pushCall pushes LR on stack, restore it
   318  	p("MOVW.P %d(R13), R15", lfp.stack+4) // load PC, pop frame (including the space pushed by sigctxt.pushCall)
   319  	p("UNDEF")                            // shouldn't get here
   320  }
   321  
   322  func genARM64() {
   323  	// Add integer registers R0-R26
   324  	// R27 (REGTMP), R28 (g), R29 (FP), R30 (LR), R31 (SP) are special
   325  	// and not saved here.
   326  	var l = layout{sp: "RSP", stack: 8} // add slot to save PC of interrupted instruction
   327  	for i := 0; i <= 26; i++ {
   328  		if i == 18 {
   329  			continue // R18 is not used, skip
   330  		}
   331  		reg := fmt.Sprintf("R%d", i)
   332  		l.add("MOVD", reg, 8)
   333  	}
   334  	// Add flag registers.
   335  	l.addSpecial(
   336  		"MOVD NZCV, R0\nMOVD R0, %d(RSP)",
   337  		"MOVD %d(RSP), R0\nMOVD R0, NZCV",
   338  		8)
   339  	l.addSpecial(
   340  		"MOVD FPSR, R0\nMOVD R0, %d(RSP)",
   341  		"MOVD %d(RSP), R0\nMOVD R0, FPSR",
   342  		8)
   343  	// TODO: FPCR? I don't think we'll change it, so no need to save.
   344  	// Add floating point registers F0-F31.
   345  	for i := 0; i <= 31; i++ {
   346  		reg := fmt.Sprintf("F%d", i)
   347  		l.add("FMOVD", reg, 8)
   348  	}
   349  	if l.stack%16 != 0 {
   350  		l.stack += 8 // SP needs 16-byte alignment
   351  	}
   352  
   353  	// allocate frame, save PC of interrupted instruction (in LR)
   354  	p("MOVD R30, %d(RSP)", -l.stack)
   355  	p("SUB $%d, RSP", l.stack)
   356  	p("#ifdef GOOS_linux")
   357  	p("MOVD R29, -8(RSP)") // save frame pointer (only used on Linux)
   358  	p("SUB $8, RSP, R29")  // set up new frame pointer
   359  	p("#endif")
   360  	// On iOS, save the LR again after decrementing SP. We run the
   361  	// signal handler on the G stack (as it doesn't support sigaltstack),
   362  	// so any writes below SP may be clobbered.
   363  	p("#ifdef GOOS_ios")
   364  	p("MOVD R30, (RSP)")
   365  	p("#endif")
   366  
   367  	l.save()
   368  	p("CALL ·asyncPreempt2(SB)")
   369  	l.restore()
   370  
   371  	p("MOVD %d(RSP), R30", l.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it
   372  	p("#ifdef GOOS_linux")
   373  	p("MOVD -8(RSP), R29") // restore frame pointer
   374  	p("#endif")
   375  	p("MOVD (RSP), R27")          // load PC to REGTMP
   376  	p("ADD $%d, RSP", l.stack+16) // pop frame (including the space pushed by sigctxt.pushCall)
   377  	p("JMP (R27)")
   378  }
   379  
   380  func genMIPS(_64bit bool) {
   381  	mov := "MOVW"
   382  	movf := "MOVF"
   383  	add := "ADD"
   384  	sub := "SUB"
   385  	r28 := "R28"
   386  	regsize := 4
   387  	softfloat := "GOMIPS_softfloat"
   388  	if _64bit {
   389  		mov = "MOVV"
   390  		movf = "MOVD"
   391  		add = "ADDV"
   392  		sub = "SUBV"
   393  		r28 = "RSB"
   394  		regsize = 8
   395  		softfloat = "GOMIPS64_softfloat"
   396  	}
   397  
   398  	// Add integer registers R1-R22, R24-R25, R28
   399  	// R0 (zero), R23 (REGTMP), R29 (SP), R30 (g), R31 (LR) are special,
   400  	// and not saved here. R26 and R27 are reserved by kernel and not used.
   401  	var l = layout{sp: "R29", stack: regsize} // add slot to save PC of interrupted instruction (in LR)
   402  	for i := 1; i <= 25; i++ {
   403  		if i == 23 {
   404  			continue // R23 is REGTMP
   405  		}
   406  		reg := fmt.Sprintf("R%d", i)
   407  		l.add(mov, reg, regsize)
   408  	}
   409  	l.add(mov, r28, regsize)
   410  	l.addSpecial(
   411  		mov+" HI, R1\n"+mov+" R1, %d(R29)",
   412  		mov+" %d(R29), R1\n"+mov+" R1, HI",
   413  		regsize)
   414  	l.addSpecial(
   415  		mov+" LO, R1\n"+mov+" R1, %d(R29)",
   416  		mov+" %d(R29), R1\n"+mov+" R1, LO",
   417  		regsize)
   418  
   419  	// Add floating point control/status register FCR31 (FCR0-FCR30 are irrelevant)
   420  	var lfp = layout{sp: "R29", stack: l.stack}
   421  	lfp.addSpecial(
   422  		mov+" FCR31, R1\n"+mov+" R1, %d(R29)",
   423  		mov+" %d(R29), R1\n"+mov+" R1, FCR31",
   424  		regsize)
   425  	// Add floating point registers F0-F31.
   426  	for i := 0; i <= 31; i++ {
   427  		reg := fmt.Sprintf("F%d", i)
   428  		lfp.add(movf, reg, regsize)
   429  	}
   430  
   431  	// allocate frame, save PC of interrupted instruction (in LR)
   432  	p(mov+" R31, -%d(R29)", lfp.stack)
   433  	p(sub+" $%d, R29", lfp.stack)
   434  
   435  	l.save()
   436  	p("#ifndef %s", softfloat)
   437  	lfp.save()
   438  	p("#endif")
   439  	p("CALL ·asyncPreempt2(SB)")
   440  	p("#ifndef %s", softfloat)
   441  	lfp.restore()
   442  	p("#endif")
   443  	l.restore()
   444  
   445  	p(mov+" %d(R29), R31", lfp.stack)     // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it
   446  	p(mov + " (R29), R23")                // load PC to REGTMP
   447  	p(add+" $%d, R29", lfp.stack+regsize) // pop frame (including the space pushed by sigctxt.pushCall)
   448  	p("JMP (R23)")
   449  }
   450  
   451  func genPPC64() {
   452  	// Add integer registers R3-R29
   453  	// R0 (zero), R1 (SP), R30 (g) are special and not saved here.
   454  	// R2 (TOC pointer in PIC mode), R12 (function entry address in PIC mode) have been saved in sigctxt.pushCall.
   455  	// R31 (REGTMP) will be saved manually.
   456  	var l = layout{sp: "R1", stack: 32 + 8} // MinFrameSize on PPC64, plus one word for saving R31
   457  	for i := 3; i <= 29; i++ {
   458  		if i == 12 || i == 13 {
   459  			// R12 has been saved in sigctxt.pushCall.
   460  			// R13 is TLS pointer, not used by Go code. we must NOT
   461  			// restore it, otherwise if we parked and resumed on a
   462  			// different thread we'll mess up TLS addresses.
   463  			continue
   464  		}
   465  		reg := fmt.Sprintf("R%d", i)
   466  		l.add("MOVD", reg, 8)
   467  	}
   468  	l.addSpecial(
   469  		"MOVW CR, R31\nMOVW R31, %d(R1)",
   470  		"MOVW %d(R1), R31\nMOVFL R31, $0xff", // this is MOVW R31, CR
   471  		8)                                    // CR is 4-byte wide, but just keep the alignment
   472  	l.addSpecial(
   473  		"MOVD XER, R31\nMOVD R31, %d(R1)",
   474  		"MOVD %d(R1), R31\nMOVD R31, XER",
   475  		8)
   476  	// Add floating point registers F0-F31.
   477  	for i := 0; i <= 31; i++ {
   478  		reg := fmt.Sprintf("F%d", i)
   479  		l.add("FMOVD", reg, 8)
   480  	}
   481  	// Add floating point control/status register FPSCR.
   482  	l.addSpecial(
   483  		"MOVFL FPSCR, F0\nFMOVD F0, %d(R1)",
   484  		"FMOVD %d(R1), F0\nMOVFL F0, FPSCR",
   485  		8)
   486  
   487  	p("MOVD R31, -%d(R1)", l.stack-32) // save R31 first, we'll use R31 for saving LR
   488  	p("MOVD LR, R31")
   489  	p("MOVDU R31, -%d(R1)", l.stack) // allocate frame, save PC of interrupted instruction (in LR)
   490  
   491  	l.save()
   492  	p("CALL ·asyncPreempt2(SB)")
   493  	l.restore()
   494  
   495  	p("MOVD %d(R1), R31", l.stack) // sigctxt.pushCall has pushed LR, R2, R12 (at interrupt) on stack, restore them
   496  	p("MOVD R31, LR")
   497  	p("MOVD %d(R1), R2", l.stack+8)
   498  	p("MOVD %d(R1), R12", l.stack+16)
   499  	p("MOVD (R1), R31") // load PC to CTR
   500  	p("MOVD R31, CTR")
   501  	p("MOVD 32(R1), R31")        // restore R31
   502  	p("ADD $%d, R1", l.stack+32) // pop frame (including the space pushed by sigctxt.pushCall)
   503  	p("JMP (CTR)")
   504  }
   505  
   506  func genRISCV64() {
   507  	// X0 (zero), X1 (LR), X2 (SP), X3 (GP), X4 (TP), X27 (g), X31 (TMP) are special.
   508  	var l = layout{sp: "X2", stack: 8}
   509  
   510  	// Add integer registers (X5-X26, X28-30).
   511  	for i := 5; i < 31; i++ {
   512  		if i == 27 {
   513  			continue
   514  		}
   515  		reg := fmt.Sprintf("X%d", i)
   516  		l.add("MOV", reg, 8)
   517  	}
   518  
   519  	// Add floating point registers (F0-F31).
   520  	for i := 0; i <= 31; i++ {
   521  		reg := fmt.Sprintf("F%d", i)
   522  		l.add("MOVD", reg, 8)
   523  	}
   524  
   525  	p("MOV X1, -%d(X2)", l.stack)
   526  	p("ADD $-%d, X2", l.stack)
   527  	l.save()
   528  	p("CALL ·asyncPreempt2(SB)")
   529  	l.restore()
   530  	p("MOV %d(X2), X1", l.stack)
   531  	p("MOV (X2), X31")
   532  	p("ADD $%d, X2", l.stack+8)
   533  	p("JMP (X31)")
   534  }
   535  
   536  func genS390X() {
   537  	// Add integer registers R0-R12
   538  	// R13 (g), R14 (LR), R15 (SP) are special, and not saved here.
   539  	// Saving R10 (REGTMP) is not necessary, but it is saved anyway.
   540  	var l = layout{sp: "R15", stack: 16} // add slot to save PC of interrupted instruction and flags
   541  	l.addSpecial(
   542  		"STMG R0, R12, %d(R15)",
   543  		"LMG %d(R15), R0, R12",
   544  		13*8)
   545  	// Add floating point registers F0-F31.
   546  	for i := 0; i <= 15; i++ {
   547  		reg := fmt.Sprintf("F%d", i)
   548  		l.add("FMOVD", reg, 8)
   549  	}
   550  
   551  	// allocate frame, save PC of interrupted instruction (in LR) and flags (condition code)
   552  	p("IPM R10") // save flags upfront, as ADD will clobber flags
   553  	p("MOVD R14, -%d(R15)", l.stack)
   554  	p("ADD $-%d, R15", l.stack)
   555  	p("MOVW R10, 8(R15)") // save flags
   556  
   557  	l.save()
   558  	p("CALL ·asyncPreempt2(SB)")
   559  	l.restore()
   560  
   561  	p("MOVD %d(R15), R14", l.stack)    // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it
   562  	p("ADD $%d, R15", l.stack+8)       // pop frame (including the space pushed by sigctxt.pushCall)
   563  	p("MOVWZ -%d(R15), R10", l.stack)  // load flags to REGTMP
   564  	p("TMLH R10, $(3<<12)")            // restore flags
   565  	p("MOVD -%d(R15), R10", l.stack+8) // load PC to REGTMP
   566  	p("JMP (R10)")
   567  }
   568  
   569  func genWasm() {
   570  	p("// No async preemption on wasm")
   571  	p("UNDEF")
   572  }
   573  
   574  func notImplemented() {
   575  	p("// Not implemented yet")
   576  	p("JMP ·abort(SB)")
   577  }
   578  

View as plain text