buildtool.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go 1.4.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go 1.4 go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"os"
    17  	"path/filepath"
    18  	"regexp"
    19  	"runtime"
    20  	"strings"
    21  )
    22  
    23  // bootstrapDirs is a list of directories holding code that must be
    24  // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
    25  // All directories in this list are relative to and must be below $GOROOT/src.
    26  //
    27  // The list has two kinds of entries: names beginning with cmd/ with
    28  // no other slashes, which are commands, and other paths, which are packages
    29  // supporting the commands. Packages in the standard library can be listed
    30  // if a newer copy needs to be substituted for the Go 1.4 copy when used
    31  // by the command packages. Paths ending with /... automatically
    32  // include all packages within subdirectories as well.
    33  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    34  var bootstrapDirs = []string{
    35  	"cmd/asm",
    36  	"cmd/asm/internal/...",
    37  	"cmd/cgo",
    38  	"cmd/compile",
    39  	"cmd/compile/internal/...",
    40  	"cmd/internal/archive",
    41  	"cmd/internal/bio",
    42  	"cmd/internal/codesign",
    43  	"cmd/internal/dwarf",
    44  	"cmd/internal/edit",
    45  	"cmd/internal/gcprog",
    46  	"cmd/internal/goobj",
    47  	"cmd/internal/obj/...",
    48  	"cmd/internal/objabi",
    49  	"cmd/internal/pkgpath",
    50  	"cmd/internal/quoted",
    51  	"cmd/internal/src",
    52  	"cmd/internal/sys",
    53  	"cmd/link",
    54  	"cmd/link/internal/...",
    55  	"compress/flate",
    56  	"compress/zlib",
    57  	"container/heap",
    58  	"debug/dwarf",
    59  	"debug/elf",
    60  	"debug/macho",
    61  	"debug/pe",
    62  	"go/constant",
    63  	"internal/buildcfg",
    64  	"internal/goexperiment",
    65  	"internal/goversion",
    66  	"internal/race",
    67  	"internal/unsafeheader",
    68  	"internal/xcoff",
    69  	"math/big",
    70  	"math/bits",
    71  	"sort",
    72  	"strconv",
    73  }
    74  
    75  // File prefixes that are ignored by go/build anyway, and cause
    76  // problems with editor generated temporary files (#18931).
    77  var ignorePrefixes = []string{
    78  	".",
    79  	"_",
    80  	"#",
    81  }
    82  
    83  // File suffixes that use build tags introduced since Go 1.4.
    84  // These must not be copied into the bootstrap build directory.
    85  // Also ignore test files.
    86  var ignoreSuffixes = []string{
    87  	"_arm64.s",
    88  	"_arm64.go",
    89  	"_riscv64.s",
    90  	"_riscv64.go",
    91  	"_wasm.s",
    92  	"_wasm.go",
    93  	"_test.s",
    94  	"_test.go",
    95  }
    96  
    97  var tryDirs = []string{
    98  	"sdk/go1.17",
    99  	"go1.17",
   100  }
   101  
   102  func bootstrapBuildTools() {
   103  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   104  	if goroot_bootstrap == "" {
   105  		home := os.Getenv("HOME")
   106  		goroot_bootstrap = pathf("%s/go1.4", home)
   107  		for _, d := range tryDirs {
   108  			if p := pathf("%s/%s", home, d); isdir(p) {
   109  				goroot_bootstrap = p
   110  			}
   111  		}
   112  	}
   113  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   114  
   115  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   116  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   117  
   118  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   119  	// We use a subdirectory of $GOROOT/pkg because that's the
   120  	// space within $GOROOT where we store all generated objects.
   121  	// We could use a temporary directory outside $GOROOT instead,
   122  	// but it is easier to debug on failure if the files are in a known location.
   123  	workspace := pathf("%s/pkg/bootstrap", goroot)
   124  	xremoveall(workspace)
   125  	xatexit(func() { xremoveall(workspace) })
   126  	base := pathf("%s/src/bootstrap", workspace)
   127  	xmkdirall(base)
   128  
   129  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   130  	writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
   131  	for _, dir := range bootstrapDirs {
   132  		recurse := strings.HasSuffix(dir, "/...")
   133  		dir = strings.TrimSuffix(dir, "/...")
   134  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   135  			if err != nil {
   136  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   137  			}
   138  
   139  			name := filepath.Base(path)
   140  			src := pathf("%s/src/%s", goroot, path)
   141  			dst := pathf("%s/%s", base, path)
   142  
   143  			if info.IsDir() {
   144  				if !recurse && path != dir || name == "testdata" {
   145  					return filepath.SkipDir
   146  				}
   147  
   148  				xmkdirall(dst)
   149  				if path == "cmd/cgo" {
   150  					// Write to src because we need the file both for bootstrap
   151  					// and for later in the main build.
   152  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   153  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   154  				}
   155  				return nil
   156  			}
   157  
   158  			for _, pre := range ignorePrefixes {
   159  				if strings.HasPrefix(name, pre) {
   160  					return nil
   161  				}
   162  			}
   163  			for _, suf := range ignoreSuffixes {
   164  				if strings.HasSuffix(name, suf) {
   165  					return nil
   166  				}
   167  			}
   168  
   169  			text := bootstrapRewriteFile(src)
   170  			writefile(text, dst, 0)
   171  			return nil
   172  		})
   173  	}
   174  
   175  	// Set up environment for invoking Go 1.4 go command.
   176  	// GOROOT points at Go 1.4 GOROOT,
   177  	// GOPATH points at our bootstrap workspace,
   178  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   179  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   180  	// so that Go 1.4 builds whatever kind of binary it knows how to build.
   181  	// Restore GOROOT, GOPATH, and GOBIN when done.
   182  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   183  	// because setup will take care of those when bootstrapBuildTools returns.
   184  
   185  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   186  	os.Setenv("GOROOT", goroot_bootstrap)
   187  
   188  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   189  	os.Setenv("GOPATH", workspace)
   190  
   191  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   192  	os.Setenv("GOBIN", "")
   193  
   194  	os.Setenv("GOOS", "")
   195  	os.Setenv("GOHOSTOS", "")
   196  	os.Setenv("GOARCH", "")
   197  	os.Setenv("GOHOSTARCH", "")
   198  
   199  	// Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
   200  	// workaround bugs in Go 1.4's compiler. See discussion thread:
   201  	// https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
   202  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   203  	// which may contain unsupported instructions.
   204  	// Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
   205  	// only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
   206  	// or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
   207  	cmd := []string{
   208  		pathf("%s/bin/go", goroot_bootstrap),
   209  		"install",
   210  		"-gcflags=-l",
   211  		"-tags=math_big_pure_go compiler_bootstrap",
   212  	}
   213  	if vflag > 0 {
   214  		cmd = append(cmd, "-v")
   215  	}
   216  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   217  		cmd = append(cmd, "-toolexec="+tool)
   218  	}
   219  	cmd = append(cmd, "bootstrap/cmd/...")
   220  	run(base, ShowOutput|CheckExit, cmd...)
   221  
   222  	// Copy binaries into tool binary directory.
   223  	for _, name := range bootstrapDirs {
   224  		if !strings.HasPrefix(name, "cmd/") {
   225  			continue
   226  		}
   227  		name = name[len("cmd/"):]
   228  		if !strings.Contains(name, "/") {
   229  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   230  		}
   231  	}
   232  
   233  	if vflag > 0 {
   234  		xprintf("\n")
   235  	}
   236  }
   237  
   238  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   239  
   240  // isUnneededSSARewriteFile reports whether srcFile is a
   241  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   242  // architecture that isn't for the current runtime.GOARCH.
   243  //
   244  // When unneeded is true archCaps is the rewrite base filename without
   245  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   246  func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
   247  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   248  		return "", false
   249  	}
   250  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   251  	if fileArch == "" {
   252  		return "", false
   253  	}
   254  	b := fileArch[0]
   255  	if b == '_' || ('a' <= b && b <= 'z') {
   256  		return "", false
   257  	}
   258  	archCaps = fileArch
   259  	fileArch = strings.ToLower(fileArch)
   260  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   261  	if fileArch == os.Getenv("GOHOSTARCH") {
   262  		return "", false
   263  	}
   264  	if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
   265  		return "", false
   266  	}
   267  	if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
   268  		return "", false
   269  	}
   270  	return archCaps, true
   271  }
   272  
   273  func bootstrapRewriteFile(srcFile string) string {
   274  	// During bootstrap, generate dummy rewrite files for
   275  	// irrelevant architectures. We only need to build a bootstrap
   276  	// binary that works for the current runtime.GOARCH.
   277  	// This saves 6+ seconds of bootstrap.
   278  	if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
   279  		return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
   280  
   281  package ssa
   282  
   283  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   284  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   285  `, archCaps, archCaps)
   286  	}
   287  
   288  	return bootstrapFixImports(srcFile)
   289  }
   290  
   291  func bootstrapFixImports(srcFile string) string {
   292  	text := readfile(srcFile)
   293  	if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
   294  		text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
   295  	}
   296  	lines := strings.SplitAfter(text, "\n")
   297  	inBlock := false
   298  	for i, line := range lines {
   299  		if strings.HasPrefix(line, "import (") {
   300  			inBlock = true
   301  			continue
   302  		}
   303  		if inBlock && strings.HasPrefix(line, ")") {
   304  			inBlock = false
   305  			continue
   306  		}
   307  		if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
   308  			inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
   309  			line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
   310  			// During bootstrap, must use plain os/exec.
   311  			line = strings.Replace(line, `exec "internal/execabs"`, `"os/exec"`, -1)
   312  			for _, dir := range bootstrapDirs {
   313  				if strings.HasPrefix(dir, "cmd/") {
   314  					continue
   315  				}
   316  				line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
   317  			}
   318  			lines[i] = line
   319  		}
   320  	}
   321  
   322  	lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   323  
   324  	return strings.Join(lines, "")
   325  }
   326
View as plain text