Source file src/cmd/dist/build.go

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"bytes"
     9  	"encoding/json"
    10  	"flag"
    11  	"fmt"
    12  	"io/ioutil"
    13  	"log"
    14  	"os"
    15  	"os/exec"
    16  	"path/filepath"
    17  	"regexp"
    18  	"sort"
    19  	"strings"
    20  	"sync"
    21  	"time"
    22  )
    23  
    24  // Initialization for any invocation.
    25  
    26  // The usual variables.
    27  var (
    28  	goarch           string
    29  	gobin            string
    30  	gohostarch       string
    31  	gohostos         string
    32  	goos             string
    33  	goarm            string
    34  	go386            string
    35  	goamd64          string
    36  	gomips           string
    37  	gomips64         string
    38  	goppc64          string
    39  	goroot           string
    40  	goroot_final     string
    41  	goextlinkenabled string
    42  	gogcflags        string // For running built compiler
    43  	goldflags        string
    44  	goexperiment     string
    45  	workdir          string
    46  	tooldir          string
    47  	oldgoos          string
    48  	oldgoarch        string
    49  	exe              string
    50  	defaultcc        map[string]string
    51  	defaultcxx       map[string]string
    52  	defaultpkgconfig string
    53  	defaultldso      string
    54  
    55  	rebuildall   bool
    56  	defaultclang bool
    57  
    58  	vflag int // verbosity
    59  )
    60  
    61  // The known architectures.
    62  var okgoarch = []string{
    63  	"386",
    64  	"amd64",
    65  	"arm",
    66  	"arm64",
    67  	"mips",
    68  	"mipsle",
    69  	"mips64",
    70  	"mips64le",
    71  	"ppc64",
    72  	"ppc64le",
    73  	"riscv64",
    74  	"s390x",
    75  	"sparc64",
    76  	"wasm",
    77  }
    78  
    79  // The known operating systems.
    80  var okgoos = []string{
    81  	"darwin",
    82  	"dragonfly",
    83  	"illumos",
    84  	"ios",
    85  	"js",
    86  	"linux",
    87  	"android",
    88  	"solaris",
    89  	"freebsd",
    90  	"nacl", // keep;
    91  	"netbsd",
    92  	"openbsd",
    93  	"plan9",
    94  	"windows",
    95  	"aix",
    96  }
    97  
    98  // find reports the first index of p in l[0:n], or else -1.
    99  func find(p string, l []string) int {
   100  	for i, s := range l {
   101  		if p == s {
   102  			return i
   103  		}
   104  	}
   105  	return -1
   106  }
   107  
   108  // xinit handles initialization of the various global state, like goroot and goarch.
   109  func xinit() {
   110  	b := os.Getenv("GOROOT")
   111  	if b == "" {
   112  		fatalf("$GOROOT must be set")
   113  	}
   114  	goroot = filepath.Clean(b)
   115  
   116  	b = os.Getenv("GOROOT_FINAL")
   117  	if b == "" {
   118  		b = goroot
   119  	}
   120  	goroot_final = b
   121  
   122  	b = os.Getenv("GOBIN")
   123  	if b == "" {
   124  		b = pathf("%s/bin", goroot)
   125  	}
   126  	gobin = b
   127  
   128  	b = os.Getenv("GOOS")
   129  	if b == "" {
   130  		b = gohostos
   131  	}
   132  	goos = b
   133  	if find(goos, okgoos) < 0 {
   134  		fatalf("unknown $GOOS %s", goos)
   135  	}
   136  
   137  	b = os.Getenv("GOARM")
   138  	if b == "" {
   139  		b = xgetgoarm()
   140  	}
   141  	goarm = b
   142  
   143  	b = os.Getenv("GO386")
   144  	if b == "" {
   145  		b = "sse2"
   146  	}
   147  	go386 = b
   148  
   149  	b = os.Getenv("GOAMD64")
   150  	if b == "" {
   151  		b = "v1"
   152  	}
   153  	goamd64 = b
   154  
   155  	b = os.Getenv("GOMIPS")
   156  	if b == "" {
   157  		b = "hardfloat"
   158  	}
   159  	gomips = b
   160  
   161  	b = os.Getenv("GOMIPS64")
   162  	if b == "" {
   163  		b = "hardfloat"
   164  	}
   165  	gomips64 = b
   166  
   167  	b = os.Getenv("GOPPC64")
   168  	if b == "" {
   169  		b = "power8"
   170  	}
   171  	goppc64 = b
   172  
   173  	if p := pathf("%s/src/all.bash", goroot); !isfile(p) {
   174  		fatalf("$GOROOT is not set correctly or not exported\n"+
   175  			"\tGOROOT=%s\n"+
   176  			"\t%s does not exist", goroot, p)
   177  	}
   178  
   179  	b = os.Getenv("GOHOSTARCH")
   180  	if b != "" {
   181  		gohostarch = b
   182  	}
   183  	if find(gohostarch, okgoarch) < 0 {
   184  		fatalf("unknown $GOHOSTARCH %s", gohostarch)
   185  	}
   186  
   187  	b = os.Getenv("GOARCH")
   188  	if b == "" {
   189  		b = gohostarch
   190  	}
   191  	goarch = b
   192  	if find(goarch, okgoarch) < 0 {
   193  		fatalf("unknown $GOARCH %s", goarch)
   194  	}
   195  
   196  	b = os.Getenv("GO_EXTLINK_ENABLED")
   197  	if b != "" {
   198  		if b != "0" && b != "1" {
   199  			fatalf("unknown $GO_EXTLINK_ENABLED %s", b)
   200  		}
   201  		goextlinkenabled = b
   202  	}
   203  
   204  	goexperiment = os.Getenv("GOEXPERIMENT")
   205  	// TODO(mdempsky): Validate known experiments?
   206  
   207  	gogcflags = os.Getenv("BOOT_GO_GCFLAGS")
   208  	goldflags = os.Getenv("BOOT_GO_LDFLAGS")
   209  
   210  	cc, cxx := "gcc", "g++"
   211  	if defaultclang {
   212  		cc, cxx = "clang", "clang++"
   213  	}
   214  	defaultcc = compilerEnv("CC", cc)
   215  	defaultcxx = compilerEnv("CXX", cxx)
   216  
   217  	b = os.Getenv("PKG_CONFIG")
   218  	if b == "" {
   219  		b = "pkg-config"
   220  	}
   221  	defaultpkgconfig = b
   222  
   223  	defaultldso = os.Getenv("GO_LDSO")
   224  
   225  	// For tools being invoked but also for os.ExpandEnv.
   226  	os.Setenv("GO386", go386)
   227  	os.Setenv("GOAMD64", goamd64)
   228  	os.Setenv("GOARCH", goarch)
   229  	os.Setenv("GOARM", goarm)
   230  	os.Setenv("GOHOSTARCH", gohostarch)
   231  	os.Setenv("GOHOSTOS", gohostos)
   232  	os.Setenv("GOOS", goos)
   233  	os.Setenv("GOMIPS", gomips)
   234  	os.Setenv("GOMIPS64", gomips64)
   235  	os.Setenv("GOPPC64", goppc64)
   236  	os.Setenv("GOROOT", goroot)
   237  	os.Setenv("GOROOT_FINAL", goroot_final)
   238  
   239  	// Use a build cache separate from the default user one.
   240  	// Also one that will be wiped out during startup, so that
   241  	// make.bash really does start from a clean slate.
   242  	os.Setenv("GOCACHE", pathf("%s/pkg/obj/go-build", goroot))
   243  
   244  	// Make the environment more predictable.
   245  	os.Setenv("LANG", "C")
   246  	os.Setenv("LANGUAGE", "en_US.UTF8")
   247  
   248  	workdir = xworkdir()
   249  	if err := ioutil.WriteFile(pathf("%s/go.mod", workdir), []byte("module bootstrap"), 0666); err != nil {
   250  		fatalf("cannot write stub go.mod: %s", err)
   251  	}
   252  	xatexit(rmworkdir)
   253  
   254  	tooldir = pathf("%s/pkg/tool/%s_%s", goroot, gohostos, gohostarch)
   255  }
   256  
   257  // compilerEnv returns a map from "goos/goarch" to the
   258  // compiler setting to use for that platform.
   259  // The entry for key "" covers any goos/goarch not explicitly set in the map.
   260  // For example, compilerEnv("CC", "gcc") returns the C compiler settings
   261  // read from $CC, defaulting to gcc.
   262  //
   263  // The result is a map because additional environment variables
   264  // can be set to change the compiler based on goos/goarch settings.
   265  // The following applies to all envNames but CC is assumed to simplify
   266  // the presentation.
   267  //
   268  // If no environment variables are set, we use def for all goos/goarch.
   269  // $CC, if set, applies to all goos/goarch but is overridden by the following.
   270  // $CC_FOR_TARGET, if set, applies to all goos/goarch except gohostos/gohostarch,
   271  // but is overridden by the following.
   272  // If gohostos=goos and gohostarch=goarch, then $CC_FOR_TARGET applies even for gohostos/gohostarch.
   273  // $CC_FOR_goos_goarch, if set, applies only to goos/goarch.
   274  func compilerEnv(envName, def string) map[string]string {
   275  	m := map[string]string{"": def}
   276  
   277  	if env := os.Getenv(envName); env != "" {
   278  		m[""] = env
   279  	}
   280  	if env := os.Getenv(envName + "_FOR_TARGET"); env != "" {
   281  		if gohostos != goos || gohostarch != goarch {
   282  			m[gohostos+"/"+gohostarch] = m[""]
   283  		}
   284  		m[""] = env
   285  	}
   286  
   287  	for _, goos := range okgoos {
   288  		for _, goarch := range okgoarch {
   289  			if env := os.Getenv(envName + "_FOR_" + goos + "_" + goarch); env != "" {
   290  				m[goos+"/"+goarch] = env
   291  			}
   292  		}
   293  	}
   294  
   295  	return m
   296  }
   297  
   298  // compilerEnvLookup returns the compiler settings for goos/goarch in map m.
   299  func compilerEnvLookup(m map[string]string, goos, goarch string) string {
   300  	if cc := m[goos+"/"+goarch]; cc != "" {
   301  		return cc
   302  	}
   303  	return m[""]
   304  }
   305  
   306  // rmworkdir deletes the work directory.
   307  func rmworkdir() {
   308  	if vflag > 1 {
   309  		errprintf("rm -rf %s\n", workdir)
   310  	}
   311  	xremoveall(workdir)
   312  }
   313  
   314  // Remove trailing spaces.
   315  func chomp(s string) string {
   316  	return strings.TrimRight(s, " \t\r\n")
   317  }
   318  
   319  func branchtag(branch string) (tag string, precise bool) {
   320  	log := run(goroot, CheckExit, "git", "log", "--decorate=full", "--format=format:%d", "master.."+branch)
   321  	tag = branch
   322  	for row, line := range strings.Split(log, "\n") {
   323  		// Each line is either blank, or looks like
   324  		//	  (tag: refs/tags/go1.4rc2, refs/remotes/origin/release-branch.go1.4, refs/heads/release-branch.go1.4)
   325  		// We need to find an element starting with refs/tags/.
   326  		const s = " refs/tags/"
   327  		i := strings.Index(line, s)
   328  		if i < 0 {
   329  			continue
   330  		}
   331  		// Trim off known prefix.
   332  		line = line[i+len(s):]
   333  		// The tag name ends at a comma or paren.
   334  		j := strings.IndexAny(line, ",)")
   335  		if j < 0 {
   336  			continue // malformed line; ignore it
   337  		}
   338  		tag = line[:j]
   339  		if row == 0 {
   340  			precise = true // tag denotes HEAD
   341  		}
   342  		break
   343  	}
   344  	return
   345  }
   346  
   347  // findgoversion determines the Go version to use in the version string.
   348  func findgoversion() string {
   349  	// The $GOROOT/VERSION file takes priority, for distributions
   350  	// without the source repo.
   351  	path := pathf("%s/VERSION", goroot)
   352  	if isfile(path) {
   353  		b := chomp(readfile(path))
   354  		// Commands such as "dist version > VERSION" will cause
   355  		// the shell to create an empty VERSION file and set dist's
   356  		// stdout to its fd. dist in turn looks at VERSION and uses
   357  		// its content if available, which is empty at this point.
   358  		// Only use the VERSION file if it is non-empty.
   359  		if b != "" {
   360  			// Some builders cross-compile the toolchain on linux-amd64
   361  			// and then copy the toolchain to the target builder (say, linux-arm)
   362  			// for use there. But on non-release (devel) branches, the compiler
   363  			// used on linux-amd64 will be an amd64 binary, and the compiler
   364  			// shipped to linux-arm will be an arm binary, so they will have different
   365  			// content IDs (they are binaries for different architectures) and so the
   366  			// packages compiled by the running-on-amd64 compiler will appear
   367  			// stale relative to the running-on-arm compiler. Avoid this by setting
   368  			// the version string to something that doesn't begin with devel.
   369  			// Then the version string will be used in place of the content ID,
   370  			// and the packages will look up-to-date.
   371  			// TODO(rsc): Really the builders could be writing out a better VERSION file instead,
   372  			// but it is easier to change cmd/dist than to try to make changes to
   373  			// the builder while Brad is away.
   374  			if strings.HasPrefix(b, "devel") {
   375  				if hostType := os.Getenv("META_BUILDLET_HOST_TYPE"); strings.Contains(hostType, "-cross") {
   376  					fmt.Fprintf(os.Stderr, "warning: changing VERSION from %q to %q\n", b, "builder "+hostType)
   377  					b = "builder " + hostType
   378  				}
   379  			}
   380  			return b
   381  		}
   382  	}
   383  
   384  	// The $GOROOT/VERSION.cache file is a cache to avoid invoking
   385  	// git every time we run this command. Unlike VERSION, it gets
   386  	// deleted by the clean command.
   387  	path = pathf("%s/VERSION.cache", goroot)
   388  	if isfile(path) {
   389  		return chomp(readfile(path))
   390  	}
   391  
   392  	// Show a nicer error message if this isn't a Git repo.
   393  	if !isGitRepo() {
   394  		fatalf("FAILED: not a Git repo; must put a VERSION file in $GOROOT")
   395  	}
   396  
   397  	// Otherwise, use Git.
   398  	// What is the current branch?
   399  	branch := chomp(run(goroot, CheckExit, "git", "rev-parse", "--abbrev-ref", "HEAD"))
   400  
   401  	// What are the tags along the current branch?
   402  	tag := "devel"
   403  	precise := false
   404  
   405  	// If we're on a release branch, use the closest matching tag
   406  	// that is on the release branch (and not on the master branch).
   407  	if strings.HasPrefix(branch, "release-branch.") {
   408  		tag, precise = branchtag(branch)
   409  	}
   410  
   411  	if !precise {
   412  		// Tag does not point at HEAD; add 1.x base version, hash, and date to version.
   413  		//
   414  		// Note that we lightly parse internal/goversion/goversion.go to
   415  		// obtain the base version. We can't just import the package,
   416  		// because cmd/dist is built with a bootstrap GOROOT which could
   417  		// be an entirely different version of Go, like 1.4. We assume
   418  		// that the file contains "const Version = <Integer>".
   419  
   420  		goversionSource := readfile(pathf("%s/src/internal/goversion/goversion.go", goroot))
   421  		m := regexp.MustCompile(`(?m)^const Version = (\d+)`).FindStringSubmatch(goversionSource)
   422  		if m == nil {
   423  			fatalf("internal/goversion/goversion.go does not contain 'const Version = ...'")
   424  		}
   425  		tag += fmt.Sprintf(" go1.%s-", m[1])
   426  
   427  		tag += chomp(run(goroot, CheckExit, "git", "log", "-n", "1", "--format=format:%h %cd", "HEAD"))
   428  	}
   429  
   430  	// Cache version.
   431  	writefile(tag, path, 0)
   432  
   433  	return tag
   434  }
   435  
   436  // isGitRepo reports whether the working directory is inside a Git repository.
   437  func isGitRepo() bool {
   438  	// NB: simply checking the exit code of `git rev-parse --git-dir` would
   439  	// suffice here, but that requires deviating from the infrastructure
   440  	// provided by `run`.
   441  	gitDir := chomp(run(goroot, 0, "git", "rev-parse", "--git-dir"))
   442  	if !filepath.IsAbs(gitDir) {
   443  		gitDir = filepath.Join(goroot, gitDir)
   444  	}
   445  	return isdir(gitDir)
   446  }
   447  
   448  /*
   449   * Initial tree setup.
   450   */
   451  
   452  // The old tools that no longer live in $GOBIN or $GOROOT/bin.
   453  var oldtool = []string{
   454  	"5a", "5c", "5g", "5l",
   455  	"6a", "6c", "6g", "6l",
   456  	"8a", "8c", "8g", "8l",
   457  	"9a", "9c", "9g", "9l",
   458  	"6cov",
   459  	"6nm",
   460  	"6prof",
   461  	"cgo",
   462  	"ebnflint",
   463  	"goapi",
   464  	"gofix",
   465  	"goinstall",
   466  	"gomake",
   467  	"gopack",
   468  	"gopprof",
   469  	"gotest",
   470  	"gotype",
   471  	"govet",
   472  	"goyacc",
   473  	"quietgcc",
   474  }
   475  
   476  // Unreleased directories (relative to $GOROOT) that should
   477  // not be in release branches.
   478  var unreleased = []string{
   479  	"src/cmd/newlink",
   480  	"src/cmd/objwriter",
   481  	"src/debug/goobj",
   482  	"src/old",
   483  }
   484  
   485  // setup sets up the tree for the initial build.
   486  func setup() {
   487  	// Create bin directory.
   488  	if p := pathf("%s/bin", goroot); !isdir(p) {
   489  		xmkdir(p)
   490  	}
   491  
   492  	// Create package directory.
   493  	if p := pathf("%s/pkg", goroot); !isdir(p) {
   494  		xmkdir(p)
   495  	}
   496  
   497  	p := pathf("%s/pkg/%s_%s", goroot, gohostos, gohostarch)
   498  	if rebuildall {
   499  		xremoveall(p)
   500  	}
   501  	xmkdirall(p)
   502  
   503  	if goos != gohostos || goarch != gohostarch {
   504  		p := pathf("%s/pkg/%s_%s", goroot, goos, goarch)
   505  		if rebuildall {
   506  			xremoveall(p)
   507  		}
   508  		xmkdirall(p)
   509  	}
   510  
   511  	// Create object directory.
   512  	// We used to use it for C objects.
   513  	// Now we use it for the build cache, to separate dist's cache
   514  	// from any other cache the user might have.
   515  	p = pathf("%s/pkg/obj/go-build", goroot)
   516  	if rebuildall {
   517  		xremoveall(p)
   518  	}
   519  	xmkdirall(p)
   520  	xatexit(func() { xremoveall(p) })
   521  
   522  	// Create tool directory.
   523  	// We keep it in pkg/, just like the object directory above.
   524  	if rebuildall {
   525  		xremoveall(tooldir)
   526  	}
   527  	xmkdirall(tooldir)
   528  
   529  	// Remove tool binaries from before the tool/gohostos_gohostarch
   530  	xremoveall(pathf("%s/bin/tool", goroot))
   531  
   532  	// Remove old pre-tool binaries.
   533  	for _, old := range oldtool {
   534  		xremove(pathf("%s/bin/%s", goroot, old))
   535  	}
   536  
   537  	// If $GOBIN is set and has a Go compiler, it must be cleaned.
   538  	for _, char := range "56789" {
   539  		if isfile(pathf("%s/%c%s", gobin, char, "g")) {
   540  			for _, old := range oldtool {
   541  				xremove(pathf("%s/%s", gobin, old))
   542  			}
   543  			break
   544  		}
   545  	}
   546  
   547  	// For release, make sure excluded things are excluded.
   548  	goversion := findgoversion()
   549  	if strings.HasPrefix(goversion, "release.") || (strings.HasPrefix(goversion, "go") && !strings.Contains(goversion, "beta")) {
   550  		for _, dir := range unreleased {
   551  			if p := pathf("%s/%s", goroot, dir); isdir(p) {
   552  				fatalf("%s should not exist in release build", p)
   553  			}
   554  		}
   555  	}
   556  }
   557  
   558  /*
   559   * Tool building
   560   */
   561  
   562  // deptab lists changes to the default dependencies for a given prefix.
   563  // deps ending in /* read the whole directory; deps beginning with -
   564  // exclude files with that prefix.
   565  // Note that this table applies only to the build of cmd/go,
   566  // after the main compiler bootstrap.
   567  var deptab = []struct {
   568  	prefix string   // prefix of target
   569  	dep    []string // dependency tweaks for targets with that prefix
   570  }{
   571  	{"cmd/go/internal/cfg", []string{
   572  		"zdefaultcc.go",
   573  		"zosarch.go",
   574  	}},
   575  	{"runtime/internal/sys", []string{
   576  		"zversion.go",
   577  	}},
   578  	{"go/build", []string{
   579  		"zcgo.go",
   580  	}},
   581  }
   582  
   583  // depsuffix records the allowed suffixes for source files.
   584  var depsuffix = []string{
   585  	".s",
   586  	".go",
   587  }
   588  
   589  // gentab records how to generate some trivial files.
   590  var gentab = []struct {
   591  	nameprefix string
   592  	gen        func(string, string)
   593  }{
   594  	{"zdefaultcc.go", mkzdefaultcc},
   595  	{"zosarch.go", mkzosarch},
   596  	{"zversion.go", mkzversion},
   597  	{"zcgo.go", mkzcgo},
   598  
   599  	// not generated anymore, but delete the file if we see it
   600  	{"enam.c", nil},
   601  	{"anames5.c", nil},
   602  	{"anames6.c", nil},
   603  	{"anames8.c", nil},
   604  	{"anames9.c", nil},
   605  }
   606  
   607  // installed maps from a dir name (as given to install) to a chan
   608  // closed when the dir's package is installed.
   609  var installed = make(map[string]chan struct{})
   610  var installedMu sync.Mutex
   611  
   612  func install(dir string) {
   613  	<-startInstall(dir)
   614  }
   615  
   616  func startInstall(dir string) chan struct{} {
   617  	installedMu.Lock()
   618  	ch := installed[dir]
   619  	if ch == nil {
   620  		ch = make(chan struct{})
   621  		installed[dir] = ch
   622  		go runInstall(dir, ch)
   623  	}
   624  	installedMu.Unlock()
   625  	return ch
   626  }
   627  
   628  // runInstall installs the library, package, or binary associated with dir,
   629  // which is relative to $GOROOT/src.
   630  func runInstall(pkg string, ch chan struct{}) {
   631  	if pkg == "net" || pkg == "os/user" || pkg == "crypto/x509" {
   632  		fatalf("go_bootstrap cannot depend on cgo package %s", pkg)
   633  	}
   634  
   635  	defer close(ch)
   636  
   637  	if pkg == "unsafe" {
   638  		return
   639  	}
   640  
   641  	if vflag > 0 {
   642  		if goos != gohostos || goarch != gohostarch {
   643  			errprintf("%s (%s/%s)\n", pkg, goos, goarch)
   644  		} else {
   645  			errprintf("%s\n", pkg)
   646  		}
   647  	}
   648  
   649  	workdir := pathf("%s/%s", workdir, pkg)
   650  	xmkdirall(workdir)
   651  
   652  	var clean []string
   653  	defer func() {
   654  		for _, name := range clean {
   655  			xremove(name)
   656  		}
   657  	}()
   658  
   659  	// dir = full path to pkg.
   660  	dir := pathf("%s/src/%s", goroot, pkg)
   661  	name := filepath.Base(dir)
   662  
   663  	// ispkg predicts whether the package should be linked as a binary, based
   664  	// on the name. There should be no "main" packages in vendor, since
   665  	// 'go mod vendor' will only copy imported packages there.
   666  	ispkg := !strings.HasPrefix(pkg, "cmd/") || strings.Contains(pkg, "/internal/") || strings.Contains(pkg, "/vendor/")
   667  
   668  	// Start final link command line.
   669  	// Note: code below knows that link.p[targ] is the target.
   670  	var (
   671  		link      []string
   672  		targ      int
   673  		ispackcmd bool
   674  	)
   675  	if ispkg {
   676  		// Go library (package).
   677  		ispackcmd = true
   678  		link = []string{"pack", packagefile(pkg)}
   679  		targ = len(link) - 1
   680  		xmkdirall(filepath.Dir(link[targ]))
   681  	} else {
   682  		// Go command.
   683  		elem := name
   684  		if elem == "go" {
   685  			elem = "go_bootstrap"
   686  		}
   687  		link = []string{pathf("%s/link", tooldir)}
   688  		if goos == "android" {
   689  			link = append(link, "-buildmode=pie")
   690  		}
   691  		if goldflags != "" {
   692  			link = append(link, goldflags)
   693  		}
   694  		link = append(link, "-extld="+compilerEnvLookup(defaultcc, goos, goarch))
   695  		link = append(link, "-o", pathf("%s/%s%s", tooldir, elem, exe))
   696  		targ = len(link) - 1
   697  	}
   698  	ttarg := mtime(link[targ])
   699  
   700  	// Gather files that are sources for this target.
   701  	// Everything in that directory, and any target-specific
   702  	// additions.
   703  	files := xreaddir(dir)
   704  
   705  	// Remove files beginning with . or _,
   706  	// which are likely to be editor temporary files.
   707  	// This is the same heuristic build.ScanDir uses.
   708  	// There do exist real C files beginning with _,
   709  	// so limit that check to just Go files.
   710  	files = filter(files, func(p string) bool {
   711  		return !strings.HasPrefix(p, ".") && (!strings.HasPrefix(p, "_") || !strings.HasSuffix(p, ".go"))
   712  	})
   713  
   714  	for _, dt := range deptab {
   715  		if pkg == dt.prefix || strings.HasSuffix(dt.prefix, "/") && strings.HasPrefix(pkg, dt.prefix) {
   716  			for _, p := range dt.dep {
   717  				p = os.ExpandEnv(p)
   718  				files = append(files, p)
   719  			}
   720  		}
   721  	}
   722  	files = uniq(files)
   723  
   724  	// Convert to absolute paths.
   725  	for i, p := range files {
   726  		if !filepath.IsAbs(p) {
   727  			files[i] = pathf("%s/%s", dir, p)
   728  		}
   729  	}
   730  
   731  	// Is the target up-to-date?
   732  	var gofiles, sfiles, missing []string
   733  	stale := rebuildall
   734  	files = filter(files, func(p string) bool {
   735  		for _, suf := range depsuffix {
   736  			if strings.HasSuffix(p, suf) {
   737  				goto ok
   738  			}
   739  		}
   740  		return false
   741  	ok:
   742  		t := mtime(p)
   743  		if !t.IsZero() && !strings.HasSuffix(p, ".a") && !shouldbuild(p, pkg) {
   744  			return false
   745  		}
   746  		if strings.HasSuffix(p, ".go") {
   747  			gofiles = append(gofiles, p)
   748  		} else if strings.HasSuffix(p, ".s") {
   749  			sfiles = append(sfiles, p)
   750  		}
   751  		if t.After(ttarg) {
   752  			stale = true
   753  		}
   754  		if t.IsZero() {
   755  			missing = append(missing, p)
   756  		}
   757  		return true
   758  	})
   759  
   760  	// If there are no files to compile, we're done.
   761  	if len(files) == 0 {
   762  		return
   763  	}
   764  
   765  	if !stale {
   766  		return
   767  	}
   768  
   769  	// For package runtime, copy some files into the work space.
   770  	if pkg == "runtime" {
   771  		xmkdirall(pathf("%s/pkg/include", goroot))
   772  		// For use by assembly and C files.
   773  		copyfile(pathf("%s/pkg/include/textflag.h", goroot),
   774  			pathf("%s/src/runtime/textflag.h", goroot), 0)
   775  		copyfile(pathf("%s/pkg/include/funcdata.h", goroot),
   776  			pathf("%s/src/runtime/funcdata.h", goroot), 0)
   777  		copyfile(pathf("%s/pkg/include/asm_ppc64x.h", goroot),
   778  			pathf("%s/src/runtime/asm_ppc64x.h", goroot), 0)
   779  	}
   780  
   781  	// Generate any missing files; regenerate existing ones.
   782  	for _, p := range files {
   783  		elem := filepath.Base(p)
   784  		for _, gt := range gentab {
   785  			if gt.gen == nil {
   786  				continue
   787  			}
   788  			if strings.HasPrefix(elem, gt.nameprefix) {
   789  				if vflag > 1 {
   790  					errprintf("generate %s\n", p)
   791  				}
   792  				gt.gen(dir, p)
   793  				// Do not add generated file to clean list.
   794  				// In runtime, we want to be able to
   795  				// build the package with the go tool,
   796  				// and it assumes these generated files already
   797  				// exist (it does not know how to build them).
   798  				// The 'clean' command can remove
   799  				// the generated files.
   800  				goto built
   801  			}
   802  		}
   803  		// Did not rebuild p.
   804  		if find(p, missing) >= 0 {
   805  			fatalf("missing file %s", p)
   806  		}
   807  	built:
   808  	}
   809  
   810  	// Resolve imported packages to actual package paths.
   811  	// Make sure they're installed.
   812  	importMap := make(map[string]string)
   813  	for _, p := range gofiles {
   814  		for _, imp := range readimports(p) {
   815  			if imp == "C" {
   816  				fatalf("%s imports C", p)
   817  			}
   818  			importMap[imp] = resolveVendor(imp, dir)
   819  		}
   820  	}
   821  	sortedImports := make([]string, 0, len(importMap))
   822  	for imp := range importMap {
   823  		sortedImports = append(sortedImports, imp)
   824  	}
   825  	sort.Strings(sortedImports)
   826  
   827  	for _, dep := range importMap {
   828  		if dep == "C" {
   829  			fatalf("%s imports C", pkg)
   830  		}
   831  		startInstall(dep)
   832  	}
   833  	for _, dep := range importMap {
   834  		install(dep)
   835  	}
   836  
   837  	if goos != gohostos || goarch != gohostarch {
   838  		// We've generated the right files; the go command can do the build.
   839  		if vflag > 1 {
   840  			errprintf("skip build for cross-compile %s\n", pkg)
   841  		}
   842  		return
   843  	}
   844  
   845  	asmArgs := []string{
   846  		pathf("%s/asm", tooldir),
   847  		"-I", workdir,
   848  		"-I", pathf("%s/pkg/include", goroot),
   849  		"-D", "GOOS_" + goos,
   850  		"-D", "GOARCH_" + goarch,
   851  		"-D", "GOOS_GOARCH_" + goos + "_" + goarch,
   852  		"-p", pkg,
   853  	}
   854  	if goarch == "mips" || goarch == "mipsle" {
   855  		// Define GOMIPS_value from gomips.
   856  		asmArgs = append(asmArgs, "-D", "GOMIPS_"+gomips)
   857  	}
   858  	if goarch == "mips64" || goarch == "mips64le" {
   859  		// Define GOMIPS64_value from gomips64.
   860  		asmArgs = append(asmArgs, "-D", "GOMIPS64_"+gomips64)
   861  	}
   862  	goasmh := pathf("%s/go_asm.h", workdir)
   863  	if IsRuntimePackagePath(pkg) {
   864  		asmArgs = append(asmArgs, "-compiling-runtime")
   865  	}
   866  
   867  	// Collect symabis from assembly code.
   868  	var symabis string
   869  	if len(sfiles) > 0 {
   870  		symabis = pathf("%s/symabis", workdir)
   871  		var wg sync.WaitGroup
   872  		asmabis := append(asmArgs[:len(asmArgs):len(asmArgs)], "-gensymabis", "-o", symabis)
   873  		asmabis = append(asmabis, sfiles...)
   874  		if err := ioutil.WriteFile(goasmh, nil, 0666); err != nil {
   875  			fatalf("cannot write empty go_asm.h: %s", err)
   876  		}
   877  		bgrun(&wg, dir, asmabis...)
   878  		bgwait(&wg)
   879  	}
   880  
   881  	// Build an importcfg file for the compiler.
   882  	buf := &bytes.Buffer{}
   883  	for _, imp := range sortedImports {
   884  		if imp == "unsafe" {
   885  			continue
   886  		}
   887  		dep := importMap[imp]
   888  		if imp != dep {
   889  			fmt.Fprintf(buf, "importmap %s=%s\n", imp, dep)
   890  		}
   891  		fmt.Fprintf(buf, "packagefile %s=%s\n", dep, packagefile(dep))
   892  	}
   893  	importcfg := pathf("%s/importcfg", workdir)
   894  	if err := ioutil.WriteFile(importcfg, buf.Bytes(), 0666); err != nil {
   895  		fatalf("cannot write importcfg file: %v", err)
   896  	}
   897  
   898  	var archive string
   899  	// The next loop will compile individual non-Go files.
   900  	// Hand the Go files to the compiler en masse.
   901  	// For packages containing assembly, this writes go_asm.h, which
   902  	// the assembly files will need.
   903  	pkgName := pkg
   904  	if strings.HasPrefix(pkg, "cmd/") && strings.Count(pkg, "/") == 1 {
   905  		pkgName = "main"
   906  	}
   907  	b := pathf("%s/_go_.a", workdir)
   908  	clean = append(clean, b)
   909  	if !ispackcmd {
   910  		link = append(link, b)
   911  	} else {
   912  		archive = b
   913  	}
   914  
   915  	// Compile Go code.
   916  	compile := []string{pathf("%s/compile", tooldir), "-std", "-pack", "-o", b, "-p", pkgName, "-importcfg", importcfg}
   917  	if gogcflags != "" {
   918  		compile = append(compile, strings.Fields(gogcflags)...)
   919  	}
   920  	if pkg == "runtime" {
   921  		compile = append(compile, "-+")
   922  	}
   923  	if len(sfiles) > 0 {
   924  		compile = append(compile, "-asmhdr", goasmh)
   925  	}
   926  	if symabis != "" {
   927  		compile = append(compile, "-symabis", symabis)
   928  	}
   929  	if goos == "android" {
   930  		compile = append(compile, "-shared")
   931  	}
   932  
   933  	compile = append(compile, gofiles...)
   934  	var wg sync.WaitGroup
   935  	// We use bgrun and immediately wait for it instead of calling run() synchronously.
   936  	// This executes all jobs through the bgwork channel and allows the process
   937  	// to exit cleanly in case an error occurs.
   938  	bgrun(&wg, dir, compile...)
   939  	bgwait(&wg)
   940  
   941  	// Compile the files.
   942  	for _, p := range sfiles {
   943  		// Assembly file for a Go package.
   944  		compile := asmArgs[:len(asmArgs):len(asmArgs)]
   945  
   946  		doclean := true
   947  		b := pathf("%s/%s", workdir, filepath.Base(p))
   948  
   949  		// Change the last character of the output file (which was c or s).
   950  		b = b[:len(b)-1] + "o"
   951  		compile = append(compile, "-o", b, p)
   952  		bgrun(&wg, dir, compile...)
   953  
   954  		link = append(link, b)
   955  		if doclean {
   956  			clean = append(clean, b)
   957  		}
   958  	}
   959  	bgwait(&wg)
   960  
   961  	if ispackcmd {
   962  		xremove(link[targ])
   963  		dopack(link[targ], archive, link[targ+1:])
   964  		return
   965  	}
   966  
   967  	// Remove target before writing it.
   968  	xremove(link[targ])
   969  	bgrun(&wg, "", link...)
   970  	bgwait(&wg)
   971  }
   972  
   973  // packagefile returns the path to a compiled .a file for the given package
   974  // path. Paths may need to be resolved with resolveVendor first.
   975  func packagefile(pkg string) string {
   976  	return pathf("%s/pkg/%s_%s/%s.a", goroot, goos, goarch, pkg)
   977  }
   978  
   979  // matchtag reports whether the tag matches this build.
   980  func matchtag(tag string) bool {
   981  	return tag == "gc" || tag == goos || tag == goarch || tag == "cmd_go_bootstrap" || tag == "go1.1" ||
   982  		(goos == "android" && tag == "linux") ||
   983  		(goos == "illumos" && tag == "solaris") ||
   984  		(goos == "ios" && tag == "darwin")
   985  }
   986  
   987  // shouldbuild reports whether we should build this file.
   988  // It applies the same rules that are used with context tags
   989  // in package go/build, except it's less picky about the order
   990  // of GOOS and GOARCH.
   991  // We also allow the special tag cmd_go_bootstrap.
   992  // See ../go/bootstrap.go and package go/build.
   993  func shouldbuild(file, pkg string) bool {
   994  	// Check file name for GOOS or GOARCH.
   995  	name := filepath.Base(file)
   996  	excluded := func(list []string, ok string) bool {
   997  		for _, x := range list {
   998  			if x == ok || (ok == "android" && x == "linux") || (ok == "illumos" && x == "solaris") || (ok == "ios" && x == "darwin") {
   999  				continue
  1000  			}
  1001  			i := strings.Index(name, x)
  1002  			if i <= 0 || name[i-1] != '_' {
  1003  				continue
  1004  			}
  1005  			i += len(x)
  1006  			if i == len(name) || name[i] == '.' || name[i] == '_' {
  1007  				return true
  1008  			}
  1009  		}
  1010  		return false
  1011  	}
  1012  	if excluded(okgoos, goos) || excluded(okgoarch, goarch) {
  1013  		return false
  1014  	}
  1015  
  1016  	// Omit test files.
  1017  	if strings.Contains(name, "_test") {
  1018  		return false
  1019  	}
  1020  
  1021  	// Check file contents for //go:build lines.
  1022  	for _, p := range strings.Split(readfile(file), "\n") {
  1023  		p = strings.TrimSpace(p)
  1024  		if p == "" {
  1025  			continue
  1026  		}
  1027  		code := p
  1028  		i := strings.Index(code, "//")
  1029  		if i > 0 {
  1030  			code = strings.TrimSpace(code[:i])
  1031  		}
  1032  		if code == "package documentation" {
  1033  			return false
  1034  		}
  1035  		if code == "package main" && pkg != "cmd/go" && pkg != "cmd/cgo" {
  1036  			return false
  1037  		}
  1038  		if !strings.HasPrefix(p, "//") {
  1039  			break
  1040  		}
  1041  		if strings.HasPrefix(p, "//go:build ") {
  1042  			matched, err := matchexpr(p[len("//go:build "):])
  1043  			if err != nil {
  1044  				errprintf("%s: %v", file, err)
  1045  			}
  1046  			return matched
  1047  		}
  1048  	}
  1049  
  1050  	return true
  1051  }
  1052  
  1053  // copy copies the file src to dst, via memory (so only good for small files).
  1054  func copyfile(dst, src string, flag int) {
  1055  	if vflag > 1 {
  1056  		errprintf("cp %s %s\n", src, dst)
  1057  	}
  1058  	writefile(readfile(src), dst, flag)
  1059  }
  1060  
  1061  // dopack copies the package src to dst,
  1062  // appending the files listed in extra.
  1063  // The archive format is the traditional Unix ar format.
  1064  func dopack(dst, src string, extra []string) {
  1065  	bdst := bytes.NewBufferString(readfile(src))
  1066  	for _, file := range extra {
  1067  		b := readfile(file)
  1068  		// find last path element for archive member name
  1069  		i := strings.LastIndex(file, "/") + 1
  1070  		j := strings.LastIndex(file, `\`) + 1
  1071  		if i < j {
  1072  			i = j
  1073  		}
  1074  		fmt.Fprintf(bdst, "%-16.16s%-12d%-6d%-6d%-8o%-10d`\n", file[i:], 0, 0, 0, 0644, len(b))
  1075  		bdst.WriteString(b)
  1076  		if len(b)&1 != 0 {
  1077  			bdst.WriteByte(0)
  1078  		}
  1079  	}
  1080  	writefile(bdst.String(), dst, 0)
  1081  }
  1082  
  1083  var runtimegen = []string{
  1084  	"zaexperiment.h",
  1085  	"zversion.go",
  1086  }
  1087  
  1088  // cleanlist is a list of packages with generated files and commands.
  1089  var cleanlist = []string{
  1090  	"runtime/internal/sys",
  1091  	"cmd/cgo",
  1092  	"cmd/go/internal/cfg",
  1093  	"go/build",
  1094  }
  1095  
  1096  func clean() {
  1097  	for _, name := range cleanlist {
  1098  		path := pathf("%s/src/%s", goroot, name)
  1099  		// Remove generated files.
  1100  		for _, elem := range xreaddir(path) {
  1101  			for _, gt := range gentab {
  1102  				if strings.HasPrefix(elem, gt.nameprefix) {
  1103  					xremove(pathf("%s/%s", path, elem))
  1104  				}
  1105  			}
  1106  		}
  1107  		// Remove generated binary named for directory.
  1108  		if strings.HasPrefix(name, "cmd/") {
  1109  			xremove(pathf("%s/%s", path, name[4:]))
  1110  		}
  1111  	}
  1112  
  1113  	// remove runtimegen files.
  1114  	path := pathf("%s/src/runtime", goroot)
  1115  	for _, elem := range runtimegen {
  1116  		xremove(pathf("%s/%s", path, elem))
  1117  	}
  1118  
  1119  	if rebuildall {
  1120  		// Remove object tree.
  1121  		xremoveall(pathf("%s/pkg/obj/%s_%s", goroot, gohostos, gohostarch))
  1122  
  1123  		// Remove installed packages and tools.
  1124  		xremoveall(pathf("%s/pkg/%s_%s", goroot, gohostos, gohostarch))
  1125  		xremoveall(pathf("%s/pkg/%s_%s", goroot, goos, goarch))
  1126  		xremoveall(pathf("%s/pkg/%s_%s_race", goroot, gohostos, gohostarch))
  1127  		xremoveall(pathf("%s/pkg/%s_%s_race", goroot, goos, goarch))
  1128  		xremoveall(tooldir)
  1129  
  1130  		// Remove cached version info.
  1131  		xremove(pathf("%s/VERSION.cache", goroot))
  1132  	}
  1133  }
  1134  
  1135  /*
  1136   * command implementations
  1137   */
  1138  
  1139  // The env command prints the default environment.
  1140  func cmdenv() {
  1141  	path := flag.Bool("p", false, "emit updated PATH")
  1142  	plan9 := flag.Bool("9", false, "emit plan 9 syntax")
  1143  	windows := flag.Bool("w", false, "emit windows syntax")
  1144  	xflagparse(0)
  1145  
  1146  	format := "%s=\"%s\"\n"
  1147  	switch {
  1148  	case *plan9:
  1149  		format = "%s='%s'\n"
  1150  	case *windows:
  1151  		format = "set %s=%s\r\n"
  1152  	}
  1153  
  1154  	xprintf(format, "GOARCH", goarch)
  1155  	xprintf(format, "GOBIN", gobin)
  1156  	xprintf(format, "GOCACHE", os.Getenv("GOCACHE"))
  1157  	xprintf(format, "GODEBUG", os.Getenv("GODEBUG"))
  1158  	xprintf(format, "GOHOSTARCH", gohostarch)
  1159  	xprintf(format, "GOHOSTOS", gohostos)
  1160  	xprintf(format, "GOOS", goos)
  1161  	xprintf(format, "GOPROXY", os.Getenv("GOPROXY"))
  1162  	xprintf(format, "GOROOT", goroot)
  1163  	xprintf(format, "GOTMPDIR", os.Getenv("GOTMPDIR"))
  1164  	xprintf(format, "GOTOOLDIR", tooldir)
  1165  	if goarch == "arm" {
  1166  		xprintf(format, "GOARM", goarm)
  1167  	}
  1168  	if goarch == "386" {
  1169  		xprintf(format, "GO386", go386)
  1170  	}
  1171  	if goarch == "amd64" {
  1172  		xprintf(format, "GOAMD64", goamd64)
  1173  	}
  1174  	if goarch == "mips" || goarch == "mipsle" {
  1175  		xprintf(format, "GOMIPS", gomips)
  1176  	}
  1177  	if goarch == "mips64" || goarch == "mips64le" {
  1178  		xprintf(format, "GOMIPS64", gomips64)
  1179  	}
  1180  	if goarch == "ppc64" || goarch == "ppc64le" {
  1181  		xprintf(format, "GOPPC64", goppc64)
  1182  	}
  1183  
  1184  	if *path {
  1185  		sep := ":"
  1186  		if gohostos == "windows" {
  1187  			sep = ";"
  1188  		}
  1189  		xprintf(format, "PATH", fmt.Sprintf("%s%s%s", gobin, sep, os.Getenv("PATH")))
  1190  	}
  1191  }
  1192  
  1193  var (
  1194  	timeLogEnabled = os.Getenv("GOBUILDTIMELOGFILE") != ""
  1195  	timeLogMu      sync.Mutex
  1196  	timeLogFile    *os.File
  1197  	timeLogStart   time.Time
  1198  )
  1199  
  1200  func timelog(op, name string) {
  1201  	if !timeLogEnabled {
  1202  		return
  1203  	}
  1204  	timeLogMu.Lock()
  1205  	defer timeLogMu.Unlock()
  1206  	if timeLogFile == nil {
  1207  		f, err := os.OpenFile(os.Getenv("GOBUILDTIMELOGFILE"), os.O_RDWR|os.O_APPEND, 0666)
  1208  		if err != nil {
  1209  			log.Fatal(err)
  1210  		}
  1211  		buf := make([]byte, 100)
  1212  		n, _ := f.Read(buf)
  1213  		s := string(buf[:n])
  1214  		if i := strings.Index(s, "\n"); i >= 0 {
  1215  			s = s[:i]
  1216  		}
  1217  		i := strings.Index(s, " start")
  1218  		if i < 0 {
  1219  			log.Fatalf("time log %s does not begin with start line", os.Getenv("GOBUILDTIMELOGFILE"))
  1220  		}
  1221  		t, err := time.Parse(time.UnixDate, s[:i])
  1222  		if err != nil {
  1223  			log.Fatalf("cannot parse time log line %q: %v", s, err)
  1224  		}
  1225  		timeLogStart = t
  1226  		timeLogFile = f
  1227  	}
  1228  	t := time.Now()
  1229  	fmt.Fprintf(timeLogFile, "%s %+.1fs %s %s\n", t.Format(time.UnixDate), t.Sub(timeLogStart).Seconds(), op, name)
  1230  }
  1231  
  1232  var toolchain = []string{"cmd/asm", "cmd/cgo", "cmd/compile", "cmd/link"}
  1233  
  1234  // The bootstrap command runs a build from scratch,
  1235  // stopping at having installed the go_bootstrap command.
  1236  //
  1237  // WARNING: This command runs after cmd/dist is built with Go 1.4.
  1238  // It rebuilds and installs cmd/dist with the new toolchain, so other
  1239  // commands (like "go tool dist test" in run.bash) can rely on bug fixes
  1240  // made since Go 1.4, but this function cannot. In particular, the uses
  1241  // of os/exec in this function cannot assume that
  1242  //	cmd.Env = append(os.Environ(), "X=Y")
  1243  // sets $X to Y in the command's environment. That guarantee was
  1244  // added after Go 1.4, and in fact in Go 1.4 it was typically the opposite:
  1245  // if $X was already present in os.Environ(), most systems preferred
  1246  // that setting, not the new one.
  1247  func cmdbootstrap() {
  1248  	timelog("start", "dist bootstrap")
  1249  	defer timelog("end", "dist bootstrap")
  1250  
  1251  	var noBanner, noClean bool
  1252  	var debug bool
  1253  	flag.BoolVar(&rebuildall, "a", rebuildall, "rebuild all")
  1254  	flag.BoolVar(&debug, "d", debug, "enable debugging of bootstrap process")
  1255  	flag.BoolVar(&noBanner, "no-banner", noBanner, "do not print banner")
  1256  	flag.BoolVar(&noClean, "no-clean", noClean, "print deprecation warning")
  1257  
  1258  	xflagparse(0)
  1259  
  1260  	if noClean {
  1261  		xprintf("warning: --no-clean is deprecated and has no effect; use 'go install std cmd' instead\n")
  1262  	}
  1263  
  1264  	// Set GOPATH to an internal directory. We shouldn't actually
  1265  	// need to store files here, since the toolchain won't
  1266  	// depend on modules outside of vendor directories, but if
  1267  	// GOPATH points somewhere else (e.g., to GOROOT), the
  1268  	// go tool may complain.
  1269  	os.Setenv("GOPATH", pathf("%s/pkg/obj/gopath", goroot))
  1270  
  1271  	// Disable GOEXPERIMENT when building toolchain1 and
  1272  	// go_bootstrap. We don't need any experiments for the
  1273  	// bootstrap toolchain, and this lets us avoid duplicating the
  1274  	// GOEXPERIMENT-related build logic from cmd/go here. If the
  1275  	// bootstrap toolchain is < Go 1.17, it will ignore this
  1276  	// anyway since GOEXPERIMENT is baked in; otherwise it will
  1277  	// pick it up from the environment we set here. Once we're
  1278  	// using toolchain1 with dist as the build system, we need to
  1279  	// override this to keep the experiments assumed by the
  1280  	// toolchain and by dist consistent. Once go_bootstrap takes
  1281  	// over the build process, we'll set this back to the original
  1282  	// GOEXPERIMENT.
  1283  	os.Setenv("GOEXPERIMENT", "none")
  1284  
  1285  	if debug {
  1286  		// cmd/buildid is used in debug mode.
  1287  		toolchain = append(toolchain, "cmd/buildid")
  1288  	}
  1289  
  1290  	if isdir(pathf("%s/src/pkg", goroot)) {
  1291  		fatalf("\n\n"+
  1292  			"The Go package sources have moved to $GOROOT/src.\n"+
  1293  			"*** %s still exists. ***\n"+
  1294  			"It probably contains stale files that may confuse the build.\n"+
  1295  			"Please (check what's there and) remove it and try again.\n"+
  1296  			"See https://golang.org/s/go14nopkg\n",
  1297  			pathf("%s/src/pkg", goroot))
  1298  	}
  1299  
  1300  	if rebuildall {
  1301  		clean()
  1302  	}
  1303  
  1304  	setup()
  1305  
  1306  	timelog("build", "toolchain1")
  1307  	checkCC()
  1308  	bootstrapBuildTools()
  1309  
  1310  	// Remember old content of $GOROOT/bin for comparison below.
  1311  	oldBinFiles, _ := filepath.Glob(pathf("%s/bin/*", goroot))
  1312  
  1313  	// For the main bootstrap, building for host os/arch.
  1314  	oldgoos = goos
  1315  	oldgoarch = goarch
  1316  	goos = gohostos
  1317  	goarch = gohostarch
  1318  	os.Setenv("GOHOSTARCH", gohostarch)
  1319  	os.Setenv("GOHOSTOS", gohostos)
  1320  	os.Setenv("GOARCH", goarch)
  1321  	os.Setenv("GOOS", goos)
  1322  
  1323  	timelog("build", "go_bootstrap")
  1324  	xprintf("Building Go bootstrap cmd/go (go_bootstrap) using Go toolchain1.\n")
  1325  	install("runtime") // dependency not visible in sources; also sets up textflag.h
  1326  	install("cmd/go")
  1327  	if vflag > 0 {
  1328  		xprintf("\n")
  1329  	}
  1330  
  1331  	gogcflags = os.Getenv("GO_GCFLAGS") // we were using $BOOT_GO_GCFLAGS until now
  1332  	goldflags = os.Getenv("GO_LDFLAGS") // we were using $BOOT_GO_LDFLAGS until now
  1333  	goBootstrap := pathf("%s/go_bootstrap", tooldir)
  1334  	cmdGo := pathf("%s/go", gobin)
  1335  	if debug {
  1336  		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
  1337  		copyfile(pathf("%s/compile1", tooldir), pathf("%s/compile", tooldir), writeExec)
  1338  	}
  1339  
  1340  	// To recap, so far we have built the new toolchain
  1341  	// (cmd/asm, cmd/cgo, cmd/compile, cmd/link)
  1342  	// using Go 1.4's toolchain and go command.
  1343  	// Then we built the new go command (as go_bootstrap)
  1344  	// using the new toolchain and our own build logic (above).
  1345  	//
  1346  	//	toolchain1 = mk(new toolchain, go1.4 toolchain, go1.4 cmd/go)
  1347  	//	go_bootstrap = mk(new cmd/go, toolchain1, cmd/dist)
  1348  	//
  1349  	// The toolchain1 we built earlier is built from the new sources,
  1350  	// but because it was built using cmd/go it has no build IDs.
  1351  	// The eventually installed toolchain needs build IDs, so we need
  1352  	// to do another round:
  1353  	//
  1354  	//	toolchain2 = mk(new toolchain, toolchain1, go_bootstrap)
  1355  	//
  1356  	timelog("build", "toolchain2")
  1357  	if vflag > 0 {
  1358  		xprintf("\n")
  1359  	}
  1360  	xprintf("Building Go toolchain2 using go_bootstrap and Go toolchain1.\n")
  1361  	os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch))
  1362  	// Now that cmd/go is in charge of the build process, enable GOEXPERIMENT.
  1363  	os.Setenv("GOEXPERIMENT", goexperiment)
  1364  	goInstall(goBootstrap, append([]string{"-i"}, toolchain...)...)
  1365  	if debug {
  1366  		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
  1367  		run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/pkg/%s_%s/runtime/internal/sys.a", goroot, goos, goarch))
  1368  		copyfile(pathf("%s/compile2", tooldir), pathf("%s/compile", tooldir), writeExec)
  1369  	}
  1370  
  1371  	// Toolchain2 should be semantically equivalent to toolchain1,
  1372  	// but it was built using the new compilers instead of the Go 1.4 compilers,
  1373  	// so it should at the least run faster. Also, toolchain1 had no build IDs
  1374  	// in the binaries, while toolchain2 does. In non-release builds, the
  1375  	// toolchain's build IDs feed into constructing the build IDs of built targets,
  1376  	// so in non-release builds, everything now looks out-of-date due to
  1377  	// toolchain2 having build IDs - that is, due to the go command seeing
  1378  	// that there are new compilers. In release builds, the toolchain's reported
  1379  	// version is used in place of the build ID, and the go command does not
  1380  	// see that change from toolchain1 to toolchain2, so in release builds,
  1381  	// nothing looks out of date.
  1382  	// To keep the behavior the same in both non-release and release builds,
  1383  	// we force-install everything here.
  1384  	//
  1385  	//	toolchain3 = mk(new toolchain, toolchain2, go_bootstrap)
  1386  	//
  1387  	timelog("build", "toolchain3")
  1388  	if vflag > 0 {
  1389  		xprintf("\n")
  1390  	}
  1391  	xprintf("Building Go toolchain3 using go_bootstrap and Go toolchain2.\n")
  1392  	goInstall(goBootstrap, append([]string{"-a", "-i"}, toolchain...)...)
  1393  	if debug {
  1394  		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
  1395  		run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/pkg/%s_%s/runtime/internal/sys.a", goroot, goos, goarch))
  1396  		copyfile(pathf("%s/compile3", tooldir), pathf("%s/compile", tooldir), writeExec)
  1397  	}
  1398  	checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...)
  1399  
  1400  	if goos == oldgoos && goarch == oldgoarch {
  1401  		// Common case - not setting up for cross-compilation.
  1402  		timelog("build", "toolchain")
  1403  		if vflag > 0 {
  1404  			xprintf("\n")
  1405  		}
  1406  		xprintf("Building packages and commands for %s/%s.\n", goos, goarch)
  1407  	} else {
  1408  		// GOOS/GOARCH does not match GOHOSTOS/GOHOSTARCH.
  1409  		// Finish GOHOSTOS/GOHOSTARCH installation and then
  1410  		// run GOOS/GOARCH installation.
  1411  		timelog("build", "host toolchain")
  1412  		if vflag > 0 {
  1413  			xprintf("\n")
  1414  		}
  1415  		xprintf("Building packages and commands for host, %s/%s.\n", goos, goarch)
  1416  		goInstall(goBootstrap, "std", "cmd")
  1417  		checkNotStale(goBootstrap, "std", "cmd")
  1418  		checkNotStale(cmdGo, "std", "cmd")
  1419  
  1420  		timelog("build", "target toolchain")
  1421  		if vflag > 0 {
  1422  			xprintf("\n")
  1423  		}
  1424  		goos = oldgoos
  1425  		goarch = oldgoarch
  1426  		os.Setenv("GOOS", goos)
  1427  		os.Setenv("GOARCH", goarch)
  1428  		os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch))
  1429  		xprintf("Building packages and commands for target, %s/%s.\n", goos, goarch)
  1430  	}
  1431  	targets := []string{"std", "cmd"}
  1432  	if goos == "js" && goarch == "wasm" {
  1433  		// Skip the cmd tools for js/wasm. They're not usable.
  1434  		targets = targets[:1]
  1435  	}
  1436  	goInstall(goBootstrap, targets...)
  1437  	checkNotStale(goBootstrap, targets...)
  1438  	checkNotStale(cmdGo, targets...)
  1439  	if debug {
  1440  		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
  1441  		run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/pkg/%s_%s/runtime/internal/sys.a", goroot, goos, goarch))
  1442  		checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...)
  1443  		copyfile(pathf("%s/compile4", tooldir), pathf("%s/compile", tooldir), writeExec)
  1444  	}
  1445  
  1446  	// Check that there are no new files in $GOROOT/bin other than
  1447  	// go and gofmt and $GOOS_$GOARCH (target bin when cross-compiling).
  1448  	binFiles, _ := filepath.Glob(pathf("%s/bin/*", goroot))
  1449  	ok := map[string]bool{}
  1450  	for _, f := range oldBinFiles {
  1451  		ok[f] = true
  1452  	}
  1453  	for _, f := range binFiles {
  1454  		elem := strings.TrimSuffix(filepath.Base(f), ".exe")
  1455  		if !ok[f] && elem != "go" && elem != "gofmt" && elem != goos+"_"+goarch {
  1456  			fatalf("unexpected new file in $GOROOT/bin: %s", elem)
  1457  		}
  1458  	}
  1459  
  1460  	// Remove go_bootstrap now that we're done.
  1461  	xremove(pathf("%s/go_bootstrap", tooldir))
  1462  
  1463  	if goos == "android" {
  1464  		// Make sure the exec wrapper will sync a fresh $GOROOT to the device.
  1465  		xremove(pathf("%s/go_android_exec-adb-sync-status", os.TempDir()))
  1466  	}
  1467  
  1468  	if wrapperPath := wrapperPathFor(goos, goarch); wrapperPath != "" {
  1469  		oldcc := os.Getenv("CC")
  1470  		os.Setenv("GOOS", gohostos)
  1471  		os.Setenv("GOARCH", gohostarch)
  1472  		os.Setenv("CC", compilerEnvLookup(defaultcc, gohostos, gohostarch))
  1473  		goCmd(cmdGo, "build", "-o", pathf("%s/go_%s_%s_exec%s", gobin, goos, goarch, exe), wrapperPath)
  1474  		// Restore environment.
  1475  		// TODO(elias.naur): support environment variables in goCmd?
  1476  		os.Setenv("GOOS", goos)
  1477  		os.Setenv("GOARCH", goarch)
  1478  		os.Setenv("CC", oldcc)
  1479  	}
  1480  
  1481  	// Print trailing banner unless instructed otherwise.
  1482  	if !noBanner {
  1483  		banner()
  1484  	}
  1485  }
  1486  
  1487  func wrapperPathFor(goos, goarch string) string {
  1488  	switch {
  1489  	case goos == "android":
  1490  		if gohostos != "android" {
  1491  			return pathf("%s/misc/android/go_android_exec.go", goroot)
  1492  		}
  1493  	case goos == "ios":
  1494  		if gohostos != "ios" {
  1495  			return pathf("%s/misc/ios/go_ios_exec.go", goroot)
  1496  		}
  1497  	}
  1498  	return ""
  1499  }
  1500  
  1501  func goInstall(goBinary string, args ...string) {
  1502  	goCmd(goBinary, "install", args...)
  1503  }
  1504  
  1505  func goCmd(goBinary string, cmd string, args ...string) {
  1506  	goCmd := []string{goBinary, cmd, "-gcflags=all=" + gogcflags, "-ldflags=all=" + goldflags}
  1507  	if vflag > 0 {
  1508  		goCmd = append(goCmd, "-v")
  1509  	}
  1510  
  1511  	// Force only one process at a time on vx32 emulation.
  1512  	if gohostos == "plan9" && os.Getenv("sysname") == "vx32" {
  1513  		goCmd = append(goCmd, "-p=1")
  1514  	}
  1515  
  1516  	run(workdir, ShowOutput|CheckExit, append(goCmd, args...)...)
  1517  }
  1518  
  1519  func checkNotStale(goBinary string, targets ...string) {
  1520  	out := run(workdir, CheckExit,
  1521  		append([]string{
  1522  			goBinary,
  1523  			"list", "-gcflags=all=" + gogcflags, "-ldflags=all=" + goldflags,
  1524  			"-f={{if .Stale}}\tSTALE {{.ImportPath}}: {{.StaleReason}}{{end}}",
  1525  		}, targets...)...)
  1526  	if strings.Contains(out, "\tSTALE ") {
  1527  		os.Setenv("GODEBUG", "gocachehash=1")
  1528  		for _, target := range []string{"runtime/internal/sys", "cmd/dist", "cmd/link"} {
  1529  			if strings.Contains(out, "STALE "+target) {
  1530  				run(workdir, ShowOutput|CheckExit, goBinary, "list", "-f={{.ImportPath}} {{.Stale}}", target)
  1531  				break
  1532  			}
  1533  		}
  1534  		fatalf("unexpected stale targets reported by %s list -gcflags=\"%s\" -ldflags=\"%s\" for %v (consider rerunning with GOMAXPROCS=1 GODEBUG=gocachehash=1):\n%s", goBinary, gogcflags, goldflags, targets, out)
  1535  	}
  1536  }
  1537  
  1538  // Cannot use go/build directly because cmd/dist for a new release
  1539  // builds against an old release's go/build, which may be out of sync.
  1540  // To reduce duplication, we generate the list for go/build from this.
  1541  //
  1542  // We list all supported platforms in this list, so that this is the
  1543  // single point of truth for supported platforms. This list is used
  1544  // by 'go tool dist list'.
  1545  var cgoEnabled = map[string]bool{
  1546  	"aix/ppc64":       true,
  1547  	"darwin/amd64":    true,
  1548  	"darwin/arm64":    true,
  1549  	"dragonfly/amd64": true,
  1550  	"freebsd/386":     true,
  1551  	"freebsd/amd64":   true,
  1552  	"freebsd/arm":     true,
  1553  	"freebsd/arm64":   true,
  1554  	"illumos/amd64":   true,
  1555  	"linux/386":       true,
  1556  	"linux/amd64":     true,
  1557  	"linux/arm":       true,
  1558  	"linux/arm64":     true,
  1559  	"linux/ppc64":     false,
  1560  	"linux/ppc64le":   true,
  1561  	"linux/mips":      true,
  1562  	"linux/mipsle":    true,
  1563  	"linux/mips64":    true,
  1564  	"linux/mips64le":  true,
  1565  	"linux/riscv64":   true,
  1566  	"linux/s390x":     true,
  1567  	"linux/sparc64":   true,
  1568  	"android/386":     true,
  1569  	"android/amd64":   true,
  1570  	"android/arm":     true,
  1571  	"android/arm64":   true,
  1572  	"ios/arm64":       true,
  1573  	"ios/amd64":       true,
  1574  	"js/wasm":         false,
  1575  	"netbsd/386":      true,
  1576  	"netbsd/amd64":    true,
  1577  	"netbsd/arm":      true,
  1578  	"netbsd/arm64":    true,
  1579  	"openbsd/386":     true,
  1580  	"openbsd/amd64":   true,
  1581  	"openbsd/arm":     true,
  1582  	"openbsd/arm64":   true,
  1583  	"openbsd/mips64":  true,
  1584  	"plan9/386":       false,
  1585  	"plan9/amd64":     false,
  1586  	"plan9/arm":       false,
  1587  	"solaris/amd64":   true,
  1588  	"windows/386":     true,
  1589  	"windows/amd64":   true,
  1590  	"windows/arm":     false,
  1591  	"windows/arm64":   true,
  1592  }
  1593  
  1594  // List of platforms which are supported but not complete yet. These get
  1595  // filtered out of cgoEnabled for 'dist list'. See golang.org/issue/28944
  1596  var incomplete = map[string]bool{
  1597  	"linux/sparc64": true,
  1598  }
  1599  
  1600  // List of platforms which are first class ports. See golang.org/issue/38874.
  1601  var firstClass = map[string]bool{
  1602  	"darwin/amd64":  true,
  1603  	"darwin/arm64":  true,
  1604  	"linux/386":     true,
  1605  	"linux/amd64":   true,
  1606  	"linux/arm":     true,
  1607  	"linux/arm64":   true,
  1608  	"windows/386":   true,
  1609  	"windows/amd64": true,
  1610  }
  1611  
  1612  func needCC() bool {
  1613  	switch os.Getenv("CGO_ENABLED") {
  1614  	case "1":
  1615  		return true
  1616  	case "0":
  1617  		return false
  1618  	}
  1619  	return cgoEnabled[gohostos+"/"+gohostarch]
  1620  }
  1621  
  1622  func checkCC() {
  1623  	if !needCC() {
  1624  		return
  1625  	}
  1626  	if output, err := exec.Command(defaultcc[""], "--help").CombinedOutput(); err != nil {
  1627  		outputHdr := ""
  1628  		if len(output) > 0 {
  1629  			outputHdr = "\nCommand output:\n\n"
  1630  		}
  1631  		fatalf("cannot invoke C compiler %q: %v\n\n"+
  1632  			"Go needs a system C compiler for use with cgo.\n"+
  1633  			"To set a C compiler, set CC=the-compiler.\n"+
  1634  			"To disable cgo, set CGO_ENABLED=0.\n%s%s", defaultcc[""], err, outputHdr, output)
  1635  	}
  1636  }
  1637  
  1638  func defaulttarg() string {
  1639  	// xgetwd might return a path with symlinks fully resolved, and if
  1640  	// there happens to be symlinks in goroot, then the hasprefix test
  1641  	// will never succeed. Instead, we use xrealwd to get a canonical
  1642  	// goroot/src before the comparison to avoid this problem.
  1643  	pwd := xgetwd()
  1644  	src := pathf("%s/src/", goroot)
  1645  	real_src := xrealwd(src)
  1646  	if !strings.HasPrefix(pwd, real_src) {
  1647  		fatalf("current directory %s is not under %s", pwd, real_src)
  1648  	}
  1649  	pwd = pwd[len(real_src):]
  1650  	// guard against xrealwd returning the directory without the trailing /
  1651  	pwd = strings.TrimPrefix(pwd, "/")
  1652  
  1653  	return pwd
  1654  }
  1655  
  1656  // Install installs the list of packages named on the command line.
  1657  func cmdinstall() {
  1658  	xflagparse(-1)
  1659  
  1660  	if flag.NArg() == 0 {
  1661  		install(defaulttarg())
  1662  	}
  1663  
  1664  	for _, arg := range flag.Args() {
  1665  		install(arg)
  1666  	}
  1667  }
  1668  
  1669  // Clean deletes temporary objects.
  1670  func cmdclean() {
  1671  	xflagparse(0)
  1672  	clean()
  1673  }
  1674  
  1675  // Banner prints the 'now you've installed Go' banner.
  1676  func cmdbanner() {
  1677  	xflagparse(0)
  1678  	banner()
  1679  }
  1680  
  1681  func banner() {
  1682  	if vflag > 0 {
  1683  		xprintf("\n")
  1684  	}
  1685  	xprintf("---\n")
  1686  	xprintf("Installed Go for %s/%s in %s\n", goos, goarch, goroot)
  1687  	xprintf("Installed commands in %s\n", gobin)
  1688  
  1689  	if !xsamefile(goroot_final, goroot) {
  1690  		// If the files are to be moved, don't check that gobin
  1691  		// is on PATH; assume they know what they are doing.
  1692  	} else if gohostos == "plan9" {
  1693  		// Check that gobin is bound before /bin.
  1694  		pid := strings.Replace(readfile("#c/pid"), " ", "", -1)
  1695  		ns := fmt.Sprintf("/proc/%s/ns", pid)
  1696  		if !strings.Contains(readfile(ns), fmt.Sprintf("bind -b %s /bin", gobin)) {
  1697  			xprintf("*** You need to bind %s before /bin.\n", gobin)
  1698  		}
  1699  	} else {
  1700  		// Check that gobin appears in $PATH.
  1701  		pathsep := ":"
  1702  		if gohostos == "windows" {
  1703  			pathsep = ";"
  1704  		}
  1705  		if !strings.Contains(pathsep+os.Getenv("PATH")+pathsep, pathsep+gobin+pathsep) {
  1706  			xprintf("*** You need to add %s to your PATH.\n", gobin)
  1707  		}
  1708  	}
  1709  
  1710  	if !xsamefile(goroot_final, goroot) {
  1711  		xprintf("\n"+
  1712  			"The binaries expect %s to be copied or moved to %s\n",
  1713  			goroot, goroot_final)
  1714  	}
  1715  }
  1716  
  1717  // Version prints the Go version.
  1718  func cmdversion() {
  1719  	xflagparse(0)
  1720  	xprintf("%s\n", findgoversion())
  1721  }
  1722  
  1723  // cmdlist lists all supported platforms.
  1724  func cmdlist() {
  1725  	jsonFlag := flag.Bool("json", false, "produce JSON output")
  1726  	xflagparse(0)
  1727  
  1728  	var plats []string
  1729  	for p := range cgoEnabled {
  1730  		if incomplete[p] {
  1731  			continue
  1732  		}
  1733  		plats = append(plats, p)
  1734  	}
  1735  	sort.Strings(plats)
  1736  
  1737  	if !*jsonFlag {
  1738  		for _, p := range plats {
  1739  			xprintf("%s\n", p)
  1740  		}
  1741  		return
  1742  	}
  1743  
  1744  	type jsonResult struct {
  1745  		GOOS         string
  1746  		GOARCH       string
  1747  		CgoSupported bool
  1748  		FirstClass   bool
  1749  	}
  1750  	var results []jsonResult
  1751  	for _, p := range plats {
  1752  		fields := strings.Split(p, "/")
  1753  		results = append(results, jsonResult{
  1754  			GOOS:         fields[0],
  1755  			GOARCH:       fields[1],
  1756  			CgoSupported: cgoEnabled[p],
  1757  			FirstClass:   firstClass[p]})
  1758  	}
  1759  	out, err := json.MarshalIndent(results, "", "\t")
  1760  	if err != nil {
  1761  		fatalf("json marshal error: %v", err)
  1762  	}
  1763  	if _, err := os.Stdout.Write(out); err != nil {
  1764  		fatalf("write failed: %v", err)
  1765  	}
  1766  }
  1767  
  1768  // IsRuntimePackagePath examines 'pkgpath' and returns TRUE if it
  1769  // belongs to the collection of "runtime-related" packages, including
  1770  // "runtime" itself, "reflect", "syscall", and the
  1771  // "runtime/internal/*" packages.
  1772  //
  1773  // Keep in sync with cmd/internal/objabi/path.go:IsRuntimePackagePath.
  1774  func IsRuntimePackagePath(pkgpath string) bool {
  1775  	rval := false
  1776  	switch pkgpath {
  1777  	case "runtime":
  1778  		rval = true
  1779  	case "reflect":
  1780  		rval = true
  1781  	case "syscall":
  1782  		rval = true
  1783  	case "internal/bytealg":
  1784  		rval = true
  1785  	default:
  1786  		rval = strings.HasPrefix(pkgpath, "runtime/internal")
  1787  	}
  1788  	return rval
  1789  }
  1790  

View as plain text