vendor.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modcmd
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"io"
    14  	"io/fs"
    15  	"os"
    16  	"path"
    17  	"path/filepath"
    18  	"sort"
    19  	"strings"
    20  
    21  	"cmd/go/internal/base"
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/fsys"
    24  	"cmd/go/internal/imports"
    25  	"cmd/go/internal/load"
    26  	"cmd/go/internal/modload"
    27  	"cmd/go/internal/str"
    28  
    29  	"golang.org/x/mod/module"
    30  	"golang.org/x/mod/semver"
    31  )
    32  
    33  var cmdVendor = &base.Command{
    34  	UsageLine: "go mod vendor [-e] [-v] [-o outdir]",
    35  	Short:     "make vendored copy of dependencies",
    36  	Long: `
    37  Vendor resets the main module's vendor directory to include all packages
    38  needed to build and test all the main module's packages.
    39  It does not include test code for vendored packages.
    40  
    41  The -v flag causes vendor to print the names of vendored
    42  modules and packages to standard error.
    43  
    44  The -e flag causes vendor to attempt to proceed despite errors
    45  encountered while loading packages.
    46  
    47  The -o flag causes vendor to create the vendor directory at the given
    48  path instead of "vendor". The go command can only use a vendor directory
    49  named "vendor" within the module root directory, so this flag is
    50  primarily useful for other tools.
    51  
    52  See https://golang.org/ref/mod#go-mod-vendor for more about 'go mod vendor'.
    53  	`,
    54  	Run: runVendor,
    55  }
    56  
    57  var vendorE bool   // if true, report errors but proceed anyway
    58  var vendorO string // if set, overrides the default output directory
    59  
    60  func init() {
    61  	cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "")
    62  	cmdVendor.Flag.BoolVar(&vendorE, "e", false, "")
    63  	cmdVendor.Flag.StringVar(&vendorO, "o", "", "")
    64  	base.AddModCommonFlags(&cmdVendor.Flag)
    65  }
    66  
    67  func runVendor(ctx context.Context, cmd *base.Command, args []string) {
    68  	if len(args) != 0 {
    69  		base.Fatalf("go: 'go mod vendor' accepts no arguments")
    70  	}
    71  	modload.ForceUseModules = true
    72  	modload.RootMode = modload.NeedRoot
    73  
    74  	loadOpts := modload.PackageOpts{
    75  		Tags:                     imports.AnyTags(),
    76  		VendorModulesInGOROOTSrc: true,
    77  		ResolveMissingImports:    true,
    78  		UseVendorAll:             true,
    79  		AllowErrors:              vendorE,
    80  		SilenceMissingStdImports: true,
    81  	}
    82  	_, pkgs := modload.LoadPackages(ctx, loadOpts, "all")
    83  
    84  	var vdir string
    85  	switch {
    86  	case filepath.IsAbs(vendorO):
    87  		vdir = vendorO
    88  	case vendorO != "":
    89  		vdir = filepath.Join(base.Cwd(), vendorO)
    90  	default:
    91  		vdir = filepath.Join(modload.VendorDir())
    92  	}
    93  	if err := os.RemoveAll(vdir); err != nil {
    94  		base.Fatalf("go: %v", err)
    95  	}
    96  
    97  	modpkgs := make(map[module.Version][]string)
    98  	for _, pkg := range pkgs {
    99  		m := modload.PackageModule(pkg)
   100  		if m.Path == "" || m.Version == "" && modload.MainModules.Contains(m.Path) {
   101  			continue
   102  		}
   103  		modpkgs[m] = append(modpkgs[m], pkg)
   104  	}
   105  
   106  	includeAllReplacements := false
   107  	includeGoVersions := false
   108  	isExplicit := map[module.Version]bool{}
   109  	if gv := modload.ModFile().Go; gv != nil {
   110  		if semver.Compare("v"+gv.Version, "v1.14") >= 0 {
   111  			// If the Go version is at least 1.14, annotate all explicit 'require' and
   112  			// 'replace' targets found in the go.mod file so that we can perform a
   113  			// stronger consistency check when -mod=vendor is set.
   114  			for _, r := range modload.ModFile().Require {
   115  				isExplicit[r.Mod] = true
   116  			}
   117  			includeAllReplacements = true
   118  		}
   119  		if semver.Compare("v"+gv.Version, "v1.17") >= 0 {
   120  			// If the Go version is at least 1.17, annotate all modules with their
   121  			// 'go' version directives.
   122  			includeGoVersions = true
   123  		}
   124  	}
   125  
   126  	var vendorMods []module.Version
   127  	for m := range isExplicit {
   128  		vendorMods = append(vendorMods, m)
   129  	}
   130  	for m := range modpkgs {
   131  		if !isExplicit[m] {
   132  			vendorMods = append(vendorMods, m)
   133  		}
   134  	}
   135  	module.Sort(vendorMods)
   136  
   137  	var (
   138  		buf bytes.Buffer
   139  		w   io.Writer = &buf
   140  	)
   141  	if cfg.BuildV {
   142  		w = io.MultiWriter(&buf, os.Stderr)
   143  	}
   144  
   145  	for _, m := range vendorMods {
   146  		replacement := modload.Replacement(m)
   147  		line := moduleLine(m, replacement)
   148  		io.WriteString(w, line)
   149  
   150  		goVersion := ""
   151  		if includeGoVersions {
   152  			goVersion = modload.ModuleInfo(ctx, m.Path).GoVersion
   153  		}
   154  		switch {
   155  		case isExplicit[m] && goVersion != "":
   156  			fmt.Fprintf(w, "## explicit; go %s\n", goVersion)
   157  		case isExplicit[m]:
   158  			io.WriteString(w, "## explicit\n")
   159  		case goVersion != "":
   160  			fmt.Fprintf(w, "## go %s\n", goVersion)
   161  		}
   162  
   163  		pkgs := modpkgs[m]
   164  		sort.Strings(pkgs)
   165  		for _, pkg := range pkgs {
   166  			fmt.Fprintf(w, "%s\n", pkg)
   167  			vendorPkg(vdir, pkg)
   168  		}
   169  	}
   170  
   171  	if includeAllReplacements {
   172  		// Record unused and wildcard replacements at the end of the modules.txt file:
   173  		// without access to the complete build list, the consumer of the vendor
   174  		// directory can't otherwise determine that those replacements had no effect.
   175  		for _, r := range modload.ModFile().Replace {
   176  			if len(modpkgs[r.Old]) > 0 {
   177  				// We we already recorded this replacement in the entry for the replaced
   178  				// module with the packages it provides.
   179  				continue
   180  			}
   181  
   182  			line := moduleLine(r.Old, r.New)
   183  			buf.WriteString(line)
   184  			if cfg.BuildV {
   185  				os.Stderr.WriteString(line)
   186  			}
   187  		}
   188  	}
   189  
   190  	if buf.Len() == 0 {
   191  		fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
   192  		return
   193  	}
   194  
   195  	if err := os.MkdirAll(vdir, 0777); err != nil {
   196  		base.Fatalf("go: %v", err)
   197  	}
   198  
   199  	if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil {
   200  		base.Fatalf("go: %v", err)
   201  	}
   202  }
   203  
   204  func moduleLine(m, r module.Version) string {
   205  	b := new(strings.Builder)
   206  	b.WriteString("# ")
   207  	b.WriteString(m.Path)
   208  	if m.Version != "" {
   209  		b.WriteString(" ")
   210  		b.WriteString(m.Version)
   211  	}
   212  	if r.Path != "" {
   213  		b.WriteString(" => ")
   214  		b.WriteString(r.Path)
   215  		if r.Version != "" {
   216  			b.WriteString(" ")
   217  			b.WriteString(r.Version)
   218  		}
   219  	}
   220  	b.WriteString("\n")
   221  	return b.String()
   222  }
   223  
   224  func vendorPkg(vdir, pkg string) {
   225  	// TODO(#42504): Instead of calling modload.ImportMap then build.ImportDir,
   226  	// just call load.PackagesAndErrors. To do that, we need to add a good way
   227  	// to ignore build constraints.
   228  	realPath := modload.ImportMap(pkg)
   229  	if realPath != pkg && modload.ImportMap(realPath) != "" {
   230  		fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
   231  	}
   232  
   233  	copiedFiles := make(map[string]bool)
   234  	dst := filepath.Join(vdir, pkg)
   235  	src := modload.PackageDir(realPath)
   236  	if src == "" {
   237  		fmt.Fprintf(os.Stderr, "internal error: no pkg for %s -> %s\n", pkg, realPath)
   238  	}
   239  	copyDir(dst, src, matchPotentialSourceFile, copiedFiles)
   240  	if m := modload.PackageModule(realPath); m.Path != "" {
   241  		copyMetadata(m.Path, realPath, dst, src, copiedFiles)
   242  	}
   243  
   244  	ctx := build.Default
   245  	ctx.UseAllFiles = true
   246  	bp, err := ctx.ImportDir(src, build.IgnoreVendor)
   247  	// Because UseAllFiles is set on the build.Context, it's possible ta get
   248  	// a MultiplePackageError on an otherwise valid package: the package could
   249  	// have different names for GOOS=windows and GOOS=mac for example. On the
   250  	// other hand if there's a NoGoError, the package might have source files
   251  	// specifying "// +build ignore" those packages should be skipped because
   252  	// embeds from ignored files can't be used.
   253  	// TODO(#42504): Find a better way to avoid errors from ImportDir. We'll
   254  	// need to figure this out when we switch to PackagesAndErrors as per the
   255  	// TODO above.
   256  	var multiplePackageError *build.MultiplePackageError
   257  	var noGoError *build.NoGoError
   258  	if err != nil {
   259  		if errors.As(err, &noGoError) {
   260  			return // No source files in this package are built. Skip embeds in ignored files.
   261  		} else if !errors.As(err, &multiplePackageError) { // multiplePackageErrors are OK, but others are not.
   262  			base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err)
   263  		}
   264  	}
   265  	embedPatterns := str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns)
   266  	embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns)
   267  	if err != nil {
   268  		base.Fatalf("go: %v", err)
   269  	}
   270  	for _, embed := range embeds {
   271  		embedDst := filepath.Join(dst, embed)
   272  		if copiedFiles[embedDst] {
   273  			continue
   274  		}
   275  
   276  		// Copy the file as is done by copyDir below.
   277  		r, err := os.Open(filepath.Join(src, embed))
   278  		if err != nil {
   279  			base.Fatalf("go: %v", err)
   280  		}
   281  		if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil {
   282  			base.Fatalf("go: %v", err)
   283  		}
   284  		w, err := os.Create(embedDst)
   285  		if err != nil {
   286  			base.Fatalf("go: %v", err)
   287  		}
   288  		if _, err := io.Copy(w, r); err != nil {
   289  			base.Fatalf("go: %v", err)
   290  		}
   291  		r.Close()
   292  		if err := w.Close(); err != nil {
   293  			base.Fatalf("go: %v", err)
   294  		}
   295  	}
   296  }
   297  
   298  type metakey struct {
   299  	modPath string
   300  	dst     string
   301  }
   302  
   303  var copiedMetadata = make(map[metakey]bool)
   304  
   305  // copyMetadata copies metadata files from parents of src to parents of dst,
   306  // stopping after processing the src parent for modPath.
   307  func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) {
   308  	for parent := 0; ; parent++ {
   309  		if copiedMetadata[metakey{modPath, dst}] {
   310  			break
   311  		}
   312  		copiedMetadata[metakey{modPath, dst}] = true
   313  		if parent > 0 {
   314  			copyDir(dst, src, matchMetadata, copiedFiles)
   315  		}
   316  		if modPath == pkg {
   317  			break
   318  		}
   319  		pkg = path.Dir(pkg)
   320  		dst = filepath.Dir(dst)
   321  		src = filepath.Dir(src)
   322  	}
   323  }
   324  
   325  // metaPrefixes is the list of metadata file prefixes.
   326  // Vendoring copies metadata files from parents of copied directories.
   327  // Note that this list could be arbitrarily extended, and it is longer
   328  // in other tools (such as godep or dep). By using this limited set of
   329  // prefixes and also insisting on capitalized file names, we are trying
   330  // to nudge people toward more agreement on the naming
   331  // and also trying to avoid false positives.
   332  var metaPrefixes = []string{
   333  	"AUTHORS",
   334  	"CONTRIBUTORS",
   335  	"COPYLEFT",
   336  	"COPYING",
   337  	"COPYRIGHT",
   338  	"LEGAL",
   339  	"LICENSE",
   340  	"NOTICE",
   341  	"PATENTS",
   342  }
   343  
   344  // matchMetadata reports whether info is a metadata file.
   345  func matchMetadata(dir string, info fs.DirEntry) bool {
   346  	name := info.Name()
   347  	for _, p := range metaPrefixes {
   348  		if strings.HasPrefix(name, p) {
   349  			return true
   350  		}
   351  	}
   352  	return false
   353  }
   354  
   355  // matchPotentialSourceFile reports whether info may be relevant to a build operation.
   356  func matchPotentialSourceFile(dir string, info fs.DirEntry) bool {
   357  	if strings.HasSuffix(info.Name(), "_test.go") {
   358  		return false
   359  	}
   360  	if info.Name() == "go.mod" || info.Name() == "go.sum" {
   361  		if gv := modload.ModFile().Go; gv != nil && semver.Compare("v"+gv.Version, "v1.17") >= 0 {
   362  			// As of Go 1.17, we strip go.mod and go.sum files from dependency modules.
   363  			// Otherwise, 'go' commands invoked within the vendor subtree may misidentify
   364  			// an arbitrary directory within the vendor tree as a module root.
   365  			// (See https://golang.org/issue/42970.)
   366  			return false
   367  		}
   368  	}
   369  	if strings.HasSuffix(info.Name(), ".go") {
   370  		f, err := fsys.Open(filepath.Join(dir, info.Name()))
   371  		if err != nil {
   372  			base.Fatalf("go: %v", err)
   373  		}
   374  		defer f.Close()
   375  
   376  		content, err := imports.ReadImports(f, false, nil)
   377  		if err == nil && !imports.ShouldBuild(content, imports.AnyTags()) {
   378  			// The file is explicitly tagged "ignore", so it can't affect the build.
   379  			// Leave it out.
   380  			return false
   381  		}
   382  		return true
   383  	}
   384  
   385  	// We don't know anything about this file, so optimistically assume that it is
   386  	// needed.
   387  	return true
   388  }
   389  
   390  // copyDir copies all regular files satisfying match(info) from src to dst.
   391  func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) {
   392  	files, err := os.ReadDir(src)
   393  	if err != nil {
   394  		base.Fatalf("go: %v", err)
   395  	}
   396  	if err := os.MkdirAll(dst, 0777); err != nil {
   397  		base.Fatalf("go: %v", err)
   398  	}
   399  	for _, file := range files {
   400  		if file.IsDir() || !file.Type().IsRegular() || !match(src, file) {
   401  			continue
   402  		}
   403  		copiedFiles[file.Name()] = true
   404  		r, err := os.Open(filepath.Join(src, file.Name()))
   405  		if err != nil {
   406  			base.Fatalf("go: %v", err)
   407  		}
   408  		dstPath := filepath.Join(dst, file.Name())
   409  		copiedFiles[dstPath] = true
   410  		w, err := os.Create(dstPath)
   411  		if err != nil {
   412  			base.Fatalf("go: %v", err)
   413  		}
   414  		if _, err := io.Copy(w, r); err != nil {
   415  			base.Fatalf("go: %v", err)
   416  		}
   417  		r.Close()
   418  		if err := w.Close(); err != nil {
   419  			base.Fatalf("go: %v", err)
   420  		}
   421  	}
   422  }
   423
View as plain text