Source file src/cmd/go/internal/modfetch/fetch.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"context"
    11  	"crypto/sha256"
    12  	"encoding/base64"
    13  	"errors"
    14  	"fmt"
    15  	"io"
    16  	"io/fs"
    17  	"os"
    18  	"path/filepath"
    19  	"sort"
    20  	"strings"
    21  	"sync"
    22  
    23  	"cmd/go/internal/base"
    24  	"cmd/go/internal/cfg"
    25  	"cmd/go/internal/fsys"
    26  	"cmd/go/internal/lockedfile"
    27  	"cmd/go/internal/par"
    28  	"cmd/go/internal/robustio"
    29  	"cmd/go/internal/trace"
    30  
    31  	"golang.org/x/mod/module"
    32  	"golang.org/x/mod/sumdb/dirhash"
    33  	modzip "golang.org/x/mod/zip"
    34  )
    35  
    36  var downloadCache par.Cache
    37  
    38  // Download downloads the specific module version to the
    39  // local download cache and returns the name of the directory
    40  // corresponding to the root of the module's file tree.
    41  func Download(ctx context.Context, mod module.Version) (dir string, err error) {
    42  	if err := checkCacheDir(); err != nil {
    43  		base.Fatalf("go: %v", err)
    44  	}
    45  
    46  	// The par.Cache here avoids duplicate work.
    47  	type cached struct {
    48  		dir string
    49  		err error
    50  	}
    51  	c := downloadCache.Do(mod, func() any {
    52  		dir, err := download(ctx, mod)
    53  		if err != nil {
    54  			return cached{"", err}
    55  		}
    56  		checkMod(mod)
    57  		return cached{dir, nil}
    58  	}).(cached)
    59  	return c.dir, c.err
    60  }
    61  
    62  func download(ctx context.Context, mod module.Version) (dir string, err error) {
    63  	ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
    64  	defer span.Done()
    65  
    66  	dir, err = DownloadDir(mod)
    67  	if err == nil {
    68  		// The directory has already been completely extracted (no .partial file exists).
    69  		return dir, nil
    70  	} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
    71  		return "", err
    72  	}
    73  
    74  	// To avoid cluttering the cache with extraneous files,
    75  	// DownloadZip uses the same lockfile as Download.
    76  	// Invoke DownloadZip before locking the file.
    77  	zipfile, err := DownloadZip(ctx, mod)
    78  	if err != nil {
    79  		return "", err
    80  	}
    81  
    82  	unlock, err := lockVersion(mod)
    83  	if err != nil {
    84  		return "", err
    85  	}
    86  	defer unlock()
    87  
    88  	ctx, span = trace.StartSpan(ctx, "unzip "+zipfile)
    89  	defer span.Done()
    90  
    91  	// Check whether the directory was populated while we were waiting on the lock.
    92  	_, dirErr := DownloadDir(mod)
    93  	if dirErr == nil {
    94  		return dir, nil
    95  	}
    96  	_, dirExists := dirErr.(*DownloadDirPartialError)
    97  
    98  	// Clean up any remaining temporary directories created by old versions
    99  	// (before 1.16), as well as partially extracted directories (indicated by
   100  	// DownloadDirPartialError, usually because of a .partial file). This is only
   101  	// safe to do because the lock file ensures that their writers are no longer
   102  	// active.
   103  	parentDir := filepath.Dir(dir)
   104  	tmpPrefix := filepath.Base(dir) + ".tmp-"
   105  	if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil {
   106  		for _, path := range old {
   107  			RemoveAll(path) // best effort
   108  		}
   109  	}
   110  	if dirExists {
   111  		if err := RemoveAll(dir); err != nil {
   112  			return "", err
   113  		}
   114  	}
   115  
   116  	partialPath, err := CachePath(mod, "partial")
   117  	if err != nil {
   118  		return "", err
   119  	}
   120  
   121  	// Extract the module zip directory at its final location.
   122  	//
   123  	// To prevent other processes from reading the directory if we crash,
   124  	// create a .partial file before extracting the directory, and delete
   125  	// the .partial file afterward (all while holding the lock).
   126  	//
   127  	// Before Go 1.16, we extracted to a temporary directory with a random name
   128  	// then renamed it into place with os.Rename. On Windows, this failed with
   129  	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
   130  	// opened files in the temporary directory.
   131  	//
   132  	// Go 1.14.2 and higher respect .partial files. Older versions may use
   133  	// partially extracted directories. 'go mod verify' can detect this,
   134  	// and 'go clean -modcache' can fix it.
   135  	if err := os.MkdirAll(parentDir, 0777); err != nil {
   136  		return "", err
   137  	}
   138  	if err := os.WriteFile(partialPath, nil, 0666); err != nil {
   139  		return "", err
   140  	}
   141  	if err := modzip.Unzip(dir, mod, zipfile); err != nil {
   142  		fmt.Fprintf(os.Stderr, "-> %s\n", err)
   143  		if rmErr := RemoveAll(dir); rmErr == nil {
   144  			os.Remove(partialPath)
   145  		}
   146  		return "", err
   147  	}
   148  	if err := os.Remove(partialPath); err != nil {
   149  		return "", err
   150  	}
   151  
   152  	if !cfg.ModCacheRW {
   153  		makeDirsReadOnly(dir)
   154  	}
   155  	return dir, nil
   156  }
   157  
   158  var downloadZipCache par.Cache
   159  
   160  // DownloadZip downloads the specific module version to the
   161  // local zip cache and returns the name of the zip file.
   162  func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) {
   163  	// The par.Cache here avoids duplicate work.
   164  	type cached struct {
   165  		zipfile string
   166  		err     error
   167  	}
   168  	c := downloadZipCache.Do(mod, func() any {
   169  		zipfile, err := CachePath(mod, "zip")
   170  		if err != nil {
   171  			return cached{"", err}
   172  		}
   173  		ziphashfile := zipfile + "hash"
   174  
   175  		// Return without locking if the zip and ziphash files exist.
   176  		if _, err := os.Stat(zipfile); err == nil {
   177  			if _, err := os.Stat(ziphashfile); err == nil {
   178  				return cached{zipfile, nil}
   179  			}
   180  		}
   181  
   182  		// The zip or ziphash file does not exist. Acquire the lock and create them.
   183  		if cfg.CmdName != "mod download" {
   184  			fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
   185  		}
   186  		unlock, err := lockVersion(mod)
   187  		if err != nil {
   188  			return cached{"", err}
   189  		}
   190  		defer unlock()
   191  
   192  		if err := downloadZip(ctx, mod, zipfile); err != nil {
   193  			return cached{"", err}
   194  		}
   195  		return cached{zipfile, nil}
   196  	}).(cached)
   197  	return c.zipfile, c.err
   198  }
   199  
   200  func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) {
   201  	ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile)
   202  	defer span.Done()
   203  
   204  	// Double-check that the zipfile was not created while we were waiting for
   205  	// the lock in DownloadZip.
   206  	ziphashfile := zipfile + "hash"
   207  	var zipExists, ziphashExists bool
   208  	if _, err := os.Stat(zipfile); err == nil {
   209  		zipExists = true
   210  	}
   211  	if _, err := os.Stat(ziphashfile); err == nil {
   212  		ziphashExists = true
   213  	}
   214  	if zipExists && ziphashExists {
   215  		return nil
   216  	}
   217  
   218  	// Create parent directories.
   219  	if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
   220  		return err
   221  	}
   222  
   223  	// Clean up any remaining tempfiles from previous runs.
   224  	// This is only safe to do because the lock file ensures that their
   225  	// writers are no longer active.
   226  	tmpPattern := filepath.Base(zipfile) + "*.tmp"
   227  	if old, err := filepath.Glob(filepath.Join(filepath.Dir(zipfile), tmpPattern)); err == nil {
   228  		for _, path := range old {
   229  			os.Remove(path) // best effort
   230  		}
   231  	}
   232  
   233  	// If the zip file exists, the ziphash file must have been deleted
   234  	// or lost after a file system crash. Re-hash the zip without downloading.
   235  	if zipExists {
   236  		return hashZip(mod, zipfile, ziphashfile)
   237  	}
   238  
   239  	// From here to the os.Rename call below is functionally almost equivalent to
   240  	// renameio.WriteToFile, with one key difference: we want to validate the
   241  	// contents of the file (by hashing it) before we commit it. Because the file
   242  	// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
   243  	// validate it: we can't just tee the stream as we write it.
   244  	f, err := os.CreateTemp(filepath.Dir(zipfile), tmpPattern)
   245  	if err != nil {
   246  		return err
   247  	}
   248  	defer func() {
   249  		if err != nil {
   250  			f.Close()
   251  			os.Remove(f.Name())
   252  		}
   253  	}()
   254  
   255  	var unrecoverableErr error
   256  	err = TryProxies(func(proxy string) error {
   257  		if unrecoverableErr != nil {
   258  			return unrecoverableErr
   259  		}
   260  		repo := Lookup(proxy, mod.Path)
   261  		err := repo.Zip(f, mod.Version)
   262  		if err != nil {
   263  			// Zip may have partially written to f before failing.
   264  			// (Perhaps the server crashed while sending the file?)
   265  			// Since we allow fallback on error in some cases, we need to fix up the
   266  			// file to be empty again for the next attempt.
   267  			if _, err := f.Seek(0, io.SeekStart); err != nil {
   268  				unrecoverableErr = err
   269  				return err
   270  			}
   271  			if err := f.Truncate(0); err != nil {
   272  				unrecoverableErr = err
   273  				return err
   274  			}
   275  		}
   276  		return err
   277  	})
   278  	if err != nil {
   279  		return err
   280  	}
   281  
   282  	// Double-check that the paths within the zip file are well-formed.
   283  	//
   284  	// TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
   285  	fi, err := f.Stat()
   286  	if err != nil {
   287  		return err
   288  	}
   289  	z, err := zip.NewReader(f, fi.Size())
   290  	if err != nil {
   291  		return err
   292  	}
   293  	prefix := mod.Path + "@" + mod.Version + "/"
   294  	for _, f := range z.File {
   295  		if !strings.HasPrefix(f.Name, prefix) {
   296  			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
   297  		}
   298  	}
   299  
   300  	if err := f.Close(); err != nil {
   301  		return err
   302  	}
   303  
   304  	// Hash the zip file and check the sum before renaming to the final location.
   305  	if err := hashZip(mod, f.Name(), ziphashfile); err != nil {
   306  		return err
   307  	}
   308  	if err := os.Rename(f.Name(), zipfile); err != nil {
   309  		return err
   310  	}
   311  
   312  	// TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
   313  
   314  	return nil
   315  }
   316  
   317  // hashZip reads the zip file opened in f, then writes the hash to ziphashfile,
   318  // overwriting that file if it exists.
   319  //
   320  // If the hash does not match go.sum (or the sumdb if enabled), hashZip returns
   321  // an error and does not write ziphashfile.
   322  func hashZip(mod module.Version, zipfile, ziphashfile string) (err error) {
   323  	hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash)
   324  	if err != nil {
   325  		return err
   326  	}
   327  	if err := checkModSum(mod, hash); err != nil {
   328  		return err
   329  	}
   330  	hf, err := lockedfile.Create(ziphashfile)
   331  	if err != nil {
   332  		return err
   333  	}
   334  	defer func() {
   335  		if closeErr := hf.Close(); err == nil && closeErr != nil {
   336  			err = closeErr
   337  		}
   338  	}()
   339  	if err := hf.Truncate(int64(len(hash))); err != nil {
   340  		return err
   341  	}
   342  	if _, err := hf.WriteAt([]byte(hash), 0); err != nil {
   343  		return err
   344  	}
   345  	return nil
   346  }
   347  
   348  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   349  // and its transitive contents.
   350  func makeDirsReadOnly(dir string) {
   351  	type pathMode struct {
   352  		path string
   353  		mode fs.FileMode
   354  	}
   355  	var dirs []pathMode // in lexical order
   356  	filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   357  		if err == nil && d.IsDir() {
   358  			info, err := d.Info()
   359  			if err == nil && info.Mode()&0222 != 0 {
   360  				dirs = append(dirs, pathMode{path, info.Mode()})
   361  			}
   362  		}
   363  		return nil
   364  	})
   365  
   366  	// Run over list backward to chmod children before parents.
   367  	for i := len(dirs) - 1; i >= 0; i-- {
   368  		os.Chmod(dirs[i].path, dirs[i].mode&^0222)
   369  	}
   370  }
   371  
   372  // RemoveAll removes a directory written by Download or Unzip, first applying
   373  // any permission changes needed to do so.
   374  func RemoveAll(dir string) error {
   375  	// Module cache has 0555 directories; make them writable in order to remove content.
   376  	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
   377  		if err != nil {
   378  			return nil // ignore errors walking in file system
   379  		}
   380  		if info.IsDir() {
   381  			os.Chmod(path, 0777)
   382  		}
   383  		return nil
   384  	})
   385  	return robustio.RemoveAll(dir)
   386  }
   387  
   388  var GoSumFile string             // path to go.sum; set by package modload
   389  var WorkspaceGoSumFiles []string // path to module go.sums in workspace; set by package modload
   390  
   391  type modSum struct {
   392  	mod module.Version
   393  	sum string
   394  }
   395  
   396  var goSum struct {
   397  	mu        sync.Mutex
   398  	m         map[module.Version][]string            // content of go.sum file
   399  	w         map[string]map[module.Version][]string // sum file in workspace -> content of that sum file
   400  	status    map[modSum]modSumStatus                // state of sums in m
   401  	overwrite bool                                   // if true, overwrite go.sum without incorporating its contents
   402  	enabled   bool                                   // whether to use go.sum at all
   403  }
   404  
   405  type modSumStatus struct {
   406  	used, dirty bool
   407  }
   408  
   409  // Reset resets globals in the modfetch package, so previous loads don't affect
   410  // contents of go.sum files
   411  func Reset() {
   412  	GoSumFile = ""
   413  	WorkspaceGoSumFiles = nil
   414  
   415  	// Uses of lookupCache and downloadCache both can call checkModSum,
   416  	// which in turn sets the used bit on goSum.status for modules.
   417  	// Reset them so used can be computed properly.
   418  	lookupCache = par.Cache{}
   419  	downloadCache = par.Cache{}
   420  
   421  	// Clear all fields on goSum. It will be initialized later
   422  	goSum.mu.Lock()
   423  	goSum.m = nil
   424  	goSum.w = nil
   425  	goSum.status = nil
   426  	goSum.overwrite = false
   427  	goSum.enabled = false
   428  	goSum.mu.Unlock()
   429  }
   430  
   431  // initGoSum initializes the go.sum data.
   432  // The boolean it returns reports whether the
   433  // use of go.sum is now enabled.
   434  // The goSum lock must be held.
   435  func initGoSum() (bool, error) {
   436  	if GoSumFile == "" {
   437  		return false, nil
   438  	}
   439  	if goSum.m != nil {
   440  		return true, nil
   441  	}
   442  
   443  	goSum.m = make(map[module.Version][]string)
   444  	goSum.status = make(map[modSum]modSumStatus)
   445  	goSum.w = make(map[string]map[module.Version][]string)
   446  
   447  	for _, f := range WorkspaceGoSumFiles {
   448  		goSum.w[f] = make(map[module.Version][]string)
   449  		_, err := readGoSumFile(goSum.w[f], f)
   450  		if err != nil {
   451  			return false, err
   452  		}
   453  	}
   454  
   455  	enabled, err := readGoSumFile(goSum.m, GoSumFile)
   456  	goSum.enabled = enabled
   457  	return enabled, err
   458  }
   459  
   460  func readGoSumFile(dst map[module.Version][]string, file string) (bool, error) {
   461  	var (
   462  		data []byte
   463  		err  error
   464  	)
   465  	if actualSumFile, ok := fsys.OverlayPath(file); ok {
   466  		// Don't lock go.sum if it's part of the overlay.
   467  		// On Plan 9, locking requires chmod, and we don't want to modify any file
   468  		// in the overlay. See #44700.
   469  		data, err = os.ReadFile(actualSumFile)
   470  	} else {
   471  		data, err = lockedfile.Read(file)
   472  	}
   473  	if err != nil && !os.IsNotExist(err) {
   474  		return false, err
   475  	}
   476  	readGoSum(dst, file, data)
   477  
   478  	return true, nil
   479  }
   480  
   481  // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
   482  // A bug caused us to write these into go.sum files for non-modules.
   483  // We detect and remove them.
   484  const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
   485  
   486  // readGoSum parses data, which is the content of file,
   487  // and adds it to goSum.m. The goSum lock must be held.
   488  func readGoSum(dst map[module.Version][]string, file string, data []byte) error {
   489  	lineno := 0
   490  	for len(data) > 0 {
   491  		var line []byte
   492  		lineno++
   493  		i := bytes.IndexByte(data, '\n')
   494  		if i < 0 {
   495  			line, data = data, nil
   496  		} else {
   497  			line, data = data[:i], data[i+1:]
   498  		}
   499  		f := strings.Fields(string(line))
   500  		if len(f) == 0 {
   501  			// blank line; skip it
   502  			continue
   503  		}
   504  		if len(f) != 3 {
   505  			return fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
   506  		}
   507  		if f[2] == emptyGoModHash {
   508  			// Old bug; drop it.
   509  			continue
   510  		}
   511  		mod := module.Version{Path: f[0], Version: f[1]}
   512  		dst[mod] = append(dst[mod], f[2])
   513  	}
   514  	return nil
   515  }
   516  
   517  // HaveSum returns true if the go.sum file contains an entry for mod.
   518  // The entry's hash must be generated with a known hash algorithm.
   519  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   520  // .mod and .zip files.
   521  func HaveSum(mod module.Version) bool {
   522  	goSum.mu.Lock()
   523  	defer goSum.mu.Unlock()
   524  	inited, err := initGoSum()
   525  	if err != nil || !inited {
   526  		return false
   527  	}
   528  	for _, goSums := range goSum.w {
   529  		for _, h := range goSums[mod] {
   530  			if !strings.HasPrefix(h, "h1:") {
   531  				continue
   532  			}
   533  			if !goSum.status[modSum{mod, h}].dirty {
   534  				return true
   535  			}
   536  		}
   537  	}
   538  	for _, h := range goSum.m[mod] {
   539  		if !strings.HasPrefix(h, "h1:") {
   540  			continue
   541  		}
   542  		if !goSum.status[modSum{mod, h}].dirty {
   543  			return true
   544  		}
   545  	}
   546  	return false
   547  }
   548  
   549  // checkMod checks the given module's checksum.
   550  func checkMod(mod module.Version) {
   551  	// Do the file I/O before acquiring the go.sum lock.
   552  	ziphash, err := CachePath(mod, "ziphash")
   553  	if err != nil {
   554  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   555  	}
   556  	data, err := lockedfile.Read(ziphash)
   557  	if err != nil {
   558  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   559  	}
   560  	data = bytes.TrimSpace(data)
   561  	if !isValidSum(data) {
   562  		// Recreate ziphash file from zip file and use that to check the mod sum.
   563  		zip, err := CachePath(mod, "zip")
   564  		if err != nil {
   565  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   566  		}
   567  		err = hashZip(mod, zip, ziphash)
   568  		if err != nil {
   569  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   570  		}
   571  		return
   572  	}
   573  	h := string(data)
   574  	if !strings.HasPrefix(h, "h1:") {
   575  		base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
   576  	}
   577  
   578  	if err := checkModSum(mod, h); err != nil {
   579  		base.Fatalf("%s", err)
   580  	}
   581  }
   582  
   583  // goModSum returns the checksum for the go.mod contents.
   584  func goModSum(data []byte) (string, error) {
   585  	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   586  		return io.NopCloser(bytes.NewReader(data)), nil
   587  	})
   588  }
   589  
   590  // checkGoMod checks the given module's go.mod checksum;
   591  // data is the go.mod content.
   592  func checkGoMod(path, version string, data []byte) error {
   593  	h, err := goModSum(data)
   594  	if err != nil {
   595  		return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
   596  	}
   597  
   598  	return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
   599  }
   600  
   601  // checkModSum checks that the recorded checksum for mod is h.
   602  //
   603  // mod.Version may have the additional suffix "/go.mod" to request the checksum
   604  // for the module's go.mod file only.
   605  func checkModSum(mod module.Version, h string) error {
   606  	// We lock goSum when manipulating it,
   607  	// but we arrange to release the lock when calling checkSumDB,
   608  	// so that parallel calls to checkModHash can execute parallel calls
   609  	// to checkSumDB.
   610  
   611  	// Check whether mod+h is listed in go.sum already. If so, we're done.
   612  	goSum.mu.Lock()
   613  	inited, err := initGoSum()
   614  	if err != nil {
   615  		goSum.mu.Unlock()
   616  		return err
   617  	}
   618  	done := inited && haveModSumLocked(mod, h)
   619  	if inited {
   620  		st := goSum.status[modSum{mod, h}]
   621  		st.used = true
   622  		goSum.status[modSum{mod, h}] = st
   623  	}
   624  	goSum.mu.Unlock()
   625  
   626  	if done {
   627  		return nil
   628  	}
   629  
   630  	// Not listed, so we want to add them.
   631  	// Consult checksum database if appropriate.
   632  	if useSumDB(mod) {
   633  		// Calls base.Fatalf if mismatch detected.
   634  		if err := checkSumDB(mod, h); err != nil {
   635  			return err
   636  		}
   637  	}
   638  
   639  	// Add mod+h to go.sum, if it hasn't appeared already.
   640  	if inited {
   641  		goSum.mu.Lock()
   642  		addModSumLocked(mod, h)
   643  		st := goSum.status[modSum{mod, h}]
   644  		st.dirty = true
   645  		goSum.status[modSum{mod, h}] = st
   646  		goSum.mu.Unlock()
   647  	}
   648  	return nil
   649  }
   650  
   651  // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
   652  // If it finds a conflicting pair instead, it calls base.Fatalf.
   653  // goSum.mu must be locked.
   654  func haveModSumLocked(mod module.Version, h string) bool {
   655  	sumFileName := "go.sum"
   656  	if strings.HasSuffix(GoSumFile, "go.work.sum") {
   657  		sumFileName = "go.work.sum"
   658  	}
   659  	for _, vh := range goSum.m[mod] {
   660  		if h == vh {
   661  			return true
   662  		}
   663  		if strings.HasPrefix(vh, "h1:") {
   664  			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, sumFileName, vh)
   665  		}
   666  	}
   667  	// Also check workspace sums.
   668  	foundMatch := false
   669  	// Check sums from all files in case there are conflicts between
   670  	// the files.
   671  	for goSumFile, goSums := range goSum.w {
   672  		for _, vh := range goSums[mod] {
   673  			if h == vh {
   674  				foundMatch = true
   675  			} else if strings.HasPrefix(vh, "h1:") {
   676  				base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, goSumFile, vh)
   677  			}
   678  		}
   679  	}
   680  	return foundMatch
   681  }
   682  
   683  // addModSumLocked adds the pair mod,h to go.sum.
   684  // goSum.mu must be locked.
   685  func addModSumLocked(mod module.Version, h string) {
   686  	if haveModSumLocked(mod, h) {
   687  		return
   688  	}
   689  	if len(goSum.m[mod]) > 0 {
   690  		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
   691  	}
   692  	goSum.m[mod] = append(goSum.m[mod], h)
   693  }
   694  
   695  // checkSumDB checks the mod, h pair against the Go checksum database.
   696  // It calls base.Fatalf if the hash is to be rejected.
   697  func checkSumDB(mod module.Version, h string) error {
   698  	modWithoutSuffix := mod
   699  	noun := "module"
   700  	if strings.HasSuffix(mod.Version, "/go.mod") {
   701  		noun = "go.mod"
   702  		modWithoutSuffix.Version = strings.TrimSuffix(mod.Version, "/go.mod")
   703  	}
   704  
   705  	db, lines, err := lookupSumDB(mod)
   706  	if err != nil {
   707  		return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err))
   708  	}
   709  
   710  	have := mod.Path + " " + mod.Version + " " + h
   711  	prefix := mod.Path + " " + mod.Version + " h1:"
   712  	for _, line := range lines {
   713  		if line == have {
   714  			return nil
   715  		}
   716  		if strings.HasPrefix(line, prefix) {
   717  			return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):]))
   718  		}
   719  	}
   720  	return nil
   721  }
   722  
   723  // Sum returns the checksum for the downloaded copy of the given module,
   724  // if present in the download cache.
   725  func Sum(mod module.Version) string {
   726  	if cfg.GOMODCACHE == "" {
   727  		// Do not use current directory.
   728  		return ""
   729  	}
   730  
   731  	ziphash, err := CachePath(mod, "ziphash")
   732  	if err != nil {
   733  		return ""
   734  	}
   735  	data, err := lockedfile.Read(ziphash)
   736  	if err != nil {
   737  		return ""
   738  	}
   739  	data = bytes.TrimSpace(data)
   740  	if !isValidSum(data) {
   741  		return ""
   742  	}
   743  	return string(data)
   744  }
   745  
   746  // isValidSum returns true if data is the valid contents of a zip hash file.
   747  // Certain critical files are written to disk by first truncating
   748  // then writing the actual bytes, so that if the write fails
   749  // the corrupt file should contain at least one of the null
   750  // bytes written by the truncate operation.
   751  func isValidSum(data []byte) bool {
   752  	if bytes.IndexByte(data, '\000') >= 0 {
   753  		return false
   754  	}
   755  
   756  	if len(data) != len("h1:")+base64.StdEncoding.EncodedLen(sha256.Size) {
   757  		return false
   758  	}
   759  
   760  	return true
   761  }
   762  
   763  var ErrGoSumDirty = errors.New("updates to go.sum needed, disabled by -mod=readonly")
   764  
   765  // WriteGoSum writes the go.sum file if it needs to be updated.
   766  //
   767  // keep is used to check whether a newly added sum should be saved in go.sum.
   768  // It should have entries for both module content sums and go.mod sums
   769  // (version ends with "/go.mod"). Existing sums will be preserved unless they
   770  // have been marked for deletion with TrimGoSum.
   771  func WriteGoSum(keep map[module.Version]bool, readonly bool) error {
   772  	goSum.mu.Lock()
   773  	defer goSum.mu.Unlock()
   774  
   775  	// If we haven't read the go.sum file yet, don't bother writing it.
   776  	if !goSum.enabled {
   777  		return nil
   778  	}
   779  
   780  	// Check whether we need to add sums for which keep[m] is true or remove
   781  	// unused sums marked with TrimGoSum. If there are no changes to make,
   782  	// just return without opening go.sum.
   783  	dirty := false
   784  Outer:
   785  	for m, hs := range goSum.m {
   786  		for _, h := range hs {
   787  			st := goSum.status[modSum{m, h}]
   788  			if st.dirty && (!st.used || keep[m]) {
   789  				dirty = true
   790  				break Outer
   791  			}
   792  		}
   793  	}
   794  	if !dirty {
   795  		return nil
   796  	}
   797  	if readonly {
   798  		return ErrGoSumDirty
   799  	}
   800  	if _, ok := fsys.OverlayPath(GoSumFile); ok {
   801  		base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay")
   802  	}
   803  
   804  	// Make a best-effort attempt to acquire the side lock, only to exclude
   805  	// previous versions of the 'go' command from making simultaneous edits.
   806  	if unlock, err := SideLock(); err == nil {
   807  		defer unlock()
   808  	}
   809  
   810  	err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) {
   811  		if !goSum.overwrite {
   812  			// Incorporate any sums added by other processes in the meantime.
   813  			// Add only the sums that we actually checked: the user may have edited or
   814  			// truncated the file to remove erroneous hashes, and we shouldn't restore
   815  			// them without good reason.
   816  			goSum.m = make(map[module.Version][]string, len(goSum.m))
   817  			readGoSum(goSum.m, GoSumFile, data)
   818  			for ms, st := range goSum.status {
   819  				if st.used && !sumInWorkspaceModulesLocked(ms.mod) {
   820  					addModSumLocked(ms.mod, ms.sum)
   821  				}
   822  			}
   823  		}
   824  
   825  		var mods []module.Version
   826  		for m := range goSum.m {
   827  			mods = append(mods, m)
   828  		}
   829  		module.Sort(mods)
   830  
   831  		var buf bytes.Buffer
   832  		for _, m := range mods {
   833  			list := goSum.m[m]
   834  			sort.Strings(list)
   835  			for _, h := range list {
   836  				st := goSum.status[modSum{m, h}]
   837  				if (!st.dirty || (st.used && keep[m])) && !sumInWorkspaceModulesLocked(m) {
   838  					fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
   839  				}
   840  			}
   841  		}
   842  		return buf.Bytes(), nil
   843  	})
   844  
   845  	if err != nil {
   846  		return fmt.Errorf("updating go.sum: %w", err)
   847  	}
   848  
   849  	goSum.status = make(map[modSum]modSumStatus)
   850  	goSum.overwrite = false
   851  	return nil
   852  }
   853  
   854  func sumInWorkspaceModulesLocked(m module.Version) bool {
   855  	for _, goSums := range goSum.w {
   856  		if _, ok := goSums[m]; ok {
   857  			return true
   858  		}
   859  	}
   860  	return false
   861  }
   862  
   863  // TrimGoSum trims go.sum to contain only the modules needed for reproducible
   864  // builds.
   865  //
   866  // keep is used to check whether a sum should be retained in go.mod. It should
   867  // have entries for both module content sums and go.mod sums (version ends
   868  // with "/go.mod").
   869  func TrimGoSum(keep map[module.Version]bool) {
   870  	goSum.mu.Lock()
   871  	defer goSum.mu.Unlock()
   872  	inited, err := initGoSum()
   873  	if err != nil {
   874  		base.Fatalf("%s", err)
   875  	}
   876  	if !inited {
   877  		return
   878  	}
   879  
   880  	for m, hs := range goSum.m {
   881  		if !keep[m] {
   882  			for _, h := range hs {
   883  				goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true}
   884  			}
   885  			goSum.overwrite = true
   886  		}
   887  	}
   888  }
   889  
   890  const goSumMismatch = `
   891  
   892  SECURITY ERROR
   893  This download does NOT match an earlier download recorded in go.sum.
   894  The bits may have been replaced on the origin server, or an attacker may
   895  have intercepted the download attempt.
   896  
   897  For more information, see 'go help module-auth'.
   898  `
   899  
   900  const sumdbMismatch = `
   901  
   902  SECURITY ERROR
   903  This download does NOT match the one reported by the checksum server.
   904  The bits may have been replaced on the origin server, or an attacker may
   905  have intercepted the download attempt.
   906  
   907  For more information, see 'go help module-auth'.
   908  `
   909  
   910  const hashVersionMismatch = `
   911  
   912  SECURITY WARNING
   913  This download is listed in go.sum, but using an unknown hash algorithm.
   914  The download cannot be verified.
   915  
   916  For more information, see 'go help module-auth'.
   917  
   918  `
   919  
   920  var HelpModuleAuth = &base.Command{
   921  	UsageLine: "module-auth",
   922  	Short:     "module authentication using go.sum",
   923  	Long: `
   924  When the go command downloads a module zip file or go.mod file into the
   925  module cache, it computes a cryptographic hash and compares it with a known
   926  value to verify the file hasn't changed since it was first downloaded. Known
   927  hashes are stored in a file in the module root directory named go.sum. Hashes
   928  may also be downloaded from the checksum database depending on the values of
   929  GOSUMDB, GOPRIVATE, and GONOSUMDB.
   930  
   931  For details, see https://golang.org/ref/mod#authenticating.
   932  `,
   933  }
   934  
   935  var HelpPrivate = &base.Command{
   936  	UsageLine: "private",
   937  	Short:     "configuration for downloading non-public code",
   938  	Long: `
   939  The go command defaults to downloading modules from the public Go module
   940  mirror at proxy.golang.org. It also defaults to validating downloaded modules,
   941  regardless of source, against the public Go checksum database at sum.golang.org.
   942  These defaults work well for publicly available source code.
   943  
   944  The GOPRIVATE environment variable controls which modules the go command
   945  considers to be private (not available publicly) and should therefore not use
   946  the proxy or checksum database. The variable is a comma-separated list of
   947  glob patterns (in the syntax of Go's path.Match) of module path prefixes.
   948  For example,
   949  
   950  	GOPRIVATE=*.corp.example.com,rsc.io/private
   951  
   952  causes the go command to treat as private any module with a path prefix
   953  matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
   954  and rsc.io/private/quux.
   955  
   956  For fine-grained control over module download and validation, the GONOPROXY
   957  and GONOSUMDB environment variables accept the same kind of glob list
   958  and override GOPRIVATE for the specific decision of whether to use the proxy
   959  and checksum database, respectively.
   960  
   961  For example, if a company ran a module proxy serving private modules,
   962  users would configure go using:
   963  
   964  	GOPRIVATE=*.corp.example.com
   965  	GOPROXY=proxy.example.com
   966  	GONOPROXY=none
   967  
   968  The GOPRIVATE variable is also used to define the "public" and "private"
   969  patterns for the GOVCS variable; see 'go help vcs'. For that usage,
   970  GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths
   971  instead of module paths.
   972  
   973  The 'go env -w' command (see 'go help env') can be used to set these variables
   974  for future go command invocations.
   975  
   976  For more details, see https://golang.org/ref/mod#private-modules.
   977  `,
   978  }
   979  

View as plain text