codehost.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package codehost defines the interface implemented by a code hosting source,
     6  // along with support code for use by implementations.
     7  package codehost
     8  
     9  import (
    10  	"bytes"
    11  	"crypto/sha256"
    12  	"fmt"
    13  	exec "internal/execabs"
    14  	"io"
    15  	"io/fs"
    16  	"os"
    17  	"path/filepath"
    18  	"strings"
    19  	"sync"
    20  	"time"
    21  
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/lockedfile"
    24  	"cmd/go/internal/str"
    25  )
    26  
    27  // Downloaded size limits.
    28  const (
    29  	MaxGoMod   = 16 << 20  // maximum size of go.mod file
    30  	MaxLICENSE = 16 << 20  // maximum size of LICENSE file
    31  	MaxZipFile = 500 << 20 // maximum size of downloaded zip file
    32  )
    33  
    34  // A Repo represents a code hosting source.
    35  // Typical implementations include local version control repositories,
    36  // remote version control servers, and code hosting sites.
    37  // A Repo must be safe for simultaneous use by multiple goroutines.
    38  type Repo interface {
    39  	// List lists all tags with the given prefix.
    40  	Tags(prefix string) (tags []string, err error)
    41  
    42  	// Stat returns information about the revision rev.
    43  	// A revision can be any identifier known to the underlying service:
    44  	// commit hash, branch, tag, and so on.
    45  	Stat(rev string) (*RevInfo, error)
    46  
    47  	// Latest returns the latest revision on the default branch,
    48  	// whatever that means in the underlying implementation.
    49  	Latest() (*RevInfo, error)
    50  
    51  	// ReadFile reads the given file in the file tree corresponding to revision rev.
    52  	// It should refuse to read more than maxSize bytes.
    53  	//
    54  	// If the requested file does not exist it should return an error for which
    55  	// os.IsNotExist(err) returns true.
    56  	ReadFile(rev, file string, maxSize int64) (data []byte, err error)
    57  
    58  	// ReadZip downloads a zip file for the subdir subdirectory
    59  	// of the given revision to a new file in a given temporary directory.
    60  	// It should refuse to read more than maxSize bytes.
    61  	// It returns a ReadCloser for a streamed copy of the zip file.
    62  	// All files in the zip file are expected to be
    63  	// nested in a single top-level directory, whose name is not specified.
    64  	ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error)
    65  
    66  	// RecentTag returns the most recent tag on rev or one of its predecessors
    67  	// with the given prefix. allowed may be used to filter out unwanted versions.
    68  	RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error)
    69  
    70  	// DescendsFrom reports whether rev or any of its ancestors has the given tag.
    71  	//
    72  	// DescendsFrom must return true for any tag returned by RecentTag for the
    73  	// same revision.
    74  	DescendsFrom(rev, tag string) (bool, error)
    75  }
    76  
    77  // A Rev describes a single revision in a source code repository.
    78  type RevInfo struct {
    79  	Name    string    // complete ID in underlying repository
    80  	Short   string    // shortened ID, for use in pseudo-version
    81  	Version string    // version used in lookup
    82  	Time    time.Time // commit time
    83  	Tags    []string  // known tags for commit
    84  }
    85  
    86  // A FileRev describes the result of reading a file at a given revision.
    87  type FileRev struct {
    88  	Rev  string // requested revision
    89  	Data []byte // file data
    90  	Err  error  // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev
    91  }
    92  
    93  // UnknownRevisionError is an error equivalent to fs.ErrNotExist, but for a
    94  // revision rather than a file.
    95  type UnknownRevisionError struct {
    96  	Rev string
    97  }
    98  
    99  func (e *UnknownRevisionError) Error() string {
   100  	return "unknown revision " + e.Rev
   101  }
   102  func (UnknownRevisionError) Is(err error) bool {
   103  	return err == fs.ErrNotExist
   104  }
   105  
   106  // ErrNoCommits is an error equivalent to fs.ErrNotExist indicating that a given
   107  // repository or module contains no commits.
   108  var ErrNoCommits error = noCommitsError{}
   109  
   110  type noCommitsError struct{}
   111  
   112  func (noCommitsError) Error() string {
   113  	return "no commits"
   114  }
   115  func (noCommitsError) Is(err error) bool {
   116  	return err == fs.ErrNotExist
   117  }
   118  
   119  // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
   120  func AllHex(rev string) bool {
   121  	for i := 0; i < len(rev); i++ {
   122  		c := rev[i]
   123  		if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' {
   124  			continue
   125  		}
   126  		return false
   127  	}
   128  	return true
   129  }
   130  
   131  // ShortenSHA1 shortens a SHA1 hash (40 hex digits) to the canonical length
   132  // used in pseudo-versions (12 hex digits).
   133  func ShortenSHA1(rev string) string {
   134  	if AllHex(rev) && len(rev) == 40 {
   135  		return rev[:12]
   136  	}
   137  	return rev
   138  }
   139  
   140  // WorkDir returns the name of the cached work directory to use for the
   141  // given repository type and name.
   142  func WorkDir(typ, name string) (dir, lockfile string, err error) {
   143  	if cfg.GOMODCACHE == "" {
   144  		return "", "", fmt.Errorf("neither GOPATH nor GOMODCACHE are set")
   145  	}
   146  
   147  	// We name the work directory for the SHA256 hash of the type and name.
   148  	// We intentionally avoid the actual name both because of possible
   149  	// conflicts with valid file system paths and because we want to ensure
   150  	// that one checkout is never nested inside another. That nesting has
   151  	// led to security problems in the past.
   152  	if strings.Contains(typ, ":") {
   153  		return "", "", fmt.Errorf("codehost.WorkDir: type cannot contain colon")
   154  	}
   155  	key := typ + ":" + name
   156  	dir = filepath.Join(cfg.GOMODCACHE, "cache/vcs", fmt.Sprintf("%x", sha256.Sum256([]byte(key))))
   157  
   158  	if cfg.BuildX {
   159  		fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", filepath.Dir(dir), typ, name)
   160  	}
   161  	if err := os.MkdirAll(filepath.Dir(dir), 0777); err != nil {
   162  		return "", "", err
   163  	}
   164  
   165  	lockfile = dir + ".lock"
   166  	if cfg.BuildX {
   167  		fmt.Fprintf(os.Stderr, "# lock %s", lockfile)
   168  	}
   169  
   170  	unlock, err := lockedfile.MutexAt(lockfile).Lock()
   171  	if err != nil {
   172  		return "", "", fmt.Errorf("codehost.WorkDir: can't find or create lock file: %v", err)
   173  	}
   174  	defer unlock()
   175  
   176  	data, err := os.ReadFile(dir + ".info")
   177  	info, err2 := os.Stat(dir)
   178  	if err == nil && err2 == nil && info.IsDir() {
   179  		// Info file and directory both already exist: reuse.
   180  		have := strings.TrimSuffix(string(data), "\n")
   181  		if have != key {
   182  			return "", "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key)
   183  		}
   184  		if cfg.BuildX {
   185  			fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name)
   186  		}
   187  		return dir, lockfile, nil
   188  	}
   189  
   190  	// Info file or directory missing. Start from scratch.
   191  	if cfg.BuildX {
   192  		fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name)
   193  	}
   194  	os.RemoveAll(dir)
   195  	if err := os.MkdirAll(dir, 0777); err != nil {
   196  		return "", "", err
   197  	}
   198  	if err := os.WriteFile(dir+".info", []byte(key), 0666); err != nil {
   199  		os.RemoveAll(dir)
   200  		return "", "", err
   201  	}
   202  	return dir, lockfile, nil
   203  }
   204  
   205  type RunError struct {
   206  	Cmd      string
   207  	Err      error
   208  	Stderr   []byte
   209  	HelpText string
   210  }
   211  
   212  func (e *RunError) Error() string {
   213  	text := e.Cmd + ": " + e.Err.Error()
   214  	stderr := bytes.TrimRight(e.Stderr, "\n")
   215  	if len(stderr) > 0 {
   216  		text += ":\n\t" + strings.ReplaceAll(string(stderr), "\n", "\n\t")
   217  	}
   218  	if len(e.HelpText) > 0 {
   219  		text += "\n" + e.HelpText
   220  	}
   221  	return text
   222  }
   223  
   224  var dirLock sync.Map
   225  
   226  // Run runs the command line in the given directory
   227  // (an empty dir means the current directory).
   228  // It returns the standard output and, for a non-zero exit,
   229  // a *RunError indicating the command, exit status, and standard error.
   230  // Standard error is unavailable for commands that exit successfully.
   231  func Run(dir string, cmdline ...any) ([]byte, error) {
   232  	return RunWithStdin(dir, nil, cmdline...)
   233  }
   234  
   235  // bashQuoter escapes characters that have special meaning in double-quoted strings in the bash shell.
   236  // See https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html.
   237  var bashQuoter = strings.NewReplacer(`"`, `\"`, `$`, `\$`, "`", "\\`", `\`, `\\`)
   238  
   239  func RunWithStdin(dir string, stdin io.Reader, cmdline ...any) ([]byte, error) {
   240  	if dir != "" {
   241  		muIface, ok := dirLock.Load(dir)
   242  		if !ok {
   243  			muIface, _ = dirLock.LoadOrStore(dir, new(sync.Mutex))
   244  		}
   245  		mu := muIface.(*sync.Mutex)
   246  		mu.Lock()
   247  		defer mu.Unlock()
   248  	}
   249  
   250  	cmd := str.StringList(cmdline...)
   251  	if os.Getenv("TESTGOVCS") == "panic" {
   252  		panic(fmt.Sprintf("use of vcs: %v", cmd))
   253  	}
   254  	if cfg.BuildX {
   255  		text := new(strings.Builder)
   256  		if dir != "" {
   257  			text.WriteString("cd ")
   258  			text.WriteString(dir)
   259  			text.WriteString("; ")
   260  		}
   261  		for i, arg := range cmd {
   262  			if i > 0 {
   263  				text.WriteByte(' ')
   264  			}
   265  			switch {
   266  			case strings.ContainsAny(arg, "'"):
   267  				// Quote args that could be mistaken for quoted args.
   268  				text.WriteByte('"')
   269  				text.WriteString(bashQuoter.Replace(arg))
   270  				text.WriteByte('"')
   271  			case strings.ContainsAny(arg, "$`\\*?[\"\t\n\v\f\r \u0085\u00a0"):
   272  				// Quote args that contain special characters, glob patterns, or spaces.
   273  				text.WriteByte('\'')
   274  				text.WriteString(arg)
   275  				text.WriteByte('\'')
   276  			default:
   277  				text.WriteString(arg)
   278  			}
   279  		}
   280  		fmt.Fprintf(os.Stderr, "%s\n", text)
   281  		start := time.Now()
   282  		defer func() {
   283  			fmt.Fprintf(os.Stderr, "%.3fs # %s\n", time.Since(start).Seconds(), text)
   284  		}()
   285  	}
   286  	// TODO: Impose limits on command output size.
   287  	// TODO: Set environment to get English error messages.
   288  	var stderr bytes.Buffer
   289  	var stdout bytes.Buffer
   290  	c := exec.Command(cmd[0], cmd[1:]...)
   291  	c.Dir = dir
   292  	c.Stdin = stdin
   293  	c.Stderr = &stderr
   294  	c.Stdout = &stdout
   295  	err := c.Run()
   296  	if err != nil {
   297  		err = &RunError{Cmd: strings.Join(cmd, " ") + " in " + dir, Stderr: stderr.Bytes(), Err: err}
   298  	}
   299  	return stdout.Bytes(), err
   300  }
   301
View as plain text