vcs.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package codehost
     6  
     7  import (
     8  	"errors"
     9  	"fmt"
    10  	"internal/lazyregexp"
    11  	"io"
    12  	"io/fs"
    13  	"os"
    14  	"path/filepath"
    15  	"sort"
    16  	"strconv"
    17  	"strings"
    18  	"sync"
    19  	"time"
    20  
    21  	"cmd/go/internal/lockedfile"
    22  	"cmd/go/internal/par"
    23  	"cmd/go/internal/str"
    24  )
    25  
    26  // A VCSError indicates an error using a version control system.
    27  // The implication of a VCSError is that we know definitively where
    28  // to get the code, but we can't access it due to the error.
    29  // The caller should report this error instead of continuing to probe
    30  // other possible module paths.
    31  //
    32  // TODO(golang.org/issue/31730): See if we can invert this. (Return a
    33  // distinguished error for “repo not found” and treat everything else
    34  // as terminal.)
    35  type VCSError struct {
    36  	Err error
    37  }
    38  
    39  func (e *VCSError) Error() string { return e.Err.Error() }
    40  
    41  func vcsErrorf(format string, a ...any) error {
    42  	return &VCSError{Err: fmt.Errorf(format, a...)}
    43  }
    44  
    45  func NewRepo(vcs, remote string) (Repo, error) {
    46  	type key struct {
    47  		vcs    string
    48  		remote string
    49  	}
    50  	type cached struct {
    51  		repo Repo
    52  		err  error
    53  	}
    54  	c := vcsRepoCache.Do(key{vcs, remote}, func() any {
    55  		repo, err := newVCSRepo(vcs, remote)
    56  		if err != nil {
    57  			err = &VCSError{err}
    58  		}
    59  		return cached{repo, err}
    60  	}).(cached)
    61  
    62  	return c.repo, c.err
    63  }
    64  
    65  var vcsRepoCache par.Cache
    66  
    67  type vcsRepo struct {
    68  	mu lockedfile.Mutex // protects all commands, so we don't have to decide which are safe on a per-VCS basis
    69  
    70  	remote string
    71  	cmd    *vcsCmd
    72  	dir    string
    73  
    74  	tagsOnce sync.Once
    75  	tags     map[string]bool
    76  
    77  	branchesOnce sync.Once
    78  	branches     map[string]bool
    79  
    80  	fetchOnce sync.Once
    81  	fetchErr  error
    82  }
    83  
    84  func newVCSRepo(vcs, remote string) (Repo, error) {
    85  	if vcs == "git" {
    86  		return newGitRepo(remote, false)
    87  	}
    88  	cmd := vcsCmds[vcs]
    89  	if cmd == nil {
    90  		return nil, fmt.Errorf("unknown vcs: %s %s", vcs, remote)
    91  	}
    92  	if !strings.Contains(remote, "://") {
    93  		return nil, fmt.Errorf("invalid vcs remote: %s %s", vcs, remote)
    94  	}
    95  
    96  	r := &vcsRepo{remote: remote, cmd: cmd}
    97  	var err error
    98  	r.dir, r.mu.Path, err = WorkDir(vcsWorkDirType+vcs, r.remote)
    99  	if err != nil {
   100  		return nil, err
   101  	}
   102  
   103  	if cmd.init == nil {
   104  		return r, nil
   105  	}
   106  
   107  	unlock, err := r.mu.Lock()
   108  	if err != nil {
   109  		return nil, err
   110  	}
   111  	defer unlock()
   112  
   113  	if _, err := os.Stat(filepath.Join(r.dir, "."+vcs)); err != nil {
   114  		if _, err := Run(r.dir, cmd.init(r.remote)); err != nil {
   115  			os.RemoveAll(r.dir)
   116  			return nil, err
   117  		}
   118  	}
   119  	return r, nil
   120  }
   121  
   122  const vcsWorkDirType = "vcs1."
   123  
   124  type vcsCmd struct {
   125  	vcs           string                                                         // vcs name "hg"
   126  	init          func(remote string) []string                                   // cmd to init repo to track remote
   127  	tags          func(remote string) []string                                   // cmd to list local tags
   128  	tagRE         *lazyregexp.Regexp                                             // regexp to extract tag names from output of tags cmd
   129  	branches      func(remote string) []string                                   // cmd to list local branches
   130  	branchRE      *lazyregexp.Regexp                                             // regexp to extract branch names from output of tags cmd
   131  	badLocalRevRE *lazyregexp.Regexp                                             // regexp of names that must not be served out of local cache without doing fetch first
   132  	statLocal     func(rev, remote string) []string                              // cmd to stat local rev
   133  	parseStat     func(rev, out string) (*RevInfo, error)                        // cmd to parse output of statLocal
   134  	fetch         []string                                                       // cmd to fetch everything from remote
   135  	latest        string                                                         // name of latest commit on remote (tip, HEAD, etc)
   136  	readFile      func(rev, file, remote string) []string                        // cmd to read rev's file
   137  	readZip       func(rev, subdir, remote, target string) []string              // cmd to read rev's subdir as zip file
   138  	doReadZip     func(dst io.Writer, workDir, rev, subdir, remote string) error // arbitrary function to read rev's subdir as zip file
   139  }
   140  
   141  var re = lazyregexp.New
   142  
   143  var vcsCmds = map[string]*vcsCmd{
   144  	"hg": {
   145  		vcs: "hg",
   146  		init: func(remote string) []string {
   147  			return []string{"hg", "clone", "-U", "--", remote, "."}
   148  		},
   149  		tags: func(remote string) []string {
   150  			return []string{"hg", "tags", "-q"}
   151  		},
   152  		tagRE: re(`(?m)^[^\n]+$`),
   153  		branches: func(remote string) []string {
   154  			return []string{"hg", "branches", "-c", "-q"}
   155  		},
   156  		branchRE:      re(`(?m)^[^\n]+$`),
   157  		badLocalRevRE: re(`(?m)^(tip)$`),
   158  		statLocal: func(rev, remote string) []string {
   159  			return []string{"hg", "log", "-l1", "-r", rev, "--template", "{node} {date|hgdate} {tags}"}
   160  		},
   161  		parseStat: hgParseStat,
   162  		fetch:     []string{"hg", "pull", "-f"},
   163  		latest:    "tip",
   164  		readFile: func(rev, file, remote string) []string {
   165  			return []string{"hg", "cat", "-r", rev, file}
   166  		},
   167  		readZip: func(rev, subdir, remote, target string) []string {
   168  			pattern := []string{}
   169  			if subdir != "" {
   170  				pattern = []string{"-I", subdir + "/**"}
   171  			}
   172  			return str.StringList("hg", "archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/", pattern, "--", target)
   173  		},
   174  	},
   175  
   176  	"svn": {
   177  		vcs:  "svn",
   178  		init: nil, // no local checkout
   179  		tags: func(remote string) []string {
   180  			return []string{"svn", "list", "--", strings.TrimSuffix(remote, "/trunk") + "/tags"}
   181  		},
   182  		tagRE: re(`(?m)^(.*?)/?$`),
   183  		statLocal: func(rev, remote string) []string {
   184  			suffix := "@" + rev
   185  			if rev == "latest" {
   186  				suffix = ""
   187  			}
   188  			return []string{"svn", "log", "-l1", "--xml", "--", remote + suffix}
   189  		},
   190  		parseStat: svnParseStat,
   191  		latest:    "latest",
   192  		readFile: func(rev, file, remote string) []string {
   193  			return []string{"svn", "cat", "--", remote + "/" + file + "@" + rev}
   194  		},
   195  		doReadZip: svnReadZip,
   196  	},
   197  
   198  	"bzr": {
   199  		vcs: "bzr",
   200  		init: func(remote string) []string {
   201  			return []string{"bzr", "branch", "--use-existing-dir", "--", remote, "."}
   202  		},
   203  		fetch: []string{
   204  			"bzr", "pull", "--overwrite-tags",
   205  		},
   206  		tags: func(remote string) []string {
   207  			return []string{"bzr", "tags"}
   208  		},
   209  		tagRE:         re(`(?m)^\S+`),
   210  		badLocalRevRE: re(`^revno:-`),
   211  		statLocal: func(rev, remote string) []string {
   212  			return []string{"bzr", "log", "-l1", "--long", "--show-ids", "-r", rev}
   213  		},
   214  		parseStat: bzrParseStat,
   215  		latest:    "revno:-1",
   216  		readFile: func(rev, file, remote string) []string {
   217  			return []string{"bzr", "cat", "-r", rev, file}
   218  		},
   219  		readZip: func(rev, subdir, remote, target string) []string {
   220  			extra := []string{}
   221  			if subdir != "" {
   222  				extra = []string{"./" + subdir}
   223  			}
   224  			return str.StringList("bzr", "export", "--format=zip", "-r", rev, "--root=prefix/", "--", target, extra)
   225  		},
   226  	},
   227  
   228  	"fossil": {
   229  		vcs: "fossil",
   230  		init: func(remote string) []string {
   231  			return []string{"fossil", "clone", "--", remote, ".fossil"}
   232  		},
   233  		fetch: []string{"fossil", "pull", "-R", ".fossil"},
   234  		tags: func(remote string) []string {
   235  			return []string{"fossil", "tag", "-R", ".fossil", "list"}
   236  		},
   237  		tagRE: re(`XXXTODO`),
   238  		statLocal: func(rev, remote string) []string {
   239  			return []string{"fossil", "info", "-R", ".fossil", rev}
   240  		},
   241  		parseStat: fossilParseStat,
   242  		latest:    "trunk",
   243  		readFile: func(rev, file, remote string) []string {
   244  			return []string{"fossil", "cat", "-R", ".fossil", "-r", rev, file}
   245  		},
   246  		readZip: func(rev, subdir, remote, target string) []string {
   247  			extra := []string{}
   248  			if subdir != "" && !strings.ContainsAny(subdir, "*?[],") {
   249  				extra = []string{"--include", subdir}
   250  			}
   251  			// Note that vcsRepo.ReadZip below rewrites this command
   252  			// to run in a different directory, to work around a fossil bug.
   253  			return str.StringList("fossil", "zip", "-R", ".fossil", "--name", "prefix", extra, "--", rev, target)
   254  		},
   255  	},
   256  }
   257  
   258  func (r *vcsRepo) loadTags() {
   259  	out, err := Run(r.dir, r.cmd.tags(r.remote))
   260  	if err != nil {
   261  		return
   262  	}
   263  
   264  	// Run tag-listing command and extract tags.
   265  	r.tags = make(map[string]bool)
   266  	for _, tag := range r.cmd.tagRE.FindAllString(string(out), -1) {
   267  		if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(tag) {
   268  			continue
   269  		}
   270  		r.tags[tag] = true
   271  	}
   272  }
   273  
   274  func (r *vcsRepo) loadBranches() {
   275  	if r.cmd.branches == nil {
   276  		return
   277  	}
   278  
   279  	out, err := Run(r.dir, r.cmd.branches(r.remote))
   280  	if err != nil {
   281  		return
   282  	}
   283  
   284  	r.branches = make(map[string]bool)
   285  	for _, branch := range r.cmd.branchRE.FindAllString(string(out), -1) {
   286  		if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(branch) {
   287  			continue
   288  		}
   289  		r.branches[branch] = true
   290  	}
   291  }
   292  
   293  func (r *vcsRepo) Tags(prefix string) ([]string, error) {
   294  	unlock, err := r.mu.Lock()
   295  	if err != nil {
   296  		return nil, err
   297  	}
   298  	defer unlock()
   299  
   300  	r.tagsOnce.Do(r.loadTags)
   301  
   302  	tags := []string{}
   303  	for tag := range r.tags {
   304  		if strings.HasPrefix(tag, prefix) {
   305  			tags = append(tags, tag)
   306  		}
   307  	}
   308  	sort.Strings(tags)
   309  	return tags, nil
   310  }
   311  
   312  func (r *vcsRepo) Stat(rev string) (*RevInfo, error) {
   313  	unlock, err := r.mu.Lock()
   314  	if err != nil {
   315  		return nil, err
   316  	}
   317  	defer unlock()
   318  
   319  	if rev == "latest" {
   320  		rev = r.cmd.latest
   321  	}
   322  	r.branchesOnce.Do(r.loadBranches)
   323  	revOK := (r.cmd.badLocalRevRE == nil || !r.cmd.badLocalRevRE.MatchString(rev)) && !r.branches[rev]
   324  	if revOK {
   325  		if info, err := r.statLocal(rev); err == nil {
   326  			return info, nil
   327  		}
   328  	}
   329  
   330  	r.fetchOnce.Do(r.fetch)
   331  	if r.fetchErr != nil {
   332  		return nil, r.fetchErr
   333  	}
   334  	info, err := r.statLocal(rev)
   335  	if err != nil {
   336  		return nil, err
   337  	}
   338  	if !revOK {
   339  		info.Version = info.Name
   340  	}
   341  	return info, nil
   342  }
   343  
   344  func (r *vcsRepo) fetch() {
   345  	if len(r.cmd.fetch) > 0 {
   346  		_, r.fetchErr = Run(r.dir, r.cmd.fetch)
   347  	}
   348  }
   349  
   350  func (r *vcsRepo) statLocal(rev string) (*RevInfo, error) {
   351  	out, err := Run(r.dir, r.cmd.statLocal(rev, r.remote))
   352  	if err != nil {
   353  		return nil, &UnknownRevisionError{Rev: rev}
   354  	}
   355  	return r.cmd.parseStat(rev, string(out))
   356  }
   357  
   358  func (r *vcsRepo) Latest() (*RevInfo, error) {
   359  	return r.Stat("latest")
   360  }
   361  
   362  func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
   363  	if rev == "latest" {
   364  		rev = r.cmd.latest
   365  	}
   366  	_, err := r.Stat(rev) // download rev into local repo
   367  	if err != nil {
   368  		return nil, err
   369  	}
   370  
   371  	// r.Stat acquires r.mu, so lock after that.
   372  	unlock, err := r.mu.Lock()
   373  	if err != nil {
   374  		return nil, err
   375  	}
   376  	defer unlock()
   377  
   378  	out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote))
   379  	if err != nil {
   380  		return nil, fs.ErrNotExist
   381  	}
   382  	return out, nil
   383  }
   384  
   385  func (r *vcsRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) {
   386  	// We don't technically need to lock here since we're returning an error
   387  	// uncondititonally, but doing so anyway will help to avoid baking in
   388  	// lock-inversion bugs.
   389  	unlock, err := r.mu.Lock()
   390  	if err != nil {
   391  		return "", err
   392  	}
   393  	defer unlock()
   394  
   395  	return "", vcsErrorf("RecentTag not implemented")
   396  }
   397  
   398  func (r *vcsRepo) DescendsFrom(rev, tag string) (bool, error) {
   399  	unlock, err := r.mu.Lock()
   400  	if err != nil {
   401  		return false, err
   402  	}
   403  	defer unlock()
   404  
   405  	return false, vcsErrorf("DescendsFrom not implemented")
   406  }
   407  
   408  func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) {
   409  	if r.cmd.readZip == nil && r.cmd.doReadZip == nil {
   410  		return nil, vcsErrorf("ReadZip not implemented for %s", r.cmd.vcs)
   411  	}
   412  
   413  	unlock, err := r.mu.Lock()
   414  	if err != nil {
   415  		return nil, err
   416  	}
   417  	defer unlock()
   418  
   419  	if rev == "latest" {
   420  		rev = r.cmd.latest
   421  	}
   422  	f, err := os.CreateTemp("", "go-readzip-*.zip")
   423  	if err != nil {
   424  		return nil, err
   425  	}
   426  	if r.cmd.doReadZip != nil {
   427  		lw := &limitedWriter{
   428  			W:               f,
   429  			N:               maxSize,
   430  			ErrLimitReached: errors.New("ReadZip: encoded file exceeds allowed size"),
   431  		}
   432  		err = r.cmd.doReadZip(lw, r.dir, rev, subdir, r.remote)
   433  		if err == nil {
   434  			_, err = f.Seek(0, io.SeekStart)
   435  		}
   436  	} else if r.cmd.vcs == "fossil" {
   437  		// If you run
   438  		//	fossil zip -R .fossil --name prefix trunk /tmp/x.zip
   439  		// fossil fails with "unable to create directory /tmp" [sic].
   440  		// Change the command to run in /tmp instead,
   441  		// replacing the -R argument with an absolute path.
   442  		args := r.cmd.readZip(rev, subdir, r.remote, filepath.Base(f.Name()))
   443  		for i := range args {
   444  			if args[i] == ".fossil" {
   445  				args[i] = filepath.Join(r.dir, ".fossil")
   446  			}
   447  		}
   448  		_, err = Run(filepath.Dir(f.Name()), args)
   449  	} else {
   450  		_, err = Run(r.dir, r.cmd.readZip(rev, subdir, r.remote, f.Name()))
   451  	}
   452  	if err != nil {
   453  		f.Close()
   454  		os.Remove(f.Name())
   455  		return nil, err
   456  	}
   457  	return &deleteCloser{f}, nil
   458  }
   459  
   460  // deleteCloser is a file that gets deleted on Close.
   461  type deleteCloser struct {
   462  	*os.File
   463  }
   464  
   465  func (d *deleteCloser) Close() error {
   466  	defer os.Remove(d.File.Name())
   467  	return d.File.Close()
   468  }
   469  
   470  func hgParseStat(rev, out string) (*RevInfo, error) {
   471  	f := strings.Fields(string(out))
   472  	if len(f) < 3 {
   473  		return nil, vcsErrorf("unexpected response from hg log: %q", out)
   474  	}
   475  	hash := f[0]
   476  	version := rev
   477  	if strings.HasPrefix(hash, version) {
   478  		version = hash // extend to full hash
   479  	}
   480  	t, err := strconv.ParseInt(f[1], 10, 64)
   481  	if err != nil {
   482  		return nil, vcsErrorf("invalid time from hg log: %q", out)
   483  	}
   484  
   485  	var tags []string
   486  	for _, tag := range f[3:] {
   487  		if tag != "tip" {
   488  			tags = append(tags, tag)
   489  		}
   490  	}
   491  	sort.Strings(tags)
   492  
   493  	info := &RevInfo{
   494  		Name:    hash,
   495  		Short:   ShortenSHA1(hash),
   496  		Time:    time.Unix(t, 0).UTC(),
   497  		Version: version,
   498  		Tags:    tags,
   499  	}
   500  	return info, nil
   501  }
   502  
   503  func bzrParseStat(rev, out string) (*RevInfo, error) {
   504  	var revno int64
   505  	var tm time.Time
   506  	for _, line := range strings.Split(out, "\n") {
   507  		if line == "" || line[0] == ' ' || line[0] == '\t' {
   508  			// End of header, start of commit message.
   509  			break
   510  		}
   511  		if line[0] == '-' {
   512  			continue
   513  		}
   514  		i := strings.Index(line, ":")
   515  		if i < 0 {
   516  			// End of header, start of commit message.
   517  			break
   518  		}
   519  		key, val := line[:i], strings.TrimSpace(line[i+1:])
   520  		switch key {
   521  		case "revno":
   522  			if j := strings.Index(val, " "); j >= 0 {
   523  				val = val[:j]
   524  			}
   525  			i, err := strconv.ParseInt(val, 10, 64)
   526  			if err != nil {
   527  				return nil, vcsErrorf("unexpected revno from bzr log: %q", line)
   528  			}
   529  			revno = i
   530  		case "timestamp":
   531  			j := strings.Index(val, " ")
   532  			if j < 0 {
   533  				return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line)
   534  			}
   535  			t, err := time.Parse("2006-01-02 15:04:05 -0700", val[j+1:])
   536  			if err != nil {
   537  				return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line)
   538  			}
   539  			tm = t.UTC()
   540  		}
   541  	}
   542  	if revno == 0 || tm.IsZero() {
   543  		return nil, vcsErrorf("unexpected response from bzr log: %q", out)
   544  	}
   545  
   546  	info := &RevInfo{
   547  		Name:    fmt.Sprintf("%d", revno),
   548  		Short:   fmt.Sprintf("%012d", revno),
   549  		Time:    tm,
   550  		Version: rev,
   551  	}
   552  	return info, nil
   553  }
   554  
   555  func fossilParseStat(rev, out string) (*RevInfo, error) {
   556  	for _, line := range strings.Split(out, "\n") {
   557  		if strings.HasPrefix(line, "uuid:") || strings.HasPrefix(line, "hash:") {
   558  			f := strings.Fields(line)
   559  			if len(f) != 5 || len(f[1]) != 40 || f[4] != "UTC" {
   560  				return nil, vcsErrorf("unexpected response from fossil info: %q", line)
   561  			}
   562  			t, err := time.Parse("2006-01-02 15:04:05", f[2]+" "+f[3])
   563  			if err != nil {
   564  				return nil, vcsErrorf("unexpected response from fossil info: %q", line)
   565  			}
   566  			hash := f[1]
   567  			version := rev
   568  			if strings.HasPrefix(hash, version) {
   569  				version = hash // extend to full hash
   570  			}
   571  			info := &RevInfo{
   572  				Name:    hash,
   573  				Short:   ShortenSHA1(hash),
   574  				Time:    t,
   575  				Version: version,
   576  			}
   577  			return info, nil
   578  		}
   579  	}
   580  	return nil, vcsErrorf("unexpected response from fossil info: %q", out)
   581  }
   582  
   583  type limitedWriter struct {
   584  	W               io.Writer
   585  	N               int64
   586  	ErrLimitReached error
   587  }
   588  
   589  func (l *limitedWriter) Write(p []byte) (n int, err error) {
   590  	if l.N > 0 {
   591  		max := len(p)
   592  		if l.N < int64(max) {
   593  			max = int(l.N)
   594  		}
   595  		n, err = l.W.Write(p[:max])
   596  		l.N -= int64(n)
   597  		if err != nil || n >= len(p) {
   598  			return n, err
   599  		}
   600  	}
   601  
   602  	return n, l.ErrLimitReached
   603  }
   604
View as plain text