dirs.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	exec "internal/execabs"
    11  	"log"
    12  	"os"
    13  	"path/filepath"
    14  	"regexp"
    15  	"strings"
    16  	"sync"
    17  
    18  	"golang.org/x/mod/semver"
    19  )
    20  
    21  // A Dir describes a directory holding code by specifying
    22  // the expected import path and the file system directory.
    23  type Dir struct {
    24  	importPath string // import path for that dir
    25  	dir        string // file system directory
    26  	inModule   bool
    27  }
    28  
    29  // Dirs is a structure for scanning the directory tree.
    30  // Its Next method returns the next Go source directory it finds.
    31  // Although it can be used to scan the tree multiple times, it
    32  // only walks the tree once, caching the data it finds.
    33  type Dirs struct {
    34  	scan   chan Dir // Directories generated by walk.
    35  	hist   []Dir    // History of reported Dirs.
    36  	offset int      // Counter for Next.
    37  }
    38  
    39  var dirs Dirs
    40  
    41  // dirsInit starts the scanning of package directories in GOROOT and GOPATH. Any
    42  // extra paths passed to it are included in the channel.
    43  func dirsInit(extra ...Dir) {
    44  	dirs.hist = make([]Dir, 0, 1000)
    45  	dirs.hist = append(dirs.hist, extra...)
    46  	dirs.scan = make(chan Dir)
    47  	go dirs.walk(codeRoots())
    48  }
    49  
    50  // Reset puts the scan back at the beginning.
    51  func (d *Dirs) Reset() {
    52  	d.offset = 0
    53  }
    54  
    55  // Next returns the next directory in the scan. The boolean
    56  // is false when the scan is done.
    57  func (d *Dirs) Next() (Dir, bool) {
    58  	if d.offset < len(d.hist) {
    59  		dir := d.hist[d.offset]
    60  		d.offset++
    61  		return dir, true
    62  	}
    63  	dir, ok := <-d.scan
    64  	if !ok {
    65  		return Dir{}, false
    66  	}
    67  	d.hist = append(d.hist, dir)
    68  	d.offset++
    69  	return dir, ok
    70  }
    71  
    72  // walk walks the trees in GOROOT and GOPATH.
    73  func (d *Dirs) walk(roots []Dir) {
    74  	for _, root := range roots {
    75  		d.bfsWalkRoot(root)
    76  	}
    77  	close(d.scan)
    78  }
    79  
    80  // bfsWalkRoot walks a single directory hierarchy in breadth-first lexical order.
    81  // Each Go source directory it finds is delivered on d.scan.
    82  func (d *Dirs) bfsWalkRoot(root Dir) {
    83  	root.dir = filepath.Clean(root.dir) // because filepath.Join will do it anyway
    84  
    85  	// this is the queue of directories to examine in this pass.
    86  	this := []string{}
    87  	// next is the queue of directories to examine in the next pass.
    88  	next := []string{root.dir}
    89  
    90  	for len(next) > 0 {
    91  		this, next = next, this[0:0]
    92  		for _, dir := range this {
    93  			fd, err := os.Open(dir)
    94  			if err != nil {
    95  				log.Print(err)
    96  				continue
    97  			}
    98  			entries, err := fd.Readdir(0)
    99  			fd.Close()
   100  			if err != nil {
   101  				log.Print(err)
   102  				continue
   103  			}
   104  			hasGoFiles := false
   105  			for _, entry := range entries {
   106  				name := entry.Name()
   107  				// For plain files, remember if this directory contains any .go
   108  				// source files, but ignore them otherwise.
   109  				if !entry.IsDir() {
   110  					if !hasGoFiles && strings.HasSuffix(name, ".go") {
   111  						hasGoFiles = true
   112  					}
   113  					continue
   114  				}
   115  				// Entry is a directory.
   116  
   117  				// The go tool ignores directories starting with ., _, or named "testdata".
   118  				if name[0] == '.' || name[0] == '_' || name == "testdata" {
   119  					continue
   120  				}
   121  				// When in a module, ignore vendor directories and stop at module boundaries.
   122  				if root.inModule {
   123  					if name == "vendor" {
   124  						continue
   125  					}
   126  					if fi, err := os.Stat(filepath.Join(dir, name, "go.mod")); err == nil && !fi.IsDir() {
   127  						continue
   128  					}
   129  				}
   130  				// Remember this (fully qualified) directory for the next pass.
   131  				next = append(next, filepath.Join(dir, name))
   132  			}
   133  			if hasGoFiles {
   134  				// It's a candidate.
   135  				importPath := root.importPath
   136  				if len(dir) > len(root.dir) {
   137  					if importPath != "" {
   138  						importPath += "/"
   139  					}
   140  					importPath += filepath.ToSlash(dir[len(root.dir)+1:])
   141  				}
   142  				d.scan <- Dir{importPath, dir, root.inModule}
   143  			}
   144  		}
   145  
   146  	}
   147  }
   148  
   149  var testGOPATH = false // force GOPATH use for testing
   150  
   151  // codeRoots returns the code roots to search for packages.
   152  // In GOPATH mode this is GOROOT/src and GOPATH/src, with empty import paths.
   153  // In module mode, this is each module root, with an import path set to its module path.
   154  func codeRoots() []Dir {
   155  	codeRootsCache.once.Do(func() {
   156  		codeRootsCache.roots = findCodeRoots()
   157  	})
   158  	return codeRootsCache.roots
   159  }
   160  
   161  var codeRootsCache struct {
   162  	once  sync.Once
   163  	roots []Dir
   164  }
   165  
   166  var usingModules bool
   167  
   168  func findCodeRoots() []Dir {
   169  	var list []Dir
   170  	if !testGOPATH {
   171  		// Check for use of modules by 'go env GOMOD',
   172  		// which reports a go.mod file path if modules are enabled.
   173  		stdout, _ := exec.Command("go", "env", "GOMOD").Output()
   174  		gomod := string(bytes.TrimSpace(stdout))
   175  
   176  		usingModules = len(gomod) > 0
   177  		if usingModules {
   178  			list = append(list,
   179  				Dir{dir: filepath.Join(buildCtx.GOROOT, "src"), inModule: true},
   180  				Dir{importPath: "cmd", dir: filepath.Join(buildCtx.GOROOT, "src", "cmd"), inModule: true})
   181  		}
   182  
   183  		if gomod == os.DevNull {
   184  			// Modules are enabled, but the working directory is outside any module.
   185  			// We can still access std, cmd, and packages specified as source files
   186  			// on the command line, but there are no module roots.
   187  			// Avoid 'go list -m all' below, since it will not work.
   188  			return list
   189  		}
   190  	}
   191  
   192  	if !usingModules {
   193  		list = append(list, Dir{dir: filepath.Join(buildCtx.GOROOT, "src")})
   194  		for _, root := range splitGopath() {
   195  			list = append(list, Dir{dir: filepath.Join(root, "src")})
   196  		}
   197  		return list
   198  	}
   199  
   200  	// Find module root directories from go list.
   201  	// Eventually we want golang.org/x/tools/go/packages
   202  	// to handle the entire file system search and become go/packages,
   203  	// but for now enumerating the module roots lets us fit modules
   204  	// into the current code with as few changes as possible.
   205  	mainMod, vendorEnabled, err := vendorEnabled()
   206  	if err != nil {
   207  		return list
   208  	}
   209  	if vendorEnabled {
   210  		// Add the vendor directory to the search path ahead of "std".
   211  		// That way, if the main module *is* "std", we will identify the path
   212  		// without the "vendor/" prefix before the one with that prefix.
   213  		list = append([]Dir{{dir: filepath.Join(mainMod.Dir, "vendor"), inModule: false}}, list...)
   214  		if mainMod.Path != "std" {
   215  			list = append(list, Dir{importPath: mainMod.Path, dir: mainMod.Dir, inModule: true})
   216  		}
   217  		return list
   218  	}
   219  
   220  	cmd := exec.Command("go", "list", "-m", "-f={{.Path}}\t{{.Dir}}", "all")
   221  	cmd.Stderr = os.Stderr
   222  	out, _ := cmd.Output()
   223  	for _, line := range strings.Split(string(out), "\n") {
   224  		path, dir, _ := strings.Cut(line, "\t")
   225  		if dir != "" {
   226  			list = append(list, Dir{importPath: path, dir: dir, inModule: true})
   227  		}
   228  	}
   229  
   230  	return list
   231  }
   232  
   233  // The functions below are derived from x/tools/internal/imports at CL 203017.
   234  
   235  type moduleJSON struct {
   236  	Path, Dir, GoVersion string
   237  }
   238  
   239  var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`)
   240  
   241  // vendorEnabled indicates if vendoring is enabled.
   242  // Inspired by setDefaultBuildMod in modload/init.go
   243  func vendorEnabled() (*moduleJSON, bool, error) {
   244  	mainMod, go114, err := getMainModuleAnd114()
   245  	if err != nil {
   246  		return nil, false, err
   247  	}
   248  
   249  	stdout, _ := exec.Command("go", "env", "GOFLAGS").Output()
   250  	goflags := string(bytes.TrimSpace(stdout))
   251  	matches := modFlagRegexp.FindStringSubmatch(goflags)
   252  	var modFlag string
   253  	if len(matches) != 0 {
   254  		modFlag = matches[1]
   255  	}
   256  	if modFlag != "" {
   257  		// Don't override an explicit '-mod=' argument.
   258  		return mainMod, modFlag == "vendor", nil
   259  	}
   260  	if mainMod == nil || !go114 {
   261  		return mainMod, false, nil
   262  	}
   263  	// Check 1.14's automatic vendor mode.
   264  	if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() {
   265  		if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 {
   266  			// The Go version is at least 1.14, and a vendor directory exists.
   267  			// Set -mod=vendor by default.
   268  			return mainMod, true, nil
   269  		}
   270  	}
   271  	return mainMod, false, nil
   272  }
   273  
   274  // getMainModuleAnd114 gets the main module's information and whether the
   275  // go command in use is 1.14+. This is the information needed to figure out
   276  // if vendoring should be enabled.
   277  func getMainModuleAnd114() (*moduleJSON, bool, error) {
   278  	const format = `{{.Path}}
   279  {{.Dir}}
   280  {{.GoVersion}}
   281  {{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}}
   282  `
   283  	cmd := exec.Command("go", "list", "-m", "-f", format)
   284  	cmd.Stderr = os.Stderr
   285  	stdout, err := cmd.Output()
   286  	if err != nil {
   287  		return nil, false, nil
   288  	}
   289  	lines := strings.Split(string(stdout), "\n")
   290  	if len(lines) < 5 {
   291  		return nil, false, fmt.Errorf("unexpected stdout: %q", stdout)
   292  	}
   293  	mod := &moduleJSON{
   294  		Path:      lines[0],
   295  		Dir:       lines[1],
   296  		GoVersion: lines[2],
   297  	}
   298  	return mod, lines[3] == "go1.14", nil
   299  }
   300
View as plain text