gofmt.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"flag"
    11  	"fmt"
    12  	"go/ast"
    13  	"go/parser"
    14  	"go/printer"
    15  	"go/scanner"
    16  	"go/token"
    17  	"io"
    18  	"io/fs"
    19  	"os"
    20  	"path/filepath"
    21  	"runtime"
    22  	"runtime/pprof"
    23  	"strings"
    24  
    25  	"cmd/internal/diff"
    26  
    27  	"golang.org/x/sync/semaphore"
    28  )
    29  
    30  var (
    31  	// main operation modes
    32  	list        = flag.Bool("l", false, "list files whose formatting differs from gofmt's")
    33  	write       = flag.Bool("w", false, "write result to (source) file instead of stdout")
    34  	rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'a[b:len(a)] -> a[b:]')")
    35  	simplifyAST = flag.Bool("s", false, "simplify code")
    36  	doDiff      = flag.Bool("d", false, "display diffs instead of rewriting files")
    37  	allErrors   = flag.Bool("e", false, "report all errors (not just the first 10 on different lines)")
    38  
    39  	// debugging
    40  	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to this file")
    41  )
    42  
    43  // Keep these in sync with go/format/format.go.
    44  const (
    45  	tabWidth    = 8
    46  	printerMode = printer.UseSpaces | printer.TabIndent | printerNormalizeNumbers
    47  
    48  	// printerNormalizeNumbers means to canonicalize number literal prefixes
    49  	// and exponents while printing. See https://golang.org/doc/go1.13#gofmt.
    50  	//
    51  	// This value is defined in go/printer specifically for go/format and cmd/gofmt.
    52  	printerNormalizeNumbers = 1 << 30
    53  )
    54  
    55  // fdSem guards the number of concurrently-open file descriptors.
    56  //
    57  // For now, this is arbitrarily set to 200, based on the observation that many
    58  // platforms default to a kernel limit of 256. Ideally, perhaps we should derive
    59  // it from rlimit on platforms that support that system call.
    60  //
    61  // File descriptors opened from outside of this package are not tracked,
    62  // so this limit may be approximate.
    63  var fdSem = make(chan bool, 200)
    64  
    65  var (
    66  	rewrite    func(*token.FileSet, *ast.File) *ast.File
    67  	parserMode parser.Mode
    68  )
    69  
    70  func usage() {
    71  	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n")
    72  	flag.PrintDefaults()
    73  }
    74  
    75  func initParserMode() {
    76  	parserMode = parser.ParseComments
    77  	if *allErrors {
    78  		parserMode |= parser.AllErrors
    79  	}
    80  }
    81  
    82  func isGoFile(f fs.DirEntry) bool {
    83  	// ignore non-Go files
    84  	name := f.Name()
    85  	return !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go") && !f.IsDir()
    86  }
    87  
    88  // A sequencer performs concurrent tasks that may write output, but emits that
    89  // output in a deterministic order.
    90  type sequencer struct {
    91  	maxWeight int64
    92  	sem       *semaphore.Weighted   // weighted by input bytes (an approximate proxy for memory overhead)
    93  	prev      <-chan *reporterState // 1-buffered
    94  }
    95  
    96  // newSequencer returns a sequencer that allows concurrent tasks up to maxWeight
    97  // and writes tasks' output to out and err.
    98  func newSequencer(maxWeight int64, out, err io.Writer) *sequencer {
    99  	sem := semaphore.NewWeighted(maxWeight)
   100  	prev := make(chan *reporterState, 1)
   101  	prev <- &reporterState{out: out, err: err}
   102  	return &sequencer{
   103  		maxWeight: maxWeight,
   104  		sem:       sem,
   105  		prev:      prev,
   106  	}
   107  }
   108  
   109  // exclusive is a weight that can be passed to a sequencer to cause
   110  // a task to be executed without any other concurrent tasks.
   111  const exclusive = -1
   112  
   113  // Add blocks until the sequencer has enough weight to spare, then adds f as a
   114  // task to be executed concurrently.
   115  //
   116  // If the weight is either negative or larger than the sequencer's maximum
   117  // weight, Add blocks until all other tasks have completed, then the task
   118  // executes exclusively (blocking all other calls to Add until it completes).
   119  //
   120  // f may run concurrently in a goroutine, but its output to the passed-in
   121  // reporter will be sequential relative to the other tasks in the sequencer.
   122  //
   123  // If f invokes a method on the reporter, execution of that method may block
   124  // until the previous task has finished. (To maximize concurrency, f should
   125  // avoid invoking the reporter until it has finished any parallelizable work.)
   126  //
   127  // If f returns a non-nil error, that error will be reported after f's output
   128  // (if any) and will cause a nonzero final exit code.
   129  func (s *sequencer) Add(weight int64, f func(*reporter) error) {
   130  	if weight < 0 || weight > s.maxWeight {
   131  		weight = s.maxWeight
   132  	}
   133  	if err := s.sem.Acquire(context.TODO(), weight); err != nil {
   134  		// Change the task from "execute f" to "report err".
   135  		weight = 0
   136  		f = func(*reporter) error { return err }
   137  	}
   138  
   139  	r := &reporter{prev: s.prev}
   140  	next := make(chan *reporterState, 1)
   141  	s.prev = next
   142  
   143  	// Start f in parallel: it can run until it invokes a method on r, at which
   144  	// point it will block until the previous task releases the output state.
   145  	go func() {
   146  		if err := f(r); err != nil {
   147  			r.Report(err)
   148  		}
   149  		next <- r.getState() // Release the next task.
   150  		s.sem.Release(weight)
   151  	}()
   152  }
   153  
   154  // AddReport prints an error to s after the output of any previously-added
   155  // tasks, causing the final exit code to be nonzero.
   156  func (s *sequencer) AddReport(err error) {
   157  	s.Add(0, func(*reporter) error { return err })
   158  }
   159  
   160  // GetExitCode waits for all previously-added tasks to complete, then returns an
   161  // exit code for the sequence suitable for passing to os.Exit.
   162  func (s *sequencer) GetExitCode() int {
   163  	c := make(chan int, 1)
   164  	s.Add(0, func(r *reporter) error {
   165  		c <- r.ExitCode()
   166  		return nil
   167  	})
   168  	return <-c
   169  }
   170  
   171  // A reporter reports output, warnings, and errors.
   172  type reporter struct {
   173  	prev  <-chan *reporterState
   174  	state *reporterState
   175  }
   176  
   177  // reporterState carries the state of a reporter instance.
   178  //
   179  // Only one reporter at a time may have access to a reporterState.
   180  type reporterState struct {
   181  	out, err io.Writer
   182  	exitCode int
   183  }
   184  
   185  // getState blocks until any prior reporters are finished with the reporter
   186  // state, then returns the state for manipulation.
   187  func (r *reporter) getState() *reporterState {
   188  	if r.state == nil {
   189  		r.state = <-r.prev
   190  	}
   191  	return r.state
   192  }
   193  
   194  // Warnf emits a warning message to the reporter's error stream,
   195  // without changing its exit code.
   196  func (r *reporter) Warnf(format string, args ...any) {
   197  	fmt.Fprintf(r.getState().err, format, args...)
   198  }
   199  
   200  // Write emits a slice to the reporter's output stream.
   201  //
   202  // Any error is returned to the caller, and does not otherwise affect the
   203  // reporter's exit code.
   204  func (r *reporter) Write(p []byte) (int, error) {
   205  	return r.getState().out.Write(p)
   206  }
   207  
   208  // Report emits a non-nil error to the reporter's error stream,
   209  // changing its exit code to a nonzero value.
   210  func (r *reporter) Report(err error) {
   211  	if err == nil {
   212  		panic("Report with nil error")
   213  	}
   214  	st := r.getState()
   215  	scanner.PrintError(st.err, err)
   216  	st.exitCode = 2
   217  }
   218  
   219  func (r *reporter) ExitCode() int {
   220  	return r.getState().exitCode
   221  }
   222  
   223  // If info == nil, we are formatting stdin instead of a file.
   224  // If in == nil, the source is the contents of the file with the given filename.
   225  func processFile(filename string, info fs.FileInfo, in io.Reader, r *reporter) error {
   226  	src, err := readFile(filename, info, in)
   227  	if err != nil {
   228  		return err
   229  	}
   230  
   231  	fileSet := token.NewFileSet()
   232  	fragmentOk := false
   233  	if info == nil {
   234  		// If we are formatting stdin, we accept a program fragment in lieu of a
   235  		// complete source file.
   236  		fragmentOk = true
   237  	}
   238  	file, sourceAdj, indentAdj, err := parse(fileSet, filename, src, fragmentOk)
   239  	if err != nil {
   240  		return err
   241  	}
   242  
   243  	if rewrite != nil {
   244  		if sourceAdj == nil {
   245  			file = rewrite(fileSet, file)
   246  		} else {
   247  			r.Warnf("warning: rewrite ignored for incomplete programs\n")
   248  		}
   249  	}
   250  
   251  	ast.SortImports(fileSet, file)
   252  
   253  	if *simplifyAST {
   254  		simplify(file)
   255  	}
   256  
   257  	res, err := format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth})
   258  	if err != nil {
   259  		return err
   260  	}
   261  
   262  	if !bytes.Equal(src, res) {
   263  		// formatting has changed
   264  		if *list {
   265  			fmt.Fprintln(r, filename)
   266  		}
   267  		if *write {
   268  			if info == nil {
   269  				panic("-w should not have been allowed with stdin")
   270  			}
   271  			// make a temporary backup before overwriting original
   272  			perm := info.Mode().Perm()
   273  			bakname, err := backupFile(filename+".", src, perm)
   274  			if err != nil {
   275  				return err
   276  			}
   277  			fdSem <- true
   278  			err = os.WriteFile(filename, res, perm)
   279  			<-fdSem
   280  			if err != nil {
   281  				os.Rename(bakname, filename)
   282  				return err
   283  			}
   284  			err = os.Remove(bakname)
   285  			if err != nil {
   286  				return err
   287  			}
   288  		}
   289  		if *doDiff {
   290  			data, err := diffWithReplaceTempFile(src, res, filename)
   291  			if err != nil {
   292  				return fmt.Errorf("computing diff: %s", err)
   293  			}
   294  			fmt.Fprintf(r, "diff -u %s %s\n", filepath.ToSlash(filename+".orig"), filepath.ToSlash(filename))
   295  			r.Write(data)
   296  		}
   297  	}
   298  
   299  	if !*list && !*write && !*doDiff {
   300  		_, err = r.Write(res)
   301  	}
   302  
   303  	return err
   304  }
   305  
   306  // readFile reads the contents of filename, described by info.
   307  // If in is non-nil, readFile reads directly from it.
   308  // Otherwise, readFile opens and reads the file itself,
   309  // with the number of concurrently-open files limited by fdSem.
   310  func readFile(filename string, info fs.FileInfo, in io.Reader) ([]byte, error) {
   311  	if in == nil {
   312  		fdSem <- true
   313  		var err error
   314  		f, err := os.Open(filename)
   315  		if err != nil {
   316  			return nil, err
   317  		}
   318  		in = f
   319  		defer func() {
   320  			f.Close()
   321  			<-fdSem
   322  		}()
   323  	}
   324  
   325  	// Compute the file's size and read its contents with minimal allocations.
   326  	//
   327  	// If we have the FileInfo from filepath.WalkDir, use it to make
   328  	// a buffer of the right size and avoid ReadAll's reallocations.
   329  	//
   330  	// If the size is unknown (or bogus, or overflows an int), fall back to
   331  	// a size-independent ReadAll.
   332  	size := -1
   333  	if info != nil && info.Mode().IsRegular() && int64(int(info.Size())) == info.Size() {
   334  		size = int(info.Size())
   335  	}
   336  	if size+1 <= 0 {
   337  		// The file is not known to be regular, so we don't have a reliable size for it.
   338  		var err error
   339  		src, err := io.ReadAll(in)
   340  		if err != nil {
   341  			return nil, err
   342  		}
   343  		return src, nil
   344  	}
   345  
   346  	// We try to read size+1 bytes so that we can detect modifications: if we
   347  	// read more than size bytes, then the file was modified concurrently.
   348  	// (If that happens, we could, say, append to src to finish the read, or
   349  	// proceed with a truncated buffer — but the fact that it changed at all
   350  	// indicates a possible race with someone editing the file, so we prefer to
   351  	// stop to avoid corrupting it.)
   352  	src := make([]byte, size+1)
   353  	n, err := io.ReadFull(in, src)
   354  	if err != nil && err != io.ErrUnexpectedEOF {
   355  		return nil, err
   356  	}
   357  	if n < size {
   358  		return nil, fmt.Errorf("error: size of %s changed during reading (from %d to %d bytes)", filename, size, n)
   359  	} else if n > size {
   360  		return nil, fmt.Errorf("error: size of %s changed during reading (from %d to >=%d bytes)", filename, size, len(src))
   361  	}
   362  	return src[:n], nil
   363  }
   364  
   365  func main() {
   366  	// Arbitrarily limit in-flight work to 2MiB times the number of threads.
   367  	//
   368  	// The actual overhead for the parse tree and output will depend on the
   369  	// specifics of the file, but this at least keeps the footprint of the process
   370  	// roughly proportional to GOMAXPROCS.
   371  	maxWeight := (2 << 20) * int64(runtime.GOMAXPROCS(0))
   372  	s := newSequencer(maxWeight, os.Stdout, os.Stderr)
   373  
   374  	// call gofmtMain in a separate function
   375  	// so that it can use defer and have them
   376  	// run before the exit.
   377  	gofmtMain(s)
   378  	os.Exit(s.GetExitCode())
   379  }
   380  
   381  func gofmtMain(s *sequencer) {
   382  	flag.Usage = usage
   383  	flag.Parse()
   384  
   385  	if *cpuprofile != "" {
   386  		fdSem <- true
   387  		f, err := os.Create(*cpuprofile)
   388  		if err != nil {
   389  			s.AddReport(fmt.Errorf("creating cpu profile: %s", err))
   390  			return
   391  		}
   392  		defer func() {
   393  			f.Close()
   394  			<-fdSem
   395  		}()
   396  		pprof.StartCPUProfile(f)
   397  		defer pprof.StopCPUProfile()
   398  	}
   399  
   400  	initParserMode()
   401  	initRewrite()
   402  
   403  	args := flag.Args()
   404  	if len(args) == 0 {
   405  		if *write {
   406  			s.AddReport(fmt.Errorf("error: cannot use -w with standard input"))
   407  			return
   408  		}
   409  		s.Add(0, func(r *reporter) error {
   410  			return processFile("<standard input>", nil, os.Stdin, r)
   411  		})
   412  		return
   413  	}
   414  
   415  	for _, arg := range args {
   416  		switch info, err := os.Stat(arg); {
   417  		case err != nil:
   418  			s.AddReport(err)
   419  		case !info.IsDir():
   420  			// Non-directory arguments are always formatted.
   421  			arg := arg
   422  			s.Add(fileWeight(arg, info), func(r *reporter) error {
   423  				return processFile(arg, info, nil, r)
   424  			})
   425  		default:
   426  			// Directories are walked, ignoring non-Go files.
   427  			err := filepath.WalkDir(arg, func(path string, f fs.DirEntry, err error) error {
   428  				if err != nil || !isGoFile(f) {
   429  					return err
   430  				}
   431  				info, err := f.Info()
   432  				if err != nil {
   433  					s.AddReport(err)
   434  					return nil
   435  				}
   436  				s.Add(fileWeight(path, info), func(r *reporter) error {
   437  					return processFile(path, info, nil, r)
   438  				})
   439  				return nil
   440  			})
   441  			if err != nil {
   442  				s.AddReport(err)
   443  			}
   444  		}
   445  	}
   446  }
   447  
   448  func fileWeight(path string, info fs.FileInfo) int64 {
   449  	if info == nil {
   450  		return exclusive
   451  	}
   452  	if info.Mode().Type() == fs.ModeSymlink {
   453  		var err error
   454  		info, err = os.Stat(path)
   455  		if err != nil {
   456  			return exclusive
   457  		}
   458  	}
   459  	if !info.Mode().IsRegular() {
   460  		// For non-regular files, FileInfo.Size is system-dependent and thus not a
   461  		// reliable indicator of weight.
   462  		return exclusive
   463  	}
   464  	return info.Size()
   465  }
   466  
   467  func diffWithReplaceTempFile(b1, b2 []byte, filename string) ([]byte, error) {
   468  	data, err := diff.Diff("gofmt", b1, b2)
   469  	if len(data) > 0 {
   470  		return replaceTempFilename(data, filename)
   471  	}
   472  	return data, err
   473  }
   474  
   475  // replaceTempFilename replaces temporary filenames in diff with actual one.
   476  //
   477  // --- /tmp/gofmt316145376	2017-02-03 19:13:00.280468375 -0500
   478  // +++ /tmp/gofmt617882815	2017-02-03 19:13:00.280468375 -0500
   479  // ...
   480  // ->
   481  // --- path/to/file.go.orig	2017-02-03 19:13:00.280468375 -0500
   482  // +++ path/to/file.go	2017-02-03 19:13:00.280468375 -0500
   483  // ...
   484  func replaceTempFilename(diff []byte, filename string) ([]byte, error) {
   485  	bs := bytes.SplitN(diff, []byte{'\n'}, 3)
   486  	if len(bs) < 3 {
   487  		return nil, fmt.Errorf("got unexpected diff for %s", filename)
   488  	}
   489  	// Preserve timestamps.
   490  	var t0, t1 []byte
   491  	if i := bytes.LastIndexByte(bs[0], '\t'); i != -1 {
   492  		t0 = bs[0][i:]
   493  	}
   494  	if i := bytes.LastIndexByte(bs[1], '\t'); i != -1 {
   495  		t1 = bs[1][i:]
   496  	}
   497  	// Always print filepath with slash separator.
   498  	f := filepath.ToSlash(filename)
   499  	bs[0] = []byte(fmt.Sprintf("--- %s%s", f+".orig", t0))
   500  	bs[1] = []byte(fmt.Sprintf("+++ %s%s", f, t1))
   501  	return bytes.Join(bs, []byte{'\n'}), nil
   502  }
   503  
   504  const chmodSupported = runtime.GOOS != "windows"
   505  
   506  // backupFile writes data to a new file named filename<number> with permissions perm,
   507  // with <number randomly chosen such that the file name is unique. backupFile returns
   508  // the chosen file name.
   509  func backupFile(filename string, data []byte, perm fs.FileMode) (string, error) {
   510  	fdSem <- true
   511  	defer func() { <-fdSem }()
   512  
   513  	// create backup file
   514  	f, err := os.CreateTemp(filepath.Dir(filename), filepath.Base(filename))
   515  	if err != nil {
   516  		return "", err
   517  	}
   518  	bakname := f.Name()
   519  	if chmodSupported {
   520  		err = f.Chmod(perm)
   521  		if err != nil {
   522  			f.Close()
   523  			os.Remove(bakname)
   524  			return bakname, err
   525  		}
   526  	}
   527  
   528  	// write data to backup file
   529  	_, err = f.Write(data)
   530  	if err1 := f.Close(); err == nil {
   531  		err = err1
   532  	}
   533  
   534  	return bakname, err
   535  }
   536
View as plain text