example.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Extract example functions from file ASTs.
     6  
     7  package doc
     8  
     9  import (
    10  	"go/ast"
    11  	"go/token"
    12  	"internal/lazyregexp"
    13  	"path"
    14  	"sort"
    15  	"strconv"
    16  	"strings"
    17  	"unicode"
    18  	"unicode/utf8"
    19  )
    20  
    21  // An Example represents an example function found in a test source file.
    22  type Example struct {
    23  	Name        string // name of the item being exemplified (including optional suffix)
    24  	Suffix      string // example suffix, without leading '_' (only populated by NewFromFiles)
    25  	Doc         string // example function doc string
    26  	Code        ast.Node
    27  	Play        *ast.File // a whole program version of the example
    28  	Comments    []*ast.CommentGroup
    29  	Output      string // expected output
    30  	Unordered   bool
    31  	EmptyOutput bool // expect empty output
    32  	Order       int  // original source code order
    33  }
    34  
    35  // Examples returns the examples found in testFiles, sorted by Name field.
    36  // The Order fields record the order in which the examples were encountered.
    37  // The Suffix field is not populated when Examples is called directly, it is
    38  // only populated by NewFromFiles for examples it finds in _test.go files.
    39  //
    40  // Playable Examples must be in a package whose name ends in "_test".
    41  // An Example is "playable" (the Play field is non-nil) in either of these
    42  // circumstances:
    43  //   - The example function is self-contained: the function references only
    44  //     identifiers from other packages (or predeclared identifiers, such as
    45  //     "int") and the test file does not include a dot import.
    46  //   - The entire test file is the example: the file contains exactly one
    47  //     example function, zero test, fuzz test, or benchmark function, and at
    48  //     least one top-level function, type, variable, or constant declaration
    49  //     other than the example function.
    50  func Examples(testFiles ...*ast.File) []*Example {
    51  	var list []*Example
    52  	for _, file := range testFiles {
    53  		hasTests := false // file contains tests, fuzz test, or benchmarks
    54  		numDecl := 0      // number of non-import declarations in the file
    55  		var flist []*Example
    56  		for _, decl := range file.Decls {
    57  			if g, ok := decl.(*ast.GenDecl); ok && g.Tok != token.IMPORT {
    58  				numDecl++
    59  				continue
    60  			}
    61  			f, ok := decl.(*ast.FuncDecl)
    62  			if !ok || f.Recv != nil {
    63  				continue
    64  			}
    65  			numDecl++
    66  			name := f.Name.Name
    67  			if isTest(name, "Test") || isTest(name, "Benchmark") || isTest(name, "Fuzz") {
    68  				hasTests = true
    69  				continue
    70  			}
    71  			if !isTest(name, "Example") {
    72  				continue
    73  			}
    74  			if params := f.Type.Params; len(params.List) != 0 {
    75  				continue // function has params; not a valid example
    76  			}
    77  			if f.Body == nil { // ast.File.Body nil dereference (see issue 28044)
    78  				continue
    79  			}
    80  			var doc string
    81  			if f.Doc != nil {
    82  				doc = f.Doc.Text()
    83  			}
    84  			output, unordered, hasOutput := exampleOutput(f.Body, file.Comments)
    85  			flist = append(flist, &Example{
    86  				Name:        name[len("Example"):],
    87  				Doc:         doc,
    88  				Code:        f.Body,
    89  				Play:        playExample(file, f),
    90  				Comments:    file.Comments,
    91  				Output:      output,
    92  				Unordered:   unordered,
    93  				EmptyOutput: output == "" && hasOutput,
    94  				Order:       len(flist),
    95  			})
    96  		}
    97  		if !hasTests && numDecl > 1 && len(flist) == 1 {
    98  			// If this file only has one example function, some
    99  			// other top-level declarations, and no tests or
   100  			// benchmarks, use the whole file as the example.
   101  			flist[0].Code = file
   102  			flist[0].Play = playExampleFile(file)
   103  		}
   104  		list = append(list, flist...)
   105  	}
   106  	// sort by name
   107  	sort.Slice(list, func(i, j int) bool {
   108  		return list[i].Name < list[j].Name
   109  	})
   110  	return list
   111  }
   112  
   113  var outputPrefix = lazyregexp.New(`(?i)^[[:space:]]*(unordered )?output:`)
   114  
   115  // Extracts the expected output and whether there was a valid output comment
   116  func exampleOutput(b *ast.BlockStmt, comments []*ast.CommentGroup) (output string, unordered, ok bool) {
   117  	if _, last := lastComment(b, comments); last != nil {
   118  		// test that it begins with the correct prefix
   119  		text := last.Text()
   120  		if loc := outputPrefix.FindStringSubmatchIndex(text); loc != nil {
   121  			if loc[2] != -1 {
   122  				unordered = true
   123  			}
   124  			text = text[loc[1]:]
   125  			// Strip zero or more spaces followed by \n or a single space.
   126  			text = strings.TrimLeft(text, " ")
   127  			if len(text) > 0 && text[0] == '\n' {
   128  				text = text[1:]
   129  			}
   130  			return text, unordered, true
   131  		}
   132  	}
   133  	return "", false, false // no suitable comment found
   134  }
   135  
   136  // isTest tells whether name looks like a test, example, fuzz test, or
   137  // benchmark. It is a Test (say) if there is a character after Test that is not
   138  // a lower-case letter. (We don't want Testiness.)
   139  func isTest(name, prefix string) bool {
   140  	if !strings.HasPrefix(name, prefix) {
   141  		return false
   142  	}
   143  	if len(name) == len(prefix) { // "Test" is ok
   144  		return true
   145  	}
   146  	rune, _ := utf8.DecodeRuneInString(name[len(prefix):])
   147  	return !unicode.IsLower(rune)
   148  }
   149  
   150  // playExample synthesizes a new *ast.File based on the provided
   151  // file with the provided function body as the body of main.
   152  func playExample(file *ast.File, f *ast.FuncDecl) *ast.File {
   153  	body := f.Body
   154  
   155  	if !strings.HasSuffix(file.Name.Name, "_test") {
   156  		// We don't support examples that are part of the
   157  		// greater package (yet).
   158  		return nil
   159  	}
   160  
   161  	// Collect top-level declarations in the file.
   162  	topDecls := make(map[*ast.Object]ast.Decl)
   163  	typMethods := make(map[string][]ast.Decl)
   164  
   165  	for _, decl := range file.Decls {
   166  		switch d := decl.(type) {
   167  		case *ast.FuncDecl:
   168  			if d.Recv == nil {
   169  				topDecls[d.Name.Obj] = d
   170  			} else {
   171  				if len(d.Recv.List) == 1 {
   172  					t := d.Recv.List[0].Type
   173  					tname, _ := baseTypeName(t)
   174  					typMethods[tname] = append(typMethods[tname], d)
   175  				}
   176  			}
   177  		case *ast.GenDecl:
   178  			for _, spec := range d.Specs {
   179  				switch s := spec.(type) {
   180  				case *ast.TypeSpec:
   181  					topDecls[s.Name.Obj] = d
   182  				case *ast.ValueSpec:
   183  					for _, name := range s.Names {
   184  						topDecls[name.Obj] = d
   185  					}
   186  				}
   187  			}
   188  		}
   189  	}
   190  
   191  	// Find unresolved identifiers and uses of top-level declarations.
   192  	unresolved := make(map[string]bool)
   193  	var depDecls []ast.Decl
   194  	hasDepDecls := make(map[ast.Decl]bool)
   195  
   196  	var inspectFunc func(ast.Node) bool
   197  	inspectFunc = func(n ast.Node) bool {
   198  		switch e := n.(type) {
   199  		case *ast.Ident:
   200  			if e.Obj == nil && e.Name != "_" {
   201  				unresolved[e.Name] = true
   202  			} else if d := topDecls[e.Obj]; d != nil {
   203  				if !hasDepDecls[d] {
   204  					hasDepDecls[d] = true
   205  					depDecls = append(depDecls, d)
   206  				}
   207  			}
   208  			return true
   209  		case *ast.SelectorExpr:
   210  			// For selector expressions, only inspect the left hand side.
   211  			// (For an expression like fmt.Println, only add "fmt" to the
   212  			// set of unresolved names, not "Println".)
   213  			ast.Inspect(e.X, inspectFunc)
   214  			return false
   215  		case *ast.KeyValueExpr:
   216  			// For key value expressions, only inspect the value
   217  			// as the key should be resolved by the type of the
   218  			// composite literal.
   219  			ast.Inspect(e.Value, inspectFunc)
   220  			return false
   221  		}
   222  		return true
   223  	}
   224  	ast.Inspect(body, inspectFunc)
   225  	for i := 0; i < len(depDecls); i++ {
   226  		switch d := depDecls[i].(type) {
   227  		case *ast.FuncDecl:
   228  			// Inspect types of parameters and results. See #28492.
   229  			if d.Type.Params != nil {
   230  				for _, p := range d.Type.Params.List {
   231  					ast.Inspect(p.Type, inspectFunc)
   232  				}
   233  			}
   234  			if d.Type.Results != nil {
   235  				for _, r := range d.Type.Results.List {
   236  					ast.Inspect(r.Type, inspectFunc)
   237  				}
   238  			}
   239  
   240  			// Functions might not have a body. See #42706.
   241  			if d.Body != nil {
   242  				ast.Inspect(d.Body, inspectFunc)
   243  			}
   244  		case *ast.GenDecl:
   245  			for _, spec := range d.Specs {
   246  				switch s := spec.(type) {
   247  				case *ast.TypeSpec:
   248  					ast.Inspect(s.Type, inspectFunc)
   249  
   250  					depDecls = append(depDecls, typMethods[s.Name.Name]...)
   251  				case *ast.ValueSpec:
   252  					if s.Type != nil {
   253  						ast.Inspect(s.Type, inspectFunc)
   254  					}
   255  					for _, val := range s.Values {
   256  						ast.Inspect(val, inspectFunc)
   257  					}
   258  				}
   259  			}
   260  		}
   261  	}
   262  
   263  	// Remove predeclared identifiers from unresolved list.
   264  	for n := range unresolved {
   265  		if predeclaredTypes[n] || predeclaredConstants[n] || predeclaredFuncs[n] {
   266  			delete(unresolved, n)
   267  		}
   268  	}
   269  
   270  	// Use unresolved identifiers to determine the imports used by this
   271  	// example. The heuristic assumes package names match base import
   272  	// paths for imports w/o renames (should be good enough most of the time).
   273  	namedImports := make(map[string]string) // [name]path
   274  	var blankImports []ast.Spec             // _ imports
   275  	for _, s := range file.Imports {
   276  		p, err := strconv.Unquote(s.Path.Value)
   277  		if err != nil {
   278  			continue
   279  		}
   280  		if p == "syscall/js" {
   281  			// We don't support examples that import syscall/js,
   282  			// because the package syscall/js is not available in the playground.
   283  			return nil
   284  		}
   285  		n := path.Base(p)
   286  		if s.Name != nil {
   287  			n = s.Name.Name
   288  			switch n {
   289  			case "_":
   290  				blankImports = append(blankImports, s)
   291  				continue
   292  			case ".":
   293  				// We can't resolve dot imports (yet).
   294  				return nil
   295  			}
   296  		}
   297  		if unresolved[n] {
   298  			namedImports[n] = p
   299  			delete(unresolved, n)
   300  		}
   301  	}
   302  
   303  	// If there are other unresolved identifiers, give up because this
   304  	// synthesized file is not going to build.
   305  	if len(unresolved) > 0 {
   306  		return nil
   307  	}
   308  
   309  	// Include documentation belonging to blank imports.
   310  	var comments []*ast.CommentGroup
   311  	for _, s := range blankImports {
   312  		if c := s.(*ast.ImportSpec).Doc; c != nil {
   313  			comments = append(comments, c)
   314  		}
   315  	}
   316  
   317  	// Include comments that are inside the function body.
   318  	for _, c := range file.Comments {
   319  		if body.Pos() <= c.Pos() && c.End() <= body.End() {
   320  			comments = append(comments, c)
   321  		}
   322  	}
   323  
   324  	// Strip the "Output:" or "Unordered output:" comment and adjust body
   325  	// end position.
   326  	body, comments = stripOutputComment(body, comments)
   327  
   328  	// Include documentation belonging to dependent declarations.
   329  	for _, d := range depDecls {
   330  		switch d := d.(type) {
   331  		case *ast.GenDecl:
   332  			if d.Doc != nil {
   333  				comments = append(comments, d.Doc)
   334  			}
   335  		case *ast.FuncDecl:
   336  			if d.Doc != nil {
   337  				comments = append(comments, d.Doc)
   338  			}
   339  		}
   340  	}
   341  
   342  	// Synthesize import declaration.
   343  	importDecl := &ast.GenDecl{
   344  		Tok:    token.IMPORT,
   345  		Lparen: 1, // Need non-zero Lparen and Rparen so that printer
   346  		Rparen: 1, // treats this as a factored import.
   347  	}
   348  	for n, p := range namedImports {
   349  		s := &ast.ImportSpec{Path: &ast.BasicLit{Value: strconv.Quote(p)}}
   350  		if path.Base(p) != n {
   351  			s.Name = ast.NewIdent(n)
   352  		}
   353  		importDecl.Specs = append(importDecl.Specs, s)
   354  	}
   355  	importDecl.Specs = append(importDecl.Specs, blankImports...)
   356  
   357  	// Synthesize main function.
   358  	funcDecl := &ast.FuncDecl{
   359  		Name: ast.NewIdent("main"),
   360  		Type: f.Type,
   361  		Body: body,
   362  	}
   363  
   364  	decls := make([]ast.Decl, 0, 2+len(depDecls))
   365  	decls = append(decls, importDecl)
   366  	decls = append(decls, depDecls...)
   367  	decls = append(decls, funcDecl)
   368  
   369  	sort.Slice(decls, func(i, j int) bool {
   370  		return decls[i].Pos() < decls[j].Pos()
   371  	})
   372  
   373  	sort.Slice(comments, func(i, j int) bool {
   374  		return comments[i].Pos() < comments[j].Pos()
   375  	})
   376  
   377  	// Synthesize file.
   378  	return &ast.File{
   379  		Name:     ast.NewIdent("main"),
   380  		Decls:    decls,
   381  		Comments: comments,
   382  	}
   383  }
   384  
   385  // playExampleFile takes a whole file example and synthesizes a new *ast.File
   386  // such that the example is function main in package main.
   387  func playExampleFile(file *ast.File) *ast.File {
   388  	// Strip copyright comment if present.
   389  	comments := file.Comments
   390  	if len(comments) > 0 && strings.HasPrefix(comments[0].Text(), "Copyright") {
   391  		comments = comments[1:]
   392  	}
   393  
   394  	// Copy declaration slice, rewriting the ExampleX function to main.
   395  	var decls []ast.Decl
   396  	for _, d := range file.Decls {
   397  		if f, ok := d.(*ast.FuncDecl); ok && isTest(f.Name.Name, "Example") {
   398  			// Copy the FuncDecl, as it may be used elsewhere.
   399  			newF := *f
   400  			newF.Name = ast.NewIdent("main")
   401  			newF.Body, comments = stripOutputComment(f.Body, comments)
   402  			d = &newF
   403  		}
   404  		decls = append(decls, d)
   405  	}
   406  
   407  	// Copy the File, as it may be used elsewhere.
   408  	f := *file
   409  	f.Name = ast.NewIdent("main")
   410  	f.Decls = decls
   411  	f.Comments = comments
   412  	return &f
   413  }
   414  
   415  // stripOutputComment finds and removes the "Output:" or "Unordered output:"
   416  // comment from body and comments, and adjusts the body block's end position.
   417  func stripOutputComment(body *ast.BlockStmt, comments []*ast.CommentGroup) (*ast.BlockStmt, []*ast.CommentGroup) {
   418  	// Do nothing if there is no "Output:" or "Unordered output:" comment.
   419  	i, last := lastComment(body, comments)
   420  	if last == nil || !outputPrefix.MatchString(last.Text()) {
   421  		return body, comments
   422  	}
   423  
   424  	// Copy body and comments, as the originals may be used elsewhere.
   425  	newBody := &ast.BlockStmt{
   426  		Lbrace: body.Lbrace,
   427  		List:   body.List,
   428  		Rbrace: last.Pos(),
   429  	}
   430  	newComments := make([]*ast.CommentGroup, len(comments)-1)
   431  	copy(newComments, comments[:i])
   432  	copy(newComments[i:], comments[i+1:])
   433  	return newBody, newComments
   434  }
   435  
   436  // lastComment returns the last comment inside the provided block.
   437  func lastComment(b *ast.BlockStmt, c []*ast.CommentGroup) (i int, last *ast.CommentGroup) {
   438  	if b == nil {
   439  		return
   440  	}
   441  	pos, end := b.Pos(), b.End()
   442  	for j, cg := range c {
   443  		if cg.Pos() < pos {
   444  			continue
   445  		}
   446  		if cg.End() > end {
   447  			break
   448  		}
   449  		i, last = j, cg
   450  	}
   451  	return
   452  }
   453  
   454  // classifyExamples classifies examples and assigns them to the Examples field
   455  // of the relevant Func, Type, or Package that the example is associated with.
   456  //
   457  // The classification process is ambiguous in some cases:
   458  //
   459  // 	- ExampleFoo_Bar matches a type named Foo_Bar
   460  // 	  or a method named Foo.Bar.
   461  // 	- ExampleFoo_bar matches a type named Foo_bar
   462  // 	  or Foo (with a "bar" suffix).
   463  //
   464  // Examples with malformed names are not associated with anything.
   465  //
   466  func classifyExamples(p *Package, examples []*Example) {
   467  	if len(examples) == 0 {
   468  		return
   469  	}
   470  
   471  	// Mapping of names for funcs, types, and methods to the example listing.
   472  	ids := make(map[string]*[]*Example)
   473  	ids[""] = &p.Examples // package-level examples have an empty name
   474  	for _, f := range p.Funcs {
   475  		if !token.IsExported(f.Name) {
   476  			continue
   477  		}
   478  		ids[f.Name] = &f.Examples
   479  	}
   480  	for _, t := range p.Types {
   481  		if !token.IsExported(t.Name) {
   482  			continue
   483  		}
   484  		ids[t.Name] = &t.Examples
   485  		for _, f := range t.Funcs {
   486  			if !token.IsExported(f.Name) {
   487  				continue
   488  			}
   489  			ids[f.Name] = &f.Examples
   490  		}
   491  		for _, m := range t.Methods {
   492  			if !token.IsExported(m.Name) {
   493  				continue
   494  			}
   495  			ids[strings.TrimPrefix(m.Recv, "*")+"_"+m.Name] = &m.Examples
   496  		}
   497  	}
   498  
   499  	// Group each example with the associated func, type, or method.
   500  	for _, ex := range examples {
   501  		// Consider all possible split points for the suffix
   502  		// by starting at the end of string (no suffix case),
   503  		// then trying all positions that contain a '_' character.
   504  		//
   505  		// An association is made on the first successful match.
   506  		// Examples with malformed names that match nothing are skipped.
   507  		for i := len(ex.Name); i >= 0; i = strings.LastIndexByte(ex.Name[:i], '_') {
   508  			prefix, suffix, ok := splitExampleName(ex.Name, i)
   509  			if !ok {
   510  				continue
   511  			}
   512  			exs, ok := ids[prefix]
   513  			if !ok {
   514  				continue
   515  			}
   516  			ex.Suffix = suffix
   517  			*exs = append(*exs, ex)
   518  			break
   519  		}
   520  	}
   521  
   522  	// Sort list of example according to the user-specified suffix name.
   523  	for _, exs := range ids {
   524  		sort.Slice((*exs), func(i, j int) bool {
   525  			return (*exs)[i].Suffix < (*exs)[j].Suffix
   526  		})
   527  	}
   528  }
   529  
   530  // splitExampleName attempts to split example name s at index i,
   531  // and reports if that produces a valid split. The suffix may be
   532  // absent. Otherwise, it must start with a lower-case letter and
   533  // be preceded by '_'.
   534  //
   535  // One of i == len(s) or s[i] == '_' must be true.
   536  func splitExampleName(s string, i int) (prefix, suffix string, ok bool) {
   537  	if i == len(s) {
   538  		return s, "", true
   539  	}
   540  	if i == len(s)-1 {
   541  		return "", "", false
   542  	}
   543  	prefix, suffix = s[:i], s[i+1:]
   544  	return prefix, suffix, isExampleSuffix(suffix)
   545  }
   546  
   547  func isExampleSuffix(s string) bool {
   548  	r, size := utf8.DecodeRuneInString(s)
   549  	return size > 0 && unicode.IsLower(r)
   550  }
   551
View as plain text