test2json.go

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package test2json implements conversion of test binary output to JSON.
     6  // It is used by cmd/test2json and cmd/go.
     7  //
     8  // See the cmd/test2json documentation for details of the JSON encoding.
     9  package test2json
    10  
    11  import (
    12  	"bytes"
    13  	"encoding/json"
    14  	"fmt"
    15  	"io"
    16  	"strconv"
    17  	"strings"
    18  	"time"
    19  	"unicode"
    20  	"unicode/utf8"
    21  )
    22  
    23  // Mode controls details of the conversion.
    24  type Mode int
    25  
    26  const (
    27  	Timestamp Mode = 1 << iota // include Time in events
    28  )
    29  
    30  // event is the JSON struct we emit.
    31  type event struct {
    32  	Time    *time.Time `json:",omitempty"`
    33  	Action  string
    34  	Package string     `json:",omitempty"`
    35  	Test    string     `json:",omitempty"`
    36  	Elapsed *float64   `json:",omitempty"`
    37  	Output  *textBytes `json:",omitempty"`
    38  }
    39  
    40  // textBytes is a hack to get JSON to emit a []byte as a string
    41  // without actually copying it to a string.
    42  // It implements encoding.TextMarshaler, which returns its text form as a []byte,
    43  // and then json encodes that text form as a string (which was our goal).
    44  type textBytes []byte
    45  
    46  func (b textBytes) MarshalText() ([]byte, error) { return b, nil }
    47  
    48  // A Converter holds the state of a test-to-JSON conversion.
    49  // It implements io.WriteCloser; the caller writes test output in,
    50  // and the converter writes JSON output to w.
    51  type Converter struct {
    52  	w        io.Writer  // JSON output stream
    53  	pkg      string     // package to name in events
    54  	mode     Mode       // mode bits
    55  	start    time.Time  // time converter started
    56  	testName string     // name of current test, for output attribution
    57  	report   []*event   // pending test result reports (nested for subtests)
    58  	result   string     // overall test result if seen
    59  	input    lineBuffer // input buffer
    60  	output   lineBuffer // output buffer
    61  }
    62  
    63  // inBuffer and outBuffer are the input and output buffer sizes.
    64  // They're variables so that they can be reduced during testing.
    65  //
    66  // The input buffer needs to be able to hold any single test
    67  // directive line we want to recognize, like:
    68  //
    69  //     <many spaces> --- PASS: very/nested/s/u/b/t/e/s/t
    70  //
    71  // If anyone reports a test directive line > 4k not working, it will
    72  // be defensible to suggest they restructure their test or test names.
    73  //
    74  // The output buffer must be >= utf8.UTFMax, so that it can
    75  // accumulate any single UTF8 sequence. Lines that fit entirely
    76  // within the output buffer are emitted in single output events.
    77  // Otherwise they are split into multiple events.
    78  // The output buffer size therefore limits the size of the encoding
    79  // of a single JSON output event. 1k seems like a reasonable balance
    80  // between wanting to avoid splitting an output line and not wanting to
    81  // generate enormous output events.
    82  var (
    83  	inBuffer  = 4096
    84  	outBuffer = 1024
    85  )
    86  
    87  // NewConverter returns a "test to json" converter.
    88  // Writes on the returned writer are written as JSON to w,
    89  // with minimal delay.
    90  //
    91  // The writes to w are whole JSON events ending in \n,
    92  // so that it is safe to run multiple tests writing to multiple converters
    93  // writing to a single underlying output stream w.
    94  // As long as the underlying output w can handle concurrent writes
    95  // from multiple goroutines, the result will be a JSON stream
    96  // describing the relative ordering of execution in all the concurrent tests.
    97  //
    98  // The mode flag adjusts the behavior of the converter.
    99  // Passing ModeTime includes event timestamps and elapsed times.
   100  //
   101  // The pkg string, if present, specifies the import path to
   102  // report in the JSON stream.
   103  func NewConverter(w io.Writer, pkg string, mode Mode) *Converter {
   104  	c := new(Converter)
   105  	*c = Converter{
   106  		w:     w,
   107  		pkg:   pkg,
   108  		mode:  mode,
   109  		start: time.Now(),
   110  		input: lineBuffer{
   111  			b:    make([]byte, 0, inBuffer),
   112  			line: c.handleInputLine,
   113  			part: c.output.write,
   114  		},
   115  		output: lineBuffer{
   116  			b:    make([]byte, 0, outBuffer),
   117  			line: c.writeOutputEvent,
   118  			part: c.writeOutputEvent,
   119  		},
   120  	}
   121  	return c
   122  }
   123  
   124  // Write writes the test input to the converter.
   125  func (c *Converter) Write(b []byte) (int, error) {
   126  	c.input.write(b)
   127  	return len(b), nil
   128  }
   129  
   130  // Exited marks the test process as having exited with the given error.
   131  func (c *Converter) Exited(err error) {
   132  	if err == nil {
   133  		c.result = "pass"
   134  	} else {
   135  		c.result = "fail"
   136  	}
   137  }
   138  
   139  var (
   140  	// printed by test on successful run.
   141  	bigPass = []byte("PASS\n")
   142  
   143  	// printed by test after a normal test failure.
   144  	bigFail = []byte("FAIL\n")
   145  
   146  	// printed by 'go test' along with an error if the test binary terminates
   147  	// with an error.
   148  	bigFailErrorPrefix = []byte("FAIL\t")
   149  
   150  	updates = [][]byte{
   151  		[]byte("=== RUN   "),
   152  		[]byte("=== PAUSE "),
   153  		[]byte("=== CONT  "),
   154  	}
   155  
   156  	reports = [][]byte{
   157  		[]byte("--- PASS: "),
   158  		[]byte("--- FAIL: "),
   159  		[]byte("--- SKIP: "),
   160  		[]byte("--- BENCH: "),
   161  	}
   162  
   163  	fourSpace = []byte("    ")
   164  
   165  	skipLinePrefix = []byte("?   \t")
   166  	skipLineSuffix = []byte("\t[no test files]\n")
   167  )
   168  
   169  // handleInputLine handles a single whole test output line.
   170  // It must write the line to c.output but may choose to do so
   171  // before or after emitting other events.
   172  func (c *Converter) handleInputLine(line []byte) {
   173  	// Final PASS or FAIL.
   174  	if bytes.Equal(line, bigPass) || bytes.Equal(line, bigFail) || bytes.HasPrefix(line, bigFailErrorPrefix) {
   175  		c.flushReport(0)
   176  		c.output.write(line)
   177  		if bytes.Equal(line, bigPass) {
   178  			c.result = "pass"
   179  		} else {
   180  			c.result = "fail"
   181  		}
   182  		return
   183  	}
   184  
   185  	// Special case for entirely skipped test binary: "?   \tpkgname\t[no test files]\n" is only line.
   186  	// Report it as plain output but remember to say skip in the final summary.
   187  	if bytes.HasPrefix(line, skipLinePrefix) && bytes.HasSuffix(line, skipLineSuffix) && len(c.report) == 0 {
   188  		c.result = "skip"
   189  	}
   190  
   191  	// "=== RUN   "
   192  	// "=== PAUSE "
   193  	// "=== CONT  "
   194  	actionColon := false
   195  	origLine := line
   196  	ok := false
   197  	indent := 0
   198  	for _, magic := range updates {
   199  		if bytes.HasPrefix(line, magic) {
   200  			ok = true
   201  			break
   202  		}
   203  	}
   204  	if !ok {
   205  		// "--- PASS: "
   206  		// "--- FAIL: "
   207  		// "--- SKIP: "
   208  		// "--- BENCH: "
   209  		// but possibly indented.
   210  		for bytes.HasPrefix(line, fourSpace) {
   211  			line = line[4:]
   212  			indent++
   213  		}
   214  		for _, magic := range reports {
   215  			if bytes.HasPrefix(line, magic) {
   216  				actionColon = true
   217  				ok = true
   218  				break
   219  			}
   220  		}
   221  	}
   222  
   223  	// Not a special test output line.
   224  	if !ok {
   225  		// Lookup the name of the test which produced the output using the
   226  		// indentation of the output as an index into the stack of the current
   227  		// subtests.
   228  		// If the indentation is greater than the number of current subtests
   229  		// then the output must have included extra indentation. We can't
   230  		// determine which subtest produced this output, so we default to the
   231  		// old behaviour of assuming the most recently run subtest produced it.
   232  		if indent > 0 && indent <= len(c.report) {
   233  			c.testName = c.report[indent-1].Test
   234  		}
   235  		c.output.write(origLine)
   236  		return
   237  	}
   238  
   239  	// Parse out action and test name.
   240  	i := 0
   241  	if actionColon {
   242  		i = bytes.IndexByte(line, ':') + 1
   243  	}
   244  	if i == 0 {
   245  		i = len(updates[0])
   246  	}
   247  	action := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(string(line[4:i])), ":"))
   248  	name := strings.TrimSpace(string(line[i:]))
   249  
   250  	e := &event{Action: action}
   251  	if line[0] == '-' { // PASS or FAIL report
   252  		// Parse out elapsed time.
   253  		if i := strings.Index(name, " ("); i >= 0 {
   254  			if strings.HasSuffix(name, "s)") {
   255  				t, err := strconv.ParseFloat(name[i+2:len(name)-2], 64)
   256  				if err == nil {
   257  					if c.mode&Timestamp != 0 {
   258  						e.Elapsed = &t
   259  					}
   260  				}
   261  			}
   262  			name = name[:i]
   263  		}
   264  		if len(c.report) < indent {
   265  			// Nested deeper than expected.
   266  			// Treat this line as plain output.
   267  			c.output.write(origLine)
   268  			return
   269  		}
   270  		// Flush reports at this indentation level or deeper.
   271  		c.flushReport(indent)
   272  		e.Test = name
   273  		c.testName = name
   274  		c.report = append(c.report, e)
   275  		c.output.write(origLine)
   276  		return
   277  	}
   278  	// === update.
   279  	// Finish any pending PASS/FAIL reports.
   280  	c.flushReport(0)
   281  	c.testName = name
   282  
   283  	if action == "pause" {
   284  		// For a pause, we want to write the pause notification before
   285  		// delivering the pause event, just so it doesn't look like the test
   286  		// is generating output immediately after being paused.
   287  		c.output.write(origLine)
   288  	}
   289  	c.writeEvent(e)
   290  	if action != "pause" {
   291  		c.output.write(origLine)
   292  	}
   293  
   294  	return
   295  }
   296  
   297  // flushReport flushes all pending PASS/FAIL reports at levels >= depth.
   298  func (c *Converter) flushReport(depth int) {
   299  	c.testName = ""
   300  	for len(c.report) > depth {
   301  		e := c.report[len(c.report)-1]
   302  		c.report = c.report[:len(c.report)-1]
   303  		c.writeEvent(e)
   304  	}
   305  }
   306  
   307  // Close marks the end of the go test output.
   308  // It flushes any pending input and then output (only partial lines at this point)
   309  // and then emits the final overall package-level pass/fail event.
   310  func (c *Converter) Close() error {
   311  	c.input.flush()
   312  	c.output.flush()
   313  	if c.result != "" {
   314  		e := &event{Action: c.result}
   315  		if c.mode&Timestamp != 0 {
   316  			dt := time.Since(c.start).Round(1 * time.Millisecond).Seconds()
   317  			e.Elapsed = &dt
   318  		}
   319  		c.writeEvent(e)
   320  	}
   321  	return nil
   322  }
   323  
   324  // writeOutputEvent writes a single output event with the given bytes.
   325  func (c *Converter) writeOutputEvent(out []byte) {
   326  	c.writeEvent(&event{
   327  		Action: "output",
   328  		Output: (*textBytes)(&out),
   329  	})
   330  }
   331  
   332  // writeEvent writes a single event.
   333  // It adds the package, time (if requested), and test name (if needed).
   334  func (c *Converter) writeEvent(e *event) {
   335  	e.Package = c.pkg
   336  	if c.mode&Timestamp != 0 {
   337  		t := time.Now()
   338  		e.Time = &t
   339  	}
   340  	if e.Test == "" {
   341  		e.Test = c.testName
   342  	}
   343  	js, err := json.Marshal(e)
   344  	if err != nil {
   345  		// Should not happen - event is valid for json.Marshal.
   346  		c.w.Write([]byte(fmt.Sprintf("testjson internal error: %v\n", err)))
   347  		return
   348  	}
   349  	js = append(js, '\n')
   350  	c.w.Write(js)
   351  }
   352  
   353  // A lineBuffer is an I/O buffer that reacts to writes by invoking
   354  // input-processing callbacks on whole lines or (for long lines that
   355  // have been split) line fragments.
   356  //
   357  // It should be initialized with b set to a buffer of length 0 but non-zero capacity,
   358  // and line and part set to the desired input processors.
   359  // The lineBuffer will call line(x) for any whole line x (including the final newline)
   360  // that fits entirely in cap(b). It will handle input lines longer than cap(b) by
   361  // calling part(x) for sections of the line. The line will be split at UTF8 boundaries,
   362  // and the final call to part for a long line includes the final newline.
   363  type lineBuffer struct {
   364  	b    []byte       // buffer
   365  	mid  bool         // whether we're in the middle of a long line
   366  	line func([]byte) // line callback
   367  	part func([]byte) // partial line callback
   368  }
   369  
   370  // write writes b to the buffer.
   371  func (l *lineBuffer) write(b []byte) {
   372  	for len(b) > 0 {
   373  		// Copy what we can into b.
   374  		m := copy(l.b[len(l.b):cap(l.b)], b)
   375  		l.b = l.b[:len(l.b)+m]
   376  		b = b[m:]
   377  
   378  		// Process lines in b.
   379  		i := 0
   380  		for i < len(l.b) {
   381  			j := bytes.IndexByte(l.b[i:], '\n')
   382  			if j < 0 {
   383  				if !l.mid {
   384  					if j := bytes.IndexByte(l.b[i:], '\t'); j >= 0 {
   385  						if isBenchmarkName(bytes.TrimRight(l.b[i:i+j], " ")) {
   386  							l.part(l.b[i : i+j+1])
   387  							l.mid = true
   388  							i += j + 1
   389  						}
   390  					}
   391  				}
   392  				break
   393  			}
   394  			e := i + j + 1
   395  			if l.mid {
   396  				// Found the end of a partial line.
   397  				l.part(l.b[i:e])
   398  				l.mid = false
   399  			} else {
   400  				// Found a whole line.
   401  				l.line(l.b[i:e])
   402  			}
   403  			i = e
   404  		}
   405  
   406  		// Whatever's left in l.b is a line fragment.
   407  		if i == 0 && len(l.b) == cap(l.b) {
   408  			// The whole buffer is a fragment.
   409  			// Emit it as the beginning (or continuation) of a partial line.
   410  			t := trimUTF8(l.b)
   411  			l.part(l.b[:t])
   412  			l.b = l.b[:copy(l.b, l.b[t:])]
   413  			l.mid = true
   414  		}
   415  
   416  		// There's room for more input.
   417  		// Slide it down in hope of completing the line.
   418  		if i > 0 {
   419  			l.b = l.b[:copy(l.b, l.b[i:])]
   420  		}
   421  	}
   422  }
   423  
   424  // flush flushes the line buffer.
   425  func (l *lineBuffer) flush() {
   426  	if len(l.b) > 0 {
   427  		// Must be a line without a \n, so a partial line.
   428  		l.part(l.b)
   429  		l.b = l.b[:0]
   430  	}
   431  }
   432  
   433  var benchmark = []byte("Benchmark")
   434  
   435  // isBenchmarkName reports whether b is a valid benchmark name
   436  // that might appear as the first field in a benchmark result line.
   437  func isBenchmarkName(b []byte) bool {
   438  	if !bytes.HasPrefix(b, benchmark) {
   439  		return false
   440  	}
   441  	if len(b) == len(benchmark) { // just "Benchmark"
   442  		return true
   443  	}
   444  	r, _ := utf8.DecodeRune(b[len(benchmark):])
   445  	return !unicode.IsLower(r)
   446  }
   447  
   448  // trimUTF8 returns a length t as close to len(b) as possible such that b[:t]
   449  // does not end in the middle of a possibly-valid UTF-8 sequence.
   450  //
   451  // If a large text buffer must be split before position i at the latest,
   452  // splitting at position trimUTF(b[:i]) avoids splitting a UTF-8 sequence.
   453  func trimUTF8(b []byte) int {
   454  	// Scan backward to find non-continuation byte.
   455  	for i := 1; i < utf8.UTFMax && i <= len(b); i++ {
   456  		if c := b[len(b)-i]; c&0xc0 != 0x80 {
   457  			switch {
   458  			case c&0xe0 == 0xc0:
   459  				if i < 2 {
   460  					return len(b) - i
   461  				}
   462  			case c&0xf0 == 0xe0:
   463  				if i < 3 {
   464  					return len(b) - i
   465  				}
   466  			case c&0xf8 == 0xf0:
   467  				if i < 4 {
   468  					return len(b) - i
   469  				}
   470  			}
   471  			break
   472  		}
   473  	}
   474  	return len(b)
   475  }
   476
View as plain text