Source file src/text/template/parse/lex_test.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package parse
     6  
     7  import (
     8  	"fmt"
     9  	"testing"
    10  )
    11  
    12  // Make the types prettyprint.
    13  var itemName = map[itemType]string{
    14  	itemError:        "error",
    15  	itemBool:         "bool",
    16  	itemChar:         "char",
    17  	itemCharConstant: "charconst",
    18  	itemComment:      "comment",
    19  	itemComplex:      "complex",
    20  	itemDeclare:      ":=",
    21  	itemEOF:          "EOF",
    22  	itemField:        "field",
    23  	itemIdentifier:   "identifier",
    24  	itemLeftDelim:    "left delim",
    25  	itemLeftParen:    "(",
    26  	itemNumber:       "number",
    27  	itemPipe:         "pipe",
    28  	itemRawString:    "raw string",
    29  	itemRightDelim:   "right delim",
    30  	itemRightParen:   ")",
    31  	itemSpace:        "space",
    32  	itemString:       "string",
    33  	itemVariable:     "variable",
    34  
    35  	// keywords
    36  	itemDot:      ".",
    37  	itemBlock:    "block",
    38  	itemBreak:    "break",
    39  	itemContinue: "continue",
    40  	itemDefine:   "define",
    41  	itemElse:     "else",
    42  	itemIf:       "if",
    43  	itemEnd:      "end",
    44  	itemNil:      "nil",
    45  	itemRange:    "range",
    46  	itemTemplate: "template",
    47  	itemWith:     "with",
    48  }
    49  
    50  func (i itemType) String() string {
    51  	s := itemName[i]
    52  	if s == "" {
    53  		return fmt.Sprintf("item%d", int(i))
    54  	}
    55  	return s
    56  }
    57  
    58  type lexTest struct {
    59  	name  string
    60  	input string
    61  	items []item
    62  }
    63  
    64  func mkItem(typ itemType, text string) item {
    65  	return item{
    66  		typ: typ,
    67  		val: text,
    68  	}
    69  }
    70  
    71  var (
    72  	tDot        = mkItem(itemDot, ".")
    73  	tBlock      = mkItem(itemBlock, "block")
    74  	tEOF        = mkItem(itemEOF, "")
    75  	tFor        = mkItem(itemIdentifier, "for")
    76  	tLeft       = mkItem(itemLeftDelim, "{{")
    77  	tLpar       = mkItem(itemLeftParen, "(")
    78  	tPipe       = mkItem(itemPipe, "|")
    79  	tQuote      = mkItem(itemString, `"abc \n\t\" "`)
    80  	tRange      = mkItem(itemRange, "range")
    81  	tRight      = mkItem(itemRightDelim, "}}")
    82  	tRpar       = mkItem(itemRightParen, ")")
    83  	tSpace      = mkItem(itemSpace, " ")
    84  	raw         = "`" + `abc\n\t\" ` + "`"
    85  	rawNL       = "`now is{{\n}}the time`" // Contains newline inside raw quote.
    86  	tRawQuote   = mkItem(itemRawString, raw)
    87  	tRawQuoteNL = mkItem(itemRawString, rawNL)
    88  )
    89  
    90  var lexTests = []lexTest{
    91  	{"empty", "", []item{tEOF}},
    92  	{"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
    93  	{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
    94  	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
    95  		mkItem(itemText, "hello-"),
    96  		mkItem(itemComment, "/* this is a comment */"),
    97  		mkItem(itemText, "-world"),
    98  		tEOF,
    99  	}},
   100  	{"punctuation", "{{,@% }}", []item{
   101  		tLeft,
   102  		mkItem(itemChar, ","),
   103  		mkItem(itemChar, "@"),
   104  		mkItem(itemChar, "%"),
   105  		tSpace,
   106  		tRight,
   107  		tEOF,
   108  	}},
   109  	{"parens", "{{((3))}}", []item{
   110  		tLeft,
   111  		tLpar,
   112  		tLpar,
   113  		mkItem(itemNumber, "3"),
   114  		tRpar,
   115  		tRpar,
   116  		tRight,
   117  		tEOF,
   118  	}},
   119  	{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
   120  	{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
   121  	{"block", `{{block "foo" .}}`, []item{
   122  		tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
   123  	}},
   124  	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
   125  	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
   126  	{"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
   127  	{"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
   128  		tLeft,
   129  		mkItem(itemNumber, "1"),
   130  		tSpace,
   131  		mkItem(itemNumber, "02"),
   132  		tSpace,
   133  		mkItem(itemNumber, "0x14"),
   134  		tSpace,
   135  		mkItem(itemNumber, "0X14"),
   136  		tSpace,
   137  		mkItem(itemNumber, "-7.2i"),
   138  		tSpace,
   139  		mkItem(itemNumber, "1e3"),
   140  		tSpace,
   141  		mkItem(itemNumber, "1E3"),
   142  		tSpace,
   143  		mkItem(itemNumber, "+1.2e-4"),
   144  		tSpace,
   145  		mkItem(itemNumber, "4.2i"),
   146  		tSpace,
   147  		mkItem(itemComplex, "1+2i"),
   148  		tSpace,
   149  		mkItem(itemNumber, "1_2"),
   150  		tSpace,
   151  		mkItem(itemNumber, "0x1.e_fp4"),
   152  		tSpace,
   153  		mkItem(itemNumber, "0X1.E_FP4"),
   154  		tRight,
   155  		tEOF,
   156  	}},
   157  	{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
   158  		tLeft,
   159  		mkItem(itemCharConstant, `'a'`),
   160  		tSpace,
   161  		mkItem(itemCharConstant, `'\n'`),
   162  		tSpace,
   163  		mkItem(itemCharConstant, `'\''`),
   164  		tSpace,
   165  		mkItem(itemCharConstant, `'\\'`),
   166  		tSpace,
   167  		mkItem(itemCharConstant, `'\u00FF'`),
   168  		tSpace,
   169  		mkItem(itemCharConstant, `'\xFF'`),
   170  		tSpace,
   171  		mkItem(itemCharConstant, `'本'`),
   172  		tRight,
   173  		tEOF,
   174  	}},
   175  	{"bools", "{{true false}}", []item{
   176  		tLeft,
   177  		mkItem(itemBool, "true"),
   178  		tSpace,
   179  		mkItem(itemBool, "false"),
   180  		tRight,
   181  		tEOF,
   182  	}},
   183  	{"dot", "{{.}}", []item{
   184  		tLeft,
   185  		tDot,
   186  		tRight,
   187  		tEOF,
   188  	}},
   189  	{"nil", "{{nil}}", []item{
   190  		tLeft,
   191  		mkItem(itemNil, "nil"),
   192  		tRight,
   193  		tEOF,
   194  	}},
   195  	{"dots", "{{.x . .2 .x.y.z}}", []item{
   196  		tLeft,
   197  		mkItem(itemField, ".x"),
   198  		tSpace,
   199  		tDot,
   200  		tSpace,
   201  		mkItem(itemNumber, ".2"),
   202  		tSpace,
   203  		mkItem(itemField, ".x"),
   204  		mkItem(itemField, ".y"),
   205  		mkItem(itemField, ".z"),
   206  		tRight,
   207  		tEOF,
   208  	}},
   209  	{"keywords", "{{range if else end with}}", []item{
   210  		tLeft,
   211  		mkItem(itemRange, "range"),
   212  		tSpace,
   213  		mkItem(itemIf, "if"),
   214  		tSpace,
   215  		mkItem(itemElse, "else"),
   216  		tSpace,
   217  		mkItem(itemEnd, "end"),
   218  		tSpace,
   219  		mkItem(itemWith, "with"),
   220  		tRight,
   221  		tEOF,
   222  	}},
   223  	{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
   224  		tLeft,
   225  		mkItem(itemVariable, "$c"),
   226  		tSpace,
   227  		mkItem(itemDeclare, ":="),
   228  		tSpace,
   229  		mkItem(itemIdentifier, "printf"),
   230  		tSpace,
   231  		mkItem(itemVariable, "$"),
   232  		tSpace,
   233  		mkItem(itemVariable, "$hello"),
   234  		tSpace,
   235  		mkItem(itemVariable, "$23"),
   236  		tSpace,
   237  		mkItem(itemVariable, "$"),
   238  		tSpace,
   239  		mkItem(itemVariable, "$var"),
   240  		mkItem(itemField, ".Field"),
   241  		tSpace,
   242  		mkItem(itemField, ".Method"),
   243  		tRight,
   244  		tEOF,
   245  	}},
   246  	{"variable invocation", "{{$x 23}}", []item{
   247  		tLeft,
   248  		mkItem(itemVariable, "$x"),
   249  		tSpace,
   250  		mkItem(itemNumber, "23"),
   251  		tRight,
   252  		tEOF,
   253  	}},
   254  	{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
   255  		mkItem(itemText, "intro "),
   256  		tLeft,
   257  		mkItem(itemIdentifier, "echo"),
   258  		tSpace,
   259  		mkItem(itemIdentifier, "hi"),
   260  		tSpace,
   261  		mkItem(itemNumber, "1.2"),
   262  		tSpace,
   263  		tPipe,
   264  		mkItem(itemIdentifier, "noargs"),
   265  		tPipe,
   266  		mkItem(itemIdentifier, "args"),
   267  		tSpace,
   268  		mkItem(itemNumber, "1"),
   269  		tSpace,
   270  		mkItem(itemString, `"hi"`),
   271  		tRight,
   272  		mkItem(itemText, " outro"),
   273  		tEOF,
   274  	}},
   275  	{"declaration", "{{$v := 3}}", []item{
   276  		tLeft,
   277  		mkItem(itemVariable, "$v"),
   278  		tSpace,
   279  		mkItem(itemDeclare, ":="),
   280  		tSpace,
   281  		mkItem(itemNumber, "3"),
   282  		tRight,
   283  		tEOF,
   284  	}},
   285  	{"2 declarations", "{{$v , $w := 3}}", []item{
   286  		tLeft,
   287  		mkItem(itemVariable, "$v"),
   288  		tSpace,
   289  		mkItem(itemChar, ","),
   290  		tSpace,
   291  		mkItem(itemVariable, "$w"),
   292  		tSpace,
   293  		mkItem(itemDeclare, ":="),
   294  		tSpace,
   295  		mkItem(itemNumber, "3"),
   296  		tRight,
   297  		tEOF,
   298  	}},
   299  	{"field of parenthesized expression", "{{(.X).Y}}", []item{
   300  		tLeft,
   301  		tLpar,
   302  		mkItem(itemField, ".X"),
   303  		tRpar,
   304  		mkItem(itemField, ".Y"),
   305  		tRight,
   306  		tEOF,
   307  	}},
   308  	{"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
   309  		mkItem(itemText, "hello-"),
   310  		tLeft,
   311  		mkItem(itemNumber, "3"),
   312  		tRight,
   313  		mkItem(itemText, "-world"),
   314  		tEOF,
   315  	}},
   316  	{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
   317  		mkItem(itemText, "hello-"),
   318  		mkItem(itemComment, "/* hello */"),
   319  		mkItem(itemText, "-world"),
   320  		tEOF,
   321  	}},
   322  	// errors
   323  	{"badchar", "#{{\x01}}", []item{
   324  		mkItem(itemText, "#"),
   325  		tLeft,
   326  		mkItem(itemError, "unrecognized character in action: U+0001"),
   327  	}},
   328  	{"unclosed action", "{{", []item{
   329  		tLeft,
   330  		mkItem(itemError, "unclosed action"),
   331  	}},
   332  	{"EOF in action", "{{range", []item{
   333  		tLeft,
   334  		tRange,
   335  		mkItem(itemError, "unclosed action"),
   336  	}},
   337  	{"unclosed quote", "{{\"\n\"}}", []item{
   338  		tLeft,
   339  		mkItem(itemError, "unterminated quoted string"),
   340  	}},
   341  	{"unclosed raw quote", "{{`xx}}", []item{
   342  		tLeft,
   343  		mkItem(itemError, "unterminated raw quoted string"),
   344  	}},
   345  	{"unclosed char constant", "{{'\n}}", []item{
   346  		tLeft,
   347  		mkItem(itemError, "unterminated character constant"),
   348  	}},
   349  	{"bad number", "{{3k}}", []item{
   350  		tLeft,
   351  		mkItem(itemError, `bad number syntax: "3k"`),
   352  	}},
   353  	{"unclosed paren", "{{(3}}", []item{
   354  		tLeft,
   355  		tLpar,
   356  		mkItem(itemNumber, "3"),
   357  		mkItem(itemError, `unclosed left paren`),
   358  	}},
   359  	{"extra right paren", "{{3)}}", []item{
   360  		tLeft,
   361  		mkItem(itemNumber, "3"),
   362  		tRpar,
   363  		mkItem(itemError, `unexpected right paren U+0029 ')'`),
   364  	}},
   365  
   366  	// Fixed bugs
   367  	// Many elements in an action blew the lookahead until
   368  	// we made lexInsideAction not loop.
   369  	{"long pipeline deadlock", "{{|||||}}", []item{
   370  		tLeft,
   371  		tPipe,
   372  		tPipe,
   373  		tPipe,
   374  		tPipe,
   375  		tPipe,
   376  		tRight,
   377  		tEOF,
   378  	}},
   379  	{"text with bad comment", "hello-{{/*/}}-world", []item{
   380  		mkItem(itemText, "hello-"),
   381  		mkItem(itemError, `unclosed comment`),
   382  	}},
   383  	{"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
   384  		mkItem(itemText, "hello-"),
   385  		mkItem(itemError, `comment ends before closing delimiter`),
   386  	}},
   387  	// This one is an error that we can't catch because it breaks templates with
   388  	// minimized JavaScript. Should have fixed it before Go 1.1.
   389  	{"unmatched right delimiter", "hello-{.}}-world", []item{
   390  		mkItem(itemText, "hello-{.}}-world"),
   391  		tEOF,
   392  	}},
   393  }
   394  
   395  // collect gathers the emitted items into a slice.
   396  func collect(t *lexTest, left, right string) (items []item) {
   397  	l := lex(t.name, t.input, left, right, true)
   398  	for {
   399  		item := l.nextItem()
   400  		items = append(items, item)
   401  		if item.typ == itemEOF || item.typ == itemError {
   402  			break
   403  		}
   404  	}
   405  	return
   406  }
   407  
   408  func equal(i1, i2 []item, checkPos bool) bool {
   409  	if len(i1) != len(i2) {
   410  		return false
   411  	}
   412  	for k := range i1 {
   413  		if i1[k].typ != i2[k].typ {
   414  			return false
   415  		}
   416  		if i1[k].val != i2[k].val {
   417  			return false
   418  		}
   419  		if checkPos && i1[k].pos != i2[k].pos {
   420  			return false
   421  		}
   422  		if checkPos && i1[k].line != i2[k].line {
   423  			return false
   424  		}
   425  	}
   426  	return true
   427  }
   428  
   429  func TestLex(t *testing.T) {
   430  	for _, test := range lexTests {
   431  		items := collect(&test, "", "")
   432  		if !equal(items, test.items, false) {
   433  			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
   434  		}
   435  	}
   436  }
   437  
   438  // Some easy cases from above, but with delimiters $$ and @@
   439  var lexDelimTests = []lexTest{
   440  	{"punctuation", "$$,@%{{}}@@", []item{
   441  		tLeftDelim,
   442  		mkItem(itemChar, ","),
   443  		mkItem(itemChar, "@"),
   444  		mkItem(itemChar, "%"),
   445  		mkItem(itemChar, "{"),
   446  		mkItem(itemChar, "{"),
   447  		mkItem(itemChar, "}"),
   448  		mkItem(itemChar, "}"),
   449  		tRightDelim,
   450  		tEOF,
   451  	}},
   452  	{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
   453  	{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
   454  	{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
   455  	{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
   456  }
   457  
   458  var (
   459  	tLeftDelim  = mkItem(itemLeftDelim, "$$")
   460  	tRightDelim = mkItem(itemRightDelim, "@@")
   461  )
   462  
   463  func TestDelims(t *testing.T) {
   464  	for _, test := range lexDelimTests {
   465  		items := collect(&test, "$$", "@@")
   466  		if !equal(items, test.items, false) {
   467  			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
   468  		}
   469  	}
   470  }
   471  
   472  var lexPosTests = []lexTest{
   473  	{"empty", "", []item{{itemEOF, 0, "", 1}}},
   474  	{"punctuation", "{{,@%#}}", []item{
   475  		{itemLeftDelim, 0, "{{", 1},
   476  		{itemChar, 2, ",", 1},
   477  		{itemChar, 3, "@", 1},
   478  		{itemChar, 4, "%", 1},
   479  		{itemChar, 5, "#", 1},
   480  		{itemRightDelim, 6, "}}", 1},
   481  		{itemEOF, 8, "", 1},
   482  	}},
   483  	{"sample", "0123{{hello}}xyz", []item{
   484  		{itemText, 0, "0123", 1},
   485  		{itemLeftDelim, 4, "{{", 1},
   486  		{itemIdentifier, 6, "hello", 1},
   487  		{itemRightDelim, 11, "}}", 1},
   488  		{itemText, 13, "xyz", 1},
   489  		{itemEOF, 16, "", 1},
   490  	}},
   491  	{"trimafter", "{{x -}}\n{{y}}", []item{
   492  		{itemLeftDelim, 0, "{{", 1},
   493  		{itemIdentifier, 2, "x", 1},
   494  		{itemRightDelim, 5, "}}", 1},
   495  		{itemLeftDelim, 8, "{{", 2},
   496  		{itemIdentifier, 10, "y", 2},
   497  		{itemRightDelim, 11, "}}", 2},
   498  		{itemEOF, 13, "", 2},
   499  	}},
   500  	{"trimbefore", "{{x}}\n{{- y}}", []item{
   501  		{itemLeftDelim, 0, "{{", 1},
   502  		{itemIdentifier, 2, "x", 1},
   503  		{itemRightDelim, 3, "}}", 1},
   504  		{itemLeftDelim, 6, "{{", 2},
   505  		{itemIdentifier, 10, "y", 2},
   506  		{itemRightDelim, 11, "}}", 2},
   507  		{itemEOF, 13, "", 2},
   508  	}},
   509  }
   510  
   511  // The other tests don't check position, to make the test cases easier to construct.
   512  // This one does.
   513  func TestPos(t *testing.T) {
   514  	for _, test := range lexPosTests {
   515  		items := collect(&test, "", "")
   516  		if !equal(items, test.items, true) {
   517  			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
   518  			if len(items) == len(test.items) {
   519  				// Detailed print; avoid item.String() to expose the position value.
   520  				for i := range items {
   521  					if !equal(items[i:i+1], test.items[i:i+1], true) {
   522  						i1 := items[i]
   523  						i2 := test.items[i]
   524  						t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
   525  							i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
   526  					}
   527  				}
   528  			}
   529  		}
   530  	}
   531  }
   532  
   533  // Test that an error shuts down the lexing goroutine.
   534  func TestShutdown(t *testing.T) {
   535  	// We need to duplicate template.Parse here to hold on to the lexer.
   536  	const text = "erroneous{{define}}{{else}}1234"
   537  	lexer := lex("foo", text, "{{", "}}", false)
   538  	_, err := New("root").parseLexer(lexer)
   539  	if err == nil {
   540  		t.Fatalf("expected error")
   541  	}
   542  	// The error should have drained the input. Therefore, the lexer should be shut down.
   543  	token, ok := <-lexer.items
   544  	if ok {
   545  		t.Fatalf("input was not drained; got %v", token)
   546  	}
   547  }
   548  
   549  // parseLexer is a local version of parse that lets us pass in the lexer instead of building it.
   550  // We expect an error, so the tree set and funcs list are explicitly nil.
   551  func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
   552  	defer t.recover(&err)
   553  	t.ParseName = t.Name
   554  	t.startParse(nil, lex, map[string]*Tree{})
   555  	t.parse()
   556  	t.add()
   557  	t.stopParse()
   558  	return t, nil
   559  }
   560  

View as plain text