Source file src/regexp/syntax/parse.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package syntax
     6  
     7  import (
     8  	"sort"
     9  	"strings"
    10  	"unicode"
    11  	"unicode/utf8"
    12  )
    13  
    14  // An Error describes a failure to parse a regular expression
    15  // and gives the offending expression.
    16  type Error struct {
    17  	Code ErrorCode
    18  	Expr string
    19  }
    20  
    21  func (e *Error) Error() string {
    22  	return "error parsing regexp: " + e.Code.String() + ": `" + e.Expr + "`"
    23  }
    24  
    25  // An ErrorCode describes a failure to parse a regular expression.
    26  type ErrorCode string
    27  
    28  const (
    29  	// Unexpected error
    30  	ErrInternalError ErrorCode = "regexp/syntax: internal error"
    31  
    32  	// Parse errors
    33  	ErrInvalidCharClass      ErrorCode = "invalid character class"
    34  	ErrInvalidCharRange      ErrorCode = "invalid character class range"
    35  	ErrInvalidEscape         ErrorCode = "invalid escape sequence"
    36  	ErrInvalidNamedCapture   ErrorCode = "invalid named capture"
    37  	ErrInvalidPerlOp         ErrorCode = "invalid or unsupported Perl syntax"
    38  	ErrInvalidRepeatOp       ErrorCode = "invalid nested repetition operator"
    39  	ErrInvalidRepeatSize     ErrorCode = "invalid repeat count"
    40  	ErrInvalidUTF8           ErrorCode = "invalid UTF-8"
    41  	ErrMissingBracket        ErrorCode = "missing closing ]"
    42  	ErrMissingParen          ErrorCode = "missing closing )"
    43  	ErrMissingRepeatArgument ErrorCode = "missing argument to repetition operator"
    44  	ErrTrailingBackslash     ErrorCode = "trailing backslash at end of expression"
    45  	ErrUnexpectedParen       ErrorCode = "unexpected )"
    46  )
    47  
    48  func (e ErrorCode) String() string {
    49  	return string(e)
    50  }
    51  
    52  // Flags control the behavior of the parser and record information about regexp context.
    53  type Flags uint16
    54  
    55  const (
    56  	FoldCase      Flags = 1 << iota // case-insensitive match
    57  	Literal                         // treat pattern as literal string
    58  	ClassNL                         // allow character classes like [^a-z] and [[:space:]] to match newline
    59  	DotNL                           // allow . to match newline
    60  	OneLine                         // treat ^ and $ as only matching at beginning and end of text
    61  	NonGreedy                       // make repetition operators default to non-greedy
    62  	PerlX                           // allow Perl extensions
    63  	UnicodeGroups                   // allow \p{Han}, \P{Han} for Unicode group and negation
    64  	WasDollar                       // regexp OpEndText was $, not \z
    65  	Simple                          // regexp contains no counted repetition
    66  
    67  	MatchNL = ClassNL | DotNL
    68  
    69  	Perl        = ClassNL | OneLine | PerlX | UnicodeGroups // as close to Perl as possible
    70  	POSIX Flags = 0                                         // POSIX syntax
    71  )
    72  
    73  // Pseudo-ops for parsing stack.
    74  const (
    75  	opLeftParen = opPseudo + iota
    76  	opVerticalBar
    77  )
    78  
    79  // maxHeight is the maximum height of a regexp parse tree.
    80  // It is somewhat arbitrarily chosen, but the idea is to be large enough
    81  // that no one will actually hit in real use but at the same time small enough
    82  // that recursion on the Regexp tree will not hit the 1GB Go stack limit.
    83  // The maximum amount of stack for a single recursive frame is probably
    84  // closer to 1kB, so this could potentially be raised, but it seems unlikely
    85  // that people have regexps nested even this deeply.
    86  // We ran a test on Google's C++ code base and turned up only
    87  // a single use case with depth > 100; it had depth 128.
    88  // Using depth 1000 should be plenty of margin.
    89  // As an optimization, we don't even bother calculating heights
    90  // until we've allocated at least maxHeight Regexp structures.
    91  const maxHeight = 1000
    92  
    93  type parser struct {
    94  	flags       Flags     // parse mode flags
    95  	stack       []*Regexp // stack of parsed expressions
    96  	free        *Regexp
    97  	numCap      int // number of capturing groups seen
    98  	wholeRegexp string
    99  	tmpClass    []rune          // temporary char class work space
   100  	numRegexp   int             // number of regexps allocated
   101  	height      map[*Regexp]int // regexp height for height limit check
   102  }
   103  
   104  func (p *parser) newRegexp(op Op) *Regexp {
   105  	re := p.free
   106  	if re != nil {
   107  		p.free = re.Sub0[0]
   108  		*re = Regexp{}
   109  	} else {
   110  		re = new(Regexp)
   111  		p.numRegexp++
   112  	}
   113  	re.Op = op
   114  	return re
   115  }
   116  
   117  func (p *parser) reuse(re *Regexp) {
   118  	if p.height != nil {
   119  		delete(p.height, re)
   120  	}
   121  	re.Sub0[0] = p.free
   122  	p.free = re
   123  }
   124  
   125  func (p *parser) checkHeight(re *Regexp) {
   126  	if p.numRegexp < maxHeight {
   127  		return
   128  	}
   129  	if p.height == nil {
   130  		p.height = make(map[*Regexp]int)
   131  		for _, re := range p.stack {
   132  			p.checkHeight(re)
   133  		}
   134  	}
   135  	if p.calcHeight(re, true) > maxHeight {
   136  		panic(ErrInternalError)
   137  	}
   138  }
   139  
   140  func (p *parser) calcHeight(re *Regexp, force bool) int {
   141  	if !force {
   142  		if h, ok := p.height[re]; ok {
   143  			return h
   144  		}
   145  	}
   146  	h := 1
   147  	for _, sub := range re.Sub {
   148  		hsub := p.calcHeight(sub, false)
   149  		if h < 1+hsub {
   150  			h = 1 + hsub
   151  		}
   152  	}
   153  	p.height[re] = h
   154  	return h
   155  }
   156  
   157  // Parse stack manipulation.
   158  
   159  // push pushes the regexp re onto the parse stack and returns the regexp.
   160  func (p *parser) push(re *Regexp) *Regexp {
   161  	if re.Op == OpCharClass && len(re.Rune) == 2 && re.Rune[0] == re.Rune[1] {
   162  		// Single rune.
   163  		if p.maybeConcat(re.Rune[0], p.flags&^FoldCase) {
   164  			return nil
   165  		}
   166  		re.Op = OpLiteral
   167  		re.Rune = re.Rune[:1]
   168  		re.Flags = p.flags &^ FoldCase
   169  	} else if re.Op == OpCharClass && len(re.Rune) == 4 &&
   170  		re.Rune[0] == re.Rune[1] && re.Rune[2] == re.Rune[3] &&
   171  		unicode.SimpleFold(re.Rune[0]) == re.Rune[2] &&
   172  		unicode.SimpleFold(re.Rune[2]) == re.Rune[0] ||
   173  		re.Op == OpCharClass && len(re.Rune) == 2 &&
   174  			re.Rune[0]+1 == re.Rune[1] &&
   175  			unicode.SimpleFold(re.Rune[0]) == re.Rune[1] &&
   176  			unicode.SimpleFold(re.Rune[1]) == re.Rune[0] {
   177  		// Case-insensitive rune like [Aa] or [Δδ].
   178  		if p.maybeConcat(re.Rune[0], p.flags|FoldCase) {
   179  			return nil
   180  		}
   181  
   182  		// Rewrite as (case-insensitive) literal.
   183  		re.Op = OpLiteral
   184  		re.Rune = re.Rune[:1]
   185  		re.Flags = p.flags | FoldCase
   186  	} else {
   187  		// Incremental concatenation.
   188  		p.maybeConcat(-1, 0)
   189  	}
   190  
   191  	p.stack = append(p.stack, re)
   192  	p.checkHeight(re)
   193  	return re
   194  }
   195  
   196  // maybeConcat implements incremental concatenation
   197  // of literal runes into string nodes. The parser calls this
   198  // before each push, so only the top fragment of the stack
   199  // might need processing. Since this is called before a push,
   200  // the topmost literal is no longer subject to operators like *
   201  // (Otherwise ab* would turn into (ab)*.)
   202  // If r >= 0 and there's a node left over, maybeConcat uses it
   203  // to push r with the given flags.
   204  // maybeConcat reports whether r was pushed.
   205  func (p *parser) maybeConcat(r rune, flags Flags) bool {
   206  	n := len(p.stack)
   207  	if n < 2 {
   208  		return false
   209  	}
   210  
   211  	re1 := p.stack[n-1]
   212  	re2 := p.stack[n-2]
   213  	if re1.Op != OpLiteral || re2.Op != OpLiteral || re1.Flags&FoldCase != re2.Flags&FoldCase {
   214  		return false
   215  	}
   216  
   217  	// Push re1 into re2.
   218  	re2.Rune = append(re2.Rune, re1.Rune...)
   219  
   220  	// Reuse re1 if possible.
   221  	if r >= 0 {
   222  		re1.Rune = re1.Rune0[:1]
   223  		re1.Rune[0] = r
   224  		re1.Flags = flags
   225  		return true
   226  	}
   227  
   228  	p.stack = p.stack[:n-1]
   229  	p.reuse(re1)
   230  	return false // did not push r
   231  }
   232  
   233  // literal pushes a literal regexp for the rune r on the stack.
   234  func (p *parser) literal(r rune) {
   235  	re := p.newRegexp(OpLiteral)
   236  	re.Flags = p.flags
   237  	if p.flags&FoldCase != 0 {
   238  		r = minFoldRune(r)
   239  	}
   240  	re.Rune0[0] = r
   241  	re.Rune = re.Rune0[:1]
   242  	p.push(re)
   243  }
   244  
   245  // minFoldRune returns the minimum rune fold-equivalent to r.
   246  func minFoldRune(r rune) rune {
   247  	if r < minFold || r > maxFold {
   248  		return r
   249  	}
   250  	min := r
   251  	r0 := r
   252  	for r = unicode.SimpleFold(r); r != r0; r = unicode.SimpleFold(r) {
   253  		if min > r {
   254  			min = r
   255  		}
   256  	}
   257  	return min
   258  }
   259  
   260  // op pushes a regexp with the given op onto the stack
   261  // and returns that regexp.
   262  func (p *parser) op(op Op) *Regexp {
   263  	re := p.newRegexp(op)
   264  	re.Flags = p.flags
   265  	return p.push(re)
   266  }
   267  
   268  // repeat replaces the top stack element with itself repeated according to op, min, max.
   269  // before is the regexp suffix starting at the repetition operator.
   270  // after is the regexp suffix following after the repetition operator.
   271  // repeat returns an updated 'after' and an error, if any.
   272  func (p *parser) repeat(op Op, min, max int, before, after, lastRepeat string) (string, error) {
   273  	flags := p.flags
   274  	if p.flags&PerlX != 0 {
   275  		if len(after) > 0 && after[0] == '?' {
   276  			after = after[1:]
   277  			flags ^= NonGreedy
   278  		}
   279  		if lastRepeat != "" {
   280  			// In Perl it is not allowed to stack repetition operators:
   281  			// a** is a syntax error, not a doubled star, and a++ means
   282  			// something else entirely, which we don't support!
   283  			return "", &Error{ErrInvalidRepeatOp, lastRepeat[:len(lastRepeat)-len(after)]}
   284  		}
   285  	}
   286  	n := len(p.stack)
   287  	if n == 0 {
   288  		return "", &Error{ErrMissingRepeatArgument, before[:len(before)-len(after)]}
   289  	}
   290  	sub := p.stack[n-1]
   291  	if sub.Op >= opPseudo {
   292  		return "", &Error{ErrMissingRepeatArgument, before[:len(before)-len(after)]}
   293  	}
   294  
   295  	re := p.newRegexp(op)
   296  	re.Min = min
   297  	re.Max = max
   298  	re.Flags = flags
   299  	re.Sub = re.Sub0[:1]
   300  	re.Sub[0] = sub
   301  	p.stack[n-1] = re
   302  	p.checkHeight(re)
   303  
   304  	if op == OpRepeat && (min >= 2 || max >= 2) && !repeatIsValid(re, 1000) {
   305  		return "", &Error{ErrInvalidRepeatSize, before[:len(before)-len(after)]}
   306  	}
   307  
   308  	return after, nil
   309  }
   310  
   311  // repeatIsValid reports whether the repetition re is valid.
   312  // Valid means that the combination of the top-level repetition
   313  // and any inner repetitions does not exceed n copies of the
   314  // innermost thing.
   315  // This function rewalks the regexp tree and is called for every repetition,
   316  // so we have to worry about inducing quadratic behavior in the parser.
   317  // We avoid this by only calling repeatIsValid when min or max >= 2.
   318  // In that case the depth of any >= 2 nesting can only get to 9 without
   319  // triggering a parse error, so each subtree can only be rewalked 9 times.
   320  func repeatIsValid(re *Regexp, n int) bool {
   321  	if re.Op == OpRepeat {
   322  		m := re.Max
   323  		if m == 0 {
   324  			return true
   325  		}
   326  		if m < 0 {
   327  			m = re.Min
   328  		}
   329  		if m > n {
   330  			return false
   331  		}
   332  		if m > 0 {
   333  			n /= m
   334  		}
   335  	}
   336  	for _, sub := range re.Sub {
   337  		if !repeatIsValid(sub, n) {
   338  			return false
   339  		}
   340  	}
   341  	return true
   342  }
   343  
   344  // concat replaces the top of the stack (above the topmost '|' or '(') with its concatenation.
   345  func (p *parser) concat() *Regexp {
   346  	p.maybeConcat(-1, 0)
   347  
   348  	// Scan down to find pseudo-operator | or (.
   349  	i := len(p.stack)
   350  	for i > 0 && p.stack[i-1].Op < opPseudo {
   351  		i--
   352  	}
   353  	subs := p.stack[i:]
   354  	p.stack = p.stack[:i]
   355  
   356  	// Empty concatenation is special case.
   357  	if len(subs) == 0 {
   358  		return p.push(p.newRegexp(OpEmptyMatch))
   359  	}
   360  
   361  	return p.push(p.collapse(subs, OpConcat))
   362  }
   363  
   364  // alternate replaces the top of the stack (above the topmost '(') with its alternation.
   365  func (p *parser) alternate() *Regexp {
   366  	// Scan down to find pseudo-operator (.
   367  	// There are no | above (.
   368  	i := len(p.stack)
   369  	for i > 0 && p.stack[i-1].Op < opPseudo {
   370  		i--
   371  	}
   372  	subs := p.stack[i:]
   373  	p.stack = p.stack[:i]
   374  
   375  	// Make sure top class is clean.
   376  	// All the others already are (see swapVerticalBar).
   377  	if len(subs) > 0 {
   378  		cleanAlt(subs[len(subs)-1])
   379  	}
   380  
   381  	// Empty alternate is special case
   382  	// (shouldn't happen but easy to handle).
   383  	if len(subs) == 0 {
   384  		return p.push(p.newRegexp(OpNoMatch))
   385  	}
   386  
   387  	return p.push(p.collapse(subs, OpAlternate))
   388  }
   389  
   390  // cleanAlt cleans re for eventual inclusion in an alternation.
   391  func cleanAlt(re *Regexp) {
   392  	switch re.Op {
   393  	case OpCharClass:
   394  		re.Rune = cleanClass(&re.Rune)
   395  		if len(re.Rune) == 2 && re.Rune[0] == 0 && re.Rune[1] == unicode.MaxRune {
   396  			re.Rune = nil
   397  			re.Op = OpAnyChar
   398  			return
   399  		}
   400  		if len(re.Rune) == 4 && re.Rune[0] == 0 && re.Rune[1] == '\n'-1 && re.Rune[2] == '\n'+1 && re.Rune[3] == unicode.MaxRune {
   401  			re.Rune = nil
   402  			re.Op = OpAnyCharNotNL
   403  			return
   404  		}
   405  		if cap(re.Rune)-len(re.Rune) > 100 {
   406  			// re.Rune will not grow any more.
   407  			// Make a copy or inline to reclaim storage.
   408  			re.Rune = append(re.Rune0[:0], re.Rune...)
   409  		}
   410  	}
   411  }
   412  
   413  // collapse returns the result of applying op to sub.
   414  // If sub contains op nodes, they all get hoisted up
   415  // so that there is never a concat of a concat or an
   416  // alternate of an alternate.
   417  func (p *parser) collapse(subs []*Regexp, op Op) *Regexp {
   418  	if len(subs) == 1 {
   419  		return subs[0]
   420  	}
   421  	re := p.newRegexp(op)
   422  	re.Sub = re.Sub0[:0]
   423  	for _, sub := range subs {
   424  		if sub.Op == op {
   425  			re.Sub = append(re.Sub, sub.Sub...)
   426  			p.reuse(sub)
   427  		} else {
   428  			re.Sub = append(re.Sub, sub)
   429  		}
   430  	}
   431  	if op == OpAlternate {
   432  		re.Sub = p.factor(re.Sub)
   433  		if len(re.Sub) == 1 {
   434  			old := re
   435  			re = re.Sub[0]
   436  			p.reuse(old)
   437  		}
   438  	}
   439  	return re
   440  }
   441  
   442  // factor factors common prefixes from the alternation list sub.
   443  // It returns a replacement list that reuses the same storage and
   444  // frees (passes to p.reuse) any removed *Regexps.
   445  //
   446  // For example,
   447  //     ABC|ABD|AEF|BCX|BCY
   448  // simplifies by literal prefix extraction to
   449  //     A(B(C|D)|EF)|BC(X|Y)
   450  // which simplifies by character class introduction to
   451  //     A(B[CD]|EF)|BC[XY]
   452  //
   453  func (p *parser) factor(sub []*Regexp) []*Regexp {
   454  	if len(sub) < 2 {
   455  		return sub
   456  	}
   457  
   458  	// Round 1: Factor out common literal prefixes.
   459  	var str []rune
   460  	var strflags Flags
   461  	start := 0
   462  	out := sub[:0]
   463  	for i := 0; i <= len(sub); i++ {
   464  		// Invariant: the Regexps that were in sub[0:start] have been
   465  		// used or marked for reuse, and the slice space has been reused
   466  		// for out (len(out) <= start).
   467  		//
   468  		// Invariant: sub[start:i] consists of regexps that all begin
   469  		// with str as modified by strflags.
   470  		var istr []rune
   471  		var iflags Flags
   472  		if i < len(sub) {
   473  			istr, iflags = p.leadingString(sub[i])
   474  			if iflags == strflags {
   475  				same := 0
   476  				for same < len(str) && same < len(istr) && str[same] == istr[same] {
   477  					same++
   478  				}
   479  				if same > 0 {
   480  					// Matches at least one rune in current range.
   481  					// Keep going around.
   482  					str = str[:same]
   483  					continue
   484  				}
   485  			}
   486  		}
   487  
   488  		// Found end of a run with common leading literal string:
   489  		// sub[start:i] all begin with str[0:len(str)], but sub[i]
   490  		// does not even begin with str[0].
   491  		//
   492  		// Factor out common string and append factored expression to out.
   493  		if i == start {
   494  			// Nothing to do - run of length 0.
   495  		} else if i == start+1 {
   496  			// Just one: don't bother factoring.
   497  			out = append(out, sub[start])
   498  		} else {
   499  			// Construct factored form: prefix(suffix1|suffix2|...)
   500  			prefix := p.newRegexp(OpLiteral)
   501  			prefix.Flags = strflags
   502  			prefix.Rune = append(prefix.Rune[:0], str...)
   503  
   504  			for j := start; j < i; j++ {
   505  				sub[j] = p.removeLeadingString(sub[j], len(str))
   506  			}
   507  			suffix := p.collapse(sub[start:i], OpAlternate) // recurse
   508  
   509  			re := p.newRegexp(OpConcat)
   510  			re.Sub = append(re.Sub[:0], prefix, suffix)
   511  			out = append(out, re)
   512  		}
   513  
   514  		// Prepare for next iteration.
   515  		start = i
   516  		str = istr
   517  		strflags = iflags
   518  	}
   519  	sub = out
   520  
   521  	// Round 2: Factor out common simple prefixes,
   522  	// just the first piece of each concatenation.
   523  	// This will be good enough a lot of the time.
   524  	//
   525  	// Complex subexpressions (e.g. involving quantifiers)
   526  	// are not safe to factor because that collapses their
   527  	// distinct paths through the automaton, which affects
   528  	// correctness in some cases.
   529  	start = 0
   530  	out = sub[:0]
   531  	var first *Regexp
   532  	for i := 0; i <= len(sub); i++ {
   533  		// Invariant: the Regexps that were in sub[0:start] have been
   534  		// used or marked for reuse, and the slice space has been reused
   535  		// for out (len(out) <= start).
   536  		//
   537  		// Invariant: sub[start:i] consists of regexps that all begin with ifirst.
   538  		var ifirst *Regexp
   539  		if i < len(sub) {
   540  			ifirst = p.leadingRegexp(sub[i])
   541  			if first != nil && first.Equal(ifirst) &&
   542  				// first must be a character class OR a fixed repeat of a character class.
   543  				(isCharClass(first) || (first.Op == OpRepeat && first.Min == first.Max && isCharClass(first.Sub[0]))) {
   544  				continue
   545  			}
   546  		}
   547  
   548  		// Found end of a run with common leading regexp:
   549  		// sub[start:i] all begin with first but sub[i] does not.
   550  		//
   551  		// Factor out common regexp and append factored expression to out.
   552  		if i == start {
   553  			// Nothing to do - run of length 0.
   554  		} else if i == start+1 {
   555  			// Just one: don't bother factoring.
   556  			out = append(out, sub[start])
   557  		} else {
   558  			// Construct factored form: prefix(suffix1|suffix2|...)
   559  			prefix := first
   560  			for j := start; j < i; j++ {
   561  				reuse := j != start // prefix came from sub[start]
   562  				sub[j] = p.removeLeadingRegexp(sub[j], reuse)
   563  			}
   564  			suffix := p.collapse(sub[start:i], OpAlternate) // recurse
   565  
   566  			re := p.newRegexp(OpConcat)
   567  			re.Sub = append(re.Sub[:0], prefix, suffix)
   568  			out = append(out, re)
   569  		}
   570  
   571  		// Prepare for next iteration.
   572  		start = i
   573  		first = ifirst
   574  	}
   575  	sub = out
   576  
   577  	// Round 3: Collapse runs of single literals into character classes.
   578  	start = 0
   579  	out = sub[:0]
   580  	for i := 0; i <= len(sub); i++ {
   581  		// Invariant: the Regexps that were in sub[0:start] have been
   582  		// used or marked for reuse, and the slice space has been reused
   583  		// for out (len(out) <= start).
   584  		//
   585  		// Invariant: sub[start:i] consists of regexps that are either
   586  		// literal runes or character classes.
   587  		if i < len(sub) && isCharClass(sub[i]) {
   588  			continue
   589  		}
   590  
   591  		// sub[i] is not a char or char class;
   592  		// emit char class for sub[start:i]...
   593  		if i == start {
   594  			// Nothing to do - run of length 0.
   595  		} else if i == start+1 {
   596  			out = append(out, sub[start])
   597  		} else {
   598  			// Make new char class.
   599  			// Start with most complex regexp in sub[start].
   600  			max := start
   601  			for j := start + 1; j < i; j++ {
   602  				if sub[max].Op < sub[j].Op || sub[max].Op == sub[j].Op && len(sub[max].Rune) < len(sub[j].Rune) {
   603  					max = j
   604  				}
   605  			}
   606  			sub[start], sub[max] = sub[max], sub[start]
   607  
   608  			for j := start + 1; j < i; j++ {
   609  				mergeCharClass(sub[start], sub[j])
   610  				p.reuse(sub[j])
   611  			}
   612  			cleanAlt(sub[start])
   613  			out = append(out, sub[start])
   614  		}
   615  
   616  		// ... and then emit sub[i].
   617  		if i < len(sub) {
   618  			out = append(out, sub[i])
   619  		}
   620  		start = i + 1
   621  	}
   622  	sub = out
   623  
   624  	// Round 4: Collapse runs of empty matches into a single empty match.
   625  	start = 0
   626  	out = sub[:0]
   627  	for i := range sub {
   628  		if i+1 < len(sub) && sub[i].Op == OpEmptyMatch && sub[i+1].Op == OpEmptyMatch {
   629  			continue
   630  		}
   631  		out = append(out, sub[i])
   632  	}
   633  	sub = out
   634  
   635  	return sub
   636  }
   637  
   638  // leadingString returns the leading literal string that re begins with.
   639  // The string refers to storage in re or its children.
   640  func (p *parser) leadingString(re *Regexp) ([]rune, Flags) {
   641  	if re.Op == OpConcat && len(re.Sub) > 0 {
   642  		re = re.Sub[0]
   643  	}
   644  	if re.Op != OpLiteral {
   645  		return nil, 0
   646  	}
   647  	return re.Rune, re.Flags & FoldCase
   648  }
   649  
   650  // removeLeadingString removes the first n leading runes
   651  // from the beginning of re. It returns the replacement for re.
   652  func (p *parser) removeLeadingString(re *Regexp, n int) *Regexp {
   653  	if re.Op == OpConcat && len(re.Sub) > 0 {
   654  		// Removing a leading string in a concatenation
   655  		// might simplify the concatenation.
   656  		sub := re.Sub[0]
   657  		sub = p.removeLeadingString(sub, n)
   658  		re.Sub[0] = sub
   659  		if sub.Op == OpEmptyMatch {
   660  			p.reuse(sub)
   661  			switch len(re.Sub) {
   662  			case 0, 1:
   663  				// Impossible but handle.
   664  				re.Op = OpEmptyMatch
   665  				re.Sub = nil
   666  			case 2:
   667  				old := re
   668  				re = re.Sub[1]
   669  				p.reuse(old)
   670  			default:
   671  				copy(re.Sub, re.Sub[1:])
   672  				re.Sub = re.Sub[:len(re.Sub)-1]
   673  			}
   674  		}
   675  		return re
   676  	}
   677  
   678  	if re.Op == OpLiteral {
   679  		re.Rune = re.Rune[:copy(re.Rune, re.Rune[n:])]
   680  		if len(re.Rune) == 0 {
   681  			re.Op = OpEmptyMatch
   682  		}
   683  	}
   684  	return re
   685  }
   686  
   687  // leadingRegexp returns the leading regexp that re begins with.
   688  // The regexp refers to storage in re or its children.
   689  func (p *parser) leadingRegexp(re *Regexp) *Regexp {
   690  	if re.Op == OpEmptyMatch {
   691  		return nil
   692  	}
   693  	if re.Op == OpConcat && len(re.Sub) > 0 {
   694  		sub := re.Sub[0]
   695  		if sub.Op == OpEmptyMatch {
   696  			return nil
   697  		}
   698  		return sub
   699  	}
   700  	return re
   701  }
   702  
   703  // removeLeadingRegexp removes the leading regexp in re.
   704  // It returns the replacement for re.
   705  // If reuse is true, it passes the removed regexp (if no longer needed) to p.reuse.
   706  func (p *parser) removeLeadingRegexp(re *Regexp, reuse bool) *Regexp {
   707  	if re.Op == OpConcat && len(re.Sub) > 0 {
   708  		if reuse {
   709  			p.reuse(re.Sub[0])
   710  		}
   711  		re.Sub = re.Sub[:copy(re.Sub, re.Sub[1:])]
   712  		switch len(re.Sub) {
   713  		case 0:
   714  			re.Op = OpEmptyMatch
   715  			re.Sub = nil
   716  		case 1:
   717  			old := re
   718  			re = re.Sub[0]
   719  			p.reuse(old)
   720  		}
   721  		return re
   722  	}
   723  	if reuse {
   724  		p.reuse(re)
   725  	}
   726  	return p.newRegexp(OpEmptyMatch)
   727  }
   728  
   729  func literalRegexp(s string, flags Flags) *Regexp {
   730  	re := &Regexp{Op: OpLiteral}
   731  	re.Flags = flags
   732  	re.Rune = re.Rune0[:0] // use local storage for small strings
   733  	for _, c := range s {
   734  		if len(re.Rune) >= cap(re.Rune) {
   735  			// string is too long to fit in Rune0.  let Go handle it
   736  			re.Rune = []rune(s)
   737  			break
   738  		}
   739  		re.Rune = append(re.Rune, c)
   740  	}
   741  	return re
   742  }
   743  
   744  // Parsing.
   745  
   746  // Parse parses a regular expression string s, controlled by the specified
   747  // Flags, and returns a regular expression parse tree. The syntax is
   748  // described in the top-level comment.
   749  func Parse(s string, flags Flags) (*Regexp, error) {
   750  	return parse(s, flags)
   751  }
   752  
   753  func parse(s string, flags Flags) (_ *Regexp, err error) {
   754  	defer func() {
   755  		switch r := recover(); r {
   756  		default:
   757  			panic(r)
   758  		case nil:
   759  			// ok
   760  		case ErrInternalError:
   761  			err = &Error{Code: ErrInternalError, Expr: s}
   762  		}
   763  	}()
   764  
   765  	if flags&Literal != 0 {
   766  		// Trivial parser for literal string.
   767  		if err := checkUTF8(s); err != nil {
   768  			return nil, err
   769  		}
   770  		return literalRegexp(s, flags), nil
   771  	}
   772  
   773  	// Otherwise, must do real work.
   774  	var (
   775  		p          parser
   776  		c          rune
   777  		op         Op
   778  		lastRepeat string
   779  	)
   780  	p.flags = flags
   781  	p.wholeRegexp = s
   782  	t := s
   783  	for t != "" {
   784  		repeat := ""
   785  	BigSwitch:
   786  		switch t[0] {
   787  		default:
   788  			if c, t, err = nextRune(t); err != nil {
   789  				return nil, err
   790  			}
   791  			p.literal(c)
   792  
   793  		case '(':
   794  			if p.flags&PerlX != 0 && len(t) >= 2 && t[1] == '?' {
   795  				// Flag changes and non-capturing groups.
   796  				if t, err = p.parsePerlFlags(t); err != nil {
   797  					return nil, err
   798  				}
   799  				break
   800  			}
   801  			p.numCap++
   802  			p.op(opLeftParen).Cap = p.numCap
   803  			t = t[1:]
   804  		case '|':
   805  			if err = p.parseVerticalBar(); err != nil {
   806  				return nil, err
   807  			}
   808  			t = t[1:]
   809  		case ')':
   810  			if err = p.parseRightParen(); err != nil {
   811  				return nil, err
   812  			}
   813  			t = t[1:]
   814  		case '^':
   815  			if p.flags&OneLine != 0 {
   816  				p.op(OpBeginText)
   817  			} else {
   818  				p.op(OpBeginLine)
   819  			}
   820  			t = t[1:]
   821  		case '$':
   822  			if p.flags&OneLine != 0 {
   823  				p.op(OpEndText).Flags |= WasDollar
   824  			} else {
   825  				p.op(OpEndLine)
   826  			}
   827  			t = t[1:]
   828  		case '.':
   829  			if p.flags&DotNL != 0 {
   830  				p.op(OpAnyChar)
   831  			} else {
   832  				p.op(OpAnyCharNotNL)
   833  			}
   834  			t = t[1:]
   835  		case '[':
   836  			if t, err = p.parseClass(t); err != nil {
   837  				return nil, err
   838  			}
   839  		case '*', '+', '?':
   840  			before := t
   841  			switch t[0] {
   842  			case '*':
   843  				op = OpStar
   844  			case '+':
   845  				op = OpPlus
   846  			case '?':
   847  				op = OpQuest
   848  			}
   849  			after := t[1:]
   850  			if after, err = p.repeat(op, 0, 0, before, after, lastRepeat); err != nil {
   851  				return nil, err
   852  			}
   853  			repeat = before
   854  			t = after
   855  		case '{':
   856  			op = OpRepeat
   857  			before := t
   858  			min, max, after, ok := p.parseRepeat(t)
   859  			if !ok {
   860  				// If the repeat cannot be parsed, { is a literal.
   861  				p.literal('{')
   862  				t = t[1:]
   863  				break
   864  			}
   865  			if min < 0 || min > 1000 || max > 1000 || max >= 0 && min > max {
   866  				// Numbers were too big, or max is present and min > max.
   867  				return nil, &Error{ErrInvalidRepeatSize, before[:len(before)-len(after)]}
   868  			}
   869  			if after, err = p.repeat(op, min, max, before, after, lastRepeat); err != nil {
   870  				return nil, err
   871  			}
   872  			repeat = before
   873  			t = after
   874  		case '\\':
   875  			if p.flags&PerlX != 0 && len(t) >= 2 {
   876  				switch t[1] {
   877  				case 'A':
   878  					p.op(OpBeginText)
   879  					t = t[2:]
   880  					break BigSwitch
   881  				case 'b':
   882  					p.op(OpWordBoundary)
   883  					t = t[2:]
   884  					break BigSwitch
   885  				case 'B':
   886  					p.op(OpNoWordBoundary)
   887  					t = t[2:]
   888  					break BigSwitch
   889  				case 'C':
   890  					// any byte; not supported
   891  					return nil, &Error{ErrInvalidEscape, t[:2]}
   892  				case 'Q':
   893  					// \Q ... \E: the ... is always literals
   894  					var lit string
   895  					lit, t, _ = strings.Cut(t[2:], `\E`)
   896  					for lit != "" {
   897  						c, rest, err := nextRune(lit)
   898  						if err != nil {
   899  							return nil, err
   900  						}
   901  						p.literal(c)
   902  						lit = rest
   903  					}
   904  					break BigSwitch
   905  				case 'z':
   906  					p.op(OpEndText)
   907  					t = t[2:]
   908  					break BigSwitch
   909  				}
   910  			}
   911  
   912  			re := p.newRegexp(OpCharClass)
   913  			re.Flags = p.flags
   914  
   915  			// Look for Unicode character group like \p{Han}
   916  			if len(t) >= 2 && (t[1] == 'p' || t[1] == 'P') {
   917  				r, rest, err := p.parseUnicodeClass(t, re.Rune0[:0])
   918  				if err != nil {
   919  					return nil, err
   920  				}
   921  				if r != nil {
   922  					re.Rune = r
   923  					t = rest
   924  					p.push(re)
   925  					break BigSwitch
   926  				}
   927  			}
   928  
   929  			// Perl character class escape.
   930  			if r, rest := p.parsePerlClassEscape(t, re.Rune0[:0]); r != nil {
   931  				re.Rune = r
   932  				t = rest
   933  				p.push(re)
   934  				break BigSwitch
   935  			}
   936  			p.reuse(re)
   937  
   938  			// Ordinary single-character escape.
   939  			if c, t, err = p.parseEscape(t); err != nil {
   940  				return nil, err
   941  			}
   942  			p.literal(c)
   943  		}
   944  		lastRepeat = repeat
   945  	}
   946  
   947  	p.concat()
   948  	if p.swapVerticalBar() {
   949  		// pop vertical bar
   950  		p.stack = p.stack[:len(p.stack)-1]
   951  	}
   952  	p.alternate()
   953  
   954  	n := len(p.stack)
   955  	if n != 1 {
   956  		return nil, &Error{ErrMissingParen, s}
   957  	}
   958  	return p.stack[0], nil
   959  }
   960  
   961  // parseRepeat parses {min} (max=min) or {min,} (max=-1) or {min,max}.
   962  // If s is not of that form, it returns ok == false.
   963  // If s has the right form but the values are too big, it returns min == -1, ok == true.
   964  func (p *parser) parseRepeat(s string) (min, max int, rest string, ok bool) {
   965  	if s == "" || s[0] != '{' {
   966  		return
   967  	}
   968  	s = s[1:]
   969  	var ok1 bool
   970  	if min, s, ok1 = p.parseInt(s); !ok1 {
   971  		return
   972  	}
   973  	if s == "" {
   974  		return
   975  	}
   976  	if s[0] != ',' {
   977  		max = min
   978  	} else {
   979  		s = s[1:]
   980  		if s == "" {
   981  			return
   982  		}
   983  		if s[0] == '}' {
   984  			max = -1
   985  		} else if max, s, ok1 = p.parseInt(s); !ok1 {
   986  			return
   987  		} else if max < 0 {
   988  			// parseInt found too big a number
   989  			min = -1
   990  		}
   991  	}
   992  	if s == "" || s[0] != '}' {
   993  		return
   994  	}
   995  	rest = s[1:]
   996  	ok = true
   997  	return
   998  }
   999  
  1000  // parsePerlFlags parses a Perl flag setting or non-capturing group or both,
  1001  // like (?i) or (?: or (?i:.  It removes the prefix from s and updates the parse state.
  1002  // The caller must have ensured that s begins with "(?".
  1003  func (p *parser) parsePerlFlags(s string) (rest string, err error) {
  1004  	t := s
  1005  
  1006  	// Check for named captures, first introduced in Python's regexp library.
  1007  	// As usual, there are three slightly different syntaxes:
  1008  	//
  1009  	//   (?P<name>expr)   the original, introduced by Python
  1010  	//   (?<name>expr)    the .NET alteration, adopted by Perl 5.10
  1011  	//   (?'name'expr)    another .NET alteration, adopted by Perl 5.10
  1012  	//
  1013  	// Perl 5.10 gave in and implemented the Python version too,
  1014  	// but they claim that the last two are the preferred forms.
  1015  	// PCRE and languages based on it (specifically, PHP and Ruby)
  1016  	// support all three as well. EcmaScript 4 uses only the Python form.
  1017  	//
  1018  	// In both the open source world (via Code Search) and the
  1019  	// Google source tree, (?P<expr>name) is the dominant form,
  1020  	// so that's the one we implement. One is enough.
  1021  	if len(t) > 4 && t[2] == 'P' && t[3] == '<' {
  1022  		// Pull out name.
  1023  		end := strings.IndexRune(t, '>')
  1024  		if end < 0 {
  1025  			if err = checkUTF8(t); err != nil {
  1026  				return "", err
  1027  			}
  1028  			return "", &Error{ErrInvalidNamedCapture, s}
  1029  		}
  1030  
  1031  		capture := t[:end+1] // "(?P<name>"
  1032  		name := t[4:end]     // "name"
  1033  		if err = checkUTF8(name); err != nil {
  1034  			return "", err
  1035  		}
  1036  		if !isValidCaptureName(name) {
  1037  			return "", &Error{ErrInvalidNamedCapture, capture}
  1038  		}
  1039  
  1040  		// Like ordinary capture, but named.
  1041  		p.numCap++
  1042  		re := p.op(opLeftParen)
  1043  		re.Cap = p.numCap
  1044  		re.Name = name
  1045  		return t[end+1:], nil
  1046  	}
  1047  
  1048  	// Non-capturing group. Might also twiddle Perl flags.
  1049  	var c rune
  1050  	t = t[2:] // skip (?
  1051  	flags := p.flags
  1052  	sign := +1
  1053  	sawFlag := false
  1054  Loop:
  1055  	for t != "" {
  1056  		if c, t, err = nextRune(t); err != nil {
  1057  			return "", err
  1058  		}
  1059  		switch c {
  1060  		default:
  1061  			break Loop
  1062  
  1063  		// Flags.
  1064  		case 'i':
  1065  			flags |= FoldCase
  1066  			sawFlag = true
  1067  		case 'm':
  1068  			flags &^= OneLine
  1069  			sawFlag = true
  1070  		case 's':
  1071  			flags |= DotNL
  1072  			sawFlag = true
  1073  		case 'U':
  1074  			flags |= NonGreedy
  1075  			sawFlag = true
  1076  
  1077  		// Switch to negation.
  1078  		case '-':
  1079  			if sign < 0 {
  1080  				break Loop
  1081  			}
  1082  			sign = -1
  1083  			// Invert flags so that | above turn into &^ and vice versa.
  1084  			// We'll invert flags again before using it below.
  1085  			flags = ^flags
  1086  			sawFlag = false
  1087  
  1088  		// End of flags, starting group or not.
  1089  		case ':', ')':
  1090  			if sign < 0 {
  1091  				if !sawFlag {
  1092  					break Loop
  1093  				}
  1094  				flags = ^flags
  1095  			}
  1096  			if c == ':' {
  1097  				// Open new group
  1098  				p.op(opLeftParen)
  1099  			}
  1100  			p.flags = flags
  1101  			return t, nil
  1102  		}
  1103  	}
  1104  
  1105  	return "", &Error{ErrInvalidPerlOp, s[:len(s)-len(t)]}
  1106  }
  1107  
  1108  // isValidCaptureName reports whether name
  1109  // is a valid capture name: [A-Za-z0-9_]+.
  1110  // PCRE limits names to 32 bytes.
  1111  // Python rejects names starting with digits.
  1112  // We don't enforce either of those.
  1113  func isValidCaptureName(name string) bool {
  1114  	if name == "" {
  1115  		return false
  1116  	}
  1117  	for _, c := range name {
  1118  		if c != '_' && !isalnum(c) {
  1119  			return false
  1120  		}
  1121  	}
  1122  	return true
  1123  }
  1124  
  1125  // parseInt parses a decimal integer.
  1126  func (p *parser) parseInt(s string) (n int, rest string, ok bool) {
  1127  	if s == "" || s[0] < '0' || '9' < s[0] {
  1128  		return
  1129  	}
  1130  	// Disallow leading zeros.
  1131  	if len(s) >= 2 && s[0] == '0' && '0' <= s[1] && s[1] <= '9' {
  1132  		return
  1133  	}
  1134  	t := s
  1135  	for s != "" && '0' <= s[0] && s[0] <= '9' {
  1136  		s = s[1:]
  1137  	}
  1138  	rest = s
  1139  	ok = true
  1140  	// Have digits, compute value.
  1141  	t = t[:len(t)-len(s)]
  1142  	for i := 0; i < len(t); i++ {
  1143  		// Avoid overflow.
  1144  		if n >= 1e8 {
  1145  			n = -1
  1146  			break
  1147  		}
  1148  		n = n*10 + int(t[i]) - '0'
  1149  	}
  1150  	return
  1151  }
  1152  
  1153  // can this be represented as a character class?
  1154  // single-rune literal string, char class, ., and .|\n.
  1155  func isCharClass(re *Regexp) bool {
  1156  	return re.Op == OpLiteral && len(re.Rune) == 1 ||
  1157  		re.Op == OpCharClass ||
  1158  		re.Op == OpAnyCharNotNL ||
  1159  		re.Op == OpAnyChar
  1160  }
  1161  
  1162  // does re match r?
  1163  func matchRune(re *Regexp, r rune) bool {
  1164  	switch re.Op {
  1165  	case OpLiteral:
  1166  		return len(re.Rune) == 1 && re.Rune[0] == r
  1167  	case OpCharClass:
  1168  		for i := 0; i < len(re.Rune); i += 2 {
  1169  			if re.Rune[i] <= r && r <= re.Rune[i+1] {
  1170  				return true
  1171  			}
  1172  		}
  1173  		return false
  1174  	case OpAnyCharNotNL:
  1175  		return r != '\n'
  1176  	case OpAnyChar:
  1177  		return true
  1178  	}
  1179  	return false
  1180  }
  1181  
  1182  // parseVerticalBar handles a | in the input.
  1183  func (p *parser) parseVerticalBar() error {
  1184  	p.concat()
  1185  
  1186  	// The concatenation we just parsed is on top of the stack.
  1187  	// If it sits above an opVerticalBar, swap it below
  1188  	// (things below an opVerticalBar become an alternation).
  1189  	// Otherwise, push a new vertical bar.
  1190  	if !p.swapVerticalBar() {
  1191  		p.op(opVerticalBar)
  1192  	}
  1193  
  1194  	return nil
  1195  }
  1196  
  1197  // mergeCharClass makes dst = dst|src.
  1198  // The caller must ensure that dst.Op >= src.Op,
  1199  // to reduce the amount of copying.
  1200  func mergeCharClass(dst, src *Regexp) {
  1201  	switch dst.Op {
  1202  	case OpAnyChar:
  1203  		// src doesn't add anything.
  1204  	case OpAnyCharNotNL:
  1205  		// src might add \n
  1206  		if matchRune(src, '\n') {
  1207  			dst.Op = OpAnyChar
  1208  		}
  1209  	case OpCharClass:
  1210  		// src is simpler, so either literal or char class
  1211  		if src.Op == OpLiteral {
  1212  			dst.Rune = appendLiteral(dst.Rune, src.Rune[0], src.Flags)
  1213  		} else {
  1214  			dst.Rune = appendClass(dst.Rune, src.Rune)
  1215  		}
  1216  	case OpLiteral:
  1217  		// both literal
  1218  		if src.Rune[0] == dst.Rune[0] && src.Flags == dst.Flags {
  1219  			break
  1220  		}
  1221  		dst.Op = OpCharClass
  1222  		dst.Rune = appendLiteral(dst.Rune[:0], dst.Rune[0], dst.Flags)
  1223  		dst.Rune = appendLiteral(dst.Rune, src.Rune[0], src.Flags)
  1224  	}
  1225  }
  1226  
  1227  // If the top of the stack is an element followed by an opVerticalBar
  1228  // swapVerticalBar swaps the two and returns true.
  1229  // Otherwise it returns false.
  1230  func (p *parser) swapVerticalBar() bool {
  1231  	// If above and below vertical bar are literal or char class,
  1232  	// can merge into a single char class.
  1233  	n := len(p.stack)
  1234  	if n >= 3 && p.stack[n-2].Op == opVerticalBar && isCharClass(p.stack[n-1]) && isCharClass(p.stack[n-3]) {
  1235  		re1 := p.stack[n-1]
  1236  		re3 := p.stack[n-3]
  1237  		// Make re3 the more complex of the two.
  1238  		if re1.Op > re3.Op {
  1239  			re1, re3 = re3, re1
  1240  			p.stack[n-3] = re3
  1241  		}
  1242  		mergeCharClass(re3, re1)
  1243  		p.reuse(re1)
  1244  		p.stack = p.stack[:n-1]
  1245  		return true
  1246  	}
  1247  
  1248  	if n >= 2 {
  1249  		re1 := p.stack[n-1]
  1250  		re2 := p.stack[n-2]
  1251  		if re2.Op == opVerticalBar {
  1252  			if n >= 3 {
  1253  				// Now out of reach.
  1254  				// Clean opportunistically.
  1255  				cleanAlt(p.stack[n-3])
  1256  			}
  1257  			p.stack[n-2] = re1
  1258  			p.stack[n-1] = re2
  1259  			return true
  1260  		}
  1261  	}
  1262  	return false
  1263  }
  1264  
  1265  // parseRightParen handles a ) in the input.
  1266  func (p *parser) parseRightParen() error {
  1267  	p.concat()
  1268  	if p.swapVerticalBar() {
  1269  		// pop vertical bar
  1270  		p.stack = p.stack[:len(p.stack)-1]
  1271  	}
  1272  	p.alternate()
  1273  
  1274  	n := len(p.stack)
  1275  	if n < 2 {
  1276  		return &Error{ErrUnexpectedParen, p.wholeRegexp}
  1277  	}
  1278  	re1 := p.stack[n-1]
  1279  	re2 := p.stack[n-2]
  1280  	p.stack = p.stack[:n-2]
  1281  	if re2.Op != opLeftParen {
  1282  		return &Error{ErrUnexpectedParen, p.wholeRegexp}
  1283  	}
  1284  	// Restore flags at time of paren.
  1285  	p.flags = re2.Flags
  1286  	if re2.Cap == 0 {
  1287  		// Just for grouping.
  1288  		p.push(re1)
  1289  	} else {
  1290  		re2.Op = OpCapture
  1291  		re2.Sub = re2.Sub0[:1]
  1292  		re2.Sub[0] = re1
  1293  		p.push(re2)
  1294  	}
  1295  	return nil
  1296  }
  1297  
  1298  // parseEscape parses an escape sequence at the beginning of s
  1299  // and returns the rune.
  1300  func (p *parser) parseEscape(s string) (r rune, rest string, err error) {
  1301  	t := s[1:]
  1302  	if t == "" {
  1303  		return 0, "", &Error{ErrTrailingBackslash, ""}
  1304  	}
  1305  	c, t, err := nextRune(t)
  1306  	if err != nil {
  1307  		return 0, "", err
  1308  	}
  1309  
  1310  Switch:
  1311  	switch c {
  1312  	default:
  1313  		if c < utf8.RuneSelf && !isalnum(c) {
  1314  			// Escaped non-word characters are always themselves.
  1315  			// PCRE is not quite so rigorous: it accepts things like
  1316  			// \q, but we don't. We once rejected \_, but too many
  1317  			// programs and people insist on using it, so allow \_.
  1318  			return c, t, nil
  1319  		}
  1320  
  1321  	// Octal escapes.
  1322  	case '1', '2', '3', '4', '5', '6', '7':
  1323  		// Single non-zero digit is a backreference; not supported
  1324  		if t == "" || t[0] < '0' || t[0] > '7' {
  1325  			break
  1326  		}
  1327  		fallthrough
  1328  	case '0':
  1329  		// Consume up to three octal digits; already have one.
  1330  		r = c - '0'
  1331  		for i := 1; i < 3; i++ {
  1332  			if t == "" || t[0] < '0' || t[0] > '7' {
  1333  				break
  1334  			}
  1335  			r = r*8 + rune(t[0]) - '0'
  1336  			t = t[1:]
  1337  		}
  1338  		return r, t, nil
  1339  
  1340  	// Hexadecimal escapes.
  1341  	case 'x':
  1342  		if t == "" {
  1343  			break
  1344  		}
  1345  		if c, t, err = nextRune(t); err != nil {
  1346  			return 0, "", err
  1347  		}
  1348  		if c == '{' {
  1349  			// Any number of digits in braces.
  1350  			// Perl accepts any text at all; it ignores all text
  1351  			// after the first non-hex digit. We require only hex digits,
  1352  			// and at least one.
  1353  			nhex := 0
  1354  			r = 0
  1355  			for {
  1356  				if t == "" {
  1357  					break Switch
  1358  				}
  1359  				if c, t, err = nextRune(t); err != nil {
  1360  					return 0, "", err
  1361  				}
  1362  				if c == '}' {
  1363  					break
  1364  				}
  1365  				v := unhex(c)
  1366  				if v < 0 {
  1367  					break Switch
  1368  				}
  1369  				r = r*16 + v
  1370  				if r > unicode.MaxRune {
  1371  					break Switch
  1372  				}
  1373  				nhex++
  1374  			}
  1375  			if nhex == 0 {
  1376  				break Switch
  1377  			}
  1378  			return r, t, nil
  1379  		}
  1380  
  1381  		// Easy case: two hex digits.
  1382  		x := unhex(c)
  1383  		if c, t, err = nextRune(t); err != nil {
  1384  			return 0, "", err
  1385  		}
  1386  		y := unhex(c)
  1387  		if x < 0 || y < 0 {
  1388  			break
  1389  		}
  1390  		return x*16 + y, t, nil
  1391  
  1392  	// C escapes. There is no case 'b', to avoid misparsing
  1393  	// the Perl word-boundary \b as the C backspace \b
  1394  	// when in POSIX mode. In Perl, /\b/ means word-boundary
  1395  	// but /[\b]/ means backspace. We don't support that.
  1396  	// If you want a backspace, embed a literal backspace
  1397  	// character or use \x08.
  1398  	case 'a':
  1399  		return '\a', t, err
  1400  	case 'f':
  1401  		return '\f', t, err
  1402  	case 'n':
  1403  		return '\n', t, err
  1404  	case 'r':
  1405  		return '\r', t, err
  1406  	case 't':
  1407  		return '\t', t, err
  1408  	case 'v':
  1409  		return '\v', t, err
  1410  	}
  1411  	return 0, "", &Error{ErrInvalidEscape, s[:len(s)-len(t)]}
  1412  }
  1413  
  1414  // parseClassChar parses a character class character at the beginning of s
  1415  // and returns it.
  1416  func (p *parser) parseClassChar(s, wholeClass string) (r rune, rest string, err error) {
  1417  	if s == "" {
  1418  		return 0, "", &Error{Code: ErrMissingBracket, Expr: wholeClass}
  1419  	}
  1420  
  1421  	// Allow regular escape sequences even though
  1422  	// many need not be escaped in this context.
  1423  	if s[0] == '\\' {
  1424  		return p.parseEscape(s)
  1425  	}
  1426  
  1427  	return nextRune(s)
  1428  }
  1429  
  1430  type charGroup struct {
  1431  	sign  int
  1432  	class []rune
  1433  }
  1434  
  1435  // parsePerlClassEscape parses a leading Perl character class escape like \d
  1436  // from the beginning of s. If one is present, it appends the characters to r
  1437  // and returns the new slice r and the remainder of the string.
  1438  func (p *parser) parsePerlClassEscape(s string, r []rune) (out []rune, rest string) {
  1439  	if p.flags&PerlX == 0 || len(s) < 2 || s[0] != '\\' {
  1440  		return
  1441  	}
  1442  	g := perlGroup[s[0:2]]
  1443  	if g.sign == 0 {
  1444  		return
  1445  	}
  1446  	return p.appendGroup(r, g), s[2:]
  1447  }
  1448  
  1449  // parseNamedClass parses a leading POSIX named character class like [:alnum:]
  1450  // from the beginning of s. If one is present, it appends the characters to r
  1451  // and returns the new slice r and the remainder of the string.
  1452  func (p *parser) parseNamedClass(s string, r []rune) (out []rune, rest string, err error) {
  1453  	if len(s) < 2 || s[0] != '[' || s[1] != ':' {
  1454  		return
  1455  	}
  1456  
  1457  	i := strings.Index(s[2:], ":]")
  1458  	if i < 0 {
  1459  		return
  1460  	}
  1461  	i += 2
  1462  	name, s := s[0:i+2], s[i+2:]
  1463  	g := posixGroup[name]
  1464  	if g.sign == 0 {
  1465  		return nil, "", &Error{ErrInvalidCharRange, name}
  1466  	}
  1467  	return p.appendGroup(r, g), s, nil
  1468  }
  1469  
  1470  func (p *parser) appendGroup(r []rune, g charGroup) []rune {
  1471  	if p.flags&FoldCase == 0 {
  1472  		if g.sign < 0 {
  1473  			r = appendNegatedClass(r, g.class)
  1474  		} else {
  1475  			r = appendClass(r, g.class)
  1476  		}
  1477  	} else {
  1478  		tmp := p.tmpClass[:0]
  1479  		tmp = appendFoldedClass(tmp, g.class)
  1480  		p.tmpClass = tmp
  1481  		tmp = cleanClass(&p.tmpClass)
  1482  		if g.sign < 0 {
  1483  			r = appendNegatedClass(r, tmp)
  1484  		} else {
  1485  			r = appendClass(r, tmp)
  1486  		}
  1487  	}
  1488  	return r
  1489  }
  1490  
  1491  var anyTable = &unicode.RangeTable{
  1492  	R16: []unicode.Range16{{Lo: 0, Hi: 1<<16 - 1, Stride: 1}},
  1493  	R32: []unicode.Range32{{Lo: 1 << 16, Hi: unicode.MaxRune, Stride: 1}},
  1494  }
  1495  
  1496  // unicodeTable returns the unicode.RangeTable identified by name
  1497  // and the table of additional fold-equivalent code points.
  1498  func unicodeTable(name string) (*unicode.RangeTable, *unicode.RangeTable) {
  1499  	// Special case: "Any" means any.
  1500  	if name == "Any" {
  1501  		return anyTable, anyTable
  1502  	}
  1503  	if t := unicode.Categories[name]; t != nil {
  1504  		return t, unicode.FoldCategory[name]
  1505  	}
  1506  	if t := unicode.Scripts[name]; t != nil {
  1507  		return t, unicode.FoldScript[name]
  1508  	}
  1509  	return nil, nil
  1510  }
  1511  
  1512  // parseUnicodeClass parses a leading Unicode character class like \p{Han}
  1513  // from the beginning of s. If one is present, it appends the characters to r
  1514  // and returns the new slice r and the remainder of the string.
  1515  func (p *parser) parseUnicodeClass(s string, r []rune) (out []rune, rest string, err error) {
  1516  	if p.flags&UnicodeGroups == 0 || len(s) < 2 || s[0] != '\\' || s[1] != 'p' && s[1] != 'P' {
  1517  		return
  1518  	}
  1519  
  1520  	// Committed to parse or return error.
  1521  	sign := +1
  1522  	if s[1] == 'P' {
  1523  		sign = -1
  1524  	}
  1525  	t := s[2:]
  1526  	c, t, err := nextRune(t)
  1527  	if err != nil {
  1528  		return
  1529  	}
  1530  	var seq, name string
  1531  	if c != '{' {
  1532  		// Single-letter name.
  1533  		seq = s[:len(s)-len(t)]
  1534  		name = seq[2:]
  1535  	} else {
  1536  		// Name is in braces.
  1537  		end := strings.IndexRune(s, '}')
  1538  		if end < 0 {
  1539  			if err = checkUTF8(s); err != nil {
  1540  				return
  1541  			}
  1542  			return nil, "", &Error{ErrInvalidCharRange, s}
  1543  		}
  1544  		seq, t = s[:end+1], s[end+1:]
  1545  		name = s[3:end]
  1546  		if err = checkUTF8(name); err != nil {
  1547  			return
  1548  		}
  1549  	}
  1550  
  1551  	// Group can have leading negation too.  \p{^Han} == \P{Han}, \P{^Han} == \p{Han}.
  1552  	if name != "" && name[0] == '^' {
  1553  		sign = -sign
  1554  		name = name[1:]
  1555  	}
  1556  
  1557  	tab, fold := unicodeTable(name)
  1558  	if tab == nil {
  1559  		return nil, "", &Error{ErrInvalidCharRange, seq}
  1560  	}
  1561  
  1562  	if p.flags&FoldCase == 0 || fold == nil {
  1563  		if sign > 0 {
  1564  			r = appendTable(r, tab)
  1565  		} else {
  1566  			r = appendNegatedTable(r, tab)
  1567  		}
  1568  	} else {
  1569  		// Merge and clean tab and fold in a temporary buffer.
  1570  		// This is necessary for the negative case and just tidy
  1571  		// for the positive case.
  1572  		tmp := p.tmpClass[:0]
  1573  		tmp = appendTable(tmp, tab)
  1574  		tmp = appendTable(tmp, fold)
  1575  		p.tmpClass = tmp
  1576  		tmp = cleanClass(&p.tmpClass)
  1577  		if sign > 0 {
  1578  			r = appendClass(r, tmp)
  1579  		} else {
  1580  			r = appendNegatedClass(r, tmp)
  1581  		}
  1582  	}
  1583  	return r, t, nil
  1584  }
  1585  
  1586  // parseClass parses a character class at the beginning of s
  1587  // and pushes it onto the parse stack.
  1588  func (p *parser) parseClass(s string) (rest string, err error) {
  1589  	t := s[1:] // chop [
  1590  	re := p.newRegexp(OpCharClass)
  1591  	re.Flags = p.flags
  1592  	re.Rune = re.Rune0[:0]
  1593  
  1594  	sign := +1
  1595  	if t != "" && t[0] == '^' {
  1596  		sign = -1
  1597  		t = t[1:]
  1598  
  1599  		// If character class does not match \n, add it here,
  1600  		// so that negation later will do the right thing.
  1601  		if p.flags&ClassNL == 0 {
  1602  			re.Rune = append(re.Rune, '\n', '\n')
  1603  		}
  1604  	}
  1605  
  1606  	class := re.Rune
  1607  	first := true // ] and - are okay as first char in class
  1608  	for t == "" || t[0] != ']' || first {
  1609  		// POSIX: - is only okay unescaped as first or last in class.
  1610  		// Perl: - is okay anywhere.
  1611  		if t != "" && t[0] == '-' && p.flags&PerlX == 0 && !first && (len(t) == 1 || t[1] != ']') {
  1612  			_, size := utf8.DecodeRuneInString(t[1:])
  1613  			return "", &Error{Code: ErrInvalidCharRange, Expr: t[:1+size]}
  1614  		}
  1615  		first = false
  1616  
  1617  		// Look for POSIX [:alnum:] etc.
  1618  		if len(t) > 2 && t[0] == '[' && t[1] == ':' {
  1619  			nclass, nt, err := p.parseNamedClass(t, class)
  1620  			if err != nil {
  1621  				return "", err
  1622  			}
  1623  			if nclass != nil {
  1624  				class, t = nclass, nt
  1625  				continue
  1626  			}
  1627  		}
  1628  
  1629  		// Look for Unicode character group like \p{Han}.
  1630  		nclass, nt, err := p.parseUnicodeClass(t, class)
  1631  		if err != nil {
  1632  			return "", err
  1633  		}
  1634  		if nclass != nil {
  1635  			class, t = nclass, nt
  1636  			continue
  1637  		}
  1638  
  1639  		// Look for Perl character class symbols (extension).
  1640  		if nclass, nt := p.parsePerlClassEscape(t, class); nclass != nil {
  1641  			class, t = nclass, nt
  1642  			continue
  1643  		}
  1644  
  1645  		// Single character or simple range.
  1646  		rng := t
  1647  		var lo, hi rune
  1648  		if lo, t, err = p.parseClassChar(t, s); err != nil {
  1649  			return "", err
  1650  		}
  1651  		hi = lo
  1652  		// [a-] means (a|-) so check for final ].
  1653  		if len(t) >= 2 && t[0] == '-' && t[1] != ']' {
  1654  			t = t[1:]
  1655  			if hi, t, err = p.parseClassChar(t, s); err != nil {
  1656  				return "", err
  1657  			}
  1658  			if hi < lo {
  1659  				rng = rng[:len(rng)-len(t)]
  1660  				return "", &Error{Code: ErrInvalidCharRange, Expr: rng}
  1661  			}
  1662  		}
  1663  		if p.flags&FoldCase == 0 {
  1664  			class = appendRange(class, lo, hi)
  1665  		} else {
  1666  			class = appendFoldedRange(class, lo, hi)
  1667  		}
  1668  	}
  1669  	t = t[1:] // chop ]
  1670  
  1671  	// Use &re.Rune instead of &class to avoid allocation.
  1672  	re.Rune = class
  1673  	class = cleanClass(&re.Rune)
  1674  	if sign < 0 {
  1675  		class = negateClass(class)
  1676  	}
  1677  	re.Rune = class
  1678  	p.push(re)
  1679  	return t, nil
  1680  }
  1681  
  1682  // cleanClass sorts the ranges (pairs of elements of r),
  1683  // merges them, and eliminates duplicates.
  1684  func cleanClass(rp *[]rune) []rune {
  1685  
  1686  	// Sort by lo increasing, hi decreasing to break ties.
  1687  	sort.Sort(ranges{rp})
  1688  
  1689  	r := *rp
  1690  	if len(r) < 2 {
  1691  		return r
  1692  	}
  1693  
  1694  	// Merge abutting, overlapping.
  1695  	w := 2 // write index
  1696  	for i := 2; i < len(r); i += 2 {
  1697  		lo, hi := r[i], r[i+1]
  1698  		if lo <= r[w-1]+1 {
  1699  			// merge with previous range
  1700  			if hi > r[w-1] {
  1701  				r[w-1] = hi
  1702  			}
  1703  			continue
  1704  		}
  1705  		// new disjoint range
  1706  		r[w] = lo
  1707  		r[w+1] = hi
  1708  		w += 2
  1709  	}
  1710  
  1711  	return r[:w]
  1712  }
  1713  
  1714  // appendLiteral returns the result of appending the literal x to the class r.
  1715  func appendLiteral(r []rune, x rune, flags Flags) []rune {
  1716  	if flags&FoldCase != 0 {
  1717  		return appendFoldedRange(r, x, x)
  1718  	}
  1719  	return appendRange(r, x, x)
  1720  }
  1721  
  1722  // appendRange returns the result of appending the range lo-hi to the class r.
  1723  func appendRange(r []rune, lo, hi rune) []rune {
  1724  	// Expand last range or next to last range if it overlaps or abuts.
  1725  	// Checking two ranges helps when appending case-folded
  1726  	// alphabets, so that one range can be expanding A-Z and the
  1727  	// other expanding a-z.
  1728  	n := len(r)
  1729  	for i := 2; i <= 4; i += 2 { // twice, using i=2, i=4
  1730  		if n >= i {
  1731  			rlo, rhi := r[n-i], r[n-i+1]
  1732  			if lo <= rhi+1 && rlo <= hi+1 {
  1733  				if lo < rlo {
  1734  					r[n-i] = lo
  1735  				}
  1736  				if hi > rhi {
  1737  					r[n-i+1] = hi
  1738  				}
  1739  				return r
  1740  			}
  1741  		}
  1742  	}
  1743  
  1744  	return append(r, lo, hi)
  1745  }
  1746  
  1747  const (
  1748  	// minimum and maximum runes involved in folding.
  1749  	// checked during test.
  1750  	minFold = 0x0041
  1751  	maxFold = 0x1e943
  1752  )
  1753  
  1754  // appendFoldedRange returns the result of appending the range lo-hi
  1755  // and its case folding-equivalent runes to the class r.
  1756  func appendFoldedRange(r []rune, lo, hi rune) []rune {
  1757  	// Optimizations.
  1758  	if lo <= minFold && hi >= maxFold {
  1759  		// Range is full: folding can't add more.
  1760  		return appendRange(r, lo, hi)
  1761  	}
  1762  	if hi < minFold || lo > maxFold {
  1763  		// Range is outside folding possibilities.
  1764  		return appendRange(r, lo, hi)
  1765  	}
  1766  	if lo < minFold {
  1767  		// [lo, minFold-1] needs no folding.
  1768  		r = appendRange(r, lo, minFold-1)
  1769  		lo = minFold
  1770  	}
  1771  	if hi > maxFold {
  1772  		// [maxFold+1, hi] needs no folding.
  1773  		r = appendRange(r, maxFold+1, hi)
  1774  		hi = maxFold
  1775  	}
  1776  
  1777  	// Brute force. Depend on appendRange to coalesce ranges on the fly.
  1778  	for c := lo; c <= hi; c++ {
  1779  		r = appendRange(r, c, c)
  1780  		f := unicode.SimpleFold(c)
  1781  		for f != c {
  1782  			r = appendRange(r, f, f)
  1783  			f = unicode.SimpleFold(f)
  1784  		}
  1785  	}
  1786  	return r
  1787  }
  1788  
  1789  // appendClass returns the result of appending the class x to the class r.
  1790  // It assume x is clean.
  1791  func appendClass(r []rune, x []rune) []rune {
  1792  	for i := 0; i < len(x); i += 2 {
  1793  		r = appendRange(r, x[i], x[i+1])
  1794  	}
  1795  	return r
  1796  }
  1797  
  1798  // appendFolded returns the result of appending the case folding of the class x to the class r.
  1799  func appendFoldedClass(r []rune, x []rune) []rune {
  1800  	for i := 0; i < len(x); i += 2 {
  1801  		r = appendFoldedRange(r, x[i], x[i+1])
  1802  	}
  1803  	return r
  1804  }
  1805  
  1806  // appendNegatedClass returns the result of appending the negation of the class x to the class r.
  1807  // It assumes x is clean.
  1808  func appendNegatedClass(r []rune, x []rune) []rune {
  1809  	nextLo := '\u0000'
  1810  	for i := 0; i < len(x); i += 2 {
  1811  		lo, hi := x[i], x[i+1]
  1812  		if nextLo <= lo-1 {
  1813  			r = appendRange(r, nextLo, lo-1)
  1814  		}
  1815  		nextLo = hi + 1
  1816  	}
  1817  	if nextLo <= unicode.MaxRune {
  1818  		r = appendRange(r, nextLo, unicode.MaxRune)
  1819  	}
  1820  	return r
  1821  }
  1822  
  1823  // appendTable returns the result of appending x to the class r.
  1824  func appendTable(r []rune, x *unicode.RangeTable) []rune {
  1825  	for _, xr := range x.R16 {
  1826  		lo, hi, stride := rune(xr.Lo), rune(xr.Hi), rune(xr.Stride)
  1827  		if stride == 1 {
  1828  			r = appendRange(r, lo, hi)
  1829  			continue
  1830  		}
  1831  		for c := lo; c <= hi; c += stride {
  1832  			r = appendRange(r, c, c)
  1833  		}
  1834  	}
  1835  	for _, xr := range x.R32 {
  1836  		lo, hi, stride := rune(xr.Lo), rune(xr.Hi), rune(xr.Stride)
  1837  		if stride == 1 {
  1838  			r = appendRange(r, lo, hi)
  1839  			continue
  1840  		}
  1841  		for c := lo; c <= hi; c += stride {
  1842  			r = appendRange(r, c, c)
  1843  		}
  1844  	}
  1845  	return r
  1846  }
  1847  
  1848  // appendNegatedTable returns the result of appending the negation of x to the class r.
  1849  func appendNegatedTable(r []rune, x *unicode.RangeTable) []rune {
  1850  	nextLo := '\u0000' // lo end of next class to add
  1851  	for _, xr := range x.R16 {
  1852  		lo, hi, stride := rune(xr.Lo), rune(xr.Hi), rune(xr.Stride)
  1853  		if stride == 1 {
  1854  			if nextLo <= lo-1 {
  1855  				r = appendRange(r, nextLo, lo-1)
  1856  			}
  1857  			nextLo = hi + 1
  1858  			continue
  1859  		}
  1860  		for c := lo; c <= hi; c += stride {
  1861  			if nextLo <= c-1 {
  1862  				r = appendRange(r, nextLo, c-1)
  1863  			}
  1864  			nextLo = c + 1
  1865  		}
  1866  	}
  1867  	for _, xr := range x.R32 {
  1868  		lo, hi, stride := rune(xr.Lo), rune(xr.Hi), rune(xr.Stride)
  1869  		if stride == 1 {
  1870  			if nextLo <= lo-1 {
  1871  				r = appendRange(r, nextLo, lo-1)
  1872  			}
  1873  			nextLo = hi + 1
  1874  			continue
  1875  		}
  1876  		for c := lo; c <= hi; c += stride {
  1877  			if nextLo <= c-1 {
  1878  				r = appendRange(r, nextLo, c-1)
  1879  			}
  1880  			nextLo = c + 1
  1881  		}
  1882  	}
  1883  	if nextLo <= unicode.MaxRune {
  1884  		r = appendRange(r, nextLo, unicode.MaxRune)
  1885  	}
  1886  	return r
  1887  }
  1888  
  1889  // negateClass overwrites r and returns r's negation.
  1890  // It assumes the class r is already clean.
  1891  func negateClass(r []rune) []rune {
  1892  	nextLo := '\u0000' // lo end of next class to add
  1893  	w := 0             // write index
  1894  	for i := 0; i < len(r); i += 2 {
  1895  		lo, hi := r[i], r[i+1]
  1896  		if nextLo <= lo-1 {
  1897  			r[w] = nextLo
  1898  			r[w+1] = lo - 1
  1899  			w += 2
  1900  		}
  1901  		nextLo = hi + 1
  1902  	}
  1903  	r = r[:w]
  1904  	if nextLo <= unicode.MaxRune {
  1905  		// It's possible for the negation to have one more
  1906  		// range - this one - than the original class, so use append.
  1907  		r = append(r, nextLo, unicode.MaxRune)
  1908  	}
  1909  	return r
  1910  }
  1911  
  1912  // ranges implements sort.Interface on a []rune.
  1913  // The choice of receiver type definition is strange
  1914  // but avoids an allocation since we already have
  1915  // a *[]rune.
  1916  type ranges struct {
  1917  	p *[]rune
  1918  }
  1919  
  1920  func (ra ranges) Less(i, j int) bool {
  1921  	p := *ra.p
  1922  	i *= 2
  1923  	j *= 2
  1924  	return p[i] < p[j] || p[i] == p[j] && p[i+1] > p[j+1]
  1925  }
  1926  
  1927  func (ra ranges) Len() int {
  1928  	return len(*ra.p) / 2
  1929  }
  1930  
  1931  func (ra ranges) Swap(i, j int) {
  1932  	p := *ra.p
  1933  	i *= 2
  1934  	j *= 2
  1935  	p[i], p[i+1], p[j], p[j+1] = p[j], p[j+1], p[i], p[i+1]
  1936  }
  1937  
  1938  func checkUTF8(s string) error {
  1939  	for s != "" {
  1940  		rune, size := utf8.DecodeRuneInString(s)
  1941  		if rune == utf8.RuneError && size == 1 {
  1942  			return &Error{Code: ErrInvalidUTF8, Expr: s}
  1943  		}
  1944  		s = s[size:]
  1945  	}
  1946  	return nil
  1947  }
  1948  
  1949  func nextRune(s string) (c rune, t string, err error) {
  1950  	c, size := utf8.DecodeRuneInString(s)
  1951  	if c == utf8.RuneError && size == 1 {
  1952  		return 0, "", &Error{Code: ErrInvalidUTF8, Expr: s}
  1953  	}
  1954  	return c, s[size:], nil
  1955  }
  1956  
  1957  func isalnum(c rune) bool {
  1958  	return '0' <= c && c <= '9' || 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z'
  1959  }
  1960  
  1961  func unhex(c rune) rune {
  1962  	if '0' <= c && c <= '9' {
  1963  		return c - '0'
  1964  	}
  1965  	if 'a' <= c && c <= 'f' {
  1966  		return c - 'a' + 10
  1967  	}
  1968  	if 'A' <= c && c <= 'F' {
  1969  		return c - 'A' + 10
  1970  	}
  1971  	return -1
  1972  }
  1973  

View as plain text