token.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package token defines constants representing the lexical tokens of the Go
     6  // programming language and basic operations on tokens (printing, predicates).
     7  //
     8  package token
     9  
    10  import (
    11  	"strconv"
    12  	"unicode"
    13  	"unicode/utf8"
    14  )
    15  
    16  // Token is the set of lexical tokens of the Go programming language.
    17  type Token int
    18  
    19  // The list of tokens.
    20  const (
    21  	// Special tokens
    22  	ILLEGAL Token = iota
    23  	EOF
    24  	COMMENT
    25  
    26  	literal_beg
    27  	// Identifiers and basic type literals
    28  	// (these tokens stand for classes of literals)
    29  	IDENT  // main
    30  	INT    // 12345
    31  	FLOAT  // 123.45
    32  	IMAG   // 123.45i
    33  	CHAR   // 'a'
    34  	STRING // "abc"
    35  	literal_end
    36  
    37  	operator_beg
    38  	// Operators and delimiters
    39  	ADD // +
    40  	SUB // -
    41  	MUL // *
    42  	QUO // /
    43  	REM // %
    44  
    45  	AND     // &
    46  	OR      // |
    47  	XOR     // ^
    48  	SHL     // <<
    49  	SHR     // >>
    50  	AND_NOT // &^
    51  
    52  	ADD_ASSIGN // +=
    53  	SUB_ASSIGN // -=
    54  	MUL_ASSIGN // *=
    55  	QUO_ASSIGN // /=
    56  	REM_ASSIGN // %=
    57  
    58  	AND_ASSIGN     // &=
    59  	OR_ASSIGN      // |=
    60  	XOR_ASSIGN     // ^=
    61  	SHL_ASSIGN     // <<=
    62  	SHR_ASSIGN     // >>=
    63  	AND_NOT_ASSIGN // &^=
    64  
    65  	LAND  // &&
    66  	LOR   // ||
    67  	ARROW // <-
    68  	INC   // ++
    69  	DEC   // --
    70  
    71  	EQL    // ==
    72  	LSS    // <
    73  	GTR    // >
    74  	ASSIGN // =
    75  	NOT    // !
    76  
    77  	NEQ      // !=
    78  	LEQ      // <=
    79  	GEQ      // >=
    80  	DEFINE   // :=
    81  	ELLIPSIS // ...
    82  
    83  	LPAREN // (
    84  	LBRACK // [
    85  	LBRACE // {
    86  	COMMA  // ,
    87  	PERIOD // .
    88  
    89  	RPAREN    // )
    90  	RBRACK    // ]
    91  	RBRACE    // }
    92  	SEMICOLON // ;
    93  	COLON     // :
    94  	operator_end
    95  
    96  	keyword_beg
    97  	// Keywords
    98  	BREAK
    99  	CASE
   100  	CHAN
   101  	CONST
   102  	CONTINUE
   103  
   104  	DEFAULT
   105  	DEFER
   106  	ELSE
   107  	FALLTHROUGH
   108  	FOR
   109  
   110  	FUNC
   111  	GO
   112  	GOTO
   113  	IF
   114  	IMPORT
   115  
   116  	INTERFACE
   117  	MAP
   118  	PACKAGE
   119  	RANGE
   120  	RETURN
   121  
   122  	SELECT
   123  	STRUCT
   124  	SWITCH
   125  	TYPE
   126  	VAR
   127  	keyword_end
   128  
   129  	additional_beg
   130  	// additional tokens, handled in an ad-hoc manner
   131  	TILDE
   132  	additional_end
   133  )
   134  
   135  var tokens = [...]string{
   136  	ILLEGAL: "ILLEGAL",
   137  
   138  	EOF:     "EOF",
   139  	COMMENT: "COMMENT",
   140  
   141  	IDENT:  "IDENT",
   142  	INT:    "INT",
   143  	FLOAT:  "FLOAT",
   144  	IMAG:   "IMAG",
   145  	CHAR:   "CHAR",
   146  	STRING: "STRING",
   147  
   148  	ADD: "+",
   149  	SUB: "-",
   150  	MUL: "*",
   151  	QUO: "/",
   152  	REM: "%",
   153  
   154  	AND:     "&",
   155  	OR:      "|",
   156  	XOR:     "^",
   157  	SHL:     "<<",
   158  	SHR:     ">>",
   159  	AND_NOT: "&^",
   160  
   161  	ADD_ASSIGN: "+=",
   162  	SUB_ASSIGN: "-=",
   163  	MUL_ASSIGN: "*=",
   164  	QUO_ASSIGN: "/=",
   165  	REM_ASSIGN: "%=",
   166  
   167  	AND_ASSIGN:     "&=",
   168  	OR_ASSIGN:      "|=",
   169  	XOR_ASSIGN:     "^=",
   170  	SHL_ASSIGN:     "<<=",
   171  	SHR_ASSIGN:     ">>=",
   172  	AND_NOT_ASSIGN: "&^=",
   173  
   174  	LAND:  "&&",
   175  	LOR:   "||",
   176  	ARROW: "<-",
   177  	INC:   "++",
   178  	DEC:   "--",
   179  
   180  	EQL:    "==",
   181  	LSS:    "<",
   182  	GTR:    ">",
   183  	ASSIGN: "=",
   184  	NOT:    "!",
   185  
   186  	NEQ:      "!=",
   187  	LEQ:      "<=",
   188  	GEQ:      ">=",
   189  	DEFINE:   ":=",
   190  	ELLIPSIS: "...",
   191  
   192  	LPAREN: "(",
   193  	LBRACK: "[",
   194  	LBRACE: "{",
   195  	COMMA:  ",",
   196  	PERIOD: ".",
   197  
   198  	RPAREN:    ")",
   199  	RBRACK:    "]",
   200  	RBRACE:    "}",
   201  	SEMICOLON: ";",
   202  	COLON:     ":",
   203  
   204  	BREAK:    "break",
   205  	CASE:     "case",
   206  	CHAN:     "chan",
   207  	CONST:    "const",
   208  	CONTINUE: "continue",
   209  
   210  	DEFAULT:     "default",
   211  	DEFER:       "defer",
   212  	ELSE:        "else",
   213  	FALLTHROUGH: "fallthrough",
   214  	FOR:         "for",
   215  
   216  	FUNC:   "func",
   217  	GO:     "go",
   218  	GOTO:   "goto",
   219  	IF:     "if",
   220  	IMPORT: "import",
   221  
   222  	INTERFACE: "interface",
   223  	MAP:       "map",
   224  	PACKAGE:   "package",
   225  	RANGE:     "range",
   226  	RETURN:    "return",
   227  
   228  	SELECT: "select",
   229  	STRUCT: "struct",
   230  	SWITCH: "switch",
   231  	TYPE:   "type",
   232  	VAR:    "var",
   233  
   234  	TILDE: "~",
   235  }
   236  
   237  // String returns the string corresponding to the token tok.
   238  // For operators, delimiters, and keywords the string is the actual
   239  // token character sequence (e.g., for the token ADD, the string is
   240  // "+"). For all other tokens the string corresponds to the token
   241  // constant name (e.g. for the token IDENT, the string is "IDENT").
   242  //
   243  func (tok Token) String() string {
   244  	s := ""
   245  	if 0 <= tok && tok < Token(len(tokens)) {
   246  		s = tokens[tok]
   247  	}
   248  	if s == "" {
   249  		s = "token(" + strconv.Itoa(int(tok)) + ")"
   250  	}
   251  	return s
   252  }
   253  
   254  // A set of constants for precedence-based expression parsing.
   255  // Non-operators have lowest precedence, followed by operators
   256  // starting with precedence 1 up to unary operators. The highest
   257  // precedence serves as "catch-all" precedence for selector,
   258  // indexing, and other operator and delimiter tokens.
   259  //
   260  const (
   261  	LowestPrec  = 0 // non-operators
   262  	UnaryPrec   = 6
   263  	HighestPrec = 7
   264  )
   265  
   266  // Precedence returns the operator precedence of the binary
   267  // operator op. If op is not a binary operator, the result
   268  // is LowestPrecedence.
   269  //
   270  func (op Token) Precedence() int {
   271  	switch op {
   272  	case LOR:
   273  		return 1
   274  	case LAND:
   275  		return 2
   276  	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
   277  		return 3
   278  	case ADD, SUB, OR, XOR:
   279  		return 4
   280  	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
   281  		return 5
   282  	}
   283  	return LowestPrec
   284  }
   285  
   286  var keywords map[string]Token
   287  
   288  func init() {
   289  	keywords = make(map[string]Token)
   290  	for i := keyword_beg + 1; i < keyword_end; i++ {
   291  		keywords[tokens[i]] = i
   292  	}
   293  }
   294  
   295  // Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
   296  //
   297  func Lookup(ident string) Token {
   298  	if tok, is_keyword := keywords[ident]; is_keyword {
   299  		return tok
   300  	}
   301  	return IDENT
   302  }
   303  
   304  // Predicates
   305  
   306  // IsLiteral returns true for tokens corresponding to identifiers
   307  // and basic type literals; it returns false otherwise.
   308  //
   309  func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }
   310  
   311  // IsOperator returns true for tokens corresponding to operators and
   312  // delimiters; it returns false otherwise.
   313  //
   314  func (tok Token) IsOperator() bool {
   315  	return (operator_beg < tok && tok < operator_end) || tok == TILDE
   316  }
   317  
   318  // IsKeyword returns true for tokens corresponding to keywords;
   319  // it returns false otherwise.
   320  //
   321  func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }
   322  
   323  // IsExported reports whether name starts with an upper-case letter.
   324  //
   325  func IsExported(name string) bool {
   326  	ch, _ := utf8.DecodeRuneInString(name)
   327  	return unicode.IsUpper(ch)
   328  }
   329  
   330  // IsKeyword reports whether name is a Go keyword, such as "func" or "return".
   331  //
   332  func IsKeyword(name string) bool {
   333  	// TODO: opt: use a perfect hash function instead of a global map.
   334  	_, ok := keywords[name]
   335  	return ok
   336  }
   337  
   338  // IsIdentifier reports whether name is a Go identifier, that is, a non-empty
   339  // string made up of letters, digits, and underscores, where the first character
   340  // is not a digit. Keywords are not identifiers.
   341  //
   342  func IsIdentifier(name string) bool {
   343  	for i, c := range name {
   344  		if !unicode.IsLetter(c) && c != '_' && (i == 0 || !unicode.IsDigit(c)) {
   345  			return false
   346  		}
   347  	}
   348  	return name != "" && !IsKeyword(name)
   349  }
   350
View as plain text