Source file src/pkg/go/token/token.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Package token defines constants representing the lexical tokens of the Go
     6	// programming language and basic operations on tokens (printing, predicates).
     7	//
     8	package token
     9	
    10	import "strconv"
    11	
    12	// Token is the set of lexical tokens of the Go programming language.
    13	type Token int
    14	
    15	// The list of tokens.
    16	const (
    17		// Special tokens
    18		ILLEGAL Token = iota
    19		EOF
    20		COMMENT
    21	
    22		literal_beg
    23		// Identifiers and basic type literals
    24		// (these tokens stand for classes of literals)
    25		IDENT  // main
    26		INT    // 12345
    27		FLOAT  // 123.45
    28		IMAG   // 123.45i
    29		CHAR   // 'a'
    30		STRING // "abc"
    31		literal_end
    32	
    33		operator_beg
    34		// Operators and delimiters
    35		ADD // +
    36		SUB // -
    37		MUL // *
    38		QUO // /
    39		REM // %
    40	
    41		AND     // &
    42		OR      // |
    43		XOR     // ^
    44		SHL     // <<
    45		SHR     // >>
    46		AND_NOT // &^
    47	
    48		ADD_ASSIGN // +=
    49		SUB_ASSIGN // -=
    50		MUL_ASSIGN // *=
    51		QUO_ASSIGN // /=
    52		REM_ASSIGN // %=
    53	
    54		AND_ASSIGN     // &=
    55		OR_ASSIGN      // |=
    56		XOR_ASSIGN     // ^=
    57		SHL_ASSIGN     // <<=
    58		SHR_ASSIGN     // >>=
    59		AND_NOT_ASSIGN // &^=
    60	
    61		LAND  // &&
    62		LOR   // ||
    63		ARROW // <-
    64		INC   // ++
    65		DEC   // --
    66	
    67		EQL    // ==
    68		LSS    // <
    69		GTR    // >
    70		ASSIGN // =
    71		NOT    // !
    72	
    73		NEQ      // !=
    74		LEQ      // <=
    75		GEQ      // >=
    76		DEFINE   // :=
    77		ELLIPSIS // ...
    78	
    79		LPAREN // (
    80		LBRACK // [
    81		LBRACE // {
    82		COMMA  // ,
    83		PERIOD // .
    84	
    85		RPAREN    // )
    86		RBRACK    // ]
    87		RBRACE    // }
    88		SEMICOLON // ;
    89		COLON     // :
    90		operator_end
    91	
    92		keyword_beg
    93		// Keywords
    94		BREAK
    95		CASE
    96		CHAN
    97		CONST
    98		CONTINUE
    99	
   100		DEFAULT
   101		DEFER
   102		ELSE
   103		FALLTHROUGH
   104		FOR
   105	
   106		FUNC
   107		GO
   108		GOTO
   109		IF
   110		IMPORT
   111	
   112		INTERFACE
   113		MAP
   114		PACKAGE
   115		RANGE
   116		RETURN
   117	
   118		SELECT
   119		STRUCT
   120		SWITCH
   121		TYPE
   122		VAR
   123		keyword_end
   124	)
   125	
   126	var tokens = [...]string{
   127		ILLEGAL: "ILLEGAL",
   128	
   129		EOF:     "EOF",
   130		COMMENT: "COMMENT",
   131	
   132		IDENT:  "IDENT",
   133		INT:    "INT",
   134		FLOAT:  "FLOAT",
   135		IMAG:   "IMAG",
   136		CHAR:   "CHAR",
   137		STRING: "STRING",
   138	
   139		ADD: "+",
   140		SUB: "-",
   141		MUL: "*",
   142		QUO: "/",
   143		REM: "%",
   144	
   145		AND:     "&",
   146		OR:      "|",
   147		XOR:     "^",
   148		SHL:     "<<",
   149		SHR:     ">>",
   150		AND_NOT: "&^",
   151	
   152		ADD_ASSIGN: "+=",
   153		SUB_ASSIGN: "-=",
   154		MUL_ASSIGN: "*=",
   155		QUO_ASSIGN: "/=",
   156		REM_ASSIGN: "%=",
   157	
   158		AND_ASSIGN:     "&=",
   159		OR_ASSIGN:      "|=",
   160		XOR_ASSIGN:     "^=",
   161		SHL_ASSIGN:     "<<=",
   162		SHR_ASSIGN:     ">>=",
   163		AND_NOT_ASSIGN: "&^=",
   164	
   165		LAND:  "&&",
   166		LOR:   "||",
   167		ARROW: "<-",
   168		INC:   "++",
   169		DEC:   "--",
   170	
   171		EQL:    "==",
   172		LSS:    "<",
   173		GTR:    ">",
   174		ASSIGN: "=",
   175		NOT:    "!",
   176	
   177		NEQ:      "!=",
   178		LEQ:      "<=",
   179		GEQ:      ">=",
   180		DEFINE:   ":=",
   181		ELLIPSIS: "...",
   182	
   183		LPAREN: "(",
   184		LBRACK: "[",
   185		LBRACE: "{",
   186		COMMA:  ",",
   187		PERIOD: ".",
   188	
   189		RPAREN:    ")",
   190		RBRACK:    "]",
   191		RBRACE:    "}",
   192		SEMICOLON: ";",
   193		COLON:     ":",
   194	
   195		BREAK:    "break",
   196		CASE:     "case",
   197		CHAN:     "chan",
   198		CONST:    "const",
   199		CONTINUE: "continue",
   200	
   201		DEFAULT:     "default",
   202		DEFER:       "defer",
   203		ELSE:        "else",
   204		FALLTHROUGH: "fallthrough",
   205		FOR:         "for",
   206	
   207		FUNC:   "func",
   208		GO:     "go",
   209		GOTO:   "goto",
   210		IF:     "if",
   211		IMPORT: "import",
   212	
   213		INTERFACE: "interface",
   214		MAP:       "map",
   215		PACKAGE:   "package",
   216		RANGE:     "range",
   217		RETURN:    "return",
   218	
   219		SELECT: "select",
   220		STRUCT: "struct",
   221		SWITCH: "switch",
   222		TYPE:   "type",
   223		VAR:    "var",
   224	}
   225	
   226	// String returns the string corresponding to the token tok.
   227	// For operators, delimiters, and keywords the string is the actual
   228	// token character sequence (e.g., for the token ADD, the string is
   229	// "+"). For all other tokens the string corresponds to the token
   230	// constant name (e.g. for the token IDENT, the string is "IDENT").
   231	//
   232	func (tok Token) String() string {
   233		s := ""
   234		if 0 <= tok && tok < Token(len(tokens)) {
   235			s = tokens[tok]
   236		}
   237		if s == "" {
   238			s = "token(" + strconv.Itoa(int(tok)) + ")"
   239		}
   240		return s
   241	}
   242	
   243	// A set of constants for precedence-based expression parsing.
   244	// Non-operators have lowest precedence, followed by operators
   245	// starting with precedence 1 up to unary operators. The highest
   246	// precedence corresponds serves as "catch-all" precedence for
   247	// selector, indexing, and other operator and delimiter tokens.
   248	//
   249	const (
   250		LowestPrec  = 0 // non-operators
   251		UnaryPrec   = 6
   252		HighestPrec = 7
   253	)
   254	
   255	// Precedence returns the operator precedence of the binary
   256	// operator op. If op is not a binary operator, the result
   257	// is LowestPrecedence.
   258	//
   259	func (op Token) Precedence() int {
   260		switch op {
   261		case LOR:
   262			return 1
   263		case LAND:
   264			return 2
   265		case EQL, NEQ, LSS, LEQ, GTR, GEQ:
   266			return 3
   267		case ADD, SUB, OR, XOR:
   268			return 4
   269		case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
   270			return 5
   271		}
   272		return LowestPrec
   273	}
   274	
   275	var keywords map[string]Token
   276	
   277	func init() {
   278		keywords = make(map[string]Token)
   279		for i := keyword_beg + 1; i < keyword_end; i++ {
   280			keywords[tokens[i]] = i
   281		}
   282	}
   283	
   284	// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
   285	//
   286	func Lookup(ident string) Token {
   287		if tok, is_keyword := keywords[ident]; is_keyword {
   288			return tok
   289		}
   290		return IDENT
   291	}
   292	
   293	// Predicates
   294	
   295	// IsLiteral returns true for tokens corresponding to identifiers
   296	// and basic type literals; it returns false otherwise.
   297	//
   298	func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }
   299	
   300	// IsOperator returns true for tokens corresponding to operators and
   301	// delimiters; it returns false otherwise.
   302	//
   303	func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
   304	
   305	// IsKeyword returns true for tokens corresponding to keywords;
   306	// it returns false otherwise.
   307	//
   308	func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }