2023-08-17 05:15:28 +00:00
|
|
|
// Copyright (c) 2012-2022 The ANTLR Project. All rights reserved.
|
2023-05-29 21:03:29 +00:00
|
|
|
// Use of this file is governed by the BSD 3-clause license that
|
|
|
|
// can be found in the LICENSE.txt file in the project root.
|
|
|
|
|
|
|
|
package antlr
|
|
|
|
|
|
|
|
// TokenFactory creates CommonToken objects.
|
|
|
|
type TokenFactory interface {
|
|
|
|
Create(source *TokenSourceCharStreamPair, ttype int, text string, channel, start, stop, line, column int) Token
|
|
|
|
}
|
|
|
|
|
|
|
|
// CommonTokenFactory is the default TokenFactory implementation.
|
|
|
|
type CommonTokenFactory struct {
|
|
|
|
// copyText indicates whether CommonToken.setText should be called after
|
|
|
|
// constructing tokens to explicitly set the text. This is useful for cases
|
|
|
|
// where the input stream might not be able to provide arbitrary substrings of
|
|
|
|
// text from the input after the lexer creates a token (e.g. the
|
|
|
|
// implementation of CharStream.GetText in UnbufferedCharStream panics an
|
|
|
|
// UnsupportedOperationException). Explicitly setting the token text allows
|
|
|
|
// Token.GetText to be called at any time regardless of the input stream
|
|
|
|
// implementation.
|
|
|
|
//
|
|
|
|
// The default value is false to avoid the performance and memory overhead of
|
|
|
|
// copying text for every token unless explicitly requested.
|
|
|
|
copyText bool
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewCommonTokenFactory(copyText bool) *CommonTokenFactory {
|
|
|
|
return &CommonTokenFactory{copyText: copyText}
|
|
|
|
}
|
|
|
|
|
|
|
|
// CommonTokenFactoryDEFAULT is the default CommonTokenFactory. It does not
|
|
|
|
// explicitly copy token text when constructing tokens.
|
|
|
|
var CommonTokenFactoryDEFAULT = NewCommonTokenFactory(false)
|
|
|
|
|
|
|
|
func (c *CommonTokenFactory) Create(source *TokenSourceCharStreamPair, ttype int, text string, channel, start, stop, line, column int) Token {
|
|
|
|
t := NewCommonToken(source, ttype, channel, start, stop)
|
|
|
|
|
|
|
|
t.line = line
|
|
|
|
t.column = column
|
|
|
|
|
|
|
|
if text != "" {
|
|
|
|
t.SetText(text)
|
|
|
|
} else if c.copyText && source.charStream != nil {
|
|
|
|
t.SetText(source.charStream.GetTextFromInterval(NewInterval(start, stop)))
|
|
|
|
}
|
|
|
|
|
|
|
|
return t
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenFactory) createThin(ttype int, text string) Token {
|
|
|
|
t := NewCommonToken(nil, ttype, TokenDefaultChannel, -1, -1)
|
|
|
|
t.SetText(text)
|
|
|
|
|
|
|
|
return t
|
|
|
|
}
|