2023-08-17 05:15:28 +00:00
|
|
|
// Copyright (c) 2012-2022 The ANTLR Project. All rights reserved.
|
2023-05-29 21:03:29 +00:00
|
|
|
// Use of this file is governed by the BSD 3-clause license that
|
|
|
|
// can be found in the LICENSE.txt file in the project root.
|
|
|
|
|
|
|
|
package antlr
|
|
|
|
|
|
|
|
import (
|
|
|
|
"strconv"
|
|
|
|
)
|
|
|
|
|
|
|
|
// CommonTokenStream is an implementation of TokenStream that loads tokens from
|
|
|
|
// a TokenSource on-demand and places the tokens in a buffer to provide access
|
|
|
|
// to any previous token by index. This token stream ignores the value of
|
|
|
|
// Token.getChannel. If your parser requires the token stream filter tokens to
|
|
|
|
// only those on a particular channel, such as Token.DEFAULT_CHANNEL or
|
|
|
|
// Token.HIDDEN_CHANNEL, use a filtering token stream such a CommonTokenStream.
|
|
|
|
type CommonTokenStream struct {
|
|
|
|
channel int
|
|
|
|
|
|
|
|
// fetchedEOF indicates whether the Token.EOF token has been fetched from
|
|
|
|
// tokenSource and added to tokens. This field improves performance for the
|
|
|
|
// following cases:
|
|
|
|
//
|
|
|
|
// consume: The lookahead check in consume to preven consuming the EOF symbol is
|
|
|
|
// optimized by checking the values of fetchedEOF and p instead of calling LA.
|
|
|
|
//
|
|
|
|
// fetch: The check to prevent adding multiple EOF symbols into tokens is
|
|
|
|
// trivial with bt field.
|
|
|
|
fetchedEOF bool
|
|
|
|
|
2024-08-19 08:01:33 +00:00
|
|
|
// index into [tokens] of the current token (next token to consume).
|
2023-05-29 21:03:29 +00:00
|
|
|
// tokens[p] should be LT(1). It is set to -1 when the stream is first
|
|
|
|
// constructed or when SetTokenSource is called, indicating that the first token
|
|
|
|
// has not yet been fetched from the token source. For additional information,
|
2024-08-19 08:01:33 +00:00
|
|
|
// see the documentation of [IntStream] for a description of initializing methods.
|
2023-05-29 21:03:29 +00:00
|
|
|
index int
|
|
|
|
|
2024-08-19 08:01:33 +00:00
|
|
|
// tokenSource is the [TokenSource] from which tokens for the bt stream are
|
2023-05-29 21:03:29 +00:00
|
|
|
// fetched.
|
|
|
|
tokenSource TokenSource
|
|
|
|
|
2024-08-19 08:01:33 +00:00
|
|
|
// tokens contains all tokens fetched from the token source. The list is considered a
|
2023-05-29 21:03:29 +00:00
|
|
|
// complete view of the input once fetchedEOF is set to true.
|
|
|
|
tokens []Token
|
|
|
|
}
|
|
|
|
|
2024-08-19 08:01:33 +00:00
|
|
|
// NewCommonTokenStream creates a new CommonTokenStream instance using the supplied lexer to produce
|
|
|
|
// tokens and will pull tokens from the given lexer channel.
|
2023-05-29 21:03:29 +00:00
|
|
|
func NewCommonTokenStream(lexer Lexer, channel int) *CommonTokenStream {
|
|
|
|
return &CommonTokenStream{
|
|
|
|
channel: channel,
|
|
|
|
index: -1,
|
|
|
|
tokenSource: lexer,
|
|
|
|
tokens: make([]Token, 0),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-08-19 08:01:33 +00:00
|
|
|
// GetAllTokens returns all tokens currently pulled from the token source.
|
2023-05-29 21:03:29 +00:00
|
|
|
func (c *CommonTokenStream) GetAllTokens() []Token {
|
|
|
|
return c.tokens
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) Mark() int {
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
2024-08-19 08:01:33 +00:00
|
|
|
func (c *CommonTokenStream) Release(_ int) {}
|
2023-05-29 21:03:29 +00:00
|
|
|
|
2024-08-19 08:01:33 +00:00
|
|
|
func (c *CommonTokenStream) Reset() {
|
|
|
|
c.fetchedEOF = false
|
|
|
|
c.tokens = make([]Token, 0)
|
2023-05-29 21:03:29 +00:00
|
|
|
c.Seek(0)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) Seek(index int) {
|
|
|
|
c.lazyInit()
|
|
|
|
c.index = c.adjustSeekIndex(index)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) Get(index int) Token {
|
|
|
|
c.lazyInit()
|
|
|
|
|
|
|
|
return c.tokens[index]
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) Consume() {
|
|
|
|
SkipEOFCheck := false
|
|
|
|
|
|
|
|
if c.index >= 0 {
|
|
|
|
if c.fetchedEOF {
|
|
|
|
// The last token in tokens is EOF. Skip the check if p indexes any fetched.
|
|
|
|
// token except the last.
|
|
|
|
SkipEOFCheck = c.index < len(c.tokens)-1
|
|
|
|
} else {
|
|
|
|
// No EOF token in tokens. Skip the check if p indexes a fetched token.
|
|
|
|
SkipEOFCheck = c.index < len(c.tokens)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// Not yet initialized
|
|
|
|
SkipEOFCheck = false
|
|
|
|
}
|
|
|
|
|
|
|
|
if !SkipEOFCheck && c.LA(1) == TokenEOF {
|
|
|
|
panic("cannot consume EOF")
|
|
|
|
}
|
|
|
|
|
|
|
|
if c.Sync(c.index + 1) {
|
|
|
|
c.index = c.adjustSeekIndex(c.index + 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Sync makes sure index i in tokens has a token and returns true if a token is
|
|
|
|
// located at index i and otherwise false.
|
|
|
|
func (c *CommonTokenStream) Sync(i int) bool {
|
2024-08-19 08:01:33 +00:00
|
|
|
n := i - len(c.tokens) + 1 // How many more elements do we need?
|
2023-05-29 21:03:29 +00:00
|
|
|
|
|
|
|
if n > 0 {
|
|
|
|
fetched := c.fetch(n)
|
|
|
|
return fetched >= n
|
|
|
|
}
|
|
|
|
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
// fetch adds n elements to buffer and returns the actual number of elements
|
|
|
|
// added to the buffer.
|
|
|
|
func (c *CommonTokenStream) fetch(n int) int {
|
|
|
|
if c.fetchedEOF {
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
for i := 0; i < n; i++ {
|
|
|
|
t := c.tokenSource.NextToken()
|
|
|
|
|
|
|
|
t.SetTokenIndex(len(c.tokens))
|
|
|
|
c.tokens = append(c.tokens, t)
|
|
|
|
|
|
|
|
if t.GetTokenType() == TokenEOF {
|
|
|
|
c.fetchedEOF = true
|
|
|
|
|
|
|
|
return i + 1
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return n
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetTokens gets all tokens from start to stop inclusive.
|
|
|
|
func (c *CommonTokenStream) GetTokens(start int, stop int, types *IntervalSet) []Token {
|
|
|
|
if start < 0 || stop < 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
c.lazyInit()
|
|
|
|
|
|
|
|
subset := make([]Token, 0)
|
|
|
|
|
|
|
|
if stop >= len(c.tokens) {
|
|
|
|
stop = len(c.tokens) - 1
|
|
|
|
}
|
|
|
|
|
|
|
|
for i := start; i < stop; i++ {
|
|
|
|
t := c.tokens[i]
|
|
|
|
|
|
|
|
if t.GetTokenType() == TokenEOF {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
if types == nil || types.contains(t.GetTokenType()) {
|
|
|
|
subset = append(subset, t)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return subset
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) LA(i int) int {
|
|
|
|
return c.LT(i).GetTokenType()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) lazyInit() {
|
|
|
|
if c.index == -1 {
|
|
|
|
c.setup()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) setup() {
|
|
|
|
c.Sync(0)
|
|
|
|
c.index = c.adjustSeekIndex(0)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) GetTokenSource() TokenSource {
|
|
|
|
return c.tokenSource
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetTokenSource resets the c token stream by setting its token source.
|
|
|
|
func (c *CommonTokenStream) SetTokenSource(tokenSource TokenSource) {
|
|
|
|
c.tokenSource = tokenSource
|
|
|
|
c.tokens = make([]Token, 0)
|
|
|
|
c.index = -1
|
2024-08-19 08:01:33 +00:00
|
|
|
c.fetchedEOF = false
|
2023-05-29 21:03:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// NextTokenOnChannel returns the index of the next token on channel given a
|
|
|
|
// starting index. Returns i if tokens[i] is on channel. Returns -1 if there are
|
2024-08-19 08:01:33 +00:00
|
|
|
// no tokens on channel between 'i' and [TokenEOF].
|
|
|
|
func (c *CommonTokenStream) NextTokenOnChannel(i, _ int) int {
|
2023-05-29 21:03:29 +00:00
|
|
|
c.Sync(i)
|
|
|
|
|
|
|
|
if i >= len(c.tokens) {
|
|
|
|
return -1
|
|
|
|
}
|
|
|
|
|
|
|
|
token := c.tokens[i]
|
|
|
|
|
|
|
|
for token.GetChannel() != c.channel {
|
|
|
|
if token.GetTokenType() == TokenEOF {
|
|
|
|
return -1
|
|
|
|
}
|
|
|
|
|
|
|
|
i++
|
|
|
|
c.Sync(i)
|
|
|
|
token = c.tokens[i]
|
|
|
|
}
|
|
|
|
|
|
|
|
return i
|
|
|
|
}
|
|
|
|
|
|
|
|
// previousTokenOnChannel returns the index of the previous token on channel
|
|
|
|
// given a starting index. Returns i if tokens[i] is on channel. Returns -1 if
|
|
|
|
// there are no tokens on channel between i and 0.
|
|
|
|
func (c *CommonTokenStream) previousTokenOnChannel(i, channel int) int {
|
|
|
|
for i >= 0 && c.tokens[i].GetChannel() != channel {
|
|
|
|
i--
|
|
|
|
}
|
|
|
|
|
|
|
|
return i
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetHiddenTokensToRight collects all tokens on a specified channel to the
|
|
|
|
// right of the current token up until we see a token on DEFAULT_TOKEN_CHANNEL
|
|
|
|
// or EOF. If channel is -1, it finds any non-default channel token.
|
|
|
|
func (c *CommonTokenStream) GetHiddenTokensToRight(tokenIndex, channel int) []Token {
|
|
|
|
c.lazyInit()
|
|
|
|
|
|
|
|
if tokenIndex < 0 || tokenIndex >= len(c.tokens) {
|
|
|
|
panic(strconv.Itoa(tokenIndex) + " not in 0.." + strconv.Itoa(len(c.tokens)-1))
|
|
|
|
}
|
|
|
|
|
|
|
|
nextOnChannel := c.NextTokenOnChannel(tokenIndex+1, LexerDefaultTokenChannel)
|
|
|
|
from := tokenIndex + 1
|
|
|
|
|
2024-08-19 08:01:33 +00:00
|
|
|
// If no onChannel to the right, then nextOnChannel == -1, so set 'to' to the last token
|
2023-05-29 21:03:29 +00:00
|
|
|
var to int
|
|
|
|
|
|
|
|
if nextOnChannel == -1 {
|
|
|
|
to = len(c.tokens) - 1
|
|
|
|
} else {
|
|
|
|
to = nextOnChannel
|
|
|
|
}
|
|
|
|
|
|
|
|
return c.filterForChannel(from, to, channel)
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetHiddenTokensToLeft collects all tokens on channel to the left of the
|
|
|
|
// current token until we see a token on DEFAULT_TOKEN_CHANNEL. If channel is
|
|
|
|
// -1, it finds any non default channel token.
|
|
|
|
func (c *CommonTokenStream) GetHiddenTokensToLeft(tokenIndex, channel int) []Token {
|
|
|
|
c.lazyInit()
|
|
|
|
|
|
|
|
if tokenIndex < 0 || tokenIndex >= len(c.tokens) {
|
|
|
|
panic(strconv.Itoa(tokenIndex) + " not in 0.." + strconv.Itoa(len(c.tokens)-1))
|
|
|
|
}
|
|
|
|
|
|
|
|
prevOnChannel := c.previousTokenOnChannel(tokenIndex-1, LexerDefaultTokenChannel)
|
|
|
|
|
|
|
|
if prevOnChannel == tokenIndex-1 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// If there are none on channel to the left and prevOnChannel == -1 then from = 0
|
|
|
|
from := prevOnChannel + 1
|
|
|
|
to := tokenIndex - 1
|
|
|
|
|
|
|
|
return c.filterForChannel(from, to, channel)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) filterForChannel(left, right, channel int) []Token {
|
|
|
|
hidden := make([]Token, 0)
|
|
|
|
|
|
|
|
for i := left; i < right+1; i++ {
|
|
|
|
t := c.tokens[i]
|
|
|
|
|
|
|
|
if channel == -1 {
|
|
|
|
if t.GetChannel() != LexerDefaultTokenChannel {
|
|
|
|
hidden = append(hidden, t)
|
|
|
|
}
|
|
|
|
} else if t.GetChannel() == channel {
|
|
|
|
hidden = append(hidden, t)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(hidden) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return hidden
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) GetSourceName() string {
|
|
|
|
return c.tokenSource.GetSourceName()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) Size() int {
|
|
|
|
return len(c.tokens)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) Index() int {
|
|
|
|
return c.index
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) GetAllText() string {
|
2024-08-19 08:01:33 +00:00
|
|
|
c.Fill()
|
|
|
|
return c.GetTextFromInterval(NewInterval(0, len(c.tokens)-1))
|
2023-05-29 21:03:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) GetTextFromTokens(start, end Token) string {
|
|
|
|
if start == nil || end == nil {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
|
|
|
return c.GetTextFromInterval(NewInterval(start.GetTokenIndex(), end.GetTokenIndex()))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) GetTextFromRuleContext(interval RuleContext) string {
|
|
|
|
return c.GetTextFromInterval(interval.GetSourceInterval())
|
|
|
|
}
|
|
|
|
|
2024-08-19 08:01:33 +00:00
|
|
|
func (c *CommonTokenStream) GetTextFromInterval(interval Interval) string {
|
2023-05-29 21:03:29 +00:00
|
|
|
c.lazyInit()
|
2024-08-19 08:01:33 +00:00
|
|
|
c.Sync(interval.Stop)
|
2023-05-29 21:03:29 +00:00
|
|
|
|
|
|
|
start := interval.Start
|
|
|
|
stop := interval.Stop
|
|
|
|
|
|
|
|
if start < 0 || stop < 0 {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
|
|
|
if stop >= len(c.tokens) {
|
|
|
|
stop = len(c.tokens) - 1
|
|
|
|
}
|
|
|
|
|
|
|
|
s := ""
|
|
|
|
|
|
|
|
for i := start; i < stop+1; i++ {
|
|
|
|
t := c.tokens[i]
|
|
|
|
|
|
|
|
if t.GetTokenType() == TokenEOF {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
s += t.GetText()
|
|
|
|
}
|
|
|
|
|
|
|
|
return s
|
|
|
|
}
|
|
|
|
|
|
|
|
// Fill gets all tokens from the lexer until EOF.
|
|
|
|
func (c *CommonTokenStream) Fill() {
|
|
|
|
c.lazyInit()
|
|
|
|
|
|
|
|
for c.fetch(1000) == 1000 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) adjustSeekIndex(i int) int {
|
|
|
|
return c.NextTokenOnChannel(i, c.channel)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) LB(k int) Token {
|
|
|
|
if k == 0 || c.index-k < 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
i := c.index
|
|
|
|
n := 1
|
|
|
|
|
|
|
|
// Find k good tokens looking backward
|
|
|
|
for n <= k {
|
|
|
|
// Skip off-channel tokens
|
|
|
|
i = c.previousTokenOnChannel(i-1, c.channel)
|
|
|
|
n++
|
|
|
|
}
|
|
|
|
|
|
|
|
if i < 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return c.tokens[i]
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *CommonTokenStream) LT(k int) Token {
|
|
|
|
c.lazyInit()
|
|
|
|
|
|
|
|
if k == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
if k < 0 {
|
|
|
|
return c.LB(-k)
|
|
|
|
}
|
|
|
|
|
|
|
|
i := c.index
|
|
|
|
n := 1 // We know tokens[n] is valid
|
|
|
|
|
|
|
|
// Find k good tokens
|
|
|
|
for n < k {
|
|
|
|
// Skip off-channel tokens, but make sure to not look past EOF
|
|
|
|
if c.Sync(i + 1) {
|
|
|
|
i = c.NextTokenOnChannel(i+1, c.channel)
|
|
|
|
}
|
|
|
|
|
|
|
|
n++
|
|
|
|
}
|
|
|
|
|
|
|
|
return c.tokens[i]
|
|
|
|
}
|
|
|
|
|
|
|
|
// getNumberOfOnChannelTokens counts EOF once.
|
|
|
|
func (c *CommonTokenStream) getNumberOfOnChannelTokens() int {
|
|
|
|
var n int
|
|
|
|
|
|
|
|
c.Fill()
|
|
|
|
|
|
|
|
for i := 0; i < len(c.tokens); i++ {
|
|
|
|
t := c.tokens[i]
|
|
|
|
|
|
|
|
if t.GetChannel() == c.channel {
|
|
|
|
n++
|
|
|
|
}
|
|
|
|
|
|
|
|
if t.GetTokenType() == TokenEOF {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return n
|
|
|
|
}
|