TokenSourceTokenFactory.ceylon
import org.antlr.runtime {
CommonToken,
TokenSource,
Token
}
import ceylon.collection {
LinkedList,
Queue
}
import com.redhat.ceylon.compiler.typechecker.parser {
CeylonLexer
}
"A [[TokenFactory]] implementation that also serves as a [[TokenSource]],
from which some [[org.antlr.runtime::TokenStream]] may be constructed.
The [[token index|CommonToken.tokenIndex]]
and [[start index|CommonToken.startIndex]]
is incremented with each token.
In addition, the tokens are stored in a queue, from which they may be taken
with [[nextToken]].
Usage of this class is intended to be separated into two phases:
1. Create tokens with the [[token]] method;
2. Obtain tokens with the [[nextToken]] method.
I. e., you should only construct a token stream from this token source
when you’re done with using it as a token factory.
If you mix the two usages, the token source might return an EOF token
too early."
shared class TokenSourceTokenFactory(initialIndex = 0, initialStartIndex = 0)
satisfies TokenFactory & TokenSource {
"The [[token index|CommonToken.tokenIndex]] of the first token."
Integer initialIndex;
variable Integer index = initialIndex;
"The [[start index|CommonToken.startIndex]] of the first token."
Integer initialStartIndex;
variable Integer startIndex = initialStartIndex;
Queue<CommonToken> tokens = LinkedList<CommonToken>();
shared actual CommonToken token(String text, Integer type, Integer length) {
value token = CommonToken(type, text);
token.tokenIndex = index++;
token.startIndex = startIndex;
token.stopIndex = (startIndex += length) - 1;
token.charPositionInLine = 0; // initialized to -1
tokens.offer(token);
return token;
}
Token eof => token("", CeylonLexer.\iEOF);
"Returns the next token, or an EOF token if there is no next token."
shared actual Token nextToken()
=> tokens.accept() else eof;
shared actual String sourceName => "TokenSourceTokenFactory(``initialIndex``, ``initialStartIndex``)";
}