/*
* Token.cs
*
* This work is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published
* by the Free Software Foundation; either version 2 of the License,
* or (at your option) any later version.
*
* This work is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA
*
* As a special exception, the copyright holders of this library give
* you permission to link this library with independent modules to
* produce an executable, regardless of the license terms of these
* independent modules, and to copy and distribute the resulting
* executable under terms of your choice, provided that you also meet,
* for each linked independent module, the terms and conditions of the
* license of that module. An independent module is a module which is
* not derived from or based on this library. If you modify this
* library, you may extend this exception to your version of the
* library, but you are not obligated to do so. If you do not wish to
* do so, delete this exception statement from your version.
*
* Copyright (c) 2003 Per Cederberg. All rights reserved.
*/
using System.Text;
namespace PerCederberg.Grammatica.Parser {
/**
* A token node. This class represents a token (i.e. a set of adjacent
* characters) in a parse tree. The tokens are created by a tokenizer,
* that groups characters together into tokens according to a set of
* token patterns.
*
* @author Per Cederberg, <per at percederberg dot net>
* @version 1.4
*/
public class Token : Node {
/**
* The token pattern used for this token.
*/
private TokenPattern pattern;
/**
* The characters that constitute this token. This is normally
* referred to as the token image.
*/
private string image;
/**
* The line number of the first character in the token image.
*/
private int startLine;
/**
* The column number of the first character in the token image.
*/
private int startColumn;
/**
* The line number of the last character in the token image.
*/
private int endLine;
/**
* The column number of the last character in the token image.
*/
private int endColumn;
/**
* The previous token in the list of tokens.
*/
private Token previous = null;
/**
* The next token in the list of tokens.
*/
private Token next = null;
/**
* Creates a new token.
*
* @param pattern the token pattern
* @param image the token image (i.e. characters)
* @param line the line number of the first character
* @param col the column number of the first character
*/
public Token(TokenPattern pattern, string image, int line, int col) {
this.pattern = pattern;
this.image = image;
this.startLine = line;
this.startColumn = col;
this.endLine = line;
this.endColumn = col + image.Length -1;
for (int pos = 0; image.IndexOf('\n', pos) >= 0;) {
pos = image.IndexOf('\n', pos) + 1;
this.endLine++;
endColumn = image.Length -pos;
}
}
/**
* Returns the token (pattern) id. This value is set as a unique
* identifier when creating the token pattern to simplify later
* identification.
*
* @return the token id
*/
public override int GetId() {
return pattern.GetId();
}
/**
* Returns the token node name.
*
* @return the token node name
*/
public override string GetName() {
return pattern.GetName();
}
/**
* Returns the token image (i.e. the characters).
*
* @return the token characters
*/
public string GetImage() {
return image;
}
/**
* The line number of the first character in the token image.
*
* @return the line number of the first token character
*/
public override int GetStartLine() {
return startLine;
}
/**
* The column number of the first character in the token image.
*
* @return the column number of the first token character
*/
public override int GetStartColumn() {
return startColumn;
}
/**
* The line number of the last character in the token image.
*
* @return the line number of the last token character
*/
public override int GetEndLine() {
return endLine;
}
/**
* The column number of the last character in the token image.
*
* @return the column number of the last token character
*/
public override int GetEndColumn() {
return endColumn;
}
/**
* Returns the token pattern.
*
* @return the token pattern
*/
internal TokenPattern GetPattern() {
return pattern;
}
/**
* Returns the previuos token. The previous token may be a token
* that has been ignored in the parsing. Note that if the token
* list feature hasn't been used in the tokenizer, this method
* will always return null. By default the token list feature is
* not used.
*
* @return the previous token, or
* null if no such token is available
*
* @see #getNextToken
* @see Tokenizer#getUseTokenList
* @see Tokenizer#setUseTokenList
*
* @since 1.4
*/
public Token GetPreviousToken() {
return previous;
}
/**
* Sets the previous token in the token list. This method will
* also modify the token specified to have this token as it's
* next token.
*
* @param previous the previous token, or null for none
*
* @since 1.4
*/
internal void SetPreviousToken(Token previous) {
if (this.previous != null) {
this.previous.next = null;
}
this.previous = previous;
if (previous != null) {
previous.next = this;
}
}
/**
* Returns the next token. The next token may be a token that has
* been ignored in the parsing. Note that if the token list
* feature hasn't been used in the tokenizer, this method will
* always return null. By default the token list feature is not
* used.
*
* @return the next token, or
* null if no such token is available
*
* @see #getPreviousToken
* @see Tokenizer#getUseTokenList
* @see Tokenizer#setUseTokenList
*
* @since 1.4
*/
public Token GetNextToken() {
return next;
}
/**
* Sets the next token in the token list. This method will also
* modify the token specified to have this token as it's
* previous token.
*
* @param next the next token, or null for none
*
* @since 1.4
*/
internal void SetNextToken(Token next) {
if (this.next != null) {
this.next.previous = null;
}
this.next = next;
if (next != null) {
next.previous = this;
}
}
/**
* Returns a string representation of this token.
*
* @return a string representation of this token
*/
public override string ToString() {
StringBuilder buffer = new StringBuilder();
int newline = image.IndexOf('\n');
buffer.Append(pattern.GetName());
buffer.Append("(");
buffer.Append(pattern.GetId());
buffer.Append("): \"");
if (newline >= 0) {
if (newline > 0 && image[newline -1] == '\r') {
newline--;
}
buffer.Append(image.Substring(0, newline));
buffer.Append("(...)");
} else {
buffer.Append(image);
}
buffer.Append("\", line: ");
buffer.Append(startLine);
buffer.Append(", col: ");
buffer.Append(startColumn);
return buffer.ToString();
}
/**
* Returns a short string representation of this token. The
* string will only contain the token image and possibly the
* token pattern name.
*
* @return a short string representation of this token
*/
public string ToShortString() {
StringBuilder buffer = new StringBuilder();
int newline = image.IndexOf('\n');
buffer.Append('"');
if (newline >= 0) {
if (newline > 0 && image[newline -1] == '\r') {
newline--;
}
buffer.Append(image.Substring(0, newline));
buffer.Append("(...)");
} else {
buffer.Append(image);
}
buffer.Append('"');
if (pattern.GetPatternType() == TokenPattern.PatternType.REGEXP) {
buffer.Append(" <");
buffer.Append(pattern.GetName());
buffer.Append(">");
}
return buffer.ToString();
}
}
}