View Javadoc

1   package org.devaki.nextobjects.ui.editor.syntax;
2   /*
3    * This program is free software; you can redistribute it and/or modify
4    * it under the terms of the GNU General Public License as published by
5    * the Free Software Foundation; either version 2 of the License, or
6    * (at your option) any later version.
7    *
8    * This program is distributed in the hope that it will be useful,
9    * but WITHOUT ANY WARRANTY; without even the implied warranty of
10   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11   * GNU General Public License for more details.
12   *
13   * See COPYING.TXT for details.
14   */
15  /***
16   * A generic token class.
17   */
18  public abstract class Token
19  {
20  
21    /***
22     * A unique ID for this type of token.
23     * Typically, ID numbers for each type will
24     * be static variables of the Token class.
25     *
26     * @return an ID for this type of token.
27     */
28    public abstract int getID();
29  
30    /***
31     * A description of this token.  The description should
32     * be appropriate for syntax highlighting.  For example
33     * "comment" might be returned for a comment.  This should
34     * make it easy to do html syntax highlighting.  Just use
35     * stylesheets to define classes with the same name as
36     * the desription and write the token in the html file
37     * with that css class name.
38     *
39     * @return a description of this token.
40     */
41    public abstract String getDescription();
42  
43    /***
44     * The actual meat of the token.
45     *
46     * @return a string representing the text of the token.
47     */
48    public abstract String getContents();
49  
50    /***
51     * Determine if this token is a comment.  Sometimes comments should be
52     * ignored (compiling code) other times they should be used
53     * (syntax highlighting).  This provides a method to check
54     * in case you feel you should ignore comments.
55     *
56     * @return true if this token represents a comment.
57     */
58    public abstract boolean isComment();
59  
60    /***
61     * Determine if this token is whitespace.  Sometimes whitespace should be
62     * ignored (compiling code) other times they should be used
63     * (code beautification).  This provides a method to check
64     * in case you feel you should ignore whitespace.
65     *
66     * @return true if this token represents whitespace.
67     */
68    public abstract boolean isWhiteSpace();
69  
70    /***
71     * Determine if this token is an error.  Lets face it, not all code
72     * conforms to spec. The lexer might know about an error
73     * if a string literal is not closed, for example.
74     *
75     * @return true if this token is an error.
76     */
77    public abstract boolean isError();
78  
79    /***
80     * get the line number of the input on which this token started
81     *
82     * @return the line number of the input on which this token started
83     */
84    public abstract int getLineNumber();
85  
86    /***
87     * get the offset into the input in characters at which this token started
88     *
89     * @return the offset into the input in characters at which this token started
90     */
91    public abstract int getCharBegin();
92  
93    /***
94     * get the offset into the input in characters at which this token ended
95     *
96     * @return the offset into the input in characters at which this token ended
97     */
98    public abstract int getCharEnd();
99  
100   /***
101    * get a String that explains the error, if this token is an error.
102    *
103    * @return a  the error, if this token is an error, null otherwise.
104    */
105   public abstract String errorString();
106 
107   /***
108    * The state of the tokenizer is undefined.
109    */
110   public static final int UNDEFINED_STATE = -1;
111 
112   /***
113    * The initial state of the tokenizer.
114    * Anytime the tokenizer returns to this state,
115    * the tokenizer could be restarted from that point
116    * with side effects.
117    */
118   public static final int INITIAL_STATE = 0;
119 
120   /***
121    * Get an integer representing the state the tokenizer is in after
122    * returning this token.
123    * Those who are interested in incremental tokenizing for performance
124    * reasons will want to use this method to figure out where the tokenizer
125    * may be restarted.  The tokenizer starts in Token.INITIAL_STATE, so
126    * any time that it reports that it has returned to this state, the
127    * tokenizer may be restarted from there.
128    * @return state
129    */
130   public abstract int getState();
131 }