View Javadoc

1   /*
2    * Token.java - Generic token
3    * Copyright (C) 1998, 1999 Slava Pestov
4    *
5    * You may use and modify this package for any purpose. Redistribution is
6    * permitted, in both source and binary form, provided that this notice
7    * remains intact in all source distributions of this package.
8    */
9   
10  package org.syntax.jedit.tokenmarker;
11  
12  /***
13   * A linked list of tokens. Each token has three fields - a token identifier,
14   * which is a byte value that can be looked up in the array returned by
15   * <code>SyntaxDocument.getColors()</code> to get a color value, a length value
16   * which is the length of the token in the text, and a pointer to the next token
17   * in the list.
18   * 
19   * @author Slava Pestov
20   * @version $Id$
21   */
22  
23  public class Token
24  {
25  	/***
26  	 * Normal text token id. This should be used to mark normal text.
27  	 */
28  	public static final byte NULL = 0;
29  
30  	/***
31  	 * Comment 1 token id. This can be used to mark a comment.
32  	 */
33  	public static final byte COMMENT1 = 1;
34  
35  	/***
36  	 * Comment 2 token id. This can be used to mark a comment.
37  	 */
38  	public static final byte COMMENT2 = 2;
39  
40  	/***
41  	 * Literal 1 token id. This can be used to mark a string literal (eg, C mode
42  	 * uses this to mark "..." literals)
43  	 */
44  	public static final byte LITERAL1 = 3;
45  
46  	/***
47  	 * Literal 2 token id. This can be used to mark an object literal (eg, Java
48  	 * mode uses this to mark true, false, etc)
49  	 */
50  	public static final byte LITERAL2 = 4;
51  
52  	/***
53  	 * Label token id. This can be used to mark labels (eg, C mode uses this to
54  	 * mark ...: sequences)
55  	 */
56  	public static final byte LABEL = 5;
57  
58  	/***
59  	 * Keyword 1 token id. This can be used to mark a keyword. This should be
60  	 * used for general language constructs.
61  	 */
62  	public static final byte KEYWORD1 = 6;
63  
64  	/***
65  	 * Keyword 2 token id. This can be used to mark a keyword. This should be
66  	 * used for preprocessor commands, or variables.
67  	 */
68  	public static final byte KEYWORD2 = 7;
69  
70  	/***
71  	 * Keyword 3 token id. This can be used to mark a keyword. This should be
72  	 * used for data types.
73  	 */
74  	public static final byte KEYWORD3 = 8;
75  
76  	/***
77  	 * Operator token id. This can be used to mark an operator. (eg, SQL mode
78  	 * marks +, -, etc with this token type)
79  	 */
80  	public static final byte OPERATOR = 9;
81  
82  	/***
83  	 * Invalid token id. This can be used to mark invalid or incomplete tokens,
84  	 * so the user can easily spot syntax errors.
85  	 */
86  	public static final byte INVALID = 10;
87  
88  	/***
89  	 * The total number of defined token ids.
90  	 */
91  	public static final byte ID_COUNT = 11;
92  
93  	/***
94  	 * The first id that can be used for internal state in a token marker.
95  	 */
96  	public static final byte INTERNAL_FIRST = 100;
97  
98  	/***
99  	 * The last id that can be used for internal state in a token marker.
100 	 */
101 	public static final byte INTERNAL_LAST = 126;
102 
103 	/***
104 	 * The token type, that along with a length of 0 marks the end of the token
105 	 * list.
106 	 */
107 	public static final byte END = 127;
108 
109 	/***
110 	 * The length of this token.
111 	 */
112 	public int length;
113 
114 	/***
115 	 * The id of this token.
116 	 */
117 	public byte id;
118 
119 	/***
120 	 * The next token in the linked list.
121 	 */
122 	public Token next;
123 
124 	/***
125 	 * Creates a new token.
126 	 * 
127 	 * @param length
128 	 *           The length of the token
129 	 * @param id
130 	 *           The id of the token
131 	 */
132 	public Token( int length, byte id )
133 	{
134 		this.length = length;
135 		this.id = id;
136 	}
137 
138 	/***
139 	 * Returns a string representation of this token.
140 	 */
141 	public String toString()
142 	{
143 		return "[id=" + id + ",length=" + length + "]";
144 	}
145 }