1
2
3
4
5
6
7
8
9
10
11 package org.syntax.jedit.tokenmarker;
12
13 import javax.swing.text.Segment;
14
15 import org.syntax.jedit.KeywordMap;
16 import org.syntax.jedit.SyntaxUtilities;
17
18 /***
19 * Python token marker.
20 *
21 * @author Jonathan Revusky
22 * @version $Id: PythonTokenMarker.java,v 1.3 1999/12/14 04:20:35 sp Exp $
23 */
24 public class PythonTokenMarker extends TokenMarker
25 {
26 private static final byte TRIPLEQUOTE1 = Token.INTERNAL_FIRST;
27 private static final byte TRIPLEQUOTE2 = Token.INTERNAL_LAST;
28
29 public PythonTokenMarker()
30 {
31 this.keywords = getKeywords();
32 }
33
34 public byte markTokensImpl( byte token, Segment line, int lineIndex )
35 {
36 char[] array = line.array;
37 int offset = line.offset;
38 lastOffset = offset;
39 lastKeyword = offset;
40 int length = line.count + offset;
41 boolean backslash = false;
42
43 loop : for( int i = offset; i < length; i++ )
44 {
45 int i1 = ( i + 1 );
46
47 char c = array[i];
48 if( c == '//' )
49 {
50 backslash = !backslash;
51 continue;
52 }
53
54 switch( token )
55 {
56 case Token.NULL :
57 switch( c )
58 {
59 case '#' :
60 if( backslash )
61 backslash = false;
62 else
63 {
64 doKeyword( line, i, c );
65 addToken( i - lastOffset, token );
66 addToken( length - i, Token.COMMENT1 );
67 lastOffset = lastKeyword = length;
68 break loop;
69 }
70 break;
71 case '"' :
72 doKeyword( line, i, c );
73 if( backslash )
74 backslash = false;
75 else
76 {
77 addToken( i - lastOffset, token );
78 if( SyntaxUtilities.regionMatches( false, line, i1, "\"\"" ) )
79 {
80 token = TRIPLEQUOTE1;
81 }
82 else
83 {
84 token = Token.LITERAL1;
85 }
86 lastOffset = lastKeyword = i;
87 }
88 break;
89 case '\'' :
90 doKeyword( line, i, c );
91 if( backslash )
92 backslash = false;
93 else
94 {
95 addToken( i - lastOffset, token );
96 if( SyntaxUtilities.regionMatches( false, line, i1, "''" ) )
97 {
98 token = TRIPLEQUOTE2;
99 }
100 else
101 {
102 token = Token.LITERAL2;
103 }
104 lastOffset = lastKeyword = i;
105 }
106 break;
107 default :
108 backslash = false;
109 if( !Character.isLetterOrDigit( c ) && c != '_' )
110 doKeyword( line, i, c );
111 break;
112 }
113 break;
114 case Token.LITERAL1 :
115 if( backslash )
116 backslash = false;
117 else if( c == '"' )
118 {
119 addToken( i1 - lastOffset, token );
120 token = Token.NULL;
121 lastOffset = lastKeyword = i1;
122 }
123 break;
124 case Token.LITERAL2 :
125 if( backslash )
126 backslash = false;
127 else if( c == '\'' )
128 {
129 addToken( i1 - lastOffset, Token.LITERAL1 );
130 token = Token.NULL;
131 lastOffset = lastKeyword = i1;
132 }
133 break;
134 case TRIPLEQUOTE1 :
135 if( backslash )
136 backslash = false;
137 else if( SyntaxUtilities.regionMatches( false, line, i, "\"\"\"" ) )
138 {
139 addToken( ( i += 4 ) - lastOffset, Token.LITERAL1 );
140 token = Token.NULL;
141 lastOffset = lastKeyword = i;
142 }
143 break;
144 case TRIPLEQUOTE2 :
145 if( backslash )
146 backslash = false;
147 else if( SyntaxUtilities.regionMatches( false, line, i, "'''" ) )
148 {
149 addToken( ( i += 4 ) - lastOffset, Token.LITERAL1 );
150 token = Token.NULL;
151 lastOffset = lastKeyword = i;
152 }
153 break;
154 default :
155 throw new InternalError( "Invalid state: " + token );
156 }
157 }
158
159 switch( token )
160 {
161 case TRIPLEQUOTE1 :
162 case TRIPLEQUOTE2 :
163 addToken( length - lastOffset, Token.LITERAL1 );
164 break;
165 case Token.NULL :
166 doKeyword( line, length, '\0' );
167 default :
168 addToken( length - lastOffset, token );
169 break;
170 }
171
172 return token;
173 }
174
175 public static KeywordMap getKeywords()
176 {
177 if( pyKeywords == null )
178 {
179 pyKeywords = new KeywordMap( false );
180 pyKeywords.add( "and", Token.KEYWORD3 );
181 pyKeywords.add( "not", Token.KEYWORD3 );
182 pyKeywords.add( "or", Token.KEYWORD3 );
183 pyKeywords.add( "if", Token.KEYWORD1 );
184 pyKeywords.add( "for", Token.KEYWORD1 );
185 pyKeywords.add( "assert", Token.KEYWORD1 );
186 pyKeywords.add( "break", Token.KEYWORD1 );
187 pyKeywords.add( "continue", Token.KEYWORD1 );
188 pyKeywords.add( "elif", Token.KEYWORD1 );
189 pyKeywords.add( "else", Token.KEYWORD1 );
190 pyKeywords.add( "except", Token.KEYWORD1 );
191 pyKeywords.add( "exec", Token.KEYWORD1 );
192 pyKeywords.add( "finally", Token.KEYWORD1 );
193 pyKeywords.add( "raise", Token.KEYWORD1 );
194 pyKeywords.add( "return", Token.KEYWORD1 );
195 pyKeywords.add( "try", Token.KEYWORD1 );
196 pyKeywords.add( "while", Token.KEYWORD1 );
197 pyKeywords.add( "def", Token.KEYWORD2 );
198 pyKeywords.add( "class", Token.KEYWORD2 );
199 pyKeywords.add( "del", Token.KEYWORD2 );
200 pyKeywords.add( "from", Token.KEYWORD2 );
201 pyKeywords.add( "global", Token.KEYWORD2 );
202 pyKeywords.add( "import", Token.KEYWORD2 );
203 pyKeywords.add( "in", Token.KEYWORD2 );
204 pyKeywords.add( "is", Token.KEYWORD2 );
205 pyKeywords.add( "lambda", Token.KEYWORD2 );
206 pyKeywords.add( "pass", Token.KEYWORD2 );
207 pyKeywords.add( "print", Token.KEYWORD2 );
208 }
209 return pyKeywords;
210 }
211
212
213 private static KeywordMap pyKeywords;
214
215 private KeywordMap keywords;
216 private int lastOffset;
217 private int lastKeyword;
218
219 private boolean doKeyword( Segment line, int i, char c )
220 {
221 int i1 = i + 1;
222
223 int len = i - lastKeyword;
224 byte id = keywords.lookup( line, lastKeyword, len );
225 if( id != Token.NULL )
226 {
227 if( lastKeyword != lastOffset )
228 addToken( lastKeyword - lastOffset, Token.NULL );
229 addToken( len, id );
230 lastOffset = i;
231 }
232 lastKeyword = i1;
233 return false;
234 }
235 }