1
2
3
4
5
6
7
8
9
10
11 package org.syntax.jedit.tokenmarker;
12
13 import javax.swing.text.Segment;
14
15 import org.syntax.jedit.KeywordMap;
16 import org.syntax.jedit.SyntaxUtilities;
17
18 /***
19 * Python token marker.
20 *
21 * @author Jonathan Revusky
22 * @version $Id: PythonTokenMarker.java,v 1.3 1999/12/14 04:20:35 sp Exp $
23 */
24 public class PythonTokenMarker extends TokenMarker
25 {
26 private static final byte TRIPLEQUOTE1 = Token.INTERNAL_FIRST;
27 private static final byte TRIPLEQUOTE2 = Token.INTERNAL_LAST;
28
29 public PythonTokenMarker()
30 {
31 this.keywords = getKeywords();
32 }
33
34 public byte markTokensImpl(byte token, Segment line, int lineIndex)
35 {
36 char[] array = line.array;
37 int offset = line.offset;
38 lastOffset = offset;
39 lastKeyword = offset;
40 int length = line.count + offset;
41 boolean backslash = false;
42
43 loop: for(int i = offset; i < length; i++)
44 {
45 int i1 = (i+1);
46
47 char c = array[i];
48 if(c == '//')
49 {
50 backslash = !backslash;
51 continue;
52 }
53
54 switch(token)
55 {
56 case Token.NULL:
57 switch(c)
58 {
59 case '#':
60 if(backslash)
61 backslash = false;
62 else
63 {
64 doKeyword(line,i,c);
65 addToken(i - lastOffset,token);
66 addToken(length - i,Token.COMMENT1);
67 lastOffset = lastKeyword = length;
68 break loop;
69 }
70 break;
71 case '"':
72 doKeyword(line,i,c);
73 if(backslash)
74 backslash = false;
75 else
76 {
77 addToken(i - lastOffset,token);
78 if(SyntaxUtilities.regionMatches(false,
79 line,i1,"\"\""))
80 {
81 token = TRIPLEQUOTE1;
82 }
83 else
84 {
85 token = Token.LITERAL1;
86 }
87 lastOffset = lastKeyword = i;
88 }
89 break;
90 case '\'':
91 doKeyword(line,i,c);
92 if(backslash)
93 backslash = false;
94 else
95 {
96 addToken(i - lastOffset,token);
97 if(SyntaxUtilities.regionMatches(false,
98 line,i1,"''"))
99 {
100 token = TRIPLEQUOTE2;
101 }
102 else
103 {
104 token = Token.LITERAL2;
105 }
106 lastOffset = lastKeyword = i;
107 }
108 break;
109 default:
110 backslash = false;
111 if(!Character.isLetterOrDigit(c)
112 && c != '_')
113 doKeyword(line,i,c);
114 break;
115 }
116 break;
117 case Token.LITERAL1:
118 if(backslash)
119 backslash = false;
120 else if(c == '"')
121 {
122 addToken(i1 - lastOffset,token);
123 token = Token.NULL;
124 lastOffset = lastKeyword = i1;
125 }
126 break;
127 case Token.LITERAL2:
128 if(backslash)
129 backslash = false;
130 else if(c == '\'')
131 {
132 addToken(i1 - lastOffset,Token.LITERAL1);
133 token = Token.NULL;
134 lastOffset = lastKeyword = i1;
135 }
136 break;
137 case TRIPLEQUOTE1:
138 if(backslash)
139 backslash = false;
140 else if(SyntaxUtilities.regionMatches(false,
141 line,i,"\"\"\""))
142 {
143 addToken((i+=4) - lastOffset,
144 Token.LITERAL1);
145 token = Token.NULL;
146 lastOffset = lastKeyword = i;
147 }
148 break;
149 case TRIPLEQUOTE2:
150 if(backslash)
151 backslash = false;
152 else if(SyntaxUtilities.regionMatches(false,
153 line,i,"'''"))
154 {
155 addToken((i+=4) - lastOffset,
156 Token.LITERAL1);
157 token = Token.NULL;
158 lastOffset = lastKeyword = i;
159 }
160 break;
161 default:
162 throw new InternalError("Invalid state: "
163 + token);
164 }
165 }
166
167 switch(token)
168 {
169 case TRIPLEQUOTE1:
170 case TRIPLEQUOTE2:
171 addToken(length - lastOffset,Token.LITERAL1);
172 break;
173 case Token.NULL:
174 doKeyword(line,length,'\0');
175 default:
176 addToken(length - lastOffset,token);
177 break;
178 }
179
180 return token;
181 }
182
183 public static KeywordMap getKeywords()
184 {
185 if (pyKeywords == null)
186 {
187 pyKeywords = new KeywordMap(false);
188 pyKeywords.add("and",Token.KEYWORD3);
189 pyKeywords.add("not",Token.KEYWORD3);
190 pyKeywords.add("or",Token.KEYWORD3);
191 pyKeywords.add("if",Token.KEYWORD1);
192 pyKeywords.add("for",Token.KEYWORD1);
193 pyKeywords.add("assert",Token.KEYWORD1);
194 pyKeywords.add("break",Token.KEYWORD1);
195 pyKeywords.add("continue",Token.KEYWORD1);
196 pyKeywords.add("elif",Token.KEYWORD1);
197 pyKeywords.add("else",Token.KEYWORD1);
198 pyKeywords.add("except",Token.KEYWORD1);
199 pyKeywords.add("exec",Token.KEYWORD1);
200 pyKeywords.add("finally",Token.KEYWORD1);
201 pyKeywords.add("raise",Token.KEYWORD1);
202 pyKeywords.add("return",Token.KEYWORD1);
203 pyKeywords.add("try",Token.KEYWORD1);
204 pyKeywords.add("while",Token.KEYWORD1);
205 pyKeywords.add("def",Token.KEYWORD2);
206 pyKeywords.add("class",Token.KEYWORD2);
207 pyKeywords.add("del",Token.KEYWORD2);
208 pyKeywords.add("from",Token.KEYWORD2);
209 pyKeywords.add("global",Token.KEYWORD2);
210 pyKeywords.add("import",Token.KEYWORD2);
211 pyKeywords.add("in",Token.KEYWORD2);
212 pyKeywords.add("is",Token.KEYWORD2);
213 pyKeywords.add("lambda",Token.KEYWORD2);
214 pyKeywords.add("pass",Token.KEYWORD2);
215 pyKeywords.add("print",Token.KEYWORD2);
216 }
217 return pyKeywords;
218 }
219
220
221 private static KeywordMap pyKeywords;
222
223 private KeywordMap keywords;
224 private int lastOffset;
225 private int lastKeyword;
226
227 private boolean doKeyword(Segment line, int i, char c)
228 {
229 int i1 = i+1;
230
231 int len = i - lastKeyword;
232 byte id = keywords.lookup(line,lastKeyword,len);
233 if(id != Token.NULL)
234 {
235 if(lastKeyword != lastOffset)
236 addToken(lastKeyword - lastOffset,Token.NULL);
237 addToken(len,id);
238 lastOffset = i;
239 }
240 lastKeyword = i1;
241 return false;
242 }
243 }