View Javadoc

1   /*
2    * PythonTokenMarker.java - Python token marker
3    * Copyright (C) 1999 Jonathan Revusky
4    * Copyright (C) 1998, 1999 Slava Pestov
5    *
6    * You may use and modify this package for any purpose. Redistribution is
7    * permitted, in both source and binary form, provided that this notice
8    * remains intact in all source distributions of this package.
9    */
10  
11  package org.syntax.jedit.tokenmarker;
12  
13  import javax.swing.text.Segment;
14  
15  import org.syntax.jedit.KeywordMap;
16  import org.syntax.jedit.SyntaxUtilities;
17  
18  /***
19   * Python token marker.
20   *
21   * @author Jonathan Revusky
22   * @version $Id: PythonTokenMarker.java,v 1.3 1999/12/14 04:20:35 sp Exp $
23   */
24  public class PythonTokenMarker extends TokenMarker
25  {
26  	private static final byte TRIPLEQUOTE1 = Token.INTERNAL_FIRST;
27  	private static final byte TRIPLEQUOTE2 = Token.INTERNAL_LAST;
28  
29  	public PythonTokenMarker()
30  	{
31  		this.keywords = getKeywords();
32  	}
33  
34  	public byte markTokensImpl(byte token, Segment line, int lineIndex)
35  	{
36  		char[] array = line.array;
37  		int offset = line.offset;
38  		lastOffset = offset;
39  		lastKeyword = offset;
40  		int length = line.count + offset;
41  		boolean backslash = false;
42  
43  loop:		for(int i = offset; i < length; i++)
44  		{
45  			int i1 = (i+1);
46  
47  			char c = array[i];
48  			if(c == '//')
49  			{
50  				backslash = !backslash;
51  				continue;
52  			}
53  
54  			switch(token)
55  			{
56  			case Token.NULL:
57  				switch(c)
58  				{
59  				case '#':
60  					if(backslash)
61  						backslash = false;
62  					else
63  					{
64  						doKeyword(line,i,c);
65  						addToken(i - lastOffset,token);
66  						addToken(length - i,Token.COMMENT1);
67  						lastOffset = lastKeyword = length;
68  						break loop;
69  					}
70  					break;
71  				case '"':
72  					doKeyword(line,i,c);
73  					if(backslash)
74  						backslash = false;
75  					else
76  					{
77  						addToken(i - lastOffset,token);
78  						if(SyntaxUtilities.regionMatches(false,
79  							line,i1,"\"\""))
80  						{
81  							token = TRIPLEQUOTE1;
82  						}
83  						else
84  						{
85  							token = Token.LITERAL1;
86  						}
87  						lastOffset = lastKeyword = i;
88  					}
89  					break;
90  				case '\'':
91  					doKeyword(line,i,c);
92  					if(backslash)
93  						backslash = false;
94  					else
95  					{
96  						addToken(i - lastOffset,token);
97  						if(SyntaxUtilities.regionMatches(false,
98  							line,i1,"''"))
99  						{
100 							token = TRIPLEQUOTE2;
101 						}
102 						else
103 						{
104 							token = Token.LITERAL2;
105 						}
106 						lastOffset = lastKeyword = i;
107 					}
108 					break;
109 				default:
110 					backslash = false;
111 					if(!Character.isLetterOrDigit(c)
112 						&& c != '_')
113 						doKeyword(line,i,c);
114 					break;
115 				}
116 				break;
117 			case Token.LITERAL1:
118 				if(backslash)
119 					backslash = false;
120 				else if(c == '"')
121 				{
122 					addToken(i1 - lastOffset,token);
123 					token = Token.NULL;
124 					lastOffset = lastKeyword = i1;
125 				}
126 				break;
127 			case Token.LITERAL2:
128 				if(backslash)
129 					backslash = false;
130 				else if(c == '\'')
131 				{
132 					addToken(i1 - lastOffset,Token.LITERAL1);
133 					token = Token.NULL;
134 					lastOffset = lastKeyword = i1;
135 				}
136 				break;
137 			case TRIPLEQUOTE1:
138 				if(backslash)
139 					backslash = false;
140 				else if(SyntaxUtilities.regionMatches(false,
141 					line,i,"\"\"\""))
142 				{
143 					addToken((i+=4) - lastOffset,
144 						Token.LITERAL1);
145 					token = Token.NULL;
146 					lastOffset = lastKeyword = i;
147 				}
148 				break;
149 			case TRIPLEQUOTE2:
150 				if(backslash)
151 					backslash = false;
152 				else if(SyntaxUtilities.regionMatches(false,
153 					line,i,"'''"))
154 				{
155 					addToken((i+=4) - lastOffset,
156 						Token.LITERAL1);
157 					token = Token.NULL;
158 					lastOffset = lastKeyword = i;
159 				}
160 				break;
161 			default:
162 				throw new InternalError("Invalid state: "
163 					+ token);
164 			}
165 		}
166 
167 		switch(token)
168 		{
169 			case TRIPLEQUOTE1:
170 			case TRIPLEQUOTE2:
171 				addToken(length - lastOffset,Token.LITERAL1);
172 				break;
173 			case Token.NULL:
174 				doKeyword(line,length,'\0');
175 			default:
176 				addToken(length - lastOffset,token);
177 				break;
178 		}
179 
180 		return token;
181 	}
182 
183 	public static KeywordMap getKeywords()
184 	{
185 		if (pyKeywords == null)
186 		{
187 			pyKeywords = new KeywordMap(false);
188 			pyKeywords.add("and",Token.KEYWORD3);
189 			pyKeywords.add("not",Token.KEYWORD3);
190 			pyKeywords.add("or",Token.KEYWORD3);
191 			pyKeywords.add("if",Token.KEYWORD1);
192 			pyKeywords.add("for",Token.KEYWORD1);
193 			pyKeywords.add("assert",Token.KEYWORD1);
194 			pyKeywords.add("break",Token.KEYWORD1);
195 			pyKeywords.add("continue",Token.KEYWORD1);
196 			pyKeywords.add("elif",Token.KEYWORD1);
197 			pyKeywords.add("else",Token.KEYWORD1);
198 			pyKeywords.add("except",Token.KEYWORD1);
199 			pyKeywords.add("exec",Token.KEYWORD1);
200 			pyKeywords.add("finally",Token.KEYWORD1);
201 			pyKeywords.add("raise",Token.KEYWORD1);
202 			pyKeywords.add("return",Token.KEYWORD1);
203 			pyKeywords.add("try",Token.KEYWORD1);
204 			pyKeywords.add("while",Token.KEYWORD1);
205 			pyKeywords.add("def",Token.KEYWORD2);
206 			pyKeywords.add("class",Token.KEYWORD2);
207 			pyKeywords.add("del",Token.KEYWORD2);
208 			pyKeywords.add("from",Token.KEYWORD2);
209 			pyKeywords.add("global",Token.KEYWORD2);
210 			pyKeywords.add("import",Token.KEYWORD2);
211 			pyKeywords.add("in",Token.KEYWORD2);
212 			pyKeywords.add("is",Token.KEYWORD2);
213 			pyKeywords.add("lambda",Token.KEYWORD2);
214 			pyKeywords.add("pass",Token.KEYWORD2);
215 			pyKeywords.add("print",Token.KEYWORD2);
216 		}
217 		return pyKeywords;
218 	}
219 
220 	// private members
221 	private static KeywordMap pyKeywords;
222 
223 	private KeywordMap keywords;
224 	private int lastOffset;
225 	private int lastKeyword;
226 
227 	private boolean doKeyword(Segment line, int i, char c)
228 	{
229 		int i1 = i+1;
230 
231 		int len = i - lastKeyword;
232 		byte id = keywords.lookup(line,lastKeyword,len);
233 		if(id != Token.NULL)
234 		{
235 			if(lastKeyword != lastOffset)
236 				addToken(lastKeyword - lastOffset,Token.NULL);
237 			addToken(len,id);
238 			lastOffset = i;
239 		}
240 		lastKeyword = i1;
241 		return false;
242 	}
243 }