View Javadoc

1   /*
2    * PythonTokenMarker.java - Python token marker
3    * Copyright (C) 1999 Jonathan Revusky
4    * Copyright (C) 1998, 1999 Slava Pestov
5    *
6    * You may use and modify this package for any purpose. Redistribution is
7    * permitted, in both source and binary form, provided that this notice
8    * remains intact in all source distributions of this package.
9    */
10  
11  package org.syntax.jedit.tokenmarker;
12  
13  import javax.swing.text.Segment;
14  
15  import org.syntax.jedit.KeywordMap;
16  import org.syntax.jedit.SyntaxUtilities;
17  
18  /***
19   * Python token marker.
20   * 
21   * @author Jonathan Revusky
22   * @version $Id: PythonTokenMarker.java,v 1.3 1999/12/14 04:20:35 sp Exp $
23   */
24  public class PythonTokenMarker extends TokenMarker
25  {
26  	private static final byte TRIPLEQUOTE1 = Token.INTERNAL_FIRST;
27  	private static final byte TRIPLEQUOTE2 = Token.INTERNAL_LAST;
28  
29  	public PythonTokenMarker()
30  	{
31  		this.keywords = getKeywords();
32  	}
33  
34  	public byte markTokensImpl( byte token, Segment line, int lineIndex )
35  	{
36  		char[] array = line.array;
37  		int offset = line.offset;
38  		lastOffset = offset;
39  		lastKeyword = offset;
40  		int length = line.count + offset;
41  		boolean backslash = false;
42  
43  		loop : for( int i = offset; i < length; i++ )
44  		{
45  			int i1 = ( i + 1 );
46  
47  			char c = array[i];
48  			if( c == '//' )
49  			{
50  				backslash = !backslash;
51  				continue;
52  			}
53  
54  			switch( token )
55  			{
56  			case Token.NULL :
57  				switch( c )
58  				{
59  				case '#' :
60  					if( backslash )
61  						backslash = false;
62  					else
63  					{
64  						doKeyword( line, i, c );
65  						addToken( i - lastOffset, token );
66  						addToken( length - i, Token.COMMENT1 );
67  						lastOffset = lastKeyword = length;
68  						break loop;
69  					}
70  					break;
71  				case '"' :
72  					doKeyword( line, i, c );
73  					if( backslash )
74  						backslash = false;
75  					else
76  					{
77  						addToken( i - lastOffset, token );
78  						if( SyntaxUtilities.regionMatches( false, line, i1, "\"\"" ) )
79  						{
80  							token = TRIPLEQUOTE1;
81  						}
82  						else
83  						{
84  							token = Token.LITERAL1;
85  						}
86  						lastOffset = lastKeyword = i;
87  					}
88  					break;
89  				case '\'' :
90  					doKeyword( line, i, c );
91  					if( backslash )
92  						backslash = false;
93  					else
94  					{
95  						addToken( i - lastOffset, token );
96  						if( SyntaxUtilities.regionMatches( false, line, i1, "''" ) )
97  						{
98  							token = TRIPLEQUOTE2;
99  						}
100 						else
101 						{
102 							token = Token.LITERAL2;
103 						}
104 						lastOffset = lastKeyword = i;
105 					}
106 					break;
107 				default :
108 					backslash = false;
109 					if( !Character.isLetterOrDigit( c ) && c != '_' )
110 						doKeyword( line, i, c );
111 					break;
112 				}
113 				break;
114 			case Token.LITERAL1 :
115 				if( backslash )
116 					backslash = false;
117 				else if( c == '"' )
118 				{
119 					addToken( i1 - lastOffset, token );
120 					token = Token.NULL;
121 					lastOffset = lastKeyword = i1;
122 				}
123 				break;
124 			case Token.LITERAL2 :
125 				if( backslash )
126 					backslash = false;
127 				else if( c == '\'' )
128 				{
129 					addToken( i1 - lastOffset, Token.LITERAL1 );
130 					token = Token.NULL;
131 					lastOffset = lastKeyword = i1;
132 				}
133 				break;
134 			case TRIPLEQUOTE1 :
135 				if( backslash )
136 					backslash = false;
137 				else if( SyntaxUtilities.regionMatches( false, line, i, "\"\"\"" ) )
138 				{
139 					addToken( ( i += 4 ) - lastOffset, Token.LITERAL1 );
140 					token = Token.NULL;
141 					lastOffset = lastKeyword = i;
142 				}
143 				break;
144 			case TRIPLEQUOTE2 :
145 				if( backslash )
146 					backslash = false;
147 				else if( SyntaxUtilities.regionMatches( false, line, i, "'''" ) )
148 				{
149 					addToken( ( i += 4 ) - lastOffset, Token.LITERAL1 );
150 					token = Token.NULL;
151 					lastOffset = lastKeyword = i;
152 				}
153 				break;
154 			default :
155 				throw new InternalError( "Invalid state: " + token );
156 			}
157 		}
158 
159 		switch( token )
160 		{
161 		case TRIPLEQUOTE1 :
162 		case TRIPLEQUOTE2 :
163 			addToken( length - lastOffset, Token.LITERAL1 );
164 			break;
165 		case Token.NULL :
166 			doKeyword( line, length, '\0' );
167 		default :
168 			addToken( length - lastOffset, token );
169 			break;
170 		}
171 
172 		return token;
173 	}
174 
175 	public static KeywordMap getKeywords()
176 	{
177 		if( pyKeywords == null )
178 		{
179 			pyKeywords = new KeywordMap( false );
180 			pyKeywords.add( "and", Token.KEYWORD3 );
181 			pyKeywords.add( "not", Token.KEYWORD3 );
182 			pyKeywords.add( "or", Token.KEYWORD3 );
183 			pyKeywords.add( "if", Token.KEYWORD1 );
184 			pyKeywords.add( "for", Token.KEYWORD1 );
185 			pyKeywords.add( "assert", Token.KEYWORD1 );
186 			pyKeywords.add( "break", Token.KEYWORD1 );
187 			pyKeywords.add( "continue", Token.KEYWORD1 );
188 			pyKeywords.add( "elif", Token.KEYWORD1 );
189 			pyKeywords.add( "else", Token.KEYWORD1 );
190 			pyKeywords.add( "except", Token.KEYWORD1 );
191 			pyKeywords.add( "exec", Token.KEYWORD1 );
192 			pyKeywords.add( "finally", Token.KEYWORD1 );
193 			pyKeywords.add( "raise", Token.KEYWORD1 );
194 			pyKeywords.add( "return", Token.KEYWORD1 );
195 			pyKeywords.add( "try", Token.KEYWORD1 );
196 			pyKeywords.add( "while", Token.KEYWORD1 );
197 			pyKeywords.add( "def", Token.KEYWORD2 );
198 			pyKeywords.add( "class", Token.KEYWORD2 );
199 			pyKeywords.add( "del", Token.KEYWORD2 );
200 			pyKeywords.add( "from", Token.KEYWORD2 );
201 			pyKeywords.add( "global", Token.KEYWORD2 );
202 			pyKeywords.add( "import", Token.KEYWORD2 );
203 			pyKeywords.add( "in", Token.KEYWORD2 );
204 			pyKeywords.add( "is", Token.KEYWORD2 );
205 			pyKeywords.add( "lambda", Token.KEYWORD2 );
206 			pyKeywords.add( "pass", Token.KEYWORD2 );
207 			pyKeywords.add( "print", Token.KEYWORD2 );
208 		}
209 		return pyKeywords;
210 	}
211 
212 	// private members
213 	private static KeywordMap pyKeywords;
214 
215 	private KeywordMap keywords;
216 	private int lastOffset;
217 	private int lastKeyword;
218 
219 	private boolean doKeyword( Segment line, int i, char c )
220 	{
221 		int i1 = i + 1;
222 
223 		int len = i - lastKeyword;
224 		byte id = keywords.lookup( line, lastKeyword, len );
225 		if( id != Token.NULL )
226 		{
227 			if( lastKeyword != lastOffset )
228 				addToken( lastKeyword - lastOffset, Token.NULL );
229 			addToken( len, id );
230 			lastOffset = i;
231 		}
232 		lastKeyword = i1;
233 		return false;
234 	}
235 }