|
129
|
1 /*******************************************************************************
|
|
|
2 * Copyright (c) 2000, 2008 IBM Corporation and others.
|
|
|
3 * All rights reserved. This program and the accompanying materials
|
|
|
4 * are made available under the terms of the Eclipse Public License v1.0
|
|
|
5 * which accompanies this distribution, and is available at
|
|
|
6 * http://www.eclipse.org/legal/epl-v10.html
|
|
|
7 *
|
|
|
8 * Contributors:
|
|
|
9 * IBM Corporation - initial API and implementation
|
|
|
10 * Port to the D programming language:
|
|
|
11 * Frank Benoit <benoit@tionex.de>
|
|
|
12 *******************************************************************************/
|
|
|
13
|
|
|
14 module dwtx.jface.text.rules.RuleBasedScanner;
|
|
|
15
|
|
|
16 import dwt.dwthelper.utils;
|
|
|
17
|
|
|
18
|
|
|
19
|
|
|
20 import dwtx.core.runtime.Assert;
|
|
|
21 import dwtx.jface.text.BadLocationException;
|
|
|
22 import dwtx.jface.text.IDocument;
|
|
|
23
|
|
|
24
|
|
|
25 /**
|
|
|
26 * A generic scanner which can be "programmed" with a sequence of rules.
|
|
|
27 * The scanner is used to get the next token by evaluating its rule in sequence until
|
|
|
28 * one is successful. If a rule returns a token which is undefined, the scanner will proceed to
|
|
|
29 * the next rule. Otherwise the token provided by the rule will be returned by
|
|
|
30 * the scanner. If no rule returned a defined token, this scanner returns a token
|
|
|
31 * which returns <code>true</code> when calling <code>isOther</code>, unless the end
|
|
|
32 * of the file is reached. In this case the token returns <code>true</code> when calling
|
|
|
33 * <code>isEOF</code>.
|
|
|
34 *
|
|
|
35 * @see IRule
|
|
|
36 */
|
|
|
37 public class RuleBasedScanner : ICharacterScanner, ITokenScanner {
|
|
|
38
|
|
|
39 /** The list of rules of this scanner */
|
|
|
40 protected IRule[] fRules;
|
|
|
41 /** The token to be returned by default if no rule fires */
|
|
|
42 protected IToken fDefaultReturnToken;
|
|
|
43 /** The document to be scanned */
|
|
|
44 protected IDocument fDocument;
|
|
|
45 /** The cached legal line delimiters of the document */
|
|
|
46 protected char[][] fDelimiters;
|
|
|
47 /** The offset of the next character to be read */
|
|
|
48 protected int fOffset;
|
|
|
49 /** The end offset of the range to be scanned */
|
|
|
50 protected int fRangeEnd;
|
|
|
51 /** The offset of the last read token */
|
|
|
52 protected int fTokenOffset;
|
|
|
53 /** The cached column of the current scanner position */
|
|
|
54 protected int fColumn;
|
|
|
55 /** Internal setting for the un-initialized column cache. */
|
|
|
56 protected static final int UNDEFINED= -1;
|
|
|
57
|
|
|
58 /**
|
|
|
59 * Creates a new rule based scanner which does not have any rule.
|
|
|
60 */
|
|
|
61 public RuleBasedScanner() {
|
|
|
62 }
|
|
|
63
|
|
|
64 /**
|
|
|
65 * Configures the scanner with the given sequence of rules.
|
|
|
66 *
|
|
|
67 * @param rules the sequence of rules controlling this scanner
|
|
|
68 */
|
|
|
69 public void setRules(IRule[] rules) {
|
|
|
70 if (rules !is null) {
|
|
|
71 fRules= new IRule[rules.length];
|
|
|
72 System.arraycopy(rules, 0, fRules, 0, rules.length);
|
|
|
73 } else
|
|
|
74 fRules= null;
|
|
|
75 }
|
|
|
76
|
|
|
77 /**
|
|
|
78 * Configures the scanner's default return token. This is the token
|
|
|
79 * which is returned when none of the rules fired and EOF has not been
|
|
|
80 * reached.
|
|
|
81 *
|
|
|
82 * @param defaultReturnToken the default return token
|
|
|
83 * @since 2.0
|
|
|
84 */
|
|
|
85 public void setDefaultReturnToken(IToken defaultReturnToken) {
|
|
|
86 Assert.isNotNull(defaultReturnToken.getData());
|
|
|
87 fDefaultReturnToken= defaultReturnToken;
|
|
|
88 }
|
|
|
89
|
|
|
90 /*
|
|
|
91 * @see ITokenScanner#setRange(IDocument, int, int)
|
|
|
92 */
|
|
|
93 public void setRange(final IDocument document, int offset, int length) {
|
|
|
94 Assert.isLegal(document !is null);
|
|
|
95 final int documentLength= document.getLength();
|
|
|
96 checkRange(offset, length, documentLength);
|
|
|
97
|
|
|
98 fDocument= document;
|
|
|
99 fOffset= offset;
|
|
|
100 fColumn= UNDEFINED;
|
|
|
101 fRangeEnd= offset + length;
|
|
|
102
|
|
|
103 String[] delimiters= fDocument.getLegalLineDelimiters();
|
|
|
104 fDelimiters= new char[delimiters.length][];
|
|
|
105 for (int i= 0; i < delimiters.length; i++)
|
|
|
106 fDelimiters[i]= delimiters[i].toCharArray();
|
|
|
107
|
|
|
108 if (fDefaultReturnToken is null)
|
|
|
109 fDefaultReturnToken= new Token(null);
|
|
|
110 }
|
|
|
111
|
|
|
112 /**
|
|
|
113 * Checks that the given range is valid.
|
|
|
114 * See https://bugs.eclipse.org/bugs/show_bug.cgi?id=69292
|
|
|
115 *
|
|
|
116 * @param offset the offset of the document range to scan
|
|
|
117 * @param length the length of the document range to scan
|
|
|
118 * @param documentLength the document's length
|
|
|
119 * @since 3.3
|
|
|
120 */
|
|
|
121 private void checkRange(int offset, int length, int documentLength) {
|
|
|
122 Assert.isLegal(offset > -1);
|
|
|
123 Assert.isLegal(length > -1);
|
|
|
124 Assert.isLegal(offset + length <= documentLength);
|
|
|
125 }
|
|
|
126
|
|
|
127 /*
|
|
|
128 * @see ITokenScanner#getTokenOffset()
|
|
|
129 */
|
|
|
130 public int getTokenOffset() {
|
|
|
131 return fTokenOffset;
|
|
|
132 }
|
|
|
133
|
|
|
134 /*
|
|
|
135 * @see ITokenScanner#getTokenLength()
|
|
|
136 */
|
|
|
137 public int getTokenLength() {
|
|
|
138 if (fOffset < fRangeEnd)
|
|
|
139 return fOffset - getTokenOffset();
|
|
|
140 return fRangeEnd - getTokenOffset();
|
|
|
141 }
|
|
|
142
|
|
|
143
|
|
|
144 /*
|
|
|
145 * @see ICharacterScanner#getColumn()
|
|
|
146 */
|
|
|
147 public int getColumn() {
|
|
|
148 if (fColumn is UNDEFINED) {
|
|
|
149 try {
|
|
|
150 int line= fDocument.getLineOfOffset(fOffset);
|
|
|
151 int start= fDocument.getLineOffset(line);
|
|
|
152
|
|
|
153 fColumn= fOffset - start;
|
|
|
154
|
|
|
155 } catch (BadLocationException ex) {
|
|
|
156 }
|
|
|
157 }
|
|
|
158 return fColumn;
|
|
|
159 }
|
|
|
160
|
|
|
161 /*
|
|
|
162 * @see ICharacterScanner#getLegalLineDelimiters()
|
|
|
163 */
|
|
|
164 public char[][] getLegalLineDelimiters() {
|
|
|
165 return fDelimiters;
|
|
|
166 }
|
|
|
167
|
|
|
168 /*
|
|
|
169 * @see ITokenScanner#nextToken()
|
|
|
170 */
|
|
|
171 public IToken nextToken() {
|
|
|
172
|
|
|
173 fTokenOffset= fOffset;
|
|
|
174 fColumn= UNDEFINED;
|
|
|
175
|
|
|
176 if (fRules !is null) {
|
|
|
177 for (int i= 0; i < fRules.length; i++) {
|
|
|
178 IToken token= (fRules[i].evaluate(this));
|
|
|
179 if (!token.isUndefined())
|
|
|
180 return token;
|
|
|
181 }
|
|
|
182 }
|
|
|
183
|
|
|
184 if (read() is EOF)
|
|
|
185 return Token.EOF;
|
|
|
186 return fDefaultReturnToken;
|
|
|
187 }
|
|
|
188
|
|
|
189 /*
|
|
|
190 * @see ICharacterScanner#read()
|
|
|
191 */
|
|
|
192 public int read() {
|
|
|
193
|
|
|
194 try {
|
|
|
195
|
|
|
196 if (fOffset < fRangeEnd) {
|
|
|
197 try {
|
|
|
198 return fDocument.getChar(fOffset);
|
|
|
199 } catch (BadLocationException e) {
|
|
|
200 }
|
|
|
201 }
|
|
|
202
|
|
|
203 return EOF;
|
|
|
204
|
|
|
205 } finally {
|
|
|
206 ++ fOffset;
|
|
|
207 fColumn= UNDEFINED;
|
|
|
208 }
|
|
|
209 }
|
|
|
210
|
|
|
211 /*
|
|
|
212 * @see ICharacterScanner#unread()
|
|
|
213 */
|
|
|
214 public void unread() {
|
|
|
215 --fOffset;
|
|
|
216 fColumn= UNDEFINED;
|
|
|
217 }
|
|
|
218 }
|
|
|
219
|
|
|
220
|
