|
129
|
1 /*******************************************************************************
|
|
|
2 * Copyright (c) 2000, 2006 IBM Corporation and others.
|
|
|
3 * All rights reserved. This program and the accompanying materials
|
|
|
4 * are made available under the terms of the Eclipse Public License v1.0
|
|
|
5 * which accompanies this distribution, and is available at
|
|
|
6 * http://www.eclipse.org/legal/epl-v10.html
|
|
|
7 *
|
|
|
8 * Contributors:
|
|
|
9 * IBM Corporation - initial API and implementation
|
|
|
10 * Port to the D programming language:
|
|
|
11 * Frank Benoit <benoit@tionex.de>
|
|
|
12 *******************************************************************************/
|
|
|
13
|
|
131
|
14
|
|
|
15 import dwtx.jface.text.rules.FastPartitioner; // packageimport
|
|
|
16 import dwtx.jface.text.rules.ITokenScanner; // packageimport
|
|
|
17 import dwtx.jface.text.rules.RuleBasedScanner; // packageimport
|
|
|
18 import dwtx.jface.text.rules.EndOfLineRule; // packageimport
|
|
|
19 import dwtx.jface.text.rules.WordRule; // packageimport
|
|
|
20 import dwtx.jface.text.rules.WhitespaceRule; // packageimport
|
|
|
21 import dwtx.jface.text.rules.WordPatternRule; // packageimport
|
|
|
22 import dwtx.jface.text.rules.IPredicateRule; // packageimport
|
|
|
23 import dwtx.jface.text.rules.DefaultPartitioner; // packageimport
|
|
|
24 import dwtx.jface.text.rules.NumberRule; // packageimport
|
|
|
25 import dwtx.jface.text.rules.SingleLineRule; // packageimport
|
|
|
26 import dwtx.jface.text.rules.PatternRule; // packageimport
|
|
|
27 import dwtx.jface.text.rules.IWordDetector; // packageimport
|
|
|
28 import dwtx.jface.text.rules.RuleBasedDamagerRepairer; // packageimport
|
|
|
29 import dwtx.jface.text.rules.ICharacterScanner; // packageimport
|
|
|
30 import dwtx.jface.text.rules.IRule; // packageimport
|
|
|
31 import dwtx.jface.text.rules.DefaultDamagerRepairer; // packageimport
|
|
|
32 import dwtx.jface.text.rules.IToken; // packageimport
|
|
|
33 import dwtx.jface.text.rules.IPartitionTokenScanner; // packageimport
|
|
|
34 import dwtx.jface.text.rules.MultiLineRule; // packageimport
|
|
|
35 import dwtx.jface.text.rules.RuleBasedPartitioner; // packageimport
|
|
|
36 import dwtx.jface.text.rules.RuleBasedPartitionScanner; // packageimport
|
|
|
37 import dwtx.jface.text.rules.BufferedRuleBasedScanner; // packageimport
|
|
|
38 import dwtx.jface.text.rules.IWhitespaceDetector; // packageimport
|
|
|
39
|
|
129
|
40 module dwtx.jface.text.rules.Token;
|
|
|
41
|
|
|
42 import dwt.dwthelper.utils;
|
|
|
43
|
|
|
44 import dwtx.core.runtime.Assert;
|
|
|
45
|
|
|
46
|
|
|
47 /**
|
|
|
48 * Standard implementation of <code>IToken</code>.
|
|
|
49 */
|
|
|
50 public class Token : IToken {
|
|
|
51
|
|
|
52 /** Internal token type: Undefined */
|
|
|
53 private static final int T_UNDEFINED= 0;
|
|
|
54 /** Internal token type: EOF */
|
|
|
55 private static final int T_EOF= 1;
|
|
|
56 /** Internal token type: Whitespace */
|
|
|
57 private static final int T_WHITESPACE= 2;
|
|
|
58 /** Internal token type: Others */
|
|
|
59 private static final int T_OTHER= 3;
|
|
|
60
|
|
|
61
|
|
|
62 /**
|
|
|
63 * Standard token: Undefined.
|
|
|
64 */
|
|
|
65 public static final IToken UNDEFINED= new Token(T_UNDEFINED);
|
|
|
66 /**
|
|
|
67 * Standard token: End Of File.
|
|
|
68 */
|
|
|
69 public static final IToken EOF= new Token(T_EOF);
|
|
|
70 /**
|
|
|
71 * Standard token: Whitespace.
|
|
|
72 */
|
|
|
73 public static final IToken WHITESPACE= new Token(T_WHITESPACE);
|
|
|
74
|
|
|
75 /**
|
|
|
76 * Standard token: Neither {@link #UNDEFINED}, {@link #WHITESPACE}, nor {@link #EOF}.
|
|
|
77 * @deprecated will be removed
|
|
|
78 */
|
|
|
79 public static final IToken OTHER= new Token(T_OTHER);
|
|
|
80
|
|
|
81 /** The type of this token */
|
|
|
82 private int fType;
|
|
|
83 /** The data associated with this token */
|
|
|
84 private Object fData;
|
|
|
85
|
|
|
86 /**
|
|
|
87 * Creates a new token according to the given specification which does not
|
|
|
88 * have any data attached to it.
|
|
|
89 *
|
|
|
90 * @param type the type of the token
|
|
|
91 * @since 2.0
|
|
|
92 */
|
|
|
93 private Token(int type) {
|
|
|
94 fType= type;
|
|
|
95 fData= null;
|
|
|
96 }
|
|
|
97
|
|
|
98 /**
|
|
|
99 * Creates a new token which represents neither undefined, whitespace, nor EOF.
|
|
|
100 * The newly created token has the given data attached to it.
|
|
|
101 *
|
|
|
102 * @param data the data attached to the newly created token
|
|
|
103 */
|
|
|
104 public Token(Object data) {
|
|
|
105 fType= T_OTHER;
|
|
|
106 fData= data;
|
|
|
107 }
|
|
|
108
|
|
|
109 /**
|
|
|
110 * Re-initializes the data of this token. The token may not represent
|
|
|
111 * undefined, whitespace, or EOF.
|
|
|
112 *
|
|
|
113 * @param data to be attached to the token
|
|
|
114 * @since 2.0
|
|
|
115 */
|
|
|
116 public void setData(Object data) {
|
|
|
117 Assert.isTrue(isOther());
|
|
|
118 fData= data;
|
|
|
119 }
|
|
|
120
|
|
|
121 /*
|
|
|
122 * @see IToken#getData()
|
|
|
123 */
|
|
|
124 public Object getData() {
|
|
|
125 return fData;
|
|
|
126 }
|
|
|
127
|
|
|
128 /*
|
|
|
129 * @see IToken#isOther()
|
|
|
130 */
|
|
|
131 public bool isOther() {
|
|
|
132 return (fType is T_OTHER);
|
|
|
133 }
|
|
|
134
|
|
|
135 /*
|
|
|
136 * @see IToken#isEOF()
|
|
|
137 */
|
|
|
138 public bool isEOF() {
|
|
|
139 return (fType is T_EOF);
|
|
|
140 }
|
|
|
141
|
|
|
142 /*
|
|
|
143 * @see IToken#isWhitespace()
|
|
|
144 */
|
|
|
145 public bool isWhitespace() {
|
|
|
146 return (fType is T_WHITESPACE);
|
|
|
147 }
|
|
|
148
|
|
|
149 /*
|
|
|
150 * @see IToken#isUndefined()
|
|
|
151 */
|
|
|
152 public bool isUndefined() {
|
|
|
153 return (fType is T_UNDEFINED);
|
|
|
154 }
|
|
|
155 }
|
