Mercurial > projects > dil

annotate trunk/src/Lexer.d @ 3:4bbce78bfb1e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Added TOK enum. - Added two methods to Lexer class.
author aziz
date Thu, 21 Jun 2007 18:36:04 +0000
parents 81c6cc33f5c8
children 92df59b1ec4a
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
1 /++
8ba2570de175 Initial import.
aziz
parents:
diff changeset
2 Author: Aziz Köksal
8ba2570de175 Initial import.
aziz
parents:
diff changeset
3 License: GPL2
8ba2570de175 Initial import.
aziz
parents:
diff changeset
4 +/
8ba2570de175 Initial import.
aziz
parents:
diff changeset
5 module Lexer;
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
6 import Token;
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
7 import std.stdio;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
8
8ba2570de175 Initial import.
aziz
parents:
diff changeset
9 /// ASCII character properties table.
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
10 static const int ptable[256] = [
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
11 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
12 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,23,23,23,23,23,23,23,23,22,22, 0, 0, 0, 0, 0, 0,
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
13 0,28,28,28,28,28,28,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24, 0, 0, 0, 0,16,
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
14 0,28,28,28,28,28,28,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24, 0, 0, 0, 0, 0,
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
15 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
16 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
17 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
18 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
19 ];
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
20
8ba2570de175 Initial import.
aziz
parents:
diff changeset
21 enum CProperty
8ba2570de175 Initial import.
aziz
parents:
diff changeset
22 {
1
f3cd3bfde4ba - Corrected some errors to make the file compile.
aziz
parents: 0
diff changeset
23 Octal = 1,
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
24 Digit = 1<<1,
8ba2570de175 Initial import.
aziz
parents:
diff changeset
25 Hex = 1<<2,
8ba2570de175 Initial import.
aziz
parents:
diff changeset
26 Alpha = 1<<3,
8ba2570de175 Initial import.
aziz
parents:
diff changeset
27 Identifier = 1<<4
8ba2570de175 Initial import.
aziz
parents:
diff changeset
28 }
8ba2570de175 Initial import.
aziz
parents:
diff changeset
29
8ba2570de175 Initial import.
aziz
parents:
diff changeset
30 int isoctal(char c) { return ptable[c] & CProperty.Octal; }
8ba2570de175 Initial import.
aziz
parents:
diff changeset
31 int isdigit(char c) { return ptable[c] & CProperty.Digit; }
8ba2570de175 Initial import.
aziz
parents:
diff changeset
32 int ishexad(char c) { return ptable[c] & CProperty.Hex; }
8ba2570de175 Initial import.
aziz
parents:
diff changeset
33 int isalpha(char c) { return ptable[c] & CProperty.Alpha; }
8ba2570de175 Initial import.
aziz
parents:
diff changeset
34 int isalnum(char c) { return ptable[c] & (CProperty.Alpha | CProperty.Digit); }
8ba2570de175 Initial import.
aziz
parents:
diff changeset
35 int isident(char c) { return ptable[c] & CProperty.Identifier; }
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
36 /+
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
37 static this()
8ba2570de175 Initial import.
aziz
parents:
diff changeset
38 {
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
39 // Initialize character properties table.
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
40 for (int i; i < ptable.length; ++i)
8ba2570de175 Initial import.
aziz
parents:
diff changeset
41 {
8ba2570de175 Initial import.
aziz
parents:
diff changeset
42 if ('0' <= i && i <= '7')
8ba2570de175 Initial import.
aziz
parents:
diff changeset
43 ptable[i] |= CProperty.Octal;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
44 if ('0' <= i && i <= '9')
8ba2570de175 Initial import.
aziz
parents:
diff changeset
45 ptable[i] |= CProperty.Digit;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
46 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F')
8ba2570de175 Initial import.
aziz
parents:
diff changeset
47 ptable[i] |= CProperty.Hex;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
48 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z')
8ba2570de175 Initial import.
aziz
parents:
diff changeset
49 ptable[i] |= CProperty.Alpha;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
50 if (isalnum(i) || i == '_')
1
f3cd3bfde4ba - Corrected some errors to make the file compile.
aziz
parents: 0
diff changeset
51 ptable[i] |= CProperty.Identifier;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
52 }
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
53 // Print a formatted array literal.
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
54 char[] array = "[\n";
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
55 for (int i; i < ptable.length; ++i)
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
56 {
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
57 char c = ptable[i];
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
58 array ~= std.string.format("%2d,", c, ((i+1) % 32) ? "":"\n");
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
59 }
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
60 array.length = array.length - 2; // remove ",\n"
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
61 array ~= "\n]";
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
62 writefln(array);
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
63 }
2
81c6cc33f5c8 - Initializing ptable with a precomputed array literal.
aziz
parents: 1
diff changeset
64 +/
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
65
8ba2570de175 Initial import.
aziz
parents:
diff changeset
66 class Lexer
8ba2570de175 Initial import.
aziz
parents:
diff changeset
67 {
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
68 Token t;
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
69 public void scan(ref Token t)
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
70 {
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
71
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
72 }
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
73 public TOK nextToken()
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
74 {
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
75 return TOK.max;
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
76 }
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
77 }