Mercurial > projects > dil
annotate trunk/src/cmd/Statistics.d @ 485:ea8c7459f1c4
Changed a lot of things in the Lexer.
Newlines are tokenized now, instead of being treated as whitespace.
Newline tokens store location info as well, which make quite a few functions
unnecessary. Added a static method getLocation() which returns a Location
instance for any given token. This will also be very useful for finding
the location of AST nodes (through Node.begin,) which is needed for reporting
parser and semantic errors and emitting documentation.
Removed rescanNewlines(), LocState, getState(), restoreState(),
evaluateHashLine() and updateErrorLoc().
Added isUnicodeNewlineChar(), isUnicodeNewline(), isNewline(), isNewlineEnd(),
isEndOfLine(), scanNewline(), getLocation() and error_().
Replaced some clunky expressions with isascii(), isNewlineEnd(), isEndOfLine(),
isUnicodeNewline(), isUnicodeNewlineChar().
Fix in scanNormalStringLiteral(): scanPostfix() must be before label Lreturn.
Fixed Lexer unittest.
Fix in parseDeclarationDefinitionsBlock(): 'token' should be 'begin'.
Added method isMultiline() to Token and added documentation comments.: File name too long
abort: file /home/aziz/dil/trunk/Changed a lot of things in the Lexer.
Newlines are tokenized now, instead of being treated as whitespace.
Newline tokens store location info as well, which make quite a few functions
unnecessary. Added a static method getLocation() which returns a Location
instance for any given token. This will also be very useful for finding
the location of AST nodes (through Node.begin,) which is needed for reporting
parser and semantic errors and emitting documentation.
Removed rescanNewlines(), LocState, getState(), restoreState(),
evaluateHashLine() and updateErrorLoc().
Added isUnicodeNewlineChar(), isUnicodeNewline(), isNewline(), isNewlineEnd(),
isEndOfLine(), scanNewline(), getLocation() and error_().
Replaced some clunky expressions with isascii(), isNewlineEnd(), isEndOfLine(),
isUnicodeNewline(), isUnicodeNewlineChar().
Fix in scanNormalStringLiteral(): scanPostfix() must be before label Lreturn.
Fixed Lexer unittest.
Fix in parseDeclarationDefinitionsBlock(): 'token' should be 'begin'.
Added method isMultiline() to Token and added documentation comments.
author | Aziz K?ksal <aziz.koeksal@gmail.com> |
---|---|
date | Fri, 30 Nov 2007 20:17:29 +0100 |
parents | fcdf7ac5ad27 |
children | 1b897a4536a4 |
rev | line source |
---|---|
363 | 1 /++ |
2 Author: Aziz Köksal | |
3 License: GPL3 | |
4 +/ | |
5 module cmd.Statistics; | |
6 import dil.Token; | |
7 import dil.File; | |
8 import dil.Lexer; | |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
364
diff
changeset
|
9 import common; |
363 | 10 |
11 struct Statistics | |
12 { | |
13 uint whitespaceCount; | |
14 uint wsTokenCount; | |
15 uint keywordCount; | |
16 uint identCount; | |
17 uint numberCount; | |
18 uint commentCount; | |
401
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
19 uint linesOfCode; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
20 |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
21 void opAddAssign(Statistics s) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
22 { |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
23 this.whitespaceCount += s.whitespaceCount; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
24 this.wsTokenCount += s.wsTokenCount; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
25 this.keywordCount += s.keywordCount; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
26 this.identCount += s.identCount; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
27 this.numberCount += s.numberCount; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
28 this.commentCount += s.commentCount; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
29 this.linesOfCode += s.linesOfCode; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
30 } |
363 | 31 } |
32 | |
401
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
33 void execute(string[] filePaths) |
363 | 34 { |
401
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
35 Statistics[] stats; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
36 foreach (filePath; filePaths) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
37 stats ~= getStatistics(filePath); |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
38 |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
39 Statistics total; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
40 |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
41 foreach (i, ref stat; stats) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
42 { |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
43 total += stat; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
44 Stdout.formatln( |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
45 "----\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
46 "File: {0}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
47 "Whitespace character count: {1}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
48 "Whitespace token count: {2}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
49 "Keyword count: {3}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
50 "Identifier count: {4}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
51 "Number count: {5}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
52 "Comment count: {6}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
53 "Lines of code: {7}", |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
54 filePaths[i], |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
55 stat.whitespaceCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
56 stat.wsTokenCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
57 stat.keywordCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
58 stat.identCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
59 stat.numberCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
60 stat.commentCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
61 stat.linesOfCode |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
62 ); |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
63 } |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
64 |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
65 if (filePaths.length > 1) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
66 Stdout.formatln( |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
67 "--------------------------------------------------------------------------------\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
68 "Total:\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
69 "Whitespace character count: {0}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
70 "Whitespace token count: {1}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
71 "Keyword count: {2}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
72 "Identifier count: {3}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
73 "Number count: {4}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
74 "Comment count: {5}\n" |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
75 "Lines of code: {6}", |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
76 total.whitespaceCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
77 total.wsTokenCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
78 total.keywordCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
79 total.identCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
80 total.numberCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
81 total.commentCount, |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
82 total.linesOfCode |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
83 ); |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
84 } |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
85 |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
86 Statistics getStatistics(string filePath) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
87 { |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
88 auto sourceText = loadFile(filePath); |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
89 auto lx = new Lexer(sourceText, filePath); |
363 | 90 |
91 auto token = lx.getTokens(); | |
92 | |
93 Statistics stats; | |
401
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
94 |
485
ea8c7459f1c4
Changed a lot of things in the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
401
diff
changeset
|
95 stats.linesOfCode = lx.lineNum; |
363 | 96 // Traverse linked list. |
97 while (token.type != TOK.EOF) | |
98 { | |
99 token = token.next; | |
100 | |
101 // Count whitespace characters | |
401
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
102 if (token.ws !is null) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
103 stats.whitespaceCount += countWhitespaceCharacters(token.ws, token.start); |
363 | 104 |
105 switch (token.type) | |
106 { | |
107 case TOK.Identifier: | |
108 stats.identCount++; | |
109 break; | |
110 case TOK.Comment: | |
111 stats.commentCount++; | |
112 break; | |
113 case TOK.Int32, TOK.Int64, TOK.Uint32, TOK.Uint64, | |
114 TOK.Float32, TOK.Float64, TOK.Float80, | |
115 TOK.Imaginary32, TOK.Imaginary64, TOK.Imaginary80: | |
116 stats.numberCount++; | |
117 break; | |
118 default: | |
119 if (token.isKeyword) | |
120 stats.keywordCount++; | |
121 } | |
122 | |
123 if (token.isWhitespace) | |
124 stats.wsTokenCount++; | |
125 } | |
401
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
126 return stats; |
363 | 127 } |
401
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
128 |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
129 /// Counts newlines, \t, \v, \f and ' ' as whitespace characters. |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
130 uint countWhitespaceCharacters(char* p, char* end) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
131 { |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
132 uint count; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
133 Loop: |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
134 while (1) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
135 { |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
136 switch (*p) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
137 { |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
138 case '\r': |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
139 if (p[1] == '\n') |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
140 ++p; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
141 case '\n': |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
142 ++p; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
143 ++count; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
144 break; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
145 case LS[0]: |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
146 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
147 { |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
148 p += 3; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
149 ++count; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
150 break; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
151 } |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
152 default: |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
153 if (!dil.Lexer.isspace(*p)) |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
154 break Loop; // Exit loop. |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
155 ++p; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
156 ++count; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
157 } |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
158 } |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
159 assert(p is end); |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
160 return count; |
fcdf7ac5ad27
Improved statistics generation.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
391
diff
changeset
|
161 } |