# HG changeset patch
# User Aziz K?ksal <aziz.koeksal@gmail.com>
# Date 1197926173 -3600
# Node ID 581b3cf3fe8f69fdebc458f527f806d2ade159a7
# Parent  2a8d0ed0d71e14f7c7ea09c93ccb02d6678d4315
Added a new field to statistics generation.

diff -r 2a8d0ed0d71e -r 581b3cf3fe8f trunk/src/cmd/Statistics.d
--- a/trunk/src/cmd/Statistics.d	Mon Dec 17 17:35:38 2007 +0100
+++ b/trunk/src/cmd/Statistics.d	Mon Dec 17 22:16:13 2007 +0100
@@ -10,13 +10,14 @@
 
 struct Statistics
 {
-  uint whitespaceCount;
-  uint wsTokenCount;
-  uint keywordCount;
-  uint identCount;
-  uint numberCount;
-  uint commentCount;
-  uint linesOfCode;
+  uint whitespaceCount; /// Counter for whitespace characters.
+  uint wsTokenCount;    /// Counter for all whitespace tokens.
+  uint keywordCount;    /// Counter for keywords.
+  uint identCount;      /// Counter for identifier.
+  uint numberCount;     /// Counter for number literals.
+  uint commentCount;    /// Counter for comments.
+  uint tokenCount;      /// Counter for all tokens produced by the Lexer.
+  uint linesOfCode;     /// Number of lines.
 
   void opAddAssign(Statistics s)
   {
@@ -26,6 +27,7 @@
     this.identCount      += s.identCount;
     this.numberCount     += s.numberCount;
     this.commentCount    += s.commentCount;
+    this.tokenCount      += s.tokenCount;
     this.linesOfCode     += s.linesOfCode;
   }
 }
@@ -43,14 +45,15 @@
     total += stat;
     Stdout.formatln(
       "----\n"
-      "File: {0}\n"
-      "Whitespace character count: {1}\n"
-      "Whitespace token count: {2}\n"
-      "Keyword count: {3}\n"
-      "Identifier count: {4}\n"
-      "Number count: {5}\n"
-      "Comment count: {6}\n"
-      "Lines of code: {7}",
+      "File: {}\n"
+      "Whitespace character count: {}\n"
+      "Whitespace token count: {}\n"
+      "Keyword count: {}\n"
+      "Identifier count: {}\n"
+      "Number count: {}\n"
+      "Comment count: {}\n"
+      "All tokens count: {}\n"
+      "Lines of code: {}",
       filePaths[i],
       stat.whitespaceCount,
       stat.wsTokenCount,
@@ -58,6 +61,7 @@
       stat.identCount,
       stat.numberCount,
       stat.commentCount,
+      stat.tokenCount,
       stat.linesOfCode
     );
   }
@@ -65,20 +69,23 @@
   if (filePaths.length > 1)
     Stdout.formatln(
       "--------------------------------------------------------------------------------\n"
-      "Total:\n"
-      "Whitespace character count: {0}\n"
-      "Whitespace token count: {1}\n"
-      "Keyword count: {2}\n"
-      "Identifier count: {3}\n"
-      "Number count: {4}\n"
-      "Comment count: {5}\n"
-      "Lines of code: {6}",
+      "Total of {} files:\n"
+      "Whitespace character count: {}\n"
+      "Whitespace token count: {}\n"
+      "Keyword count: {}\n"
+      "Identifier count: {}\n"
+      "Number count: {}\n"
+      "Comment count: {}\n"
+      "All tokens count: {}\n"
+      "Lines of code: {}",
+      filePaths.length,
       total.whitespaceCount,
       total.wsTokenCount,
       total.keywordCount,
       total.identCount,
       total.numberCount,
       total.commentCount,
+      total.tokenCount,
       total.linesOfCode
     );
 }
@@ -91,13 +98,14 @@
   auto token = lx.firstToken();
 
   Statistics stats;
-
+  // Lexer creates HEAD + Newline, which are not in the source text.
+  // No token left behind!
+  stats.tokenCount = 2;
   stats.linesOfCode = lx.lineNum;
   // Traverse linked list.
-  while (token.type != TOK.EOF)
+  while (1)
   {
-    token = token.next;
-
+    stats.tokenCount += 1;
     // Count whitespace characters
     if (token.ws !is null)
       stats.whitespaceCount += token.start - token.ws;
@@ -123,6 +131,11 @@
       else if (token.isWhitespace)
         stats.wsTokenCount++;
     }
+
+    if (token.next is null)
+      break;
+    token = token.next;
   }
+  assert(token.type == TOK.EOF);
   return stats;
 }