# HG changeset patch
# User Aziz K?ksal <aziz.koeksal@gmail.com>
# Date 1189613909 -7200
# Node ID c4bfceab7246fcb084bfe84d7dc90ecbd8b43c45
# Parent  ae154eceba6534718f8eb5be52b17d7396650df5
Applied fixes and improvements to hex float scanner.

'0x1234i' and '0x1234Li' were wrongly matched as hex floats.
Refactored scanHexReal(). It's faster now and easier to read.
Renamed MID.HexFloatMissingExpDigits to MID.HexFloatExpMustStartWithDigit.
Renamed MID.FloatExponentDigitExpected to MID.FloatExpMustStartWidhtDigit.

Fix in scanSpecialTokenSequence(): --p must come after if statement.

diff -r ae154eceba65 -r c4bfceab7246 trunk/src/cmd/Generate.d
--- a/trunk/src/cmd/Generate.d	Wed Sep 12 13:43:45 2007 +0200
+++ b/trunk/src/cmd/Generate.d	Wed Sep 12 18:18:29 2007 +0200
@@ -477,7 +477,7 @@
     auto num = token.line_num;
     if (num is null)
     {
-      writef(token.start[0 .. token.end - token.start]);
+      writef(token.srcText);
       writef(tags[DP.HLineEnd]);
       break;
     }
diff -r ae154eceba65 -r c4bfceab7246 trunk/src/dil/Lexer.d
--- a/trunk/src/dil/Lexer.d	Wed Sep 12 13:43:45 2007 +0200
+++ b/trunk/src/dil/Lexer.d	Wed Sep 12 18:18:29 2007 +0200
@@ -1370,30 +1370,25 @@
         ulong_ += *p - 'a' + 10;
     }
 
+    assert(ishexad(p[-1]) || p[-1] == '_' || p[-1] == 'x');
+    assert(!ishexad(*p) && *p != '_');
+
     switch (*p)
     {
     case '.':
-      if (p[1] != '.')
-        goto LscanHexReal;
-      break;
-    case 'L':
-      if (p[1] != 'i')
+      if (p[1] == '.')
         break;
-    case 'i', 'p', 'P':
-      goto LscanHexReal;
+    case 'p', 'P':
+      return scanHexReal(t);
     default:
     }
+
     if (digits == 0)
       error(MID.NoDigitsInHexNumber);
     else if (digits > 16)
-    {
-      // Overflow: skip following digits.
       error(MID.OverflowHexNumber);
-      while (ishexad(*++p)) {}
-    }
+
     goto Lfinalize;
-  LscanHexReal:
-    return scanHexReal(t);
 
   LscanBin:
     assert(digits == 0);
@@ -1419,9 +1414,9 @@
 
     if (digits == 0)
       error(MID.NoDigitsInBinNumber);
+    else if (digits > 64)
+      error(MID.OverflowBinaryNumber);
 
-    if (digits > 64)
-      error(MID.OverflowBinaryNumber);
     assert(p[-1] == '0' || p[-1] == '1' || p[-1] == '_', p[-1] ~ "");
     assert( !(*p == '0' || *p == '1' || *p == '_') );
     goto Lfinalize;
@@ -1588,7 +1583,7 @@
       if (*p == '-' || *p == '+')
         ++p;
       if (!isdigit(*p))
-        error(MID.FloatExponentDigitExpected);
+        error(MID.FloatExpMustStartWithDigit);
       else
         while (isdigit(*++p) || *p == '_') {}
     }
@@ -1613,48 +1608,33 @@
 
   void scanHexReal(ref Token t)
   {
-    assert(*p == '.' || *p == 'i' || *p == 'p' || *p == 'P' || (*p == 'L' && p[1] == 'i'));
+    assert(*p == '.' || *p == 'p' || *p == 'P');
     MID mid;
     if (*p == '.')
-      while (ishexad(*++p) || *p == '_') {}
+      while (ishexad(*++p) || *p == '_')
+      {}
+    // Decimal exponent is required.
     if (*p != 'p' && *p != 'P')
     {
       mid = MID.HexFloatExponentRequired;
       goto Lerr;
     }
-    // Copy mantissa to a buffer ignoring underscores.
-    char* end = p;
-    p = t.start;
-    char[] buffer;
-    do
+    // Scan exponent
+    assert(*p == 'p' || *p == 'P');
+    if (!isdigit(*++p))
     {
-      if (*p == '_')
-      {
-        ++p;
-        continue;
-      }
-      buffer ~= *p;
-      ++p;
-    } while (p != end)
-
-    assert(p == end && (*p == 'p' || *p == 'P'));
-    // Scan and copy the exponent.
-    buffer ~= 'p';
-    size_t bufflen = buffer.length;
-    while (1)
-    {
-      if (*++p == '_')
-        continue;
-      if (isdigit(*p))
-        buffer ~= *p;
-      else
-        break;
-    }
-    // When the buffer length hasn't changed, no digits were copied.
-    if (bufflen == buffer.length) {
-      mid = MID.HexFloatMissingExpDigits;
+      mid = MID.HexFloatExpMustStartWithDigit;
       goto Lerr;
     }
+    while (isdigit(*++p) || *p == '_')
+    {}
+    // Copy whole number and remove underscores from buffer.
+    char[] buffer = t.start[0..p-t.start].dup;
+    uint j;
+    foreach (c; buffer)
+      if (c != '_')
+        buffer[j++] = c;
+    buffer.length = j; // Adjust length.
     buffer ~= 0; // Terminate for C functions.
     finalizeFloat(t, buffer);
     return;
@@ -1666,6 +1646,7 @@
 
   void finalizeFloat(ref Token t, string buffer)
   {
+    assert(buffer[$-1] == 0);
     // Float number is well-formed. Check suffixes and do conversion.
     switch (*p)
     {
@@ -1747,12 +1728,12 @@
           }
           t.line_num = new Token;
           scan(*t.line_num);
-          --p;
           if (t.line_num.type != TOK.Int32 && t.line_num.type != TOK.Uint32)
           {
             mid = MID.ExpectedIntegerAfterSTLine;
             goto Lerr;
           }
+          --p; // Go one back because scan() advanced p past the integer.
           state = State.Filespec;
         }
         else if (state == State.Filespec)
diff -r ae154eceba65 -r c4bfceab7246 trunk/src/dil/Messages.d
--- a/trunk/src/dil/Messages.d	Wed Sep 12 13:43:45 2007 +0200
+++ b/trunk/src/dil/Messages.d	Wed Sep 12 18:18:29 2007 +0200
@@ -51,8 +51,8 @@
   NoDigitsInHexNumber,
   NoDigitsInBinNumber,
   HexFloatExponentRequired,
-  HexFloatMissingExpDigits,
-  FloatExponentDigitExpected,
+  HexFloatExpMustStartWithDigit,
+  FloatExpMustStartWithDigit,
 
   // Parser messages:
   ExpectedButFound,
diff -r ae154eceba65 -r c4bfceab7246 trunk/src/lang_de.d
--- a/trunk/src/lang_de.d	Wed Sep 12 13:43:45 2007 +0200
+++ b/trunk/src/lang_de.d	Wed Sep 12 18:18:29 2007 +0200
@@ -48,7 +48,7 @@
   "ungültige Hexzahl; mindestens eine Hexziffer erforderlich.",
   "ungültige Binärzahl; mindestens eine Binärziffer erforderlich.",
   "der Exponent einer hexadezimalen Fließkommazahl ist erforderlich.",
-  "fehlende Dezimalzahlen im Exponent der hexadezimalen Fließkommazahl.",
+  "Hexadezimal-Exponenten müssen mit einer Dezimalziffer anfangen.",
   "Exponenten müssen mit einer Dezimalziffer anfangen.",
 
   // Parser messages:
diff -r ae154eceba65 -r c4bfceab7246 trunk/src/lang_en.d
--- a/trunk/src/lang_en.d	Wed Sep 12 13:43:45 2007 +0200
+++ b/trunk/src/lang_en.d	Wed Sep 12 18:18:29 2007 +0200
@@ -48,8 +48,8 @@
   "invalid hex number; at least one hex digit expected.",
   "invalid binary number; at least one binary digit expected.",
   "the exponent of a hexadecimal float number is required.",
-  "missing decimal digits in hexadecimal float exponent.",
-  "exponents have to start with a digit.",
+  "hexadecimal float exponents must start with a digit.",
+  "exponents must start with a digit.",
 
   // Parser messages
   "expected '{1}', but found '{2}'.",
diff -r ae154eceba65 -r c4bfceab7246 trunk/src/lang_fi.d
--- a/trunk/src/lang_fi.d	Wed Sep 12 13:43:45 2007 +0200
+++ b/trunk/src/lang_fi.d	Wed Sep 12 18:18:29 2007 +0200
@@ -48,7 +48,7 @@
   "virheellinen heksaluku; odotettiin vähintään yhtä heksanumeroa.",
   "virheellinen binääriluku; odotettiin vähintään yhtä binäärinumeroa.",
   "heksadesimaalisen liukuluvun eksponentti vaaditaan.",
-  "heksadesimaalisen liukuluvun eksponentista puuttui numeroita.",
+  "heksadesimaalisen liukuluvun eksponentista puuttui numeroita.", // TODO: update
   "eksponenttien tulee alkaa numerolla.",
 
   // Parser messages
diff -r ae154eceba65 -r c4bfceab7246 trunk/src/lang_tr.d
--- a/trunk/src/lang_tr.d	Wed Sep 12 13:43:45 2007 +0200
+++ b/trunk/src/lang_tr.d	Wed Sep 12 18:18:29 2007 +0200
@@ -48,7 +48,7 @@
   "geçersiz heks rakam; minimum bir heks sayı gereklidir.",
   "geçersiz binari rakam; minimum bir binari sayı gereklidir.",
   "bir heksadesimal float rakamın üsü gereklidir.",
-  "heksadesimal float rakamın üsün'de desimal sayılar eksik.",
+  "heksadesimal float üsler desimal sayı ile başlamalı.",
   "üsler desimal sayı ile başlamalı.",
 
   // Parser messages