diff src/lexer/Lexer.d @ 207:e0551773a005

Added the correct version.
author Anders Johnsen <skabet@gmail.com>
date Tue, 12 Aug 2008 18:19:34 +0200
parents d3c148ca429b
children
line wrap: on
line diff
--- a/src/lexer/Lexer.d	Tue Aug 12 18:14:56 2008 +0200
+++ b/src/lexer/Lexer.d	Tue Aug 12 18:19:34 2008 +0200
@@ -37,7 +37,7 @@
         foreach (c; "0123456789")
             charTable[c] = CharType.Number;
 
-        foreach (c; "(){}[];:.,=!<>+-*/%\"`")
+        foreach (c; "(){}[];:.,=!<>+-*/%&\"`")
             charTable[c] = CharType.Symbol;
 
         foreach (c; " \n")
@@ -67,8 +67,11 @@
         symbolFunctions['*'] = &star;
         symbolFunctions['/'] = &slash;
         symbolFunctions['%'] = &percent;
+        symbolFunctions['&'] = &and;
         symbolFunctions['"'] = &string;
         symbolFunctions['`'] = &string;
+
+        last = Token(Tok.EOF, SLoc() + 1, 0);
     }
 
     /**
@@ -80,27 +83,34 @@
       */
     Token next()
     {
+        Token res;
         switch (getNextChar)
         {
             case CharType.EOF:
-                SLoc loc;
-                return Token(Tok.EOF, loc, 0); 
+                return Token(Tok.EOF, last.location, 0); 
 
             case CharType.Whitespace:
                 position += 1;
-                return this.next;
+                res = this.next;
+                break;
 
             case CharType.Symbol:
-                return lexSymbol;
+                res = lexSymbol;
+                break;
 
             case CharType.Letter:
-                return lexLetter;
+                res = lexLetter;
+                break;
 
             case CharType.Number:
-                return lexNumber;
+                res = lexNumber;
+                break;
             case CharType.Other:
                 messages.report(UnexpectedTok, Loc(position)).fatal(ExitLevel.Lexer);
         }
+        if (res.type != Tok.EOF)
+            last = res;
+        return res;
     }
 
     /**
@@ -120,7 +130,9 @@
         return t;
     }
 
+    Token last;
 private:
+
     Token eq()
     {
         if(source[position] == '=')
@@ -188,30 +200,49 @@
     {
         if(source[position] == '=')
             return Token(Tok.Le, Loc(position++ - 1), 2);
+        if(source[position] == '<')
+            return Token(Tok.LeftShift, Loc(position++ - 1), 2);
         return Token(Tok.Lt, Loc(position - 1), 1);
     }
     Token ge() 
     {
         if(source[position] == '=')
             return Token(Tok.Ge, Loc(position++ - 1), 2);
+        if(source[position] == '>')
+            if(source[position+1] == '>')
+            {
+                position += 2;
+                return Token(Tok.UnsignedRightShift, Loc(position - 1), 3);
+            }
+            else
+                return Token(Tok.RightShift, Loc(position++ - 1), 2);
         return Token(Tok.Gt, Loc(position - 1), 1);
     }
     Token plus() 
     {
+        if(source[position] == '=')
+            return Token(Tok.PlusAssign, Loc(position++ - 1), 2);
         return Token(Tok.Plus, Loc(position - 1), 1);
     }
     Token minus() 
     {
+        if(source[position] == '=')
+            return Token(Tok.MinusAssign, Loc(position++ - 1), 2);
         return Token(Tok.Minus, Loc(position - 1), 1);
     }
     Token star() 
     {
+        if(source[position] == '=')
+            return Token(Tok.StarAssign, Loc(position++ - 1), 2);
         return Token(Tok.Star, Loc(position - 1), 1);
     }
     Token slash()
     {
+        int p = position;
         switch(source[position])
         {
+            case '=':
+                return Token(Tok.SlashAssign, Loc(position++ - 1), 2);
             case '/':
                 while(getNextChar != CharType.EOF)
                 {
@@ -231,7 +262,7 @@
                             return this.next;
                         }
                 }
-                messages.report(UnexpectedEOFBlock,Loc(position));
+                messages.report(UnexpectedEOFBlock,Loc(p)).fatal(ExitLevel.Lexer);
 
             case '+':
                 position += 2;
@@ -256,15 +287,22 @@
                     if(nesting == 0)
                         return this.next;
                 }
-                messages.report(UnexpectedEOFBlock,Loc(position));
+                messages.report(
+                        UnexpectedEOFBlock,
+                        Loc(p)).fatal(ExitLevel.Lexer);
 
             default:
                 return Token(Tok.Slash, Loc(position - 1), 1);
         }
     }
-
+    Token and()
+    {
+        return Token(Tok.And, Loc(position - 1), 1);
+    }
     Token percent() 
     {
+        if(source[position] == '=')
+            return Token(Tok.PercentAssign, Loc(position++ - 1), 2);
         return Token(Tok.Percent, Loc(position - 1), 1);
     }
 
@@ -289,7 +327,15 @@
                 {
                     ++position;
                     if (source[position-1] == '"' )
+                    {
+                        if(getNextChar != CharType.EOF)
+                            if (source[position] == 'c' ||
+                                source[position] == 'w' ||
+                                source[position] == 'd')
+                                position++;
+
                         return Token(Tok.String, Loc(start), position - start);
+                    }
                     else if (source[position-1] == '\\')
                         position++;
                 }
@@ -310,12 +356,11 @@
     
     Token lexNumber ()
     {
-        bool sign = false;
-        bool dot = false;
-        bool e = false;
+        bool sign;
 
         int i = 0;
 
+
         bool end = false;
         while(!end)
         {
@@ -326,11 +371,15 @@
                 case CharType.Symbol:
                     if(this.source[position+i] == '.')
                     {
-                        if(dot)
-                            messages.report(OnlyOneDotFloating, Loc(position + i));
-                        dot = true;
                         break;
                     }
+                    if (this.source[position+i] == '+' ||
+                        this.source[position+i] == '-')
+                    {
+                        if (source[position+i-1] == 'e' ||
+                            source[position+i-1] == 'E')
+                            break;
+                    }
                     end = true;
                     continue;
                 case CharType.Letter:
@@ -339,9 +388,6 @@
                     if (this.source[position+i] == 'e' || 
                         this.source[position+i] == 'E')
                     {
-                        if (e)
-                            messages.report(OnlyOneEFloating, Loc(position + i));
-                        e = true;
                         break;
                     }
                     end = true;
@@ -354,6 +400,13 @@
             i++;
         }
 
+        while(source[position+i] == 'u' ||
+              source[position+i] == 'U' ||
+              source[position+i] == 'L')
+            i += 1;
+
+        
+
         position += i;
 
         return Token(Tok.Integer, Loc(position - i), i);