diff lexer/Lexer.d @ 104:7ff4bc2accf2 new_gen

Lexing more types of strings. Now all's left is to parse the string in the AST.
author Anders Johnsen <skabet@gmail.com>
date Wed, 21 May 2008 21:05:23 +0200
parents 857f0d530789
children c658172ca8a0
line wrap: on
line diff
--- a/lexer/Lexer.d	Thu May 08 10:54:29 2008 +0200
+++ b/lexer/Lexer.d	Wed May 21 21:05:23 2008 +0200
@@ -37,12 +37,15 @@
         foreach (c; "0123456789")
             charTable[c] = CharType.Number;
 
-        foreach (c; "(){}[];:.,=!<>+-*/%")
+        foreach (c; "(){}[];:.,=!<>+-*/%\"`")
             charTable[c] = CharType.Symbol;
 
         foreach (c; " \n")
             charTable[c] = CharType.Whitespace;
 
+        foreach (c; "'\\")
+            charTable[c] = CharType.Other;
+
         symbolFunctions.length = 256;
 
         symbolFunctions['('] = &openParentheses;
@@ -64,6 +67,8 @@
         symbolFunctions['*'] = &star;
         symbolFunctions['/'] = &slash;
         symbolFunctions['%'] = &percent;
+        symbolFunctions['"'] = &string;
+        symbolFunctions['`'] = &string;
     }
 
     /**
@@ -93,6 +98,8 @@
 
             case CharType.Number:
                 return lexNumber;
+            case CharType.Other:
+                messages.report(UnexpectedTok, Loc(position)).fatal(ExitLevel.Lexer);
         }
     }
 
@@ -201,7 +208,7 @@
     {
         return Token(Tok.Star, Loc(position - 1), 1);
     }
-    Token slash() 
+    Token slash()
     {
         switch(source[position])
         {
@@ -220,7 +227,9 @@
                     ++position;
                     if(source[position-2] == '*')
                         if(source[position-1] == '/')
+                        {
                             return this.next;
+                        }
                 }
                 messages.report(UnexpectedEOFBlock,Loc(position));
 
@@ -258,6 +267,46 @@
     {
         return Token(Tok.Percent, Loc(position - 1), 1);
     }
+
+    Token string()
+    {
+        --position;
+        int start = position;
+        if(getNextChar() == CharType.Letter)
+            position++;
+        char end = '`';
+        switch(source[position])
+        {
+            case '"':
+                if(position > 0)
+                    if(source[position-1] == 'r')
+                    {
+                        end = '"';
+                        goto string_wys;
+                    }
+                ++position;
+                while(getNextChar != CharType.EOF)
+                {
+                    ++position;
+                    if (source[position-1] == '"' )
+                        return Token(Tok.String, Loc(start), position - start);
+                    else if (source[position-1] == '\\')
+                        position++;
+                }
+                break;
+                case '`':
+string_wys:     
+                ++position;
+                while(getNextChar != CharType.EOF)
+                {
+                    ++position;
+                    if (source[position-1] == end )
+                        return Token(Tok.String, Loc(start), position - start);
+                }
+                break;
+        }
+        messages.report(UnexpectedEOFBlock, Loc(position)).fatal(ExitLevel.Lexer);
+    }
     
     Token lexNumber ()
     {
@@ -321,6 +370,12 @@
     {
         int i = 0;
         bool hasNumber = false;
+        if (source[position+1] == '"' ||
+            source[position+1] == '`')
+        {
+            ++position;
+            return string;
+        }
         while (getNextChar(++i) == CharType.Letter || 
                 getNextChar(i) == CharType.Number)
         {
@@ -385,6 +440,7 @@
     Number,
     Symbol,
     Whitespace,
+    Other,
 
     EOF
 }