# HG changeset patch # User Aziz K?ksal # Date 1199548875 -3600 # Node ID 9e811db780a63852dedf81e50d7625329dfc3031 # Parent 0df647660e761d9a559e32d12fd6058f26673592 Moved LexerFuncs.d to package 'lexer'. diff -r 0df647660e76 -r 9e811db780a6 trunk/src/dil/Converter.d --- a/trunk/src/dil/Converter.d Sat Jan 05 16:53:27 2008 +0100 +++ b/trunk/src/dil/Converter.d Sat Jan 05 17:01:15 2008 +0100 @@ -8,7 +8,7 @@ import dil.Location; import dil.Unicode; import dil.FileBOM; -import dil.LexerFuncs; +import dil.lexer.Funcs; import dil.Messages; import common; diff -r 0df647660e76 -r 9e811db780a6 trunk/src/dil/LexerFuncs.d --- a/trunk/src/dil/LexerFuncs.d Sat Jan 05 16:53:27 2008 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,82 +0,0 @@ -/++ - Author: Aziz Köksal - License: GPL3 -+/ -module dil.LexerFuncs; - -const char[3] LS = \u2028; /// Line separator. -const char[3] PS = \u2029; /// Paragraph separator. -const dchar LSd = 0x2028; -const dchar PSd = 0x2029; -static assert(LS[0] == PS[0] && LS[1] == PS[1]); - -const uint _Z_ = 26; /// Control+Z - -/// Returns true if d is a Unicode line or paragraph separator. -bool isUnicodeNewlineChar(dchar d) -{ - return d == LSd || d == PSd; -} - -/// Returns true if p points to a line or paragraph separator. -bool isUnicodeNewline(char* p) -{ - return *p == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]); -} - -/++ - Returns true if p points to the start of a Newline. - Newline: \n | \r | \r\n | LS | PS -+/ -bool isNewline(char* p) -{ - return *p == '\n' || *p == '\r' || isUnicodeNewline(p); -} - -/// Returns if c is a Newline character. -bool isNewline(dchar c) -{ - return c == '\n' || c == '\r' || isUnicodeNewlineChar(c); -} - -/++ - Returns true if p points to an EOF character. - EOF: 0 | _Z_ -+/ -bool isEOF(dchar c) -{ - return c == 0 || c == _Z_; -} - -/++ - Returns true if p points to the first character of an EndOfLine. - EndOfLine: Newline | EOF -+/ -bool isEndOfLine(char* p) -{ - return isNewline(p) || isEOF(*p); -} - -/++ - Scans a Newline and sets p one character past it. - Returns '\n' if scanned or 0 otherwise. -+/ -dchar scanNewline(ref char* p) -{ - switch (*p) - { - case '\r': - if (p[1] == '\n') - ++p; - case '\n': - ++p; - return '\n'; - default: - if (isUnicodeNewline(p)) - { - p += 3; - return '\n'; - } - } - return 0; -} diff -r 0df647660e76 -r 9e811db780a6 trunk/src/dil/Location.d --- a/trunk/src/dil/Location.d Sat Jan 05 16:53:27 2008 +0100 +++ b/trunk/src/dil/Location.d Sat Jan 05 17:01:15 2008 +0100 @@ -3,7 +3,7 @@ License: GPL3 +/ module dil.Location; -import dil.LexerFuncs; +import dil.lexer.Funcs; import dil.Unicode; import common; diff -r 0df647660e76 -r 9e811db780a6 trunk/src/dil/Token.d --- a/trunk/src/dil/Token.d Sat Jan 05 16:53:27 2008 +0100 +++ b/trunk/src/dil/Token.d Sat Jan 05 17:01:15 2008 +0100 @@ -3,8 +3,10 @@ License: GPL3 +/ module dil.Token; + import dil.Location; import dil.Identifier; +import dil.lexer.Funcs; import tango.stdc.stdlib : malloc, free; import tango.core.Exception; import common; @@ -193,7 +195,6 @@ return type == type2; } - import dil.LexerFuncs; /// Returns the Location of this token. Location getLocation() { diff -r 0df647660e76 -r 9e811db780a6 trunk/src/dil/lexer/Funcs.d --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trunk/src/dil/lexer/Funcs.d Sat Jan 05 17:01:15 2008 +0100 @@ -0,0 +1,82 @@ +/++ + Author: Aziz Köksal + License: GPL3 ++/ +module dil.lexer.Funcs; + +const char[3] LS = \u2028; /// Line separator. +const char[3] PS = \u2029; /// Paragraph separator. +const dchar LSd = 0x2028; +const dchar PSd = 0x2029; +static assert(LS[0] == PS[0] && LS[1] == PS[1]); + +const uint _Z_ = 26; /// Control+Z + +/// Returns true if d is a Unicode line or paragraph separator. +bool isUnicodeNewlineChar(dchar d) +{ + return d == LSd || d == PSd; +} + +/// Returns true if p points to a line or paragraph separator. +bool isUnicodeNewline(char* p) +{ + return *p == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]); +} + +/++ + Returns true if p points to the start of a Newline. + Newline: \n | \r | \r\n | LS | PS ++/ +bool isNewline(char* p) +{ + return *p == '\n' || *p == '\r' || isUnicodeNewline(p); +} + +/// Returns if c is a Newline character. +bool isNewline(dchar c) +{ + return c == '\n' || c == '\r' || isUnicodeNewlineChar(c); +} + +/++ + Returns true if p points to an EOF character. + EOF: 0 | _Z_ ++/ +bool isEOF(dchar c) +{ + return c == 0 || c == _Z_; +} + +/++ + Returns true if p points to the first character of an EndOfLine. + EndOfLine: Newline | EOF ++/ +bool isEndOfLine(char* p) +{ + return isNewline(p) || isEOF(*p); +} + +/++ + Scans a Newline and sets p one character past it. + Returns '\n' if scanned or 0 otherwise. ++/ +dchar scanNewline(ref char* p) +{ + switch (*p) + { + case '\r': + if (p[1] == '\n') + ++p; + case '\n': + ++p; + return '\n'; + default: + if (isUnicodeNewline(p)) + { + p += 3; + return '\n'; + } + } + return 0; +} diff -r 0df647660e76 -r 9e811db780a6 trunk/src/dil/lexer/Lexer.d --- a/trunk/src/dil/lexer/Lexer.d Sat Jan 05 16:53:27 2008 +0100 +++ b/trunk/src/dil/lexer/Lexer.d Sat Jan 05 17:01:15 2008 +0100 @@ -3,6 +3,7 @@ License: GPL3 +/ module dil.lexer.Lexer; + import dil.Token; import dil.Information; import dil.Keywords; @@ -18,7 +19,7 @@ import tango.stdc.string : strlen; import common; -public import dil.LexerFuncs; +public import dil.lexer.Funcs; /++ The Lexer analyzes the characters of a source text and