Mercurial > projects > dil
annotate trunk/src/dil/Converter.d @ 533:2a8d0ed0d71e
Improved error reporting in dil.Converter.
Added isNewline(dchar c) to dil.LexerFuncs.
Added 4 messages to struct MSG in dil.Messages.
author | Aziz K?ksal <aziz.koeksal@gmail.com> |
---|---|
date | Mon, 17 Dec 2007 17:35:38 +0100 |
parents | 50e64bab9c7a |
children | 9e811db780a6 |
rev | line source |
---|---|
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
1 /++ |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
2 Author: Aziz Köksal |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
3 License: GPL3 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
4 +/ |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
5 module dil.Converter; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
6 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
7 import dil.Information; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
8 import dil.Location; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
9 import dil.Unicode; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
10 import dil.FileBOM; |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
11 import dil.LexerFuncs; |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
12 import dil.Messages; |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
13 import common; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
14 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
15 /// Converts various Unicode encoding formats to UTF-8. |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
16 struct Converter |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
17 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
18 char[] filePath; /// For error messages. |
532
50e64bab9c7a
Renamed InformationManager to InfoManager.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
518
diff
changeset
|
19 InfoManager infoMan; |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
20 |
532
50e64bab9c7a
Renamed InformationManager to InfoManager.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
518
diff
changeset
|
21 static Converter opCall(char[] filePath, InfoManager infoMan) |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
22 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
23 Converter conv; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
24 conv.filePath = filePath; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
25 conv.infoMan = infoMan; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
26 return conv; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
27 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
28 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
29 dchar swapBytes(dchar c) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
30 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
31 return c = (c << 24) | |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
32 ((c >> 8) & 0xFF00) | |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
33 ((c << 8) & 0xFF0000) | |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
34 (c >> 24); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
35 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
36 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
37 wchar swapBytes(wchar c) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
38 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
39 return (c << 8) | (c >> 8); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
40 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
41 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
42 wchar BEtoMachineDword(dchar c) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
43 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
44 version(LittleEndian) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
45 return swapBytes(c); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
46 else |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
47 return c; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
48 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
49 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
50 wchar LEtoMachineDword(dchar c) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
51 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
52 version(LittleEndian) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
53 return c; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
54 else |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
55 return swapBytes(c); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
56 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
57 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
58 wchar BEtoMachineWord(wchar c) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
59 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
60 version(LittleEndian) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
61 return swapBytes(c); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
62 else |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
63 return c; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
64 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
65 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
66 wchar LEtoMachineWord(wchar c) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
67 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
68 version(LittleEndian) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
69 return c; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
70 else |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
71 return swapBytes(c); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
72 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
73 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
74 char[] UTF32toUTF8(bool isBigEndian)(ubyte[] data) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
75 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
76 if (data.length == 0) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
77 return null; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
78 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
79 char[] result; |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
80 uint lineNum = 1; |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
81 dchar[] text = cast(dchar[]) data[0 .. $-($%4)]; // Trim to multiple of 4. |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
82 foreach (dchar c; text) |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
83 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
84 static if (isBigEndian) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
85 c = BEtoMachineDword(c); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
86 else |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
87 c = LEtoMachineDword(c); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
88 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
89 if (!isValidChar(c)) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
90 { |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
91 infoMan ~= new LexerError( |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
92 new Location(filePath, lineNum), |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
93 Format(MSG.InvalidUTF32Character, c) |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
94 ); |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
95 c = REPLACEMENT_CHAR; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
96 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
97 |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
98 if (isNewline(c)) |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
99 ++lineNum; |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
100 dil.Unicode.encode(result, c); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
101 } |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
102 |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
103 if (data.length % 4) |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
104 infoMan ~= new LexerError( |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
105 new Location(filePath, lineNum), |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
106 MSG.UTF32FileMustBeDivisibleBy4 |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
107 ); |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
108 |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
109 return result; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
110 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
111 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
112 alias UTF32toUTF8!(true) UTF32BEtoUTF8; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
113 alias UTF32toUTF8!(false) UTF32LEtoUTF8; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
114 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
115 char[] UTF16toUTF8(bool isBigEndian)(ubyte[] data) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
116 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
117 if (data.length == 0) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
118 return null; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
119 |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
120 wchar[] text = cast(wchar[]) data[0 .. $-($%2)]; // Trim to multiple of two. |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
121 wchar* p = text.ptr, |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
122 end = text.ptr + text.length; |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
123 char[] result; |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
124 uint lineNum = 1; |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
125 dchar c = *p; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
126 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
127 do |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
128 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
129 static if (isBigEndian) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
130 c = BEtoMachineWord(c); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
131 else |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
132 c = LEtoMachineWord(c); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
133 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
134 if (c < 0xD800 || 0xDFFF > c) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
135 {} |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
136 else if (c <= 0xDBFF && p+1 < end) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
137 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
138 wchar c2 = p[1]; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
139 static if (isBigEndian) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
140 c2 = BEtoMachineWord(c2); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
141 else |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
142 c2 = LEtoMachineWord(c2); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
143 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
144 if (0xDC00 <= c2 && c2 <= 0xDFFF) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
145 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
146 c = (c - 0xD7C0) << 10; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
147 c |= (c2 & 0x3FF); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
148 ++p; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
149 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
150 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
151 else |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
152 { |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
153 infoMan ~= new LexerError( |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
154 new Location(filePath, lineNum), |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
155 Format(MSG.InvalidUTF16Character, c) |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
156 ); |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
157 c = REPLACEMENT_CHAR; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
158 } |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
159 |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
160 if (isNewline(c)) |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
161 ++lineNum; |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
162 ++p; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
163 dil.Unicode.encode(result, c); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
164 } while (p < end) |
533
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
165 |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
166 if (data.length % 2) |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
167 infoMan ~= new LexerError( |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
168 new Location(filePath, lineNum), |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
169 MSG.UTF16FileMustBeDivisibleBy2 |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
170 ); |
2a8d0ed0d71e
Improved error reporting in dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
532
diff
changeset
|
171 |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
172 return result; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
173 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
174 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
175 alias UTF16toUTF8!(true) UTF16BEtoUTF8; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
176 alias UTF16toUTF8!(false) UTF16LEtoUTF8; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
177 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
178 char[] data2UTF8(ubyte[] data) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
179 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
180 if (data.length == 0) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
181 return null; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
182 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
183 char[] text; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
184 BOM bom = tellBOM(data); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
185 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
186 switch (bom) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
187 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
188 case BOM.None: |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
189 // No BOM found. According to the specs the first character |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
190 // must be an ASCII character. |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
191 if (data.length >= 4) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
192 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
193 if (data[0..3] == cast(ubyte[3])x"00 00 00") |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
194 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
195 text = UTF32BEtoUTF8(data); // UTF-32BE: 00 00 00 XX |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
196 break; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
197 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
198 else if (data[1..4] == cast(ubyte[3])x"00 00 00") |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
199 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
200 text = UTF32LEtoUTF8(data); // UTF-32LE: XX 00 00 00 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
201 break; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
202 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
203 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
204 if (data.length >= 2) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
205 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
206 if (data[0] == 0) // UTF-16BE: 00 XX |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
207 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
208 text = UTF16BEtoUTF8(data); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
209 break; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
210 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
211 else if (data[1] == 0) // UTF-16LE: XX 00 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
212 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
213 text = UTF16LEtoUTF8(data); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
214 break; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
215 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
216 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
217 text = cast(char[])data; // UTF-8 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
218 break; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
219 case BOM.UTF8: |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
220 text = cast(char[])data[3..$]; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
221 break; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
222 case BOM.UTF16BE: |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
223 text = UTF16BEtoUTF8(data[2..$]); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
224 break; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
225 case BOM.UTF16LE: |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
226 text = UTF16LEtoUTF8(data[2..$]); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
227 break; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
228 case BOM.UTF32BE: |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
229 text = UTF32BEtoUTF8(data[4..$]); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
230 break; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
231 case BOM.UTF32LE: |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
232 text = UTF32LEtoUTF8(data[4..$]); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
233 break; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
234 default: |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
235 assert(0); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
236 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
237 return text; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
238 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
diff
changeset
|
239 } |