projects/dil: trunk/src/dil/Converter.d annotate

annotate trunk/src/dil/Converter.d @ 764:4579e8505d5e

Fixed unittests and removed dil.File. Fixed Converter.UTF16toUTF8(). Fixed an encode() function in dil.Unicode.

author	Aziz K?ksal <aziz.koeksal@gmail.com>
date	Sat, 16 Feb 2008 03:28:39 +0100
parents	90668b83ae5e
children	3b34f6a95a27

rev	line source
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	1 /++
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	2 Author: Aziz Köksal
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	3 License: GPL3
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	4 +/
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	5 module dil.Converter;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	6
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	7 import dil.Information;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	8 import dil.Location;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	9 import dil.Unicode;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	10 import dil.FileBOM;
577 9e811db780a6 Moved LexerFuncs.d to package 'lexer'. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 533 diff changeset	11 import dil.lexer.Funcs;
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	12 import dil.Messages;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	13 import common;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	14
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	15 /// Converts various Unicode encoding formats to UTF-8.
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	16 struct Converter
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	17 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	18 char[] filePath; /// For error messages.
532 50e64bab9c7a Renamed InformationManager to InfoManager. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 518 diff changeset	19 InfoManager infoMan;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	20
532 50e64bab9c7a Renamed InformationManager to InfoManager. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 518 diff changeset	21 static Converter opCall(char[] filePath, InfoManager infoMan)
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	22 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	23 Converter conv;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	24 conv.filePath = filePath;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	25 conv.infoMan = infoMan;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	26 return conv;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	27 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	28
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	29 dchar swapBytes(dchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	30 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	31 return c = (c << 24) \|
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	32 ((c >> 8) & 0xFF00) \|
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	33 ((c << 8) & 0xFF0000) \|
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	34 (c >> 24);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	35 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	36
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	37 wchar swapBytes(wchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	38 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	39 return (c << 8) \| (c >> 8);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	40 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	41
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	42 wchar BEtoMachineDword(dchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	43 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	44 version(LittleEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	45 return swapBytes(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	46 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	47 return c;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	48 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	49
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	50 wchar LEtoMachineDword(dchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	51 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	52 version(LittleEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	53 return c;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	54 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	55 return swapBytes(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	56 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	57
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	58 wchar BEtoMachineWord(wchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	59 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	60 version(LittleEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	61 return swapBytes(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	62 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	63 return c;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	64 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	65
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	66 wchar LEtoMachineWord(wchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	67 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	68 version(LittleEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	69 return c;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	70 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	71 return swapBytes(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	72 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	73
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	74 char[] UTF32toUTF8(bool isBigEndian)(ubyte[] data)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	75 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	76 if (data.length == 0)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	77 return null;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	78
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	79 char[] result;
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	80 uint lineNum = 1;
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	81 dchar[] text = cast(dchar[]) data[0 .. $-($%4)]; // Trim to multiple of 4.
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	82 foreach (dchar c; text)
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	83 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	84 static if (isBigEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	85 c = BEtoMachineDword(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	86 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	87 c = LEtoMachineDword(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	88
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	89 if (!isValidChar(c))
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	90 {
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	91 infoMan ~= new LexerError(
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	92 new Location(filePath, lineNum),
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	93 Format(MSG.InvalidUTF32Character, c)
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	94 );
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	95 c = REPLACEMENT_CHAR;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	96 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	97
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	98 if (isNewline(c))
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	99 ++lineNum;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	100 dil.Unicode.encode(result, c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	101 }
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	102
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	103 if (data.length % 4)
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	104 infoMan ~= new LexerError(
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	105 new Location(filePath, lineNum),
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	106 MSG.UTF32FileMustBeDivisibleBy4
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	107 );
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	108
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	109 return result;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	110 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	111
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	112 alias UTF32toUTF8!(true) UTF32BEtoUTF8;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	113 alias UTF32toUTF8!(false) UTF32LEtoUTF8;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	114
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	115 char[] UTF16toUTF8(bool isBigEndian)(ubyte[] data)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	116 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	117 if (data.length == 0)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	118 return null;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	119
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	120 wchar[] text = cast(wchar[]) data[0 .. $-($%2)]; // Trim to multiple of two.
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	121 wchar* p = text.ptr,
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	122 end = text.ptr + text.length;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	123 char[] result;
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	124 uint lineNum = 1;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	125
764 4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	126 for (; p < end; p++)
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	127 {
764 4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	128 dchar c = *p;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	129 static if (isBigEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	130 c = BEtoMachineWord(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	131 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	132 c = LEtoMachineWord(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	133
764 4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	134 if (0xD800 > c \|\| c > 0xDFFF)
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	135 {}
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	136 else if (c <= 0xDBFF && p+1 < end)
764 4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	137 { // Decode surrogate pairs.
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	138 wchar c2 = p[1];
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	139 static if (isBigEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	140 c2 = BEtoMachineWord(c2);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	141 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	142 c2 = LEtoMachineWord(c2);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	143
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	144 if (0xDC00 <= c2 && c2 <= 0xDFFF)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	145 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	146 c = (c - 0xD7C0) << 10;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	147 c \|= (c2 & 0x3FF);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	148 ++p;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	149 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	150 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	151 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	152 {
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	153 infoMan ~= new LexerError(
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	154 new Location(filePath, lineNum),
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	155 Format(MSG.InvalidUTF16Character, c)
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	156 );
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	157 c = REPLACEMENT_CHAR;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	158 }
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	159
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	160 if (isNewline(c))
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	161 ++lineNum;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	162 dil.Unicode.encode(result, c);
764 4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	163 }
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	164
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	165 if (data.length % 2)
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	166 infoMan ~= new LexerError(
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	167 new Location(filePath, lineNum),
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	168 MSG.UTF16FileMustBeDivisibleBy2
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	169 );
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	170 return result;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	171 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	172
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	173 alias UTF16toUTF8!(true) UTF16BEtoUTF8;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	174 alias UTF16toUTF8!(false) UTF16LEtoUTF8;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	175
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	176 char[] data2UTF8(ubyte[] data)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	177 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	178 if (data.length == 0)
755 90668b83ae5e Introduced new module dil.SourceText and class SourceText. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 739 diff changeset	179 return "";
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	180
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	181 char[] text;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	182 BOM bom = tellBOM(data);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	183
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	184 switch (bom)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	185 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	186 case BOM.None:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	187 // No BOM found. According to the specs the first character
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	188 // must be an ASCII character.
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	189 if (data.length >= 4)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	190 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	191 if (data[0..3] == cast(ubyte[3])x"00 00 00")
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	192 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	193 text = UTF32BEtoUTF8(data); // UTF-32BE: 00 00 00 XX
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	194 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	195 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	196 else if (data[1..4] == cast(ubyte[3])x"00 00 00")
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	197 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	198 text = UTF32LEtoUTF8(data); // UTF-32LE: XX 00 00 00
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	199 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	200 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	201 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	202 if (data.length >= 2)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	203 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	204 if (data[0] == 0) // UTF-16BE: 00 XX
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	205 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	206 text = UTF16BEtoUTF8(data);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	207 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	208 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	209 else if (data[1] == 0) // UTF-16LE: XX 00
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	210 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	211 text = UTF16LEtoUTF8(data);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	212 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	213 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	214 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	215 text = cast(char[])data; // UTF-8
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	216 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	217 case BOM.UTF8:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	218 text = cast(char[])data[3..$];
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	219 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	220 case BOM.UTF16BE:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	221 text = UTF16BEtoUTF8(data[2..$]);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	222 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	223 case BOM.UTF16LE:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	224 text = UTF16LEtoUTF8(data[2..$]);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	225 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	226 case BOM.UTF32BE:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	227 text = UTF32BEtoUTF8(data[4..$]);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	228 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	229 case BOM.UTF32LE:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	230 text = UTF32LEtoUTF8(data[4..$]);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	231 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	232 default:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	233 assert(0);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	234 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	235 return text;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	236 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	237 }
739 49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	238
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	239 /// Replaces invalid UTF-8 sequences with U+FFFD (if there's enough space,)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	240 /// and Newlines with '\n'.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	241 string sanitizeText(string text)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	242 {
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	243 if (!text.length)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	244 return null;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	245
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	246 char* p = text.ptr;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	247 char* end = p + text.length;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	248 char* q = p;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	249
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	250 for (; p < end; p++, q++)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	251 {
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	252 assert(q <= p);
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	253 switch (*p)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	254 {
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	255 case '\r':
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	256 if (p+1 < end && p[1] == '\n')
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	257 p++;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	258 case '\n':
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	259 *q = '\n';
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	260 continue;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	261 default:
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	262 if (isascii(*p))
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	263 break;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	264 if (p+2 < end && isUnicodeNewline(p))
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	265 {
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	266 p += 2;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	267 goto case '\n';
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	268 }
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	269 auto p2 = p; // Beginning of the UTF-8 sequence.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	270 dchar c = decode(p, end);
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	271 if (c == ERROR_CHAR)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	272 { // Skip to next ASCII character or valid UTF-8 sequence.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	273 while (++p < end && isTrailByte(*p))
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	274 {}
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	275 alias REPLACEMENT_STR R;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	276 if (q+2 < p) // Copy replacement char if there is enough space.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	277 (q = R[0]), (++q = R[1]), (*++q = R[2]);
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	278 p--;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	279 }
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	280 else
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	281 { // Copy the valid UTF-8 sequence.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	282 while (p2 <= p) // p points to the last trail byte.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	283 q++ = p2++; // Copy code units.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	284 q--;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	285 }
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	286 continue;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	287 }
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	288 assert(isascii(*p));
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	289 q = p;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	290 }
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	291 assert(p == end);
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	292 text.length = text.length - (p - q);
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	293 //text = text.ptr[0 .. q - text.ptr]; // Another way.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	294 return text;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	295 }
764 4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	296
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	297 unittest
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	298 {
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	299 Stdout("Testing function Converter.\n");
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	300 struct Data2Text
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	301 {
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	302 char[] text;
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	303 char[] expected = "source";
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	304 ubyte[] data()
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	305 { return cast(ubyte[])text; }
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	306 }
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	307 const Data2Text[] map = [
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	308 // Without BOM
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	309 {"source"},
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	310 {"s\0o\0u\0r\0c\0e\0"},
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	311 {"\0s\0o\0u\0r\0c\0e"},
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	312 {"s\0\0\0o\0\0\0u\0\0\0r\0\0\0c\0\0\0e\0\0\0"},
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	313 {"\0\0\0s\0\0\0o\0\0\0u\0\0\0r\0\0\0c\0\0\0e"},
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	314 // With BOM
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	315 {"\xEF\xBB\xBFsource"},
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	316 {"\xFE\xFF\0s\0o\0u\0r\0c\0e"},
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	317 {"\xFF\xFEs\0o\0u\0r\0c\0e\0"},
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	318 {"\x00\x00\xFE\xFF\0\0\0s\0\0\0o\0\0\0u\0\0\0r\0\0\0c\0\0\0e"},
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	319 {"\xFF\xFE\x00\x00s\0\0\0o\0\0\0u\0\0\0r\0\0\0c\0\0\0e\0\0\0"},
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	320 ];
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	321 auto converter = Converter("", new InfoManager);
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	322 foreach (i, pair; map)
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	323 assert(converter.data2UTF8(pair.data) == pair.expected, Format("failed at item {}", i));
4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 755 diff changeset	324 }

Mercurial > projects > dil

annotate trunk/src/dil/Converter.d @ 764:4579e8505d5e