projects/dil: trunk/src/dil/Converter.d annotate

annotate trunk/src/dil/Converter.d @ 739:49fe21aa387c

Added sanitizeText() to dil.Converter. Cleaned predefined.ddoc up a bit. Removed makeString() from dil.doc.Macro. Added REPLACEMENT_STR to dil.Unicode.

author	Aziz K?ksal <aziz.koeksal@gmail.com>
date	Sat, 09 Feb 2008 14:24:35 +0100
parents	9e811db780a6
children	90668b83ae5e

rev	line source
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	1 /++
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	2 Author: Aziz Köksal
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	3 License: GPL3
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	4 +/
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	5 module dil.Converter;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	6
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	7 import dil.Information;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	8 import dil.Location;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	9 import dil.Unicode;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	10 import dil.FileBOM;
577 9e811db780a6 Moved LexerFuncs.d to package 'lexer'. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 533 diff changeset	11 import dil.lexer.Funcs;
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	12 import dil.Messages;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	13 import common;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	14
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	15 /// Converts various Unicode encoding formats to UTF-8.
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	16 struct Converter
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	17 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	18 char[] filePath; /// For error messages.
532 50e64bab9c7a Renamed InformationManager to InfoManager. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 518 diff changeset	19 InfoManager infoMan;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	20
532 50e64bab9c7a Renamed InformationManager to InfoManager. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 518 diff changeset	21 static Converter opCall(char[] filePath, InfoManager infoMan)
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	22 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	23 Converter conv;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	24 conv.filePath = filePath;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	25 conv.infoMan = infoMan;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	26 return conv;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	27 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	28
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	29 dchar swapBytes(dchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	30 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	31 return c = (c << 24) \|
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	32 ((c >> 8) & 0xFF00) \|
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	33 ((c << 8) & 0xFF0000) \|
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	34 (c >> 24);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	35 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	36
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	37 wchar swapBytes(wchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	38 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	39 return (c << 8) \| (c >> 8);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	40 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	41
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	42 wchar BEtoMachineDword(dchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	43 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	44 version(LittleEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	45 return swapBytes(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	46 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	47 return c;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	48 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	49
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	50 wchar LEtoMachineDword(dchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	51 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	52 version(LittleEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	53 return c;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	54 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	55 return swapBytes(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	56 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	57
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	58 wchar BEtoMachineWord(wchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	59 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	60 version(LittleEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	61 return swapBytes(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	62 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	63 return c;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	64 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	65
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	66 wchar LEtoMachineWord(wchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	67 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	68 version(LittleEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	69 return c;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	70 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	71 return swapBytes(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	72 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	73
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	74 char[] UTF32toUTF8(bool isBigEndian)(ubyte[] data)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	75 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	76 if (data.length == 0)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	77 return null;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	78
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	79 char[] result;
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	80 uint lineNum = 1;
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	81 dchar[] text = cast(dchar[]) data[0 .. $-($%4)]; // Trim to multiple of 4.
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	82 foreach (dchar c; text)
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	83 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	84 static if (isBigEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	85 c = BEtoMachineDword(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	86 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	87 c = LEtoMachineDword(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	88
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	89 if (!isValidChar(c))
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	90 {
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	91 infoMan ~= new LexerError(
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	92 new Location(filePath, lineNum),
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	93 Format(MSG.InvalidUTF32Character, c)
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	94 );
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	95 c = REPLACEMENT_CHAR;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	96 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	97
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	98 if (isNewline(c))
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	99 ++lineNum;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	100 dil.Unicode.encode(result, c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	101 }
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	102
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	103 if (data.length % 4)
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	104 infoMan ~= new LexerError(
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	105 new Location(filePath, lineNum),
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	106 MSG.UTF32FileMustBeDivisibleBy4
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	107 );
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	108
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	109 return result;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	110 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	111
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	112 alias UTF32toUTF8!(true) UTF32BEtoUTF8;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	113 alias UTF32toUTF8!(false) UTF32LEtoUTF8;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	114
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	115 char[] UTF16toUTF8(bool isBigEndian)(ubyte[] data)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	116 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	117 if (data.length == 0)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	118 return null;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	119
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	120 wchar[] text = cast(wchar[]) data[0 .. $-($%2)]; // Trim to multiple of two.
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	121 wchar* p = text.ptr,
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	122 end = text.ptr + text.length;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	123 char[] result;
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	124 uint lineNum = 1;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	125 dchar c = *p;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	126
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	127 do
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	128 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	129 static if (isBigEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	130 c = BEtoMachineWord(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	131 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	132 c = LEtoMachineWord(c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	133
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	134 if (c < 0xD800 \|\| 0xDFFF > c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	135 {}
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	136 else if (c <= 0xDBFF && p+1 < end)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	137 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	138 wchar c2 = p[1];
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	139 static if (isBigEndian)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	140 c2 = BEtoMachineWord(c2);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	141 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	142 c2 = LEtoMachineWord(c2);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	143
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	144 if (0xDC00 <= c2 && c2 <= 0xDFFF)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	145 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	146 c = (c - 0xD7C0) << 10;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	147 c \|= (c2 & 0x3FF);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	148 ++p;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	149 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	150 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	151 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	152 {
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	153 infoMan ~= new LexerError(
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	154 new Location(filePath, lineNum),
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	155 Format(MSG.InvalidUTF16Character, c)
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	156 );
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	157 c = REPLACEMENT_CHAR;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	158 }
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	159
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	160 if (isNewline(c))
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	161 ++lineNum;
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	162 ++p;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	163 dil.Unicode.encode(result, c);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	164 } while (p < end)
533 2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	165
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	166 if (data.length % 2)
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	167 infoMan ~= new LexerError(
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	168 new Location(filePath, lineNum),
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	169 MSG.UTF16FileMustBeDivisibleBy2
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	170 );
2a8d0ed0d71e Improved error reporting in dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 532 diff changeset	171
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	172 return result;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	173 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	174
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	175 alias UTF16toUTF8!(true) UTF16BEtoUTF8;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	176 alias UTF16toUTF8!(false) UTF16LEtoUTF8;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	177
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	178 char[] data2UTF8(ubyte[] data)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	179 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	180 if (data.length == 0)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	181 return null;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	182
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	183 char[] text;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	184 BOM bom = tellBOM(data);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	185
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	186 switch (bom)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	187 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	188 case BOM.None:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	189 // No BOM found. According to the specs the first character
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	190 // must be an ASCII character.
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	191 if (data.length >= 4)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	192 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	193 if (data[0..3] == cast(ubyte[3])x"00 00 00")
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	194 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	195 text = UTF32BEtoUTF8(data); // UTF-32BE: 00 00 00 XX
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	196 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	197 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	198 else if (data[1..4] == cast(ubyte[3])x"00 00 00")
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	199 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	200 text = UTF32LEtoUTF8(data); // UTF-32LE: XX 00 00 00
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	201 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	202 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	203 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	204 if (data.length >= 2)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	205 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	206 if (data[0] == 0) // UTF-16BE: 00 XX
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	207 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	208 text = UTF16BEtoUTF8(data);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	209 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	210 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	211 else if (data[1] == 0) // UTF-16LE: XX 00
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	212 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	213 text = UTF16LEtoUTF8(data);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	214 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	215 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	216 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	217 text = cast(char[])data; // UTF-8
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	218 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	219 case BOM.UTF8:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	220 text = cast(char[])data[3..$];
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	221 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	222 case BOM.UTF16BE:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	223 text = UTF16BEtoUTF8(data[2..$]);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	224 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	225 case BOM.UTF16LE:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	226 text = UTF16LEtoUTF8(data[2..$]);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	227 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	228 case BOM.UTF32BE:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	229 text = UTF32BEtoUTF8(data[4..$]);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	230 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	231 case BOM.UTF32LE:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	232 text = UTF32LEtoUTF8(data[4..$]);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	233 break;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	234 default:
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	235 assert(0);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	236 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	237 return text;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	238 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	239 }
739 49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	240
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	241 /// Replaces invalid UTF-8 sequences with U+FFFD (if there's enough space,)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	242 /// and Newlines with '\n'.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	243 string sanitizeText(string text)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	244 {
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	245 if (!text.length)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	246 return null;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	247
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	248 char* p = text.ptr;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	249 char* end = p + text.length;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	250 char* q = p;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	251
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	252 for (; p < end; p++, q++)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	253 {
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	254 assert(q <= p);
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	255 switch (*p)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	256 {
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	257 case '\r':
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	258 if (p+1 < end && p[1] == '\n')
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	259 p++;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	260 case '\n':
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	261 *q = '\n';
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	262 continue;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	263 default:
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	264 if (isascii(*p))
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	265 break;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	266 if (p+2 < end && isUnicodeNewline(p))
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	267 {
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	268 p += 2;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	269 goto case '\n';
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	270 }
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	271 auto p2 = p; // Beginning of the UTF-8 sequence.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	272 dchar c = decode(p, end);
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	273 if (c == ERROR_CHAR)
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	274 { // Skip to next ASCII character or valid UTF-8 sequence.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	275 while (++p < end && isTrailByte(*p))
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	276 {}
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	277 alias REPLACEMENT_STR R;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	278 if (q+2 < p) // Copy replacement char if there is enough space.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	279 (q = R[0]), (++q = R[1]), (*++q = R[2]);
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	280 p--;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	281 }
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	282 else
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	283 { // Copy the valid UTF-8 sequence.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	284 while (p2 <= p) // p points to the last trail byte.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	285 q++ = p2++; // Copy code units.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	286 q--;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	287 }
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	288 continue;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	289 }
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	290 assert(isascii(*p));
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	291 q = p;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	292 }
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	293 assert(p == end);
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	294 text.length = text.length - (p - q);
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	295 //text = text.ptr[0 .. q - text.ptr]; // Another way.
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	296 return text;
49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 577 diff changeset	297 }

Mercurial > projects > dil

annotate trunk/src/dil/Converter.d @ 739:49fe21aa387c