projects/dil: src/dil/Unicode.d annotate

annotate src/dil/Unicode.d @ 806:bcb74c9b895c

Moved out files in the trunk folder to the root.

author	Aziz K?ksal <aziz.koeksal@gmail.com>
date	Sun, 09 Mar 2008 00:12:19 +0100
parents	trunk/src/dil/Unicode.d@c1d5cfd7aa44
children

rev	line source
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	1 /++
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	2 Author: Aziz Köksal
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	3 License: GPL3
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	4 +/
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	5 module dil.Unicode;
629 d050e211402b Moved files in src/std/ to src/util/. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 518 diff changeset	6 public import util.uni : isUniAlpha;
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	7
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	8 /// U+FFFD = �. Used to replace invalid Unicode characters.
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	9 const dchar REPLACEMENT_CHAR = '\uFFFD';
739 49fe21aa387c Added sanitizeText() to dil.Converter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 737 diff changeset	10 const char[3] REPLACEMENT_STR = \uFFFD; /// Ditto
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	11 /// Invalid character, returned on errors.
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	12 const dchar ERROR_CHAR = 0xD800;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	13
769 5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	14 /// Returns: true if this character is not a surrogate
5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	15 /// code point and not higher than 0x10FFFF.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	16 bool isValidChar(dchar d)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	17 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	18 return d < 0xD800 \|\| d > 0xDFFF && d <= 0x10FFFF;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	19 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	20
769 5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	21 /// There are a total of 66 noncharacters.
5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	22 /// Returns: true if this is one of them.
5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	23 /// See_also: Chapter 16.7 Noncharacters in Unicode 5.0
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	24 bool isNoncharacter(dchar d)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	25 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	26 return 0xFDD0 <= d && d <= 0xFDEF \|\| // 32
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	27 d <= 0x10FFFF && (d & 0xFFFF) >= 0xFFFE; // 34
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	28 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	29
747 00f872d949ea Added method scanCommentText() to DDocEmitter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 741 diff changeset	30 /// Returns: true if this is a trail byte of a UTF-8 sequence.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	31 bool isTrailByte(ubyte b)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	32 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	33 return (b & 0xC0) == 0x80; // 10xx_xxxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	34 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	35
747 00f872d949ea Added method scanCommentText() to DDocEmitter. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 741 diff changeset	36 /// Returns: true if this is a lead byte of a UTF-8 sequence.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	37 bool isLeadByte(ubyte b)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	38 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	39 return (b & 0xC0) == 0xC0; // 11xx_xxxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	40 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	41
722 ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	42 /// Advances ref_p only if this is a valid Unicode alpha character.
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	43 bool isUnicodeAlpha(ref char* ref_p, char* end)
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	44 in { assert(ref_p && ref_p < end); }
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	45 body
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	46 {
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	47 if (*ref_p < 0x80)
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	48 return false;
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	49 auto p = ref_p;
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	50 auto c = decode(p, end);
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	51 if (!isUniAlpha(c))
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	52 return false;
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	53 ref_p = p;
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	54 return true;
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	55 }
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	56
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	57 /// Decodes a character from str at index.
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	58 /// Params:
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	59 /// index = set to one past the ASCII char or one past the last trail byte
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	60 /// of the valid UTF-8 sequence.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	61 dchar decode(char[] str, ref size_t index)
722 ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	62 in { assert(str.length && index < str.length); }
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	63 out { assert(index <= str.length); }
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	64 body
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	65 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	66 char* p = str.ptr + index;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	67 char* end = str.ptr + str.length;
722 ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	68 dchar c = decode(p, end);
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	69 if (c != ERROR_CHAR)
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	70 index = p - str.ptr;
722 ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	71 return c;
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	72 }
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	73
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	74 /// Decodes a character starting at ref_p.
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	75 /// Params:
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	76 /// ref_p = set to one past the ASCII char or one past the last trail byte
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	77 /// of the valid UTF-8 sequence.
722 ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	78 dchar decode(ref char* ref_p, char* end)
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	79 in { assert(ref_p && ref_p < end); }
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	80 out(c) { assert(ref_p <= end && (isValidChar(c) \|\| c == ERROR_CHAR)); }
722 ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	81 body
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	82 {
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	83 char* p = ref_p;
ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	84 dchar c = *p;
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	85
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	86 if (c < 0x80)
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	87 return ref_p++, c;
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	88
722 ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	89 p++; // Move to second byte.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	90 if (!(p < end))
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	91 return ERROR_CHAR;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	92
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	93 // Error if second byte is not a trail byte.
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	94 if (!isTrailByte(*p))
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	95 return ERROR_CHAR;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	96
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	97 // Check for overlong sequences.
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	98 switch (c)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	99 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	100 case 0xE0, // 11100000 100xxxxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	101 0xF0, // 11110000 1000xxxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	102 0xF8, // 11111000 10000xxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	103 0xFC: // 11111100 100000xx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	104 if ((*p & c) == 0x80)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	105 return ERROR_CHAR;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	106 default:
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	107 if ((c & 0xFE) == 0xC0) // 1100000x
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	108 return ERROR_CHAR;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	109 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	110
722 ceaac6a24258 Added isUnicodeAlpha() for DDocParser and MacroParser. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 629 diff changeset	111 const char[] checkNextByte = "if (!(++p < end && isTrailByte(*p)))"
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	112 " return ERROR_CHAR;";
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	113 const char[] appendSixBits = "c = (c << 6) \| *p & 0b0011_1111;";
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	114
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	115 // Decode
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	116 if ((c & 0b1110_0000) == 0b1100_0000)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	117 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	118 // 110xxxxx 10xxxxxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	119 c &= 0b0001_1111;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	120 mixin(appendSixBits);
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	121 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	122 else if ((c & 0b1111_0000) == 0b1110_0000)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	123 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	124 // 1110xxxx 10xxxxxx 10xxxxxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	125 c &= 0b0000_1111;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	126 mixin(appendSixBits ~
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	127 checkNextByte ~ appendSixBits);
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	128 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	129 else if ((c & 0b1111_1000) == 0b1111_0000)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	130 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	131 // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	132 c &= 0b0000_0111;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	133 mixin(appendSixBits ~
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	134 checkNextByte ~ appendSixBits ~
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	135 checkNextByte ~ appendSixBits);
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	136 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	137 else
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	138 // 5 and 6 byte UTF-8 sequences are not allowed yet.
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	139 // 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	140 // 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	141 return ERROR_CHAR;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	142
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	143 assert(isTrailByte(*p));
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	144
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	145 if (!isValidChar(c))
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	146 return ERROR_CHAR;
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	147 ref_p = p+1;
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	148 return c;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	149 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	150
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	151 /// Encodes c and appends it to str.
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	152 void encode(ref char[] str, dchar c)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	153 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	154 assert(isValidChar(c), "check if character is valid before calling encode().");
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	155
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	156 char[6] b = void;
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	157 if (c < 0x80)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	158 str ~= c;
764 4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 747 diff changeset	159 else if (c < 0x800)
518 8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	160 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	161 b[0] = 0xC0 \| (c >> 6);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	162 b[1] = 0x80 \| (c & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	163 str ~= b[0..2];
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	164 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	165 else if (c < 0x10000)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	166 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	167 b[0] = 0xE0 \| (c >> 12);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	168 b[1] = 0x80 \| ((c >> 6) & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	169 b[2] = 0x80 \| (c & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	170 str ~= b[0..3];
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	171 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	172 else if (c < 0x200000)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	173 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	174 b[0] = 0xF0 \| (c >> 18);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	175 b[1] = 0x80 \| ((c >> 12) & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	176 b[2] = 0x80 \| ((c >> 6) & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	177 b[3] = 0x80 \| (c & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	178 str ~= b[0..4];
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	179 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	180 /+ // There are no 5 and 6 byte UTF-8 sequences yet.
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	181 else if (c < 0x4000000)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	182 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	183 b[0] = 0xF8 \| (c >> 24);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	184 b[1] = 0x80 \| ((c >> 18) & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	185 b[2] = 0x80 \| ((c >> 12) & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	186 b[3] = 0x80 \| ((c >> 6) & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	187 b[4] = 0x80 \| (c & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	188 str ~= b[0..5];
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	189 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	190 else if (c < 0x80000000)
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	191 {
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	192 b[0] = 0xFC \| (c >> 30);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	193 b[1] = 0x80 \| ((c >> 24) & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	194 b[2] = 0x80 \| ((c >> 18) & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	195 b[3] = 0x80 \| ((c >> 12) & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	196 b[4] = 0x80 \| ((c >> 6) & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	197 b[5] = 0x80 \| (c & 0x3F);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	198 str ~= b[0..6];
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	199 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	200 +/
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	201 else
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	202 assert(0);
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	203 }
8f86bb9ef715 Added module dil.Converter and dil.FileBOM. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 510 diff changeset	204
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	205 /// Encodes c and appends it to str.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	206 void encode(ref wchar[] str, dchar c)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	207 in { assert(isValidChar(c)); }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	208 body
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	209 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	210 if (c < 0x10000)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	211 str ~= cast(wchar)c;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	212 else
764 4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 747 diff changeset	213 { // Encode with surrogate pair.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	214 wchar[2] pair = void;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	215 c -= 0x10000; // c'
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	216 // higher10bits(c') \| 0b1101_10xx_xxxx_xxxx
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	217 pair[0] = (c >> 10) \| 0xD800;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	218 // lower10bits(c') \| 0b1101_11yy_yyyy_yyyy
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	219 pair[1] = (c & 0x3FF) \| 0xDC00;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	220 str ~= pair;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	221 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	222 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	223
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	224 /// Decodes a character from a UTF-16 sequence.
769 5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	225 /// Params:
5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	226 /// str = the UTF-16 sequence.
5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	227 /// index = where to start from.
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	228 /// Returns: ERROR_CHAR in case of an error in the sequence.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	229 dchar decode(wchar[] str, ref size_t index)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	230 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	231 assert(str.length && index < str.length);
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	232 dchar c = str[index];
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	233 if (0xD800 > c \|\| c > 0xDFFF)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	234 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	235 ++index;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	236 return c;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	237 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	238 if (c <= 0xDBFF && index+1 != str.length)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	239 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	240 wchar c2 = str[index+1];
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	241 if (0xDC00 <= c2 && c2 <= 0xDFFF)
764 4579e8505d5e Fixed unittests and removed dil.File. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 747 diff changeset	242 { // Decode surrogate pair.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	243 // (c - 0xD800) << 10 + 0x10000 ->
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	244 // (c - 0xD800 + 0x40) << 10 ->
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	245 c = (c - 0xD7C0) << 10;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	246 c \|= (c2 & 0x3FF);
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	247 index += 2;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	248 return c;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	249 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	250 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	251 return ERROR_CHAR;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	252 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	253
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	254 /// Decodes a character from a UTF-16 sequence.
769 5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	255 /// Params:
5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	256 /// p = start of the UTF-16 sequence.
5e3ef1b2011c Added and improved documentation. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 764 diff changeset	257 /// end = one past the end of the sequence.
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	258 /// Returns: ERROR_CHAR in case of an error in the sequence.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	259 dchar decode(ref wchar* p, wchar* end)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	260 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	261 assert(p && p < end);
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	262 dchar c = *p;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	263 if (0xD800 > c \|\| c > 0xDFFF)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	264 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	265 ++p;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	266 return c;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	267 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	268 if (c <= 0xDBFF && p+1 != end)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	269 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	270 wchar c2 = p[1];
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	271 if (0xDC00 <= c2 && c2 <= 0xDFFF)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	272 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	273 c = (c - 0xD7C0) << 10;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	274 c \|= (c2 & 0x3FF);
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	275 p += 2;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	276 return c;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	277 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	278 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	279 return ERROR_CHAR;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	280 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	281
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	282 /// Decodes a character from a zero-terminated UTF-16 string.
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	283 /// Params:
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	284 /// p = start of the UTF-16 sequence.
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	285 /// Returns: ERROR_CHAR in case of an error in the sequence.
510 dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	286 dchar decode(ref wchar* p)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	287 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	288 assert(p);
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	289 dchar c = *p;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	290 if (0xD800 > c \|\| c > 0xDFFF)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	291 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	292 ++p;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	293 return c;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	294 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	295 if (c <= 0xDBFF)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	296 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	297 wchar c2 = p[1];
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	298 if (0xDC00 <= c2 && c2 <= 0xDFFF)
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	299 {
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	300 c = (c - 0xD7C0) << 10;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	301 c \|= (c2 & 0x3FF);
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	302 p += 2;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	303 return c;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	304 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	305 }
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	306 return ERROR_CHAR;
dd3ce87b3569 Added module dil.Unicode. Aziz K?ksal <aziz.koeksal@gmail.com> parents: diff changeset	307 }
789 c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	308
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	309 /// Converts a UTF-8 string to a UTF-16 string.
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	310 wchar[] toUTF16(char[] str)
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	311 {
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	312 wchar[] result;
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	313 size_t idx;
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	314 while (idx < str.length)
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	315 {
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	316 auto c = decode(str, idx);
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	317 if (c == ERROR_CHAR)
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	318 { // Skip trail bytes.
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	319 while (++idx < str.length && isTrailByte(str[idx]))
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	320 {}
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	321 c = REPLACEMENT_CHAR;
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	322 }
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	323 encode(result, c);
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	324 }
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	325 return result;
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	326 }
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	327
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	328 /// Converts a UTF-8 string to a UTF-32 string.
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	329 dchar[] toUTF32(char[] str)
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	330 {
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	331 dchar[] result;
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	332 size_t idx;
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	333 while (idx < str.length)
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	334 {
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	335 auto c = decode(str, idx);
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	336 if (c == ERROR_CHAR)
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	337 { // Skip trail bytes.
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	338 while (++idx < str.length && isTrailByte(str[idx]))
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	339 {}
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	340 c = REPLACEMENT_CHAR;
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	341 }
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	342 result ~= c;
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	343 }
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	344 return result;
c1d5cfd7aa44 Implemented string literal conversion. Aziz K?ksal <aziz.koeksal@gmail.com> parents: 769 diff changeset	345 }

Mercurial > projects > dil

annotate src/dil/Unicode.d @ 806:bcb74c9b895c