# HG changeset patch
# User trass3r
# Date 1284517126 -7200
# Node ID 560eaedcb7a2e2bbc9bfbe93a2737151542abafe
# Parent  4092a614a9f3b56de6837409b4e51f444f6af4c5
added lots of ddoc code, still needs much care
added helpers functions in ddoc.Util

diff -r 4092a614a9f3 -r 560eaedcb7a2 dmd/ddoc/DocComment.d
--- a/dmd/ddoc/DocComment.d	Wed Sep 15 03:00:30 2010 +0200
+++ b/dmd/ddoc/DocComment.d	Wed Sep 15 04:18:46 2010 +0200
@@ -3,6 +3,8 @@
 import dmd.ddoc.Escape;
 import dmd.ddoc.Macro;
 import dmd.ddoc.Sections;
+import dmd.ddoc.Util;
+
 import dmd.Array;
 import dmd.Scope;
 import dmd.Dsymbol;
@@ -26,7 +28,7 @@
 
 	static DocComment parse(Scope sc, Dsymbol s, string comment)
 	{
-		unsigned idlen;
+		uint idlen;
 
 		// writef("parse(%s): '%s'\n", s.toChars(), comment);
 		if (sc.lastdc && isDitto(comment))
@@ -62,9 +64,120 @@
 	 *	  name2 = value2
 	 */
 	static void parseMacros(Escape** pescapetable, Macro** pmacrotable, ubyte* m, uint mlen)
-	{
-		assert(false);
-	}
+	{/+
+		unsigned char *p = m;
+		unsigned len = mlen;
+		unsigned char *pend = p + len;
+
+		unsigned char *tempstart;
+		unsigned templen;
+
+		unsigned char *namestart;
+		unsigned namelen = 0;	   // !=0 if line continuation
+
+		unsigned char *textstart;
+		unsigned textlen;
+
+		while (p < pend)
+		{
+			// Skip to start of macro
+			while (1)
+			{
+				if (p >= pend)
+					goto Ldone;
+				switch (*p)
+				{
+					case ' ':
+					case '\t':
+						p++;
+						continue;
+
+					case '\n':
+						p++;
+						goto Lcont;
+
+					default:
+						if (isIdStart(p))
+							break;
+						if (namelen)
+							goto Ltext;			 // continuation of prev macro
+						goto Lskipline;
+				}
+				break;
+			}
+			tempstart = p;
+
+			while (1)
+			{
+				if (p >= pend)
+					goto Ldone;
+				if (!isIdTail(p))
+					break;
+				p += utfStride(p);
+			}
+			templen = p - tempstart;
+
+			while (1)
+			{
+				if (p >= pend)
+					goto Ldone;
+				if (!(*p == ' ' || *p == '\t'))
+					break;
+				p++;
+			}
+
+			if (*p != '=')
+			{   if (namelen)
+					goto Ltext;			 // continuation of prev macro
+				goto Lskipline;
+			}
+			p++;
+			if (p >= pend)
+				goto Ldone;
+
+			if (namelen)
+			{   // Output existing macro
+			L1:
+				//printf("macro '%.*s' = '%.*s'\n", namelen, namestart, textlen, textstart);
+				if (icmp("ESCAPES", namestart, namelen) == 0)
+					parseEscapes(pescapetable, textstart, textlen);
+				else
+					Macro.define(pmacrotable, namestart, namelen, textstart, textlen);
+				namelen = 0;
+				if (p >= pend)
+					break;
+			}
+
+			namestart = tempstart;
+			namelen = templen;
+
+			while (p < pend && (*p == ' ' || *p == '\t'))
+				p++;
+			textstart = p;
+
+		  Ltext:
+			while (p < pend && *p != '\n')
+				p++;
+			textlen = p - textstart;
+
+			// Remove trailing \r if there is one
+			if (p > m && p[-1] == '\r')
+				textlen--;
+
+			p++;
+			//printf("p = %p, pend = %p\n", p, pend);
+
+		 Lcont:
+			continue;
+
+		 Lskipline:
+			// Ignore this line
+			while (p < pend && *p++ != '\n') {}
+		}
+	Ldone:
+		if (namelen)
+			goto L1;				// write out last one
+	+/}
 	
 	/**************************************
 	 * Parse escapes of the form:
@@ -74,9 +187,48 @@
 	 * by whitespace and/or commas.
 	 */
 	static void parseEscapes(Escape** pescapetable, ubyte* textstart, uint textlen)
-	{
-		assert(false);
-	}
+	{/+
+		   Escape *escapetable = *pescapetable;
+
+			if (!escapetable)
+			{   escapetable = new Escape;
+				*pescapetable = escapetable;
+			}
+			unsigned char *p = textstart;
+			unsigned char *pend = p + textlen;
+
+			while (1)
+			{
+				while (1)
+				{
+					if (p + 4 >= pend)
+						return;
+					if (!(*p == ' ' || *p == '\t' || *p == '\n' || *p == ','))
+						break;
+					p++;
+				}
+				if (p[0] != '/' || p[2] != '/')
+					return;
+				unsigned char c = p[1];
+				p += 3;
+				unsigned char *start = p;
+				while (1)
+				{
+					if (p >= pend)
+						return;
+					if (*p == '/')
+						break;
+					p++;
+				}
+				size_t len = p - start;
+				char *s = (char *)memcpy(mem.malloc(len + 1), start, len);
+				s[len] = 0;
+				escapetable.strings[c] = s;
+				//printf("%c = '%s'\n", c, s);
+				p++;
+			}
+	+/}
+
 	/*****************************************
 	 * Parse next paragraph out of *pcomment.
 	 * Update *pcomment to point past paragraph.
@@ -90,10 +242,10 @@
 		ubyte*pstart;
 		ubyte*pend;
 		ubyte*idstart;
-		unsigned idlen;
+		uint idlen;
 
 		ubyte*name = null;
-		unsigned namelen = 0;
+		size_t namelen = 0;
 
 		//printf("parseSections('%s')\n", comment);
 		p = comment;
@@ -218,7 +370,7 @@
 				else
 				{
 					buf.writestring("$(DDOC_SUMMARY ");
-					unsigned o = buf.offset;
+					uint o = buf.offset;
 					buf.write(sec.body_, sec.bodylen);
 					highlightText(sc, s, buf, o);
 					buf.writestring(")\n");
diff -r 4092a614a9f3 -r 560eaedcb7a2 dmd/ddoc/Sections.d
--- a/dmd/ddoc/Sections.d	Wed Sep 15 03:00:30 2010 +0200
+++ b/dmd/ddoc/Sections.d	Wed Sep 15 04:18:46 2010 +0200
@@ -2,9 +2,11 @@
 
 import dmd.common;
 import dmd.ddoc.DocComment;
+import dmd.ddoc.Util;
 import dmd.Array;
 import dmd.Scope;
 import dmd.Dsymbol;
+import dmd.HdrGenState;
 import dmd.OutBuffer;
 
 //!
@@ -13,24 +15,25 @@
 //!
 class Section
 {
-    string name;
-    string body_;
+	string name;
+	string body_;
 
-    int nooutput;
+	int nooutput;
 
 	void write(DocComment dc, Scope sc, Dsymbol s, OutBuffer buf)
 	{
-		if (namelen)
+		if (name.length)
 		{
-			static string[] table =
-			{	   "AUTHORS", "BUGS", "COPYRIGHT", "DATE",
-					"DEPRECATED", "EXAMPLES", "HISTORY", "LICENSE",
-					"RETURNS", "SEE_ALSO", "STANDARDS", "THROWS",
-					"VERSION" };
-     
-			for (int i = 0; i < table.length; i++)
+			static string[] table = [
+				"AUTHORS", "BUGS", "COPYRIGHT", "DATE",
+				"DEPRECATED", "EXAMPLES", "HISTORY", "LICENSE",
+				"RETURNS", "SEE_ALSO", "STANDARDS", "THROWS",
+				"VERSION"
+			];
+	 
+			for (uint i = 0; i < table.length; i++)
 			{
-				if (icmp(table[i], name, namelen) == 0)
+				if (icmp(table[i], name) == 0)
 				{
 					buf.printf("$(DDOC_%s ", table[i]);
 					goto L1;
@@ -41,21 +44,21 @@
 				// Replace _ characters with spaces
 				buf.writestring("$(DDOC_SECTION_H ");
 				size_t o = buf.offset;
-				for (size_t u = 0; u < namelen; u++)
+				for (size_t u = 0; u < name.length; u++)
 				{
 					char c = name[u];
 					buf.writeByte((c == '_') ? ' ' : c);
 				}
 				escapeStrayParenthesis(buf, o, s.loc);
-				buf.writestring(":)\n");
+			buf.writestring(":)\n");
 		}
 		else
 		{
 			buf.writestring("$(DDOC_DESCRIPTION ");
 		}
 	  L1:
-		unsigned o = buf.offset;
-		buf.write(body_, bodylen);
+		uint o = buf.offset;
+		buf.write(body_, body_.length);
 		escapeStrayParenthesis(buf, o, s.loc);
 		highlightText(sc, s, buf, o);
 		buf.writestring(")\n");
@@ -67,37 +70,33 @@
 {
 	override void write(DocComment dc, Scope sc, Dsymbol s, OutBuffer buf)
 	{
-		ubyte*p = body_;
-		unsigned len = bodylen;
-		ubyte*pend = p + len;
+		size_t i = 0;
+		string p = body_;
 
-		ubyte*tempstart;
-		unsigned templen;
+		size_t tempstart, templen;
 
-		ubyte*namestart;
-		unsigned namelen = 0;	   // !=0 if line continuation
+		size_t namestart, namelen = 0;	   // !=0 if line continuation
 
-		ubyte*textstart;
-		unsigned textlen;
+		size_t textstart, textlen;
 
-		unsigned o;
+		uint o;
 		Parameter *arg;
 
 		buf.writestring("$(DDOC_PARAMS \n");
-		while (p < pend)
+		while (i < _body.length)
 		{
 			// Skip to start of macro
-			while (1)
+			while (true)
 			{
-				switch (*p)
+				switch (p[i])
 				{
 					case ' ':
 					case '\t':
-						p++;
+						i++;
 						continue;
 
 					case '\n':
-						p++;
+						i++;
 						goto Lcont;
 
 					default:
@@ -109,27 +108,27 @@
 				}
 				break;
 			}
-			tempstart = p;
+			tempstart = i;
 
-			while (isIdTail(p))
-				p += utfStride(p);
-			templen = p - tempstart;
+			while (isIdTail(body_[i .. $]))
+				i += utfStride(body_[i .. $]);
+			templen = i - tempstart;
 
-			while (*p == ' ' || *p == '\t')
-				p++;
+			while (p[i] == ' ' || p[i] == '\t')
+				i++;
 
-			if (*p != '=')
+			if (p[i] != '=')
 			{   if (namelen)
 					goto Ltext;			 // continuation of prev macro
 				goto Lskipline;
 			}
-			p++;
+			i++;
 
 			if (namelen)
 			{   // Output existing param
 
 			L1:
-				//printf("param '%.*s' = '%.*s'\n", namelen, namestart, textlen, textstart);
+				// writef("param '%.*s' = '%.*s'\n", namelen, namestart, textlen, textstart);
 				HdrGenState hgs;
 				buf.writestring("$(DDOC_PARAM_ROW ");
 					buf.writestring("$(DDOC_PARAM_ID ");
@@ -145,35 +144,35 @@
 
 					buf.writestring("$(DDOC_PARAM_DESC ");
 						o = buf.offset;
-						buf.write(textstart, textlen);
+						buf.write(&p[textstart], textlen);
 						escapeStrayParenthesis(buf, o, s.loc);
 						highlightText(sc, s, buf, o);
 					buf.writestring(")");
 				buf.writestring(")\n");
 				namelen = 0;
-				if (p >= pend)
+				if (i >= p.length)
 					break;
 			}
 
 			namestart = tempstart;
 			namelen = templen;
 
-			while (*p == ' ' || *p == '\t')
-				p++;
-			textstart = p;
+			while (p[i] == ' ' || p[i] == '\t')
+				i++;
+			textstart = i;
 
 		  Ltext:
-			while (*p != '\n')
-				p++;
-			textlen = p - textstart;
-			p++;
+			while (p[i] != '\n')
+				i++;
+			textlen = i - textstart;
+			i++;
 
 		 Lcont:
 			continue;
 
 		 Lskipline:
 			// Ignore this line
-			while (*p++ != '\n') {}
+			while (p[i++] != '\n') {}
 		}
 		if (namelen)
 			goto L1;				// write out last one
@@ -187,6 +186,6 @@
 	override void write(DocComment dc, Scope sc, Dsymbol s, OutBuffer buf)
 	{
 		// writef("MacroSection.write()\n");
-		DocComment.parseMacros(dc.pescapetable, dc.pmacrotable, body_, bodylen);
+		DocComment.parseMacros(dc.pescapetable, dc.pmacrotable, body_, body_.length);
 	}
 }
\ No newline at end of file
diff -r 4092a614a9f3 -r 560eaedcb7a2 dmd/ddoc/Util.d
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/dmd/ddoc/Util.d	Wed Sep 15 04:18:46 2010 +0200
@@ -0,0 +1,446 @@
+module dmd.ddoc.Util;
+
+public import std.string : cmp, icmp;
+
+/*****************************************
+ * Return !=0 if comment consists entirely of "ditto".
+ */
+bool isDitto(string comment)
+{
+	if (comment)
+	{
+		string p = skipwhitespace(comment);
+
+		if (icmp(p, "ditto") == 0 && skipwhitespace(p[5..$])[0] == 0)
+			return true;
+	}
+	return false;
+}
+
+/**********************************************
+ * Skip white space.
+ */
+string skipwhitespace(string p)
+{
+	size_t i = 0;
+	
+	for (; true; i++)
+	{   switch (p[i])
+		{
+			case ' ':
+			case '\t':
+			case '\n':
+				continue;
+			default:
+		}
+		break;
+	}
+	return p[i .. $];
+}
+
+/+
+/************************************************
+ * Scan forward to one of:
+ *	  start of identifier
+ *	  beginning of next line
+ *	  end of buf
+ */
+unsigned skiptoident(OutBuffer buf, size_t i)
+{
+	while (i < buf.offset)
+	{   dchar_t c;
+
+		size_t oi = i;
+		if (utf_decodeChar((unsigned char *)buf.data, buf.offset, &i, &c))
+			/* Ignore UTF errors, but still consume input
+			 */
+			break;
+		if (c >= 0x80)
+		{
+			if (!isUniAlpha(c))
+				continue;
+		}
+		else if (!(isalpha(c) || c == '_' || c == '\n'))
+			continue;
+		i = oi;
+		break;
+	}
+	return i;
+}
+
+/************************************************
+ * Scan forward past end of identifier.
+ */
+
+unsigned skippastident(OutBuffer buf, size_t i)
+{
+	while (i < buf.offset)
+	{   dchar_t c;
+
+		size_t oi = i;
+		if (utf_decodeChar((unsigned char *)buf.data, buf.offset, &i, &c))
+			/* Ignore UTF errors, but still consume input
+			 */
+			break;
+		if (c >= 0x80)
+		{
+			if (isUniAlpha(c))
+				continue;
+		}
+		else if (isalnum(c) || c == '_')
+			continue;
+		i = oi;
+		break;
+	}
+	return i;
+}
+
+
+/************************************************
+ * Scan forward past URL starting at i.
+ * We don't want to highlight parts of a URL.
+ * Returns:
+ *	  i if not a URL
+ *	  index just past it if it is a URL
+ */
+
+unsigned skippastURL(OutBuffer buf, size_t i)
+{   unsigned length = buf.offset - i;
+	unsigned char *p = &buf.data[i];
+	unsigned j;
+	unsigned sawdot = 0;
+
+	if (length > 7 && memicmp((char *)p, "http://", 7) == 0)
+	{
+		j = 7;
+	}
+	else if (length > 8 && memicmp((char *)p, "https://", 8) == 0)
+	{
+		j = 8;
+	}
+	else
+		goto Lno;
+
+	for (; j < length; j++)
+	{   unsigned char c = p[j];
+		if (isalnum(c))
+			continue;
+		if (c == '-' || c == '_' || c == '?' ||
+			c == '=' || c == '%' || c == '&' ||
+			c == '/' || c == '+' || c == '#' ||
+			c == '~')
+			continue;
+		if (c == '.')
+		{
+			sawdot = 1;
+			continue;
+		}
+		break;
+	}
+	if (sawdot)
+		return i + j;
+
+Lno:
+	return i;
+}
+
+
+/****************************************************
+ */
+bool isKeyword(string p)
+{
+	static string[] table = [ "true", "false", "null" ];
+
+	for (int i = 0; i < table.length; i++)
+	{
+		if (table[i][0 .. p.length] == p)
+			return true;
+	}
+	return false;
+}
+
+/****************************************************
+ */
+Parameter isFunctionParameter(Dsymbol s, string p)
+{
+	FuncDeclaration f = s.isFuncDeclaration();
+
+	/* f.type may be NULL for template members.
+	 */
+	if (f && f.type)
+	{
+		TypeFunction *tf;
+		if (f.originalType)
+		{
+			tf = (TypeFunction *)f.originalType;
+		}
+		else
+			tf = (TypeFunction *)f.type;
+
+		if (tf.parameters)
+		{
+			for (size_t k = 0; k < tf.parameters.dim; k++)
+			{   Parameter *arg = (Parameter *)tf.parameters.data[k];
+
+				if (arg.ident && cmp(arg.ident.toChars(), p, len) == 0)
+				{
+					return arg;
+				}
+			}
+		}
+	}
+	return NULL;
+}
++/
+/+
+/**************************************************
+ * Highlight text section.
+ */
+void highlightText(Scope *sc, Dsymbol *s, OutBuffer *buf, unsigned offset)
+{
+	//printf("highlightText()\n");
+	const char *sid = s.ident.toChars();
+	FuncDeclaration *f = s.isFuncDeclaration();
+	unsigned char *p;
+	const char *se;
+
+	int leadingBlank = 1;
+	int inCode = 0;
+	int inComment = 0;				  // in <!-- ... -. comment
+	unsigned iCodeStart;				// start of code section
+
+	unsigned iLineStart = offset;
+
+	for (unsigned i = offset; i < buf.offset; i++)
+	{   unsigned char c = buf.data[i];
+
+	 Lcont:
+		switch (c)
+		{
+			case ' ':
+			case '\t':
+				break;
+
+			case '\n':
+				if (sc && !inCode && i == iLineStart && i + 1 < buf.offset)	// if "\n\n"
+				{
+					static char blankline[] = "$(DDOC_BLANKLINE)\n";
+
+					i = buf.insert(i, blankline, sizeof(blankline) - 1);
+				}
+				leadingBlank = 1;
+				iLineStart = i + 1;
+				break;
+
+			case '<':
+				leadingBlank = 0;
+				if (inCode)
+					break;
+				p = &buf.data[i];
+
+				// Skip over comments
+				if (p[1] == '!' && p[2] == '-' && p[3] == '-')
+				{   unsigned j = i + 4;
+					p += 4;
+					while (1)
+					{
+						if (j == buf.offset)
+							goto L1;
+						if (p[0] == '-' && p[1] == '-' && p[2] == '>')
+						{
+							i = j + 2;  // place on closing '>'
+							break;
+						}
+						j++;
+						p++;
+					}
+					break;
+				}
+
+				// Skip over HTML tag
+				if (isalpha(p[1]) || (p[1] == '/' && isalpha(p[2])))
+				{   unsigned j = i + 2;
+					p += 2;
+					while (1)
+					{
+						if (j == buf.offset)
+							goto L1;
+						if (p[0] == '>')
+						{
+							i = j;	  // place on closing '>'
+							break;
+						}
+						j++;
+						p++;
+					}
+					break;
+				}
+
+			L1:
+				// Replace '<' with '&lt;' character entity
+				se = Escape::escapeChar('<');
+				if (se)
+				{   size_t len = strlen(se);
+					buf.remove(i, 1);
+					i = buf.insert(i, se, len);
+					i--;		// point to ';'
+				}
+				break;
+
+			case '>':
+				leadingBlank = 0;
+				if (inCode)
+					break;
+				// Replace '>' with '&gt;' character entity
+				se = Escape::escapeChar('>');
+				if (se)
+				{   size_t len = strlen(se);
+					buf.remove(i, 1);
+					i = buf.insert(i, se, len);
+					i--;		// point to ';'
+				}
+				break;
+
+			case '&':
+				leadingBlank = 0;
+				if (inCode)
+					break;
+				p = &buf.data[i];
+				if (p[1] == '#' || isalpha(p[1]))
+					break;					  // already a character entity
+				// Replace '&' with '&amp;' character entity
+				se = Escape::escapeChar('&');
+				if (se)
+				{   size_t len = strlen(se);
+					buf.remove(i, 1);
+					i = buf.insert(i, se, len);
+					i--;		// point to ';'
+				}
+				break;
+
+			case '-':
+				/* A line beginning with --- delimits a code section.
+				 * inCode tells us if it is start or end of a code section.
+				 */
+				if (leadingBlank)
+				{   int istart = i;
+					int eollen = 0;
+
+					leadingBlank = 0;
+					while (1)
+					{
+						++i;
+						if (i >= buf.offset)
+							break;
+						c = buf.data[i];
+						if (c == '\n')
+						{   eollen = 1;
+							break;
+						}
+						if (c == '\r')
+						{
+							eollen = 1;
+							if (i + 1 >= buf.offset)
+								break;
+							if (buf.data[i + 1] == '\n')
+							{   eollen = 2;
+								break;
+							}
+						}
+						// BUG: handle UTF PS and LS too
+						if (c != '-')
+							goto Lcont;
+					}
+					if (i - istart < 3)
+						goto Lcont;
+
+					// We have the start/end of a code section
+
+					// Remove the entire --- line, including blanks and \n
+					buf.remove(iLineStart, i - iLineStart + eollen);
+					i = iLineStart;
+
+					if (inCode && (i <= iCodeStart))
+					{   // Empty code section, just remove it completely.
+						inCode = 0;
+						break;
+					}
+
+					if (inCode)
+					{
+						inCode = 0;
+						// The code section is from iCodeStart to i
+						OutBuffer codebuf;
+
+						codebuf.write(buf.data + iCodeStart, i - iCodeStart);
+						codebuf.writeByte(0);
+						highlightCode2(sc, s, &codebuf, 0);
+						buf.remove(iCodeStart, i - iCodeStart);
+						i = buf.insert(iCodeStart, codebuf.data, codebuf.offset);
+						i = buf.insert(i, ")\n", 2);
+						i--;
+					}
+					else
+					{   static char pre[] = "$(D_CODE \n";
+
+						inCode = 1;
+						i = buf.insert(i, pre, sizeof(pre) - 1);
+						iCodeStart = i;
+						i--;			// place i on >
+						leadingBlank = true;
+					}
+				}
+				break;
+
+			default:
+				leadingBlank = 0;
+				if (sc && !inCode && isIdStart(&buf.data[i]))
+				{   unsigned j;
+
+					j = skippastident(buf, i);
+					if (j > i)
+					{
+						unsigned k = skippastURL(buf, i);
+						if (k > i)
+						{   i = k - 1;
+							break;
+						}
+
+						if (buf.data[i] == '_')		// leading '_' means no highlight
+						{
+							buf.remove(i, 1);
+							i = j - 1;
+						}
+						else
+						{
+							if (cmp(sid, buf.data + i, j - i) == 0)
+							{
+								i = buf.bracket(i, "$(DDOC_PSYMBOL ", j, ")") - 1;
+								break;
+							}
+							else if (isKeyword(buf.data + i, j - i))
+							{
+								i = buf.bracket(i, "$(DDOC_KEYWORD ", j, ")") - 1;
+								break;
+							}
+							else
+							{
+								if (f && isFunctionParameter(f, buf.data + i, j - i))
+								{
+									//printf("highlighting arg '%s', i = %d, j = %d\n", arg.ident.toChars(), i, j);
+									i = buf.bracket(i, "$(DDOC_PARAM ", j, ")") - 1;
+									break;
+								}
+							}
+							i = j - 1;
+						}
+					}
+				}
+				break;
+		}
+	}
+  Ldone:
+	if (inCode)
+		s.error("unmatched --- in DDoc comment");
+	;
+}+/
\ No newline at end of file