// Copyright (c) 1999-2006 by Digital Mars
// All Rights Reserved
// written by Walter Bright
// http://www.digitalmars.com
// License for redistribution is by either the Artistic License
// in artistic.txt, or the GNU General Public License in gnu.txt.
// See the included readme.txt for details.


/* HTML parser
 */

#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <stdarg.h>
#include <errno.h>
#include <wchar.h>

#include "mars.h"
#include "html.h"

#include <assert.h>
#include "root.h"

extern int HtmlNamedEntity(unsigned char *p, int length);

static int isLineSeparator(const unsigned char* p);

/**********************************
 * Determine if beginning of tag identifier
 * or a continuation of a tag identifier.
 */

inline int istagstart(int c)
{
    return (isalpha(c) || c == '_');
}

inline int istag(int c)
{
    return (isalnum(c) || c == '_');
}

/**********************************************
 */

Html::Html(const char *sourcename, unsigned char *base, unsigned length)
{
    //printf("Html::Html()\n");
    this->sourcename = sourcename;
    this->base = base;
    p = base;
    end = base + length;
    linnum = 1;
    dbuf = NULL;
    inCode = 0;
}

/**********************************************
 * Print error & quit.
 */

void Html::error(const char *format, ...)
{
    if (!global.gag)
    {
	printf("%s(%d) : HTML Error: ", sourcename, linnum);

	va_list ap;
	va_start(ap, format);
	vprintf(format, ap);
	va_end(ap);

	printf("\n");
	fflush(stdout);
    }

    global.errors++;
}

/**********************************************
 * Extract all the code from an HTML file,
 * concatenate it all together, and store in buf.
 */

void Html::extractCode(OutBuffer *buf)
{
    //printf("Html::extractCode()\n");
    dbuf = buf;			// save for other routines
    buf->reserve(end - p);
    inCode = 0;
    while (1)
    {
	//printf("p = %p, *p = x%x\n", p, *p);
	switch (*p)
	{
#if 0 // strings are not recognized outside of tags
	    case '"':
	    case '\'':
		skipString();
		continue;
#endif
	    case '<':
		if (p[1] == '!' && isCommentStart())
		{   // Comments start with <!--
		    scanComment();
		}
  		else if(p[1] == '!' && isCDATAStart())
  		{
  		    scanCDATA();
  		}
		else if (p[1] == '/' && istagstart(*skipWhite(p + 2)))
		    skipTag();
		else if (istagstart(*skipWhite(p + 1)))
		    skipTag();
		else
		    goto Ldefault;
		continue;

	    case 0:
	    case 0x1a:
		break;		// end of file

	    case '&':
		if (inCode)
		{   // Translate character entity into ascii for D parser
		    int c;

		    c = charEntity();
		    buf->writeUTF8(c);
		}
		else
		    p++;
		continue;

	    case '\r':
		if (p[1] == '\n')
		    goto Ldefault;
	    case '\n':
		linnum++;
		// Always extract new lines, so that D lexer counts the
		// lines right.
		buf->writeByte(*p);
		p++;
		continue;

	    default:
	    Ldefault:
		if (inCode)
		    buf->writeByte(*p);
		p++;
		continue;
	}
	break;
    }
    buf->writeByte(0);				// ending sentinel
    //printf("D code is: '%s'\n", (char *)buf->data);
}

/***********************************************
 * Scan to end of <> tag.
 * Look for <code> and </code> tags to start/stop D processing.
 * Input:
 *	p is on opening '<' of tag; it's already verified that
 *	it's a tag by lookahead
 * Output:
 *	p is past closing '>' of tag
 */

void Html::skipTag()
{
    enum TagState	// what parsing state we're in
    {
	TStagstart,	// start of tag name
	TStag,		// in a tag name
	TSrest,		// following tag name
    };
    enum TagState state = TStagstart;
    int inot;
    unsigned char *tagstart = NULL;
    int taglen = 0;

    p++;
    inot = 0;
    if (*p == '/')
    {	inot = 1;
	p++;
    }
    while (1)
    {
	switch (*p)
	{
	    case '>':		// found end of tag
		p++;
		break;

	    case '"':
	    case '\'':
		state = TSrest;
		skipString();
		continue;

	    case '<':
		if (p[1] == '!' && isCommentStart())
		{   // Comments start with <!--
		    scanComment();
		}
		else if (p[1] == '/' && istagstart(*skipWhite(p + 2)))
		{   error("nested tag");
		    skipTag();
		}
		else if (istagstart(*skipWhite(p + 1)))
		{   error("nested tag");
		    skipTag();
		}
		// Treat comments as if they were whitespace
		state = TSrest;
		continue;

	    case 0:
	    case 0x1a:
		error("end of file before end of tag");
		break;		// end of file

	    case '\r':
		if (p[1] == '\n')
		    goto Ldefault;
	    case '\n':
		linnum++;
		// Always extract new lines, so that code lexer counts the
		// lines right.
		dbuf->writeByte(*p);
		state = TSrest;			// end of tag
		p++;
		continue;

	    case ' ':
	    case '\t':
	    case '\f':
	    case '\v':
		if (state == TStagstart)
		{   p++;
		    continue;
		}
	    default:
	    Ldefault:
		switch (state)
		{
		    case TStagstart:		// start of tag name
			assert(istagstart(*p));
			state = TStag;
			tagstart = p;
			taglen = 0;
			break;

		    case TStag:
			if (istag(*p))
			{   // Continuing tag name
			    taglen++;
			}
			else
			{   // End of tag name
			    state = TSrest;
			}
			break;

		    case TSrest:
			break;
		}
		p++;
		continue;
	}
	break;
    }

    // See if we parsed a <code> or </code> tag
    if (taglen && memicmp((char *) tagstart, (char *) "CODE", taglen) == 0
	&& *(p - 2) != '/') // ignore "<code />" (XHTML)
    {
	if (inot)
	{   inCode--;
	    if (inCode < 0)
		inCode = 0;		// ignore extra </code>'s
	}
	else
	    inCode++;
    }
}

/***********************************************
 * Scan to end of attribute string.
 */

void Html::skipString()
{
    int tc = *p;

    while (1)
    {
	p++;
	switch (*p)
	{
	    case '"':
	    case '\'':
		if (*p == tc)
		{   p++;
		    break;
		}
		continue;

	    case '\r':
		if (p[1] == '\n')
		    goto Ldefault;
	    case '\n':
		linnum++;
		// Always extract new lines, so that D lexer counts the
		// lines right.
		dbuf->writeByte(*p);
		continue;

	    case 0:
	    case 0x1a:
	    Leof:
		error("end of file before closing %c of string", tc);
		break;

	    default:
	    Ldefault:
		continue;
	}
	break;
    }
}

/*********************************
 * If p points to any white space, skip it
 * and return pointer just past it.
 */

unsigned char *Html::skipWhite(unsigned char *q)
{
    for (; 1; q++)
    {
	switch (*q)
	{
	    case ' ':
	    case '\t':
	    case '\f':
	    case '\v':
	    case '\r':
	    case '\n':
		continue;

	    default:
		break;
	}
	break;
    }
    return q;
}

/***************************************************
 * Scan to end of comment.
 * Comments are defined any of a number of ways.
 * IE 5.0: <!-- followed by >
 * "HTML The Definitive Guide": <!-- text with at least one space in it -->
 * Netscape: <!-- --> comments nest
 * w3c: whitespace can appear between -- and > of comment close
 */

void Html::scanComment()
{
    // Most of the complexity is dealing with the case that
    // an arbitrary amount of whitespace can appear between
    // the -- and the > of a comment close.
    int scangt = 0;

    //printf("scanComment()\n");
    if (*p == '\n')
    {	linnum++;
	// Always extract new lines, so that D lexer counts the
	// lines right.
	dbuf->writeByte(*p);
    }
    while (1)
    {
	//scangt = 1;			// IE 5.0 compatibility
	p++;
	switch (*p)
	{
	    case '-':
		if (p[1] == '-')
		{
		    if (p[2] == '>')	// optimize for most common case
		    {
			p += 3;
			break;
		    }
		    p++;
		    scangt = 1;
		}
		else
		    scangt = 0;
		continue;

	    case '>':
		if (scangt)
		{   // found -->
		    p++;
		    break;
		}
		continue;

	    case ' ':
	    case '\t':
	    case '\f':
	    case '\v':
		// skip white space
		continue;

	    case '\r':
		if (p[1] == '\n')
		    goto Ldefault;
	    case '\n':
		linnum++;		// remember to count lines
		// Always extract new lines, so that D lexer counts the
		// lines right.
		dbuf->writeByte(*p);
		continue;

	    case 0:
	    case 0x1a:
		error("end of file before closing --> of comment");
		break;

	    default:
	    Ldefault:
		scangt = 0;		// it's not -->
		continue;
	}
	break;
    }
    //printf("*p = '%c'\n", *p);
}

/********************************************
 * Determine if we are at the start of a comment.
 * Input:
 *	p is on the opening '<'
 * Returns:
 *	0 if not start of a comment
 * 	1 if start of a comment, p is adjusted to point past --
 */

int Html::isCommentStart()
#ifdef __DMC__
    __out(result)
    {
	if (result == 0)
	    ;
	else if (result == 1)
	{
	    assert(p[-2] == '-' && p[-1] == '-');
	}
	else
	    assert(0);
    }
    __body
#endif /* __DMC__ */
    {	unsigned char *s;

	if (p[0] == '<' && p[1] == '!')
	{
	    for (s = p + 2; 1; s++)
	    {
		switch (*s)
		{
		    case ' ':
		    case '\t':
		    case '\r':
		    case '\f':
		    case '\v':
			// skip white space, even though spec says no
			// white space is allowed
			continue;

		    case '-':
			if (s[1] == '-')
			{
			    p = s + 2;
			    return 1;
			}
			goto No;

		    default:
			goto No;
		}
	    }
	}
    No:
	return 0;
    }

int Html::isCDATAStart()
{
    const char * CDATA_START_MARKER = "<![CDATA[";
    size_t len = strlen(CDATA_START_MARKER);

    if (strncmp((char*)p, CDATA_START_MARKER, len) == 0)
    {
	p += len;
	return 1;
    }
    else
    {
	return 0;
    }
}

void Html::scanCDATA()
{
    while(*p && *p != 0x1A)
    {
	int lineSepLength = isLineSeparator(p);
	if (lineSepLength>0)
	{
	    /* Always extract new lines, so that D lexer counts the lines
	     * right.
	     */
	    linnum++;
	    dbuf->writeUTF8('\n');
	    p += lineSepLength;
	    continue;
        }
	else if (p[0] == ']' && p[1] == ']' && p[2] == '>')
	{
	    /* end of CDATA section */
	    p += 3;
	    return;
	}
	else if (inCode)
	{
	    /* this CDATA section contains D code */
	    dbuf->writeByte(*p);
	}

	p++;
    }
}

/********************************************
 * Convert an HTML character entity into a character.
 * Forms are:
 *	&name;		named entity
 *	&#ddd;		decimal
 *	&#xhhhh;	hex
 * Input:
 *	p is on the &
 */

int Html::charEntity()
{   int c = 0;
    int v;
    int hex;
    unsigned char *pstart = p;

    //printf("Html::charEntity('%c')\n", *p);
    if (p[1] == '#')
    {
	p++;
	if (p[1] == 'x' || p[1] == 'X')
	{   p++;
	    hex = 1;
	}
	else
	    hex = 0;
	if (p[1] == ';')
	    goto Linvalid;
	while (1)
	{
	    p++;
	    switch (*p)
	    {
		case 0:
		case 0x1a:
		    error("end of file before end of character entity");
		    goto Lignore;

		case '\n':
		case '\r':
		case '<':	// tag start
		    // Termination is assumed
		    break;

		case ';':
		    // Termination is explicit
		    p++;
		    break;

		case '0': case '1': case '2': case '3': case '4':
		case '5': case '6': case '7': case '8': case '9':
		    v = *p - '0';
		    goto Lvalue;

		case 'a': case 'b': case 'c':
		case 'd': case 'e': case 'f':
		    if (!hex)
			goto Linvalid;
		    v = (*p - 'a') + 10;
		    goto Lvalue;

		case 'A': case 'B': case 'C':
		case 'D': case 'E': case 'F':
		    if (!hex)
			goto Linvalid;
		    v = (*p - 'A') + 10;
		    goto Lvalue;

		Lvalue:
		    if (hex)
			c = (c << 4) + v;
		    else
			c = (c * 10) + v;
		    if (c > 0x10FFFF)
		    {
			error("character entity out of range");
			goto Lignore;
		    }
		    continue;

		default:
		Linvalid:
		    error("invalid numeric character reference");
		    goto Lignore;
	    }
	    break;
	}
    }
    else
    {
	// It's a named entity; gather all characters until ;
	unsigned char *idstart = p + 1;

	while (1)
	{
	    p++;
	    switch (*p)
	    {
		case 0:
		case 0x1a:
		    error("end of file before end of character entity");
		    break;

		case '\n':
		case '\r':
		case '<':	// tag start
		    // Termination is assumed
		    c = HtmlNamedEntity(idstart, p - idstart);
		    if (c == -1)
			goto Lignore;
		    break;

		case ';':
		    // Termination is explicit
		    c = HtmlNamedEntity(idstart, p - idstart);
		    if (c == -1)
			goto Lignore;
		    p++;
		    break;

		default:
		    continue;
	    }
	    break;
	}
    }

    // Kludge to convert non-breaking space to ascii space
    if (c == 160)
	c = ' ';

    return c;

Lignore:
    //printf("Lignore\n");
    p = pstart + 1;
    return '&';
}

/**
 * identify DOS, Linux, Mac, Next and Unicode line endings
 * 0 if this is no line separator
 * >0 the length of the separator
 * Note: input has to be UTF-8
 */
static int isLineSeparator(const unsigned char* p)
{
    // Linux
    if( p[0]=='\n')
	return 1;

    // Mac & Dos
    if( p[0]=='\r')
	return (p[1]=='\n') ? 2 : 1;

    // Unicode (line || paragraph sep.)
    if( p[0]==0xE2 && p[1]==0x80 && (p[2]==0xA8 || p[2]==0xA9))
	return 3;

    // Next
    if( p[0]==0xC2 && p[1]==0x85)
	return 2;

    return 0;
}