/* parse.c
 * Copyright (c) 2011, Peter Ohler
 * All rights reserved.
 */

#include <errno.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>

#include "attr.h"
#include "err.h"
#include "helper.h"
#include "intern.h"
#include "ox.h"
#include "ruby.h"
#include "special.h"

static void  mark_pi_cb(void *ptr);
static void  read_instruction(PInfo pi);
static void  read_doctype(PInfo pi);
static void  read_comment(PInfo pi);
static char *read_element(PInfo pi);
static void  read_text(PInfo pi);
/*static void	  read_reduced_text(PInfo pi); */
static void  read_cdata(PInfo pi);
static char *read_name_token(PInfo pi);
static char *read_quoted_value(PInfo pi);
static char *read_hex_uint64(char *b, uint64_t *up);
static char *read_10_uint64(char *b, uint64_t *up);
static char *read_coded_chars(PInfo pi, char *text);
static void  next_non_white(PInfo pi);
static int   collapse_special(PInfo pi, char *str);

static const rb_data_type_t ox_wrap_type = {
    "Object",
    {
        mark_pi_cb,
        NULL,
        NULL,
    },
    0,
    0,
};

/* This XML parser is a single pass, destructive, callback parser. It is a
 * single pass parse since it only make one pass over the characters in the
 * XML document string. It is destructive because it re-uses the content of
 * the string for values in the callback and places \0 characters at various
 * places to mark the end of tokens and strings. It is a callback parser like
 * a SAX parser because it uses callback when document elements are
 * encountered.
 *
 * Parsing is very tolerant. Lack of headers and even mispelled element
 * endings are passed over without raising an error. A best attempt is made in
 * all cases to parse the string.
 */

static char xml_valid_lower_chars[34] = "xxxxxxxxxooxxoxxxxxxxxxxxxxxxxxxo";

inline static int is_white(char c) {
    switch (c) {
    case ' ':
    case '\t':
    case '\f':
    case '\n':
    case '\r': return 1;
    default: return 0;
    }
}

inline static void next_non_white(PInfo pi) {
    for (; 1; pi->s++) {
        switch (*pi->s) {
        case ' ':
        case '\t':
        case '\f':
        case '\n':
        case '\r': break;
        default: return;
        }
    }
}

inline static void next_white(PInfo pi) {
    for (; 1; pi->s++) {
        switch (*pi->s) {
        case ' ':
        case '\t':
        case '\f':
        case '\n':
        case '\r':
        case '\0': return;
        default: break;
        }
    }
}

static void fix_newlines(char *buf) {
#if HAVE_INDEX
    if (NULL != index(buf, '\r')) {
#endif
        char *s = buf;
        char *d = buf;

        for (; '\0' != *s; s++) {
            if ('\r' == *s) {
                if ('\n' == *(s + 1)) {
                    continue;
                }
                *d = '\n';
            } else if (d < s) {
                *d = *s;
            }
            d++;
        }
        *d = '\0';
#if HAVE_INDEX
    }
#endif
}

static void mark_pi_cb(void *ptr) {
    if (NULL != ptr) {
        HelperStack stack = &((PInfo)ptr)->helpers;
        Helper      h;

        for (h = stack->head; h < stack->tail; h++) {
            if (NoCode != h->type) {
                rb_gc_mark(h->obj);
            }
        }
    }
}

VALUE
ox_parse(char *xml, size_t len, ParseCallbacks pcb, char **endp, Options options, Err err) {
    struct _pInfo  pi;
    int            body_read   = 0;
    int            block_given = rb_block_given_p();
    volatile VALUE wrap;

    if (0 == xml) {
        set_error(err, "Invalid arg, xml string can not be null", xml, 0);
        return Qnil;
    }
    if (DEBUG <= options->trace) {
        printf("Parsing xml:\n%s\n", xml);
    }
    // initialize parse info
    helper_stack_init(&pi.helpers);
    // Protect against GC
    wrap = TypedData_Wrap_Struct(rb_cObject, &ox_wrap_type, &pi);

    err_init(&pi.err);
    pi.str        = xml;
    pi.end        = pi.str + len;
    pi.s          = xml;
    pi.pcb        = pcb;
    pi.obj        = Qnil;
    pi.circ_array = 0;
    pi.options    = options;
    pi.marked     = NULL;
    pi.mark_size  = 0;
    pi.mark_cnt   = 0;
    while (1) {
        next_non_white(&pi);  // skip white space
        if ('\0' == *pi.s) {
            break;
        }
        if (body_read && 0 != endp) {
            *endp = pi.s;
            break;
        }
        if ('<' != *pi.s) {  // all top level entities start with <
            set_error(err, "invalid format, expected <", pi.str, pi.s);
            helper_stack_cleanup(&pi.helpers);
            return Qnil;
        }
        pi.s++;  // past <
        switch (*pi.s) {
        case '?':  // processing instruction
            pi.s++;
            read_instruction(&pi);
            break;
        case '!':  // comment or doctype
            pi.s++;
            if ('\0' == *pi.s) {
                set_error(err, "invalid format, DOCTYPE or comment not terminated", pi.str, pi.s);
                helper_stack_cleanup(&pi.helpers);
                return Qnil;
            } else if ('-' == *pi.s) {
                pi.s++;  // skip -
                if ('-' != *pi.s) {
                    set_error(err, "invalid format, bad comment format", pi.str, pi.s);
                    helper_stack_cleanup(&pi.helpers);
                    return Qnil;
                } else {
                    pi.s++;  // skip second -
                    read_comment(&pi);
                }
            } else if ((TolerantEffort == options->effort) ? 0 == strncasecmp("DOCTYPE", pi.s, 7)
                                                           : 0 == strncmp("DOCTYPE", pi.s, 7)) {
                pi.s += 7;
                read_doctype(&pi);
            } else {
                set_error(err, "invalid format, DOCTYPE or comment expected", pi.str, pi.s);
                helper_stack_cleanup(&pi.helpers);
                return Qnil;
            }
            break;
        case '\0':
            set_error(err, "invalid format, document not terminated", pi.str, pi.s);
            helper_stack_cleanup(&pi.helpers);
            return Qnil;
        default:
            read_element(&pi);
            body_read = 1;
            break;
        }
        if (err_has(&pi.err)) {
            *err = pi.err;
            helper_stack_cleanup(&pi.helpers);
            return Qnil;
        }
        if (block_given && Qnil != pi.obj && Qundef != pi.obj) {
            if (NULL != pcb->finish) {
                pcb->finish(&pi);
            }
            rb_yield(pi.obj);
        }
    }
    DATA_PTR(wrap) = NULL;
    helper_stack_cleanup(&pi.helpers);
    if (NULL != pcb->finish) {
        pcb->finish(&pi);
    }
    return pi.obj;
}

// Entered after the "<?" sequence. Ready to read the rest.
static void read_instruction(PInfo pi) {
    char              content[256];
    char             *content_ptr;
    struct _attrStack attrs;
    char             *attr_name;
    char             *attr_value;
    char             *target;
    char             *end;
    char              c;
    char             *cend;
    size_t            size;
    bool              attrs_ok = true;

    *content = '\0';
    attr_stack_init(&attrs);
    if (0 == (target = read_name_token(pi))) {
        return;
    }
    end = pi->s;
    for (; true; pi->s++) {
        switch (*pi->s) {
        case '?':
            if ('>' == *(pi->s + 1)) {
                pi->s++;
                goto DONE;
            }
            break;
        case '\0': set_error(&pi->err, "processing instruction not terminated", pi->str, pi->s); return;
        default: break;
        }
    }
DONE:
    cend  = pi->s;
    size  = cend - end - 1;
    pi->s = end;
    if (size < sizeof(content)) {
        content_ptr = content;
    } else {
        content_ptr = ALLOC_N(char, size + 1);
    }
    memcpy(content_ptr, end, size);
    content_ptr[size] = '\0';

    next_non_white(pi);
    c    = *pi->s;
    *end = '\0';  // terminate name
    if ('?' != c) {
        while ('?' != c) {
            pi->last = 0;
            if ('\0' == *pi->s) {
                attr_stack_cleanup(&attrs);
                set_error(&pi->err, "invalid format, processing instruction not terminated", pi->str, pi->s);
                return;
            }
            next_non_white(pi);
            if (0 == (attr_name = read_name_token(pi))) {
                attr_stack_cleanup(&attrs);
                return;
            }
            end = pi->s;
            next_non_white(pi);
            if ('=' != *pi->s++) {
                attrs_ok = false;
                break;
            }
            *end = '\0';  // terminate name
            // read value
            next_non_white(pi);
            if (0 == (attr_value = read_quoted_value(pi))) {
                attr_stack_cleanup(&attrs);
                return;
            }
            attr_stack_push(&attrs, attr_name, attr_value);
            next_non_white(pi);
            if ('\0' == pi->last) {
                c = *pi->s;
            } else {
                c = pi->last;
            }
        }
        if ('?' == *pi->s) {
            pi->s++;
        }
    } else {
        pi->s++;
    }
    if (attrs_ok) {
        if ('>' != *pi->s++) {
            attr_stack_cleanup(&attrs);
            set_error(&pi->err, "invalid format, processing instruction not terminated", pi->str, pi->s);
            return;
        }
    } else {
        pi->s = cend + 1;
    }
    if (0 != pi->pcb->instruct) {
        if (attrs_ok) {
            pi->pcb->instruct(pi, target, attrs.head, 0);
        } else {
            pi->pcb->instruct(pi, target, attrs.head, content_ptr);
        }
    } else {
        for (Attr a = attrs.head; a < attrs.tail; a++) {
            if (0 == strcasecmp(a->name, "encoding")) {
                strncpy(pi->options->encoding, a->value, sizeof(pi->options->encoding) - 1);
                pi->options->encoding[sizeof(pi->options->encoding) - 1] = '\0';
                pi->options->rb_enc                                      = rb_enc_find(a->value);
                break;
            }
        }
    }
    attr_stack_cleanup(&attrs);
    if (content_ptr != content) {
        xfree(content_ptr);
    }
}

static void read_delimited(PInfo pi, char end) {
    char c;

    if ('"' == end || '\'' == end) {
        for (c = *pi->s++; end != c; c = *pi->s++) {
            if ('\0' == c) {
                set_error(&pi->err, "invalid format, dectype not terminated", pi->str, pi->s);
                return;
            }
        }
    } else {
        while (1) {
            c = *pi->s++;
            if (end == c) {
                return;
            }
            switch (c) {
            case '\0': set_error(&pi->err, "invalid format, dectype not terminated", pi->str, pi->s); return;
            case '"': read_delimited(pi, c); break;
            case '\'': read_delimited(pi, c); break;
            case '[': read_delimited(pi, ']'); break;
            case '<': read_delimited(pi, '>'); break;
            default: break;
            }
        }
    }
}

// Entered after the "<!DOCTYPE" sequence plus the first character after
// that. Ready to read the rest.
static void read_doctype(PInfo pi) {
    char *doctype;

    next_non_white(pi);
    doctype = pi->s;
    read_delimited(pi, '>');
    if (err_has(&pi->err)) {
        return;
    }
    pi->s--;
    *pi->s = '\0';
    pi->s++;
    if (0 != pi->pcb->add_doctype) {
        fix_newlines(doctype);
        pi->pcb->add_doctype(pi, doctype);
    }
}

// Entered after "<!--". Returns error code.
static void read_comment(PInfo pi) {
    char *end;
    char *s;
    char *comment;
    int   done = 0;

    next_non_white(pi);
    comment = pi->s;
    end     = strstr(pi->s, "-->");
    if (0 == end) {
        set_error(&pi->err, "invalid format, comment not terminated", pi->str, pi->s);
        return;
    }
    for (s = end - 1; pi->s < s && !done; s--) {
        switch (*s) {
        case ' ':
        case '\t':
        case '\f':
        case '\n':
        case '\r': break;
        default:
            *(s + 1) = '\0';
            done     = 1;
            break;
        }
    }
    *end  = '\0';  // in case the comment was blank
    pi->s = end + 3;
    if (0 != pi->pcb->add_comment) {
        fix_newlines(comment);
        pi->pcb->add_comment(pi, comment);
    }
}

// Entered after the '<' and the first character after that. Returns stat
// code.
static char *read_element(PInfo pi) {
    struct _attrStack attrs;
    const char       *attr_name;
    const char       *attr_value;
    char             *name;
    char             *ename;
    char             *end;
    char              c;
    long              elen;
    int               hasChildren = 0;
    int               done        = 0;

    attr_stack_init(&attrs);
    if (0 == (ename = read_name_token(pi))) {
        return 0;
    }
    end  = pi->s;
    elen = end - ename;
    next_non_white(pi);
    c    = *pi->s;
    *end = '\0';
    if ('/' == c) {
        // empty element, no attributes and no children
        pi->s++;
        if ('>' != *pi->s) {
            attr_stack_cleanup(&attrs);
            set_error(&pi->err, "invalid format, element not closed", pi->str, pi->s);
            return 0;
        }
        pi->s++; /* past > */
        pi->pcb->add_element(pi, ename, attrs.head, hasChildren);
        pi->pcb->end_element(pi, ename);

        attr_stack_cleanup(&attrs);
        return 0;
    }
    /* read attribute names until the close (/ or >) is reached */
    while (!done) {
        if ('\0' == c) {
            if (pi->end <= pi->s) {
                break;
            }
            next_non_white(pi);
            c = *pi->s;
        }
        pi->last = 0;
        switch (c) {
        case '\0':
            attr_stack_cleanup(&attrs);
            set_error(&pi->err, "invalid format, document not terminated", pi->str, pi->s);
            return 0;
        case '/':
            /* Element with just attributes. */
            pi->s++;
            if ('>' != *pi->s) {
                attr_stack_cleanup(&attrs);
                set_error(&pi->err, "invalid format, element not closed", pi->str, pi->s);
                return 0;
            }
            pi->s++;
            pi->pcb->add_element(pi, ename, attrs.head, hasChildren);
            pi->pcb->end_element(pi, ename);
            attr_stack_cleanup(&attrs);

            return 0;
        case '>':
            /* has either children or a value */
            pi->s++;
            hasChildren = 1;
            done        = 1;
            pi->pcb->add_element(pi, ename, attrs.head, hasChildren);

            break;
        default:
            /* Attribute name so it's an element and the attribute will be */
            /* added to it. */
            if (0 == (attr_name = read_name_token(pi))) {
                attr_stack_cleanup(&attrs);
                return 0;
            }
            end = pi->s;
            next_non_white(pi);
            if ('=' != *pi->s++) {
                if (TolerantEffort == pi->options->effort) {
                    pi->s--;
                    pi->last   = *pi->s;
                    *end       = '\0'; /* terminate name */
                    attr_value = "";
                    attr_stack_push(&attrs, attr_name, attr_value);
                    break;
                } else {
                    attr_stack_cleanup(&attrs);
                    set_error(&pi->err, "invalid format, no attribute value", pi->str, pi->s);
                    return 0;
                }
            }
            *end = '\0'; /* terminate name */
            /* read value */
            next_non_white(pi);
            if (0 == (attr_value = read_quoted_value(pi))) {
                return 0;
            }
            if (pi->options->convert_special && 0 != strchr(attr_value, '&')) {
                if (0 != collapse_special(pi, (char *)attr_value) || err_has(&pi->err)) {
                    attr_stack_cleanup(&attrs);
                    return 0;
                }
            }
            attr_stack_push(&attrs, attr_name, attr_value);
            break;
        }
        if ('\0' == pi->last) {
            c = '\0';
        } else {
            c        = pi->last;
            pi->last = '\0';
        }
    }
    if (hasChildren) {
        char *start;
        int   first = 1;

        done = 0;
        /* read children */
        while (!done) {
            start = pi->s;
            next_non_white(pi);
            if (OffSkip == pi->options->skip && start < pi->s && '<' == *pi->s) {
                c      = *pi->s;
                *pi->s = '\0';
                pi->pcb->add_text(pi, start, 1);
                *pi->s = c;
            }
            c = *pi->s++;
            if ('\0' == c) {
                attr_stack_cleanup(&attrs);
                set_error(&pi->err, "invalid format, document not terminated", pi->str, pi->s);
                return 0;
            }
            if ('<' == c) {
                char *slash;

                switch (*pi->s) {
                case '!': /* better be a comment or CDATA */
                    pi->s++;
                    if ('-' == *pi->s && '-' == *(pi->s + 1)) {
                        pi->s += 2;
                        read_comment(pi);
                    } else if ((TolerantEffort == pi->options->effort) ? 0 == strncasecmp("[CDATA[", pi->s, 7)
                                                                       : 0 == strncmp("[CDATA[", pi->s, 7)) {
                        pi->s += 7;
                        read_cdata(pi);
                    } else {
                        attr_stack_cleanup(&attrs);
                        set_error(&pi->err, "invalid format, invalid comment or CDATA format", pi->str, pi->s);
                        return 0;
                    }
                    break;
                case '?': /* processing instruction */
                    pi->s++;
                    read_instruction(pi);
                    break;
                case '/':
                    slash = pi->s;
                    pi->s++;
                    if (0 == (name = read_name_token(pi))) {
                        attr_stack_cleanup(&attrs);
                        return 0;
                    }
                    end = pi->s;
                    next_non_white(pi);
                    c    = *pi->s;
                    *end = '\0';
                    if (0 !=
                        ((TolerantEffort == pi->options->effort) ? strcasecmp(name, ename) : strcmp(name, ename))) {
                        attr_stack_cleanup(&attrs);
                        if (TolerantEffort == pi->options->effort) {
                            pi->pcb->end_element(pi, ename);
                            return name;
                        } else {
                            set_error(&pi->err, "invalid format, elements overlap", pi->str, pi->s);
                            return 0;
                        }
                    }
                    if ('>' != c) {
                        attr_stack_cleanup(&attrs);
                        set_error(&pi->err, "invalid format, element not closed", pi->str, pi->s);
                        return 0;
                    }
                    if (first && start != slash - 1) {
                        // Some white space between start and here so add as
                        // text after checking skip.
                        *(slash - 1) = '\0';
                        switch (pi->options->skip) {
                        case CrSkip: {
                            char *s = start;
                            char *e = start;

                            for (; '\0' != *e; e++) {
                                if ('\r' != *e) {
                                    *s++ = *e;
                                }
                            }
                            *s = '\0';
                            break;
                        }
                        case SpcSkip: *start = '\0'; break;
                        case NoSkip:
                        case OffSkip:
                        default: break;
                        }
                        if ('\0' != *start) {
                            pi->pcb->add_text(pi, start, 1);
                        }
                    }
                    pi->s++;
                    pi->pcb->end_element(pi, ename);
                    attr_stack_cleanup(&attrs);
                    return 0;
                case '\0':
                    attr_stack_cleanup(&attrs);
                    if (TolerantEffort == pi->options->effort) {
                        return 0;
                    } else {
                        set_error(&pi->err, "invalid format, document not terminated", pi->str, pi->s);
                        return 0;
                    }
                default:
                    first = 0;
                    /* a child element */
                    // Child closed with mismatched name.
                    if (0 != (name = read_element(pi))) {
                        attr_stack_cleanup(&attrs);

                        if (0 ==
                            ((TolerantEffort == pi->options->effort) ? strcasecmp(name, ename) : strcmp(name, ename))) {
                            pi->s++;
                            pi->pcb->end_element(pi, ename);
                            return 0;
                        } else {  // not the correct element yet
                            pi->pcb->end_element(pi, ename);
                            return name;
                        }
                    } else if (err_has(&pi->err)) {
                        return 0;
                    }
                    break;
                }
            } else { /* read as TEXT */
                char prev = *(start - 1);

                pi->s = start;
                if ('>' != prev && (' ' <= prev || is_white(prev))) {
                    pi->s--;
                }
                read_text(pi);
                /*read_reduced_text(pi); */

                /* to exit read_text with no errors the next character must be < */
                if ('/' == *(pi->s + 1) &&
                    0 == ((TolerantEffort == pi->options->effort) ? strncasecmp(ename, pi->s + 2, elen)
                                                                  : strncmp(ename, pi->s + 2, elen)) &&
                    '>' == *(pi->s + elen + 2)) {
                    /* close tag after text so treat as a value */
                    pi->s += elen + 3;
                    pi->pcb->end_element(pi, ename);
                    attr_stack_cleanup(&attrs);
                    return 0;
                }
            }
        }
    }
    attr_stack_cleanup(&attrs);
    return 0;
}

static void read_text(PInfo pi) {
    char  buf[MAX_TEXT_LEN];
    char *b         = buf;
    char *alloc_buf = 0;
    char *end       = b + sizeof(buf) - 2;
    char  c;
    int   done = 0;

    while (!done) {
        c = *pi->s++;
        switch (c) {
        case '<':
            done = 1;
            pi->s--;
            break;
        case '\0': set_error(&pi->err, "invalid format, document not terminated", pi->str, pi->s); return;
        default:
            if (end <= (b + (('&' == c) ? 7 : 0))) { /* extra 8 for special just in case it is sequence of bytes */
                unsigned long size;

                if (0 == alloc_buf) {
                    size      = sizeof(buf) * 2;
                    alloc_buf = ALLOC_N(char, size);
                    memcpy(alloc_buf, buf, b - buf);
                    b = alloc_buf + (b - buf);
                } else {
                    unsigned long pos = b - alloc_buf;

                    size = (end - alloc_buf) * 2;
                    REALLOC_N(alloc_buf, char, size);
                    b = alloc_buf + pos;
                }
                end = alloc_buf + size - 2;
            }
            if ('&' == c) {
                if (0 == (b = read_coded_chars(pi, b))) {
                    return;
                }
            } else {
                if (0 <= c && c <= 0x20) {
                    if (StrictEffort == pi->options->effort && 'x' == xml_valid_lower_chars[(unsigned char)c]) {
                        set_error(&pi->err, "invalid character", pi->str, pi->s);
                        return;
                    }
                    switch (pi->options->skip) {
                    case CrSkip:
                        if (buf != b && '\n' == c && '\r' == *(b - 1)) {
                            *(b - 1) = '\n';
                        } else {
                            *b++ = c;
                        }
                        break;
                    case SpcSkip:
                        if (is_white(c)) {
                            if (buf == b || ' ' != *(b - 1)) {
                                *b++ = ' ';
                            }
                        } else {
                            *b++ = c;
                        }
                        break;
                    case NoSkip:
                    case OffSkip:
                    default: *b++ = c; break;
                    }
                } else {
                    *b++ = c;
                }
            }
            break;
        }
    }
    *b = '\0';
    if (0 != alloc_buf) {
        fix_newlines(alloc_buf);
        pi->pcb->add_text(pi, alloc_buf, ('/' == *(pi->s + 1)));
        xfree(alloc_buf);
    } else {
        fix_newlines(buf);
        pi->pcb->add_text(pi, buf, ('/' == *(pi->s + 1)));
    }
}

#if 0
static void
read_reduced_text(PInfo pi) {
    char	buf[MAX_TEXT_LEN];
    char	*b = buf;
    char	*alloc_buf = 0;
    char	*end = b + sizeof(buf) - 2;
    char	c;
    int		spc = 0;
    int		done = 0;

    while (!done) {
	c = *pi->s++;
	switch(c) {
	case ' ':
	case '\t':
	case '\f':
	case '\n':
	case '\r':
	    spc = 1;
	    break;
	case '<':
	    done = 1;
	    pi->s--;
	    break;
	case '\0':
	    set_error(&pi->err, "invalid format, document not terminated", pi->str, pi->s);
	    return;
	default:
	    if (end <= (b + spc + (('&' == c) ? 7 : 0))) { /* extra 8 for special just in case it is sequence of bytes */
		unsigned long	size;

		if (0 == alloc_buf) {
		    size = sizeof(buf) * 2;
		    alloc_buf = ALLOC_N(char, size);
		    memcpy(alloc_buf, buf, b - buf);
		    b = alloc_buf + (b - buf);
		} else {
		    unsigned long	pos = b - alloc_buf;

		    size = (end - alloc_buf) * 2;
		    REALLOC(alloc_buf, char, size);
		    b = alloc_buf + pos;
		}
		end = alloc_buf + size - 2;
	    }
	    if (spc) {
		*b++ = ' ';
	    }
	    spc = 0;
	    if ('&' == c) {
		if (0 == (b = read_coded_chars(pi, b))) {
		    return;
		}
	    } else {
		*b++ = c;
	    }
	    break;
	}
    }
    *b = '\0';
    if (0 != alloc_buf) {
	fix_newlines(alloc_buf);
	pi->pcb->add_text(pi, alloc_buf, ('/' == *(pi->s + 1)));
	xfree(alloc_buf);
    } else {
	fix_newlines(buf);
	pi->pcb->add_text(pi, buf, ('/' == *(pi->s + 1)));
    }
}
#endif

static char *read_name_token(PInfo pi) {
    char *start;

    next_non_white(pi);
    start = pi->s;
    for (; 1; pi->s++) {
        switch (*pi->s) {
        case ' ':
        case '\t':
        case '\f':
        case '?':
        case '=':
        case '/':
        case '>':
        case '\n':
        case '\r': return start;
        case '\0':
            /* documents never terminate after a name token */
            set_error(&pi->err, "invalid format, document not terminated", pi->str, pi->s);
            return 0;
            break; /* to avoid warnings */
        case ':':
            if ('\0' == *pi->options->strip_ns) {
                break;
            } else if ('*' == *pi->options->strip_ns && '\0' == pi->options->strip_ns[1]) {
                start = pi->s + 1;
            } else if (0 == strncmp(pi->options->strip_ns, start, pi->s - start)) {
                start = pi->s + 1;
            }
            break;
        default: break;
        }
    }
    return start;
}

static void read_cdata(PInfo pi) {
    char *start;
    char *end;

    start = pi->s;
    end   = strstr(pi->s, "]]>");
    if (end == 0) {
        set_error(&pi->err, "invalid format, CDATA not terminated", pi->str, pi->s);
        return;
    }
    *end  = '\0';
    pi->s = end + 3;
    if (0 != pi->pcb->add_cdata) {
        fix_newlines(start);
        pi->pcb->add_cdata(pi, start, end - start);
    }
}

/* Assume the value starts immediately and goes until the quote character is
 * reached again. Do not read the character after the terminating quote.
 */
static char *read_quoted_value(PInfo pi) {
    char *value = 0;

    if ('"' == *pi->s || '\'' == *pi->s) {
        char term = *pi->s;

        pi->s++; /* skip quote character */
        value = pi->s;
        for (; *pi->s != term; pi->s++) {
            if ('\0' == *pi->s) {
                set_error(&pi->err, "invalid format, document not terminated", pi->str, pi->s);
                return 0;
            }
        }
        *pi->s = '\0'; /* terminate value */
        pi->s++;       /* move past quote */
    } else if (StrictEffort == pi->options->effort) {
        set_error(&pi->err, "invalid format, expected a quote character", pi->str, pi->s);
        return 0;
    } else if (TolerantEffort == pi->options->effort) {
        value = pi->s;
        for (; 1; pi->s++) {
            switch (*pi->s) {
            case '\0': set_error(&pi->err, "invalid format, document not terminated", pi->str, pi->s); return 0;
            case ' ':
            case '/':
            case '>':
            case '?':  // for instructions
            case '\t':
            case '\n':
            case '\r':
                pi->last = *pi->s;
                *pi->s   = '\0'; /* terminate value */
                pi->s++;
                return value;
            default: break;
            }
        }
    } else {
        value = pi->s;
        next_white(pi);
        if ('\0' == *pi->s) {
            set_error(&pi->err, "invalid format, document not terminated", pi->str, pi->s);
            return 0;
        }
        *pi->s++ = '\0'; /* terminate value */
    }
    return value;
}

static char *read_hex_uint64(char *b, uint64_t *up) {
    uint64_t u = 0;
    char     c;

    for (; ';' != *b; b++) {
        c = *b;
        if ('0' <= c && c <= '9') {
            u = (u << 4) | (uint64_t)(c - '0');
        } else if ('a' <= c && c <= 'f') {
            u = (u << 4) | (uint64_t)(c - 'a' + 10);
        } else if ('A' <= c && c <= 'F') {
            u = (u << 4) | (uint64_t)(c - 'A' + 10);
        } else {
            return 0;
        }
    }
    *up = u;

    return b;
}

static char *read_10_uint64(char *b, uint64_t *up) {
    uint64_t u = 0;
    char     c;

    for (; ';' != *b; b++) {
        c = *b;
        if ('0' <= c && c <= '9') {
            u = (u * 10) + (uint64_t)(c - '0');
        } else {
            return 0;
        }
    }
    *up = u;

    return b;
}

static char *read_coded_chars(PInfo pi, char *text) {
    char *b, buf[32];
    char *end = buf + sizeof(buf) - 1;
    char *s;
    long  blen = 0;

    for (b = buf, s = pi->s; b < end; b++, s++) {
        *b = *s;
        if (';' == *s) {
            *(b + 1) = '\0';
            blen     = b - buf;
            s++;
            break;
        }
    }
    if (b > end) {
        *text++ = '&';
    } else if ('#' == *buf) {
        uint64_t u = 0;

        b = buf + 1;
        if ('x' == *b || 'X' == *b) {
            b = read_hex_uint64(b + 1, &u);
        } else {
            b = read_10_uint64(b, &u);
        }
        if (0 == b) {
            *text++ = '&';
        } else {
            if (u <= 0x000000000000007FULL) {
                *text++ = (char)u;
            } else if (ox_utf8_encoding == pi->options->rb_enc) {
                text = ox_ucs_to_utf8_chars(text, u);
            } else if (0 == pi->options->rb_enc) {
                pi->options->rb_enc = ox_utf8_encoding;
                text                = ox_ucs_to_utf8_chars(text, u);
            } else if (TolerantEffort == pi->options->effort) {
                *text++ = '&';
                return text;
            } else if (u <= 0x00000000000000FFULL) {
                *text++ = (char)u;
            } else {
                /*set_error(&pi->err, "Invalid encoding, need UTF-8 or UTF-16 encoding to parse &#nnnn; character
                 * sequences.", pi->str, pi->s); */
                set_error(&pi->err,
                          "Invalid encoding, need UTF-8 encoding to parse &#nnnn; character sequences.",
                          pi->str,
                          pi->s);
                return NULL;
            }
            pi->s = s;
        }
    } else {
        char *t2;

        buf[blen] = '\0';
        if (NULL == (t2 = ox_entity_lookup(text, buf))) {
            *text++ = '&';
        } else {
            text  = t2;
            pi->s = s;
        }
    }
    return text;
}

static int collapse_special(PInfo pi, char *str) {
    char *s = str;
    char *b = str;

    while ('\0' != *s) {
        if ('&' == *s) {
            int   c;
            char *end;

            s++;
            if ('#' == *s) {
                uint64_t u = 0;
                char     x;

                s++;
                if ('x' == *s || 'X' == *s) {
                    x = *s;
                    s++;
                    end = read_hex_uint64(s, &u);
                } else {
                    x   = '\0';
                    end = read_10_uint64(s, &u);
                }
                if (0 == end) {
                    if (TolerantEffort == pi->options->effort) {
                        *b++ = '&';
                        *b++ = '#';
                        if ('\0' != x) {
                            *b++ = x;
                        }
                        continue;
                    }
                    return EDOM;
                }
                if (u <= 0x000000000000007FULL) {
                    *b++ = (char)u;
                } else if (ox_utf8_encoding == pi->options->rb_enc) {
                    b = ox_ucs_to_utf8_chars(b, u);
                    /* TBD support UTF-16 */
                } else if (0 == pi->options->rb_enc) {
                    pi->options->rb_enc = ox_utf8_encoding;
                    b                   = ox_ucs_to_utf8_chars(b, u);
                } else {
                    /* set_error(&pi->err, "Invalid encoding, need UTF-8 or UTF-16 encoding to parse &#nnnn; character
                     * sequences.", pi->str, pi->s);*/
                    set_error(&pi->err,
                              "Invalid encoding, need UTF-8 encoding to parse &#nnnn; character sequences.",
                              pi->str,
                              pi->s);
                    return 0;
                }
                s = end + 1;
            } else {
                if (0 == strncasecmp(s, "lt;", 3)) {
                    c = '<';
                    s += 3;
                } else if (0 == strncasecmp(s, "gt;", 3)) {
                    c = '>';
                    s += 3;
                } else if (0 == strncasecmp(s, "amp;", 4)) {
                    c = '&';
                    s += 4;
                } else if (0 == strncasecmp(s, "quot;", 5)) {
                    c = '"';
                    s += 5;
                } else if (0 == strncasecmp(s, "apos;", 5)) {
                    c = '\'';
                    s += 5;
                } else if (TolerantEffort == pi->options->effort) {
                    *b++ = '&';
                    continue;
                } else {
                    char  key[16];
                    char *k    = key;
                    char *kend = key + sizeof(key) - 1;

                    *k++ = *s;
                    while (';' != *s++) {
                        if ('\0' == *s) {
                            set_error(&pi->err,
                                      "Invalid format, special character does not end with a semicolon",
                                      pi->str,
                                      pi->s);
                            return EDOM;
                        }
                        if (kend <= k) {
                            k = key + 1;  // because k-- follows
                            break;
                        }
                        *k++ = *s;
                    }
                    k--;
                    *k = '\0';
                    if ('\0' == *key || NULL == (b = ox_entity_lookup(b, key))) {
                        set_error(&pi->err, "Invalid format, invalid special character sequence", pi->str, pi->s);
                        c = '?';
                        return 0;
                    }
                    continue;
                }
                *b++ = (char)c;
            }
        } else {
            *b++ = *s++;
        }
    }
    *b = '\0';

    return 0;
}