/*
 * Basic Charset functions for QEmacs
 *
 * Copyright (c) 2000-2002 Fabrice Bellard.
 * Copyright (c) 2002-2024 Charlie Gordon.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include "charset.h"

/* XXX: Should move this to QEmacsState, and find a way for html2png */
struct QECharset *first_charset;

#define REP2(x)    x, x
#define REP4(x)    x, x, x, x
#define REP8(x)    REP4(x), REP4(x)
#define REP16(x)   REP4(x), REP4(x), REP4(x), REP4(x)
#define REP32(x)   REP16(x), REP16(x)
#define REP64(x)   REP16(x), REP16(x), REP16(x), REP16(x)
#define REP128(x)  REP64(x), REP64(x)
#define REP256(x)  REP64(x), REP64(x), REP64(x), REP64(x)

#define RUN2(x)    (x)+0, (x)+1
#define RUN4(x)    (x)+0, (x)+1, (x)+2, (x)+3
#define RUN8(x)    RUN4(x), RUN4((x)+4)
#define RUN16(x)   RUN4(x), RUN4((x)+4), RUN4((x)+8), RUN4((x)+12)
#define RUN32(x)   RUN16(x), RUN16((x)+16)
#define RUN64(x)   RUN16(x), RUN16((x)+16), RUN16((x)+32), RUN16((x)+48)
#define RUN128(x)  RUN64(x), RUN64((x)+64)
#define RUN256(x)  RUN64(x), RUN64((x)+64), RUN64((x)+128), RUN64((x)+192)

static unsigned short const table_idem[256] = { RUN256(0) };
static unsigned short const table_none[256] = { REP256(ESCAPE_CHAR) };

/********************************************************/
/* raw */

static u8 *encode_raw(qe__unused__ QECharset *charset, u8 *p, char32_t c) {
    if (c <= 0xff) {
        *p++ = c;
        return p;
    } else {
        return NULL;
    }
}

struct QECharset charset_raw = {
    "raw",
    "binary|none",
    NULL,
    NULL,
    decode_8bit,
    encode_raw,
    charset_get_pos_8bit,
    charset_get_chars_8bit,
    charset_goto_char_8bit,
    charset_goto_line_8bit,
    1, 0, 0, 10, 0, 0, table_idem, NULL, NULL,
};

/********************************************************/
/* 8859-1 */

static int probe_8859_1(qe__unused__ QECharset *charset, const u8 *buf, int size)
{
    const uint32_t magic = (1U << '\b') | (1U << '\t') | (1U << '\f') |
                           (1U << '\n') | (1U << '\r') | (1U << '\033') |
                           (1U << 0x0e) | (1U << 0x0f) | (1U << 0x1a) |
                           (1U << 0x1f);
    const u8 *p = buf;
    const u8 *p_end = p + size;
    uint32_t c;
    int count_spaces, count_lines, count_high;

    count_spaces = count_lines = count_high = 0;

    while (p < p_end) {
        c = p[0];
        p += 1;
        if (c <= 32) {
            if (c == ' ')
                count_spaces++;
            else
            if (c == '\n')
                count_lines++;
            else
            if (!(magic & (1U << c)))
                return 0;
        } else
        if (c < 0x7F) {
            continue;
        } else
        if (c < 0x80) {
            return 0;
        } else {
            count_high++;
        }
    }
    if (count_spaces | count_lines)
        return 1;
    else
        return 0;
}

static u8 *encode_8859_1(qe__unused__ QECharset *charset, u8 *p, char32_t c)
{
    if (c <= 0xff) {
        *p++ = c;
        return p;
    } else {
        return NULL;
    }
}

struct QECharset charset_8859_1 = {
    "8859-1",
    "ISO-8859-1|iso-ir-100|latin1|l1|819",
    probe_8859_1,
    NULL,
    decode_8bit,
    encode_8859_1,
    charset_get_pos_8bit,
    charset_get_chars_8bit,
    charset_goto_char_8bit,
    charset_goto_line_8bit,
    1, 0, 0, 10, 0, 0, table_idem, NULL, NULL,
};

/********************************************************/
/* vt100 */

static u8 *encode_vt100(qe__unused__ QECharset *charset, u8 *p, char32_t c)
{
    if (c <= 0xff) {
        *p++ = c;
        return p;
    } else {
        return NULL;
    }
}

struct QECharset charset_vt100 = {
    "vt100",
    NULL,
    NULL,
    NULL,
    decode_8bit,
    encode_vt100,
    charset_get_pos_8bit,
    charset_get_chars_8bit,
    charset_goto_char_8bit,
    charset_goto_line_8bit,
    1, 0, 0, 10, 0, 0, table_idem, NULL, NULL,
};

/********************************************************/
/* 7 bit */

static u8 *encode_7bit(qe__unused__ QECharset *charset, u8 *p, char32_t c)
{
    if (c <= 0x7f) {
        *p++ = c;
        return p;
    } else {
        return NULL;
    }
}

static struct QECharset charset_7bit = {
    "7bit",
    "us-ascii|ascii|7-bit|iso-ir-6|ANSI_X3.4|646",
    NULL,
    NULL,
    decode_8bit,
    encode_7bit,
    charset_get_pos_8bit,
    charset_get_chars_8bit,
    charset_goto_char_8bit,
    charset_goto_line_8bit,
    1, 0, 0, 10, 0, 0, table_idem, NULL, NULL,
};

/********************************************************/
/* UTF-8 */

static unsigned short const table_utf8[256] = {
    RUN128(0),              /* [0x00...0x7F] are self-encoding ASCII bytes */
    REP64(INVALID_CHAR),    /* [0x80...0xBF] are invalid prefix bytes */
    /* 0xC0 and 0xC1 are invalid prefix bytes because they encode 7-bit codes */
    REP32(ESCAPE_CHAR),     /* [0xC0...0xDF] leading bytes of 2 byte sequences */
    REP16(ESCAPE_CHAR),     /* [0xE0...0xEF] leading bytes of 3 byte sequences */
    REP8(ESCAPE_CHAR),      /* [0xF0...0xF7] leading bytes of 4 byte sequences */
    REP4(ESCAPE_CHAR),      /* [0xF8...0xFB] leading bytes of 5 byte sequences */
    REP2(ESCAPE_CHAR),      /* [0xFC...0xFD] leading bytes of 6 byte sequences */
    /* allow for utf8x encoding */
    ESCAPE_CHAR,//INVALID_CHAR,           /* 0xFE is invalid in UTF-8 encoding */
    ESCAPE_CHAR,//INVALID_CHAR,           /* 0xFF is invalid in UTF-8 encoding */
};

static int probe_utf8(qe__unused__ QECharset *charset, const u8 *buf, int size)
{
    /* probe initial file block for UTF-8 encoding:
       scan buffer contents for valid ASCII and UTF-8 contents */
    const uint32_t magic = (1U << '\b') | (1U << '\t') | (1U << '\f') |
                           (1U << '\n') | (1U << '\r') | (1U << '\033') |
                           (1U << 0x0e) | (1U << 0x0f) | (1U << 0x1a) |
                           (1U << 0x1f);
    const u8 *p = buf;
    const u8 *p_end = p + size;
    uint32_t c;
    int count_spaces, count_lines, count_utf8;

    count_spaces = count_lines = count_utf8 = 0;

    while (p < p_end) {
        c = *p++;
        if (c <= 32) {
            if (c == ' ')
                count_spaces++;
            else
            if (c == '\n')
                count_lines++;
            else
            if (!(magic & (1U << c)))
                return 0;
        } else
        if (c < 0x7F) { /* ASCII subset */
            continue;
        } else
        if (c < 0xc2) { /* UTF-8 trailing bytes and invalid prefix bytes */
            return 0;
        } else
        if (c < 0xe0) { /* 2 byte UTF-8 encoding for U+0080..U+07FF */
            if ((p[0] ^ 0x80) > 0x3f)
                return 0;
            count_utf8++;
            p += 1;
        } else
        if (c < 0xf0) { /* 3 byte UTF-8 encoding for U+0800..U+FFFF */
            if ((p[0] ^ 0x80) > 0x3f || (p[1] ^ 0x80) > 0x3f)
                return 0;
            count_utf8++;
            p += 2;
        } else
        if (c < 0xf8) { /* 4 byte UTF-8 encoding for U+10000..U+10FFFF */
            if ((p[0] ^ 0x80) > 0x3f || (p[1] ^ 0x80) > 0x3f
            ||  (p[2] ^ 0x80) > 0x3f)
                return 0;
            count_utf8++;
            p += 3;
        } else { /* overlong encodings and invalid lead bytes */
            return 0;
        }
    }
    if (count_spaces | count_lines | count_utf8)
        return 1;
    else
        return 0;
}

static char32_t decode_utf8_func(CharsetDecodeState *s)
{
    return utf8_decode((const char **)(void *)&s->p);
}

static u8 *encode_utf8(qe__unused__ QECharset *charset, u8 *q, char32_t c)
{
    return q + utf8_encode((char*)q, c);
}

/* return the number of lines and column position for a buffer */
static void charset_get_pos_utf8(CharsetDecodeState *s, const u8 *buf, int size,
                                 int *line_ptr, int *col_ptr)
{
    const u8 *p, *p1, *lp;
    int nl, line, col;

    QASSERT(size >= 0);

    line = 0;
    lp = p = buf;
    p1 = p + size;
    nl = s->eol_char;

    for (;;) {
        p = memchr(p, nl, p1 - p);
        if (!p)
            break;
        p++;
        lp = p;
        line++;
    }
    /* now compute number of chars (XXX: potential problem if out of
     * block, but works for correctly encoded UTF-8 contents. */
    // XXX: trailing bytes are counted at the start of the block.
    //      this is inconsistent with charset_goto_char_utf8().
    //      should instead count the number of single and leading bytes
    col = 0;
    while (lp < p1) {
        col++;
        lp += utf8_length[*lp];
    }
    *line_ptr = line;
    *col_ptr = col;
}

static int charset_get_chars_utf8(CharsetDecodeState *s,
                                  const u8 *buf, int size)
{
    int nb_chars, c;
    const u8 *buf_end, *buf_ptr;

    nb_chars = 0;
    buf_ptr = buf;
    buf_end = buf_ptr + size;
    while (buf_ptr < buf_end) {
        c = *buf_ptr++;
        if (c == '\n' && s->eol_type == EOL_DOS) {
            /* ignore \n in EOL_DOS scan, but count \r.
             * XXX: potentially incorrect if buffer contains
             * \n not preceded by \r and requires special state
             * data to handle \r\n sequence at page boundary.
             */
            continue;
        }
        /* ignoring trailing bytes: will produce incorrect
         * count on isolated and trailing bytes and overlong
         * sequences.
         */
        nb_chars += ((c ^ 0x80) > 0x3f);
    }
    /* CG: nb_chars is the number of character boundaries, trailing
     * UTF-8 sequence at start of buffer is ignored in count while
     * incomplete UTF-8 sequence at end of buffer is counted.  This may
     * cause problems when counting characters with eb_get_pos with an
     * offset falling inside a UTF-8 sequence, and will produce
     * incorrect counts on broken UTF-8 sequences spanning page
     * boundaries.
     */
    return nb_chars;
}

static int charset_goto_char_utf8(CharsetDecodeState *s,
                                  const u8 *buf, int size, int pos)
{
    int nb_chars, c;
    const u8 *buf_ptr, *buf_end;

    nb_chars = 0;
    buf_ptr = buf;
    buf_end = buf_ptr + size;
    for (; buf_ptr < buf_end; buf_ptr++) {
        c = *buf_ptr;
        if ((c ^ 0x80) <= 0x3f) {
            /* Test done here to skip initial trailing bytes if any */
            continue;
        }
        if (c == '\n' && s->eol_type == EOL_DOS) {
            /* ignore \n in EOL_DOS scan, but count \r.
             * see comment above.
             */
            continue;
        }
        if (nb_chars >= pos)
            break;
        nb_chars++;
    }
    return buf_ptr - buf;
}

struct QECharset charset_utf8 = {
    "utf8",
    "utf-8|al32utf8",
    probe_utf8,
    NULL,
    decode_utf8_func,
    encode_utf8,
    charset_get_pos_utf8,
    charset_get_chars_utf8,
    charset_goto_char_utf8,
    charset_goto_line_8bit,
    1, 1, 0, 10, 0, 0, table_utf8, NULL, NULL,
};

/* Extended UTF-8 encoding: accept any 32-bit value from 0 to 0xffffffff
   All combinations are accepted.
 */
// XXX: should also have CESU-8 where non-BMP1 glyphs are encoded as
//      surrogate pairs, themselves encoded in UTF-8.
static struct QECharset charset_utf8x = {
    "utf8x",
    "utf-8x",
    probe_utf8,
    NULL,
    decode_utf8_func,
    encode_utf8,
    charset_get_pos_utf8,
    charset_get_chars_utf8,
    charset_goto_char_utf8,
    charset_goto_line_8bit,
    1, 1, 0, 10, 0, 0, table_utf8, NULL, NULL,
};

/********************************************************/
/* UCS2/UCS4 */

static int probe_ucs2le(qe__unused__ QECharset *charset, const u8 *buf, int size)
{
    const uint32_t magic = (1U << '\b') | (1U << '\t') | (1U << '\f') |
                           (1U << '\n') | (1U << '\r') | (1U << '\033') |
                           (1U << 0x0e) | (1U << 0x0f) | (1U << 0x1a) |
                           (1U << 0x1f);
    const u8 *p = buf;
    const u8 *p_end = p + (size & ~1);
    uint32_t c;
    int count_spaces, count_lines;

    if (size & 1)
        return 0;

    count_spaces = count_lines = 0;

    while (p < p_end) {
        c = (p[0] << 0) | (p[1] << 8);
        p += 2;
        if (c <= 32) {
            if (c == ' ')
                count_spaces++;
            else
            if (c == '\n')
                count_lines++;
            else
            if (!(magic & (1U << c)))
                return 0;
        } else
        if (c >= 0x10000) {
            return 0;
        }
    }
    if (count_spaces + count_lines > size / (16 * 2))
        return 1;
    else
        return 0;
}

static char32_t decode_ucs2le(CharsetDecodeState *s) {
    /* XXX: should handle surrogates */
    const u8 *p;

    p = s->p;
    s->p += 2;
    return p[0] + ((char32_t)p[1] << 8);
}

static u8 *encode_ucs2le(qe__unused__ QECharset *charset, u8 *p, char32_t c)
{
    /* XXX: should handle surrogates */
    p[0] = c;
    p[1] = c >> 8;
    return p + 2;
}

/* return the number of lines and column position for a buffer */
static void charset_get_pos_ucs2(CharsetDecodeState *s, const u8 *buf, int size,
                                 int *line_ptr, int *col_ptr)
{
    const uint16_t *p, *p1, *lp;
    uint16_t nl, lf;
    union { uint16_t n; char c[2]; } u;
    int line, col;

    line = 0;
    lp = p = (const uint16_t *)(const void *)buf;
    p1 = p + (size >> 1);
    u.n = 0;
    u.c[s->charset == &charset_ucs2be] = s->eol_char;
    nl = u.n;
    u.c[s->charset == &charset_ucs2be] = '\n';
    lf = u.n;

    if (s->eol_type == EOL_DOS && p < p1 && *p == lf) {
        /* Skip \n at start of buffer.
         * Should check for pending skip state */
        p++;
        lp++;
    }

    /* XXX: should handle surrogates */
    while (p < p1) {
        if (*p++ == nl) {
            if (s->eol_type == EOL_DOS && p < p1 && *p == lf)
                p++;
            lp = p;
            line++;
        }
    }
    col = p1 - lp;
    *line_ptr = line;
    *col_ptr = col;
}

static int charset_goto_line_ucs2(CharsetDecodeState *s,
                                  const u8 *buf, int size, int nlines)
{
    const uint16_t *p, *p1, *lp;
    uint16_t nl, lf;
    union { uint16_t n; char c[2]; } u;

    lp = p = (const uint16_t *)(const void *)buf;
    p1 = p + (size >> 1);
    u.n = 0;
    u.c[s->charset == &charset_ucs2be] = s->eol_char;
    nl = u.n;
    u.c[s->charset == &charset_ucs2be] = '\n';
    lf = u.n;

    if (s->eol_type == EOL_DOS && p < p1 && *p == lf) {
        /* Skip \n at start of buffer.
         * Should check for pending skip state */
        p++;
        lp++;
    }

    while (nlines > 0 && p < p1) {
        while (p < p1) {
            if (*p++ == nl) {
                if (s->eol_type == EOL_DOS && p < p1 && *p == lf)
                    p++;
                lp = p;
                nlines--;
                break;
            }
        }
    }
    return (const u8 *)lp - buf;
}

static int probe_ucs2be(qe__unused__ QECharset *charset, const u8 *buf, int size)
{
    const uint32_t magic = (1U << '\b') | (1U << '\t') | (1U << '\f') |
                           (1U << '\n') | (1U << '\r') | (1U << '\033') |
                           (1U << 0x0e) | (1U << 0x0f) | (1U << 0x1a) |
                           (1U << 0x1f);
    const u8 *p = buf;
    const u8 *p_end = p + (size & ~1);
    uint32_t c;
    int count_spaces, count_lines;

    if (size & 1)
        return 0;

    count_spaces = count_lines = 0;

    while (p < p_end) {
        c = (p[0] << 8) | (p[1] << 0);
        p += 2;
        if (c <= 32) {
            if (c == ' ')
                count_spaces++;
            else
            if (c == '\n')
                count_lines++;
            else
            if (!(magic & (1U << c)))
                return 0;
        } else
        if (c >= 0x10000) {
            return 0;
        }
    }
    if (count_spaces + count_lines > size / (16 * 2))
        return 1;
    else
        return 0;
}

static char32_t decode_ucs2be(CharsetDecodeState *s) {
    /* XXX: should handle surrogates */
    const u8 *p;

    p = s->p;
    s->p += 2;
    return ((char32_t)p[0] << 8) + p[1];
}

static u8 *encode_ucs2be(qe__unused__ QECharset *charset, u8 *p, char32_t c)
{
    /* XXX: should handle surrogates */
    p[0] = c >> 8;
    p[1] = c;
    return p + 2;
}

static int charset_get_chars_ucs2(CharsetDecodeState *s,
                                  const u8 *buf, int size)
{
    /* XXX: should handle surrogates */
    int count = size >> 1;  /* convert byte count to char16 count */
    const uint16_t *buf_end, *buf_ptr;
    uint16_t nl;
    union { uint16_t n; char c[2]; } u;

    if (s->eol_type != EOL_DOS)
        return count;

    buf_ptr = (const uint16_t *)(const void *)buf;
    buf_end = buf_ptr + count;
    // XXX: undefined behavior
    u.n = 0;
    u.c[s->charset == &charset_ucs2be] = '\n';
    nl = u.n;

    while (buf_ptr < buf_end) {
        if (*buf_ptr++ == nl) {
            /* ignore \n in EOL_DOS scan, but count \r. (see above) */
            count--;
        }
    }
    return count;
}

static int charset_goto_char_ucs2(CharsetDecodeState *s,
                                  const u8 *buf, int size, int pos)
{
    /* XXX: should handle surrogates */
    int nb_chars;
    const uint16_t *buf_ptr, *buf_end;
    uint16_t nl;
    union { uint16_t n; char c[2]; } u;

    if (s->eol_type != EOL_DOS)
        return min_offset(pos << 1, size);

    nb_chars = 0;
    buf_ptr = (const uint16_t *)(const void *)buf;
    buf_end = buf_ptr + (size >> 1);
    // XXX: undefined behavior
    u.n = 0;
    u.c[s->charset == &charset_ucs2be] = '\n';
    nl = u.n;

    for (; buf_ptr < buf_end; buf_ptr++) {
        if (*buf_ptr == nl) {
            /* ignore \n in EOL_DOS scan, but count \r. (see above) */
            continue;
        }
        if (nb_chars >= pos)
            break;
        nb_chars++;
    }
    return (const u8*)buf_ptr - buf;
}

struct QECharset charset_ucs2le = {
    "ucs2le",
    "utf16le|utf-16le",
    probe_ucs2le,
    NULL,
    decode_ucs2le,
    encode_ucs2le,
    charset_get_pos_ucs2,
    charset_get_chars_ucs2,
    charset_goto_char_ucs2,
    charset_goto_line_ucs2,
    2, 0, 0, 10, 0, 0, table_none, NULL, NULL,
};

struct QECharset charset_ucs2be = {
    "ucs2be",
    "ucs2|utf16|utf-16|utf16be|utf-16be",
    probe_ucs2be,
    NULL,
    decode_ucs2be,
    encode_ucs2be,
    charset_get_pos_ucs2,
    charset_get_chars_ucs2,
    charset_goto_char_ucs2,
    charset_goto_line_ucs2,
    2, 0, 0, 10, 0, 0, table_none, NULL, NULL,
};

static int probe_ucs4le(qe__unused__ QECharset *charset, const u8 *buf, int size)
{
    const uint32_t magic = (1U << '\b') | (1U << '\t') | (1U << '\f') |
                           (1U << '\n') | (1U << '\r') | (1U << '\033') |
                           (1U << 0x0e) | (1U << 0x0f) | (1U << 0x1a) |
                           (1U << 0x1f);
    const u8 *p = buf;
    const u8 *p_end = p + (size & ~3);
    uint32_t c;
    int count_spaces, count_lines;

    if (size & 3)
        return 0;

    count_spaces = count_lines = 0;

    while (p < p_end) {
        c = (p[0] << 0) | (p[1] << 8) | (p[2] << 16) | (p[3] << 24);
        p += 4;
        if (c <= 32) {
            if (c == ' ')
                count_spaces++;
            else
            if (c == '\n')
                count_lines++;
            else
            if (!(magic & (1U << c)))
                return 0;
        } else
        if (c > 0x10FFFF) {
            return 0;
        }
    }
    if (count_spaces + count_lines > size / (16 * 4))
        return 1;
    else
        return 0;
}

static char32_t decode_ucs4le(CharsetDecodeState *s) {
    const u8 *p;

    p = s->p;
    s->p += 4;
    return (((char32_t)p[0] <<  0) + ((char32_t)p[1] <<  8) +
            ((char32_t)p[2] << 16) + ((char32_t)p[3] << 24));
}

static u8 *encode_ucs4le(qe__unused__ QECharset *charset, u8 *p, char32_t c)
{
    p[0] = c;
    p[1] = c >> 8;
    p[2] = c >> 16;
    p[3] = c >> 24;
    return p + 4;
}

/* return the number of lines and column position for a buffer */
static void charset_get_pos_ucs4(CharsetDecodeState *s, const u8 *buf, int size,
                                 int *line_ptr, int *col_ptr)
{
    const uint32_t *p, *p1, *lp;
    uint32_t nl, lf;
    union { uint32_t n; char c[4]; } u;
    int line, col;

    line = 0;
    lp = p = (const uint32_t *)(const void *)buf;
    p1 = p + (size >> 2);
    u.n = 0;
    u.c[(s->charset == &charset_ucs4be) * 3] = s->eol_char;
    nl = u.n;
    u.c[(s->charset == &charset_ucs4be) * 3] = '\n';
    lf = u.n;

    if (s->eol_type == EOL_DOS && p < p1 && *p == lf) {
        /* Skip \n at start of buffer.
         * Should check for pending skip state */
        p++;
        lp++;
    }

    while (p < p1) {
        if (*p++ == nl) {
            if (s->eol_type == EOL_DOS && p < p1 && *p == lf)
                p++;
            lp = p;
            line++;
        }
    }
    col = p1 - lp;
    *line_ptr = line;
    *col_ptr = col;
}

static int charset_goto_line_ucs4(CharsetDecodeState *s,
                                  const u8 *buf, int size, int nlines)
{
    const uint32_t *p, *p1, *lp;
    uint32_t nl, lf;
    union { uint32_t n; char c[4]; } u;

    lp = p = (const uint32_t *)(const void *)buf;
    p1 = p + (size >> 2);
    u.n = 0;
    u.c[(s->charset == &charset_ucs4be) * 3] = s->eol_char;
    nl = u.n;
    u.c[(s->charset == &charset_ucs4be) * 3] = '\n';
    lf = u.n;

    if (s->eol_type == EOL_DOS && p < p1 && *p == lf) {
        /* Skip \n at start of buffer.
         * Should check for pending skip state */
        p++;
        lp++;
    }

    while (nlines > 0 && p < p1) {
        while (p < p1) {
            if (*p++ == nl) {
                if (s->eol_type == EOL_DOS && p < p1 && *p == lf)
                    p++;
                lp = p;
                nlines--;
                break;
            }
        }
    }
    return (const u8 *)lp - buf;
}

static int probe_ucs4be(qe__unused__ QECharset *charset, const u8 *buf, int size)
{
    const uint32_t magic = (1U << '\b') | (1U << '\t') | (1U << '\f') |
                           (1U << '\n') | (1U << '\r') | (1U << '\033') |
                           (1U << 0x0e) | (1U << 0x0f) | (1U << 0x1a) |
                           (1U << 0x1f);
    const u8 *p = buf;
    const u8 *p_end = p + (size & ~3);
    uint32_t c;
    int count_spaces, count_lines;

    if (size & 3)
        return 0;

    count_spaces = count_lines = 0;

    while (p < p_end) {
        c = (p[0] << 24) | (p[1] << 16) | (p[2] << 8) | (p[3] << 0);
        p += 4;
        if (c <= 32) {
            if (c == ' ')
                count_spaces++;
            else
            if (c == '\n')
                count_lines++;
            else
            if (!(magic & (1U << c)))
                return 0;
        } else
        if (c > 0x10FFFF) {
            return 0;
        }
    }
    if (count_spaces + count_lines > size / (16 * 4))
        return 1;
    else
        return 0;
}

static char32_t decode_ucs4be(CharsetDecodeState *s) {
    const u8 *p;

    p = s->p;
    s->p += 4;
    return (((char32_t)p[0] << 24) + ((char32_t)p[1] << 16) +
            ((char32_t)p[2] <<  8) + ((char32_t)p[3] <<  0));
}

static u8 *encode_ucs4be(qe__unused__ QECharset *charset, u8 *p, char32_t c)
{
    p[0] = c >> 24;
    p[1] = c >> 16;
    p[2] = c >> 8;
    p[3] = c;
    return p + 4;
}

static int charset_get_chars_ucs4(CharsetDecodeState *s,
                                  const u8 *buf, int size)
{
    int count = size >> 2;  /* convert byte count to char32 count */
    const uint32_t *buf_end, *buf_ptr;
    uint32_t nl;
    union { uint32_t n; char c[4]; } u;

    if (s->eol_type != EOL_DOS)
        return count;

    buf_ptr = (const uint32_t *)(const void *)buf;
    buf_end = buf_ptr + count;
    // XXX: undefined behavior
    u.n = 0;
    u.c[(s->charset == &charset_ucs4be) * 3] = '\n';
    nl = u.n;

    while (buf_ptr < buf_end) {
        if (*buf_ptr++ == nl) {
            /* ignore \n in EOL_DOS scan, but count \r. (see above) */
            count--;
        }
    }
    return count;
}

static int charset_goto_char_ucs4(CharsetDecodeState *s,
                                  const u8 *buf, int size, int pos)
{
    int nb_chars;
    const uint32_t *buf_ptr, *buf_end;
    uint32_t nl;
    union { uint32_t n; char c[4]; } u;

    if (s->eol_type != EOL_DOS)
        return min_offset(pos << 2, size);

    nb_chars = 0;
    buf_ptr = (const uint32_t *)(const void *)buf;
    buf_end = buf_ptr + (size >> 2);
    // XXX: undefined behavior
    u.n = 0;
    u.c[(s->charset == &charset_ucs4be) * 3] = '\n';
    nl = u.n;

    for (; buf_ptr < buf_end; buf_ptr++) {
        if (*buf_ptr == nl) {
            /* ignore \n in EOL_DOS scan, but count \r. (see above) */
            continue;
        }
        if (nb_chars >= pos)
            break;
        nb_chars++;
    }
    return (const u8*)buf_ptr - buf;
}

struct QECharset charset_ucs4le = {
    "ucs4le",
    "utf32le|utf-32le",
    probe_ucs4le,
    NULL,
    decode_ucs4le,
    encode_ucs4le,
    charset_get_pos_ucs4,
    charset_get_chars_ucs4,
    charset_goto_char_ucs4,
    charset_goto_line_ucs4,
    4, 0, 0, 10, 0, 0, table_none, NULL, NULL,
};

struct QECharset charset_ucs4be = {
    "ucs4be",
    "ucs4|utf32|utf-32|utf32be|utf-32be",
    probe_ucs4be,
    NULL,
    decode_ucs4be,
    encode_ucs4be,
    charset_get_pos_ucs4,
    charset_get_chars_ucs4,
    charset_goto_char_ucs4,
    charset_goto_line_ucs4,
    4, 0, 0, 10, 0, 0, table_none, NULL, NULL,
};

/********************************************************/
/* generic charset functions */

void qe_register_charset(struct QEmacsState *qs, struct QECharset *charset)
{
    struct QECharset **pp;

    pp = &first_charset;
    while (*pp != NULL) {
        if (*pp == charset)
            return;
        pp = &(*pp)->next;
    }
    *pp = charset;
}

void charset_complete(CompleteState *cp, CompleteFunc enumerate) {
    QECharset *charset;
    char name[32];
    const char *p, *q;

    for (charset = first_charset; charset != NULL; charset = charset->next) {
        (*enumerate)(cp, charset->name, CT_STRX);
        if (charset->aliases) {
            for (q = p = charset->aliases;; q++) {
                if (*q == '\0' || *q == '|') {
                    if (q > p) {
                        pstrncpy(name, sizeof(name), p, q - p);
                        (*enumerate)(cp, name, CT_STRX);
                    }
                    if (*q == '\0')
                        break;
                    p = q + 1;
                }
            }
        }
    }
}

QECharset *qe_find_charset(struct QEmacsState *qs, const char *name)
{
    QECharset *charset;

    if (!name)
        return NULL;

    for (charset = first_charset; charset != NULL; charset = charset->next) {
        if (!strxcmp(charset->name, name)
        ||  (charset->aliases && strxfind(charset->aliases, name))) {
            return charset;
        }
    }
    return NULL;
}

void charset_decode_init(CharsetDecodeState *s, QECharset *charset,
                         EOLType eol_type)
{
    s->table = charset->encode_table;  /* default encode table */
    if (charset->table_alloc) {
        s->table = qe_malloc_array(unsigned short, 256);
        if (!s->table) {
            charset = &charset_8859_1;
            s->table = charset->encode_table;
        }
    }
    s->charset = charset;
    s->char_size = charset->char_size;
    s->eol_type = eol_type;
    s->eol_char = charset->eol_char;
    if (s->eol_char == '\n' && (s->eol_type == EOL_MAC || s->eol_type == EOL_DOS))
        s->eol_char = '\r';
    s->decode_func = charset->decode_func;
    s->get_pos_func = charset->get_pos_func;
    if (charset->decode_init)
        charset->decode_init(s);
}

void charset_decode_close(CharsetDecodeState *s)
{
    if (s->charset->table_alloc) {
        /* remove the const qualifier */
        qe_free(unconst(unsigned short **)&s->table);
    }
    /* safety */
    memset(s, 0, sizeof(CharsetDecodeState));
}

/* detect the end of line type. */
static void detect_eol_type_8bit(const u8 *buf, int size,
                                 QECharset *charset, EOLType *eol_typep)
{
    const u8 *p, *p1;
    int c, eol_bits;
    EOLType eol_type;

    if (!eol_typep)
        return;

    eol_type = *eol_typep;

    p = buf;
    p1 = p + size - 1;

    eol_bits = 0;
    while (p < p1) {
        c = *p++;
        if (c == '\r') {
            if (*p == '\r') {
                /* possibly spurious extra ^M in DOS file: ignore it,
                 * next iteration will determine if encoding is MAC or DOS
                 */
            } else
            if (*p == '\n') {
                p++;
                eol_bits |= 1 << EOL_DOS;
            } else {
                eol_bits |= 1 << EOL_MAC;
            }
        } else
        if (c == '\n') {
            eol_bits |= 1 << EOL_UNIX;
        }
    }
    switch (eol_bits) {
    case 0:
        /* no change, keep default value */
        break;
    case 1 << EOL_UNIX:
        eol_type = EOL_UNIX;
        break;
    case 1 << EOL_DOS:
        eol_type = EOL_DOS;
        break;
    case 1 << EOL_MAC:
        eol_type = EOL_MAC;
        break;
    default:
        /* A mixture of different styles, binary / unix */
        eol_type = EOL_UNIX;
        break;
    }
    *eol_typep = eol_type;
}

static void detect_eol_type_16bit(const u8 *buf, int size,
                                  QECharset *charset, EOLType *eol_typep)
{
    const uint16_t *p, *p1;
    uint16_t c, cr, lf;
    union { uint16_t n; char c[2]; } u;
    int eol_bits;
    EOLType eol_type;

    if (!eol_typep)
        return;

    u.n = 0;
    u.c[charset == &charset_ucs2be] = '\r';
    cr = u.n;
    u.c[charset == &charset_ucs2be] = '\n';
    lf = u.n;

    p = (const uint16_t *)(const void *)buf;
    p1 = p + (size >> 1) - 1;

    eol_bits = 0;
    while (p < p1) {
        c = *p++;
        if (c == cr) {
            if (*p == lf) {
                p++;
                eol_bits |= 1 << EOL_DOS;
            } else {
                eol_bits |= 1 << EOL_MAC;
            }
        } else
        if (c == lf) {
            eol_bits |= 1 << EOL_UNIX;
        }
    }
    switch (eol_bits) {
    case 0:
        /* no change, keep default value */
        eol_type = *eol_typep;
        break;
    case 1 << EOL_UNIX:
        eol_type = EOL_UNIX;
        break;
    case 1 << EOL_DOS:
        eol_type = EOL_DOS;
        break;
    case 1 << EOL_MAC:
        eol_type = EOL_MAC;
        break;
    default:
        /* A mixture of different styles, binary / unix */
        eol_type = EOL_UNIX;
        break;
    }
    *eol_typep = eol_type;
}

static void detect_eol_type_32bit(const u8 *buf, int size,
                                  QECharset *charset, EOLType *eol_typep)
{
    const uint32_t *p, *p1;
    uint32_t c, cr, lf;
    union { uint32_t n; char c[4]; } u;
    int eol_bits;
    EOLType eol_type;

    if (!eol_typep)
        return;

    u.n = 0;
    u.c[(charset == &charset_ucs4be) * 3] = '\r';
    cr = u.n;
    u.c[(charset == &charset_ucs4be) * 3] = '\n';
    lf = u.n;

    p = (const uint32_t *)(const void *)buf;
    p1 = p + (size >> 2) - 1;

    eol_bits = 0;
    while (p < p1) {
        c = *p++;
        if (c == cr) {
            if (*p == lf) {
                p++;
                eol_bits |= 1 << EOL_DOS;
            } else {
                eol_bits |= 1 << EOL_MAC;
            }
        } else
        if (c == lf) {
            eol_bits |= 1 << EOL_UNIX;
        }
    }
    switch (eol_bits) {
    case 0:
        /* no change, keep default value */
        eol_type = *eol_typep;
        break;
    case 1 << EOL_UNIX:
        eol_type = EOL_UNIX;
        break;
    case 1 << EOL_DOS:
        eol_type = EOL_DOS;
        break;
    case 1 << EOL_MAC:
        eol_type = EOL_MAC;
        break;
    default:
        /* A mixture of different styles, binary / unix */
        eol_type = EOL_UNIX;
        break;
    }
    *eol_typep = eol_type;
}

static QECharset *detect_eol_type(const u8 *buf, int size,
                                  QECharset *charset, EOLType *eol_typep)
{
    if (charset->char_size == 4)
        detect_eol_type_32bit(buf, size, charset, eol_typep);
    else
    if (charset->char_size == 2)
        detect_eol_type_16bit(buf, size, charset, eol_typep);
    else
        detect_eol_type_8bit(buf, size, charset, eol_typep);

    return charset;
}

QECharset *detect_charset(const u8 *buf, int size, EOLType *eol_typep)
{
#if 0
    QECharset *charset;

    /* Try and determine charset */
    /* CG: should iterate over charsets with probe function and score */
    charset = &charset_utf8;
    if (size > 0) {
        if (charset_utf8.probe_func(&charset_utf8, buf, size))
            charset = &charset_utf8;
        else
        if (charset_ucs4le.probe_func(&charset_ucs4le, buf, size))
            charset = &charset_ucs4le;
        else
        if (charset_ucs4be.probe_func(&charset_ucs4be, buf, size))
            charset = &charset_ucs4be;
        else
        if (charset_ucs2le.probe_func(&charset_ucs2le, buf, size))
            charset = &charset_ucs2le;
        else
        if (charset_ucs2be.probe_func(&charset_ucs2be, buf, size))
            charset = &charset_ucs2be;
        else
            charset = &charset_8859_1;
        /* CG: should distinguish charset_8859_1, charset_raw and
         * charset_auto */
    }
    return charset;
#else
    /* Detect the charset:
       UTF-8, UTF-16, UTF-32, Latin1, binary are detected */
    int i, j, trail, bits;
    char32_t c, min_code;
    enum {
        HAS_BINARY = 1, HAS_UTF8 = 2, TRAILING_BYTES = 4,
        INVALID_PREFIX = 8, INVALID_SEQUENCE = 16, OVERLONG_ENCODING = 32,
        HAS_SURROGATE = 64, INVALID_CODE = 128,
        UTF8_MASK = 255,
    };
    const uint32_t magic = ((1U << '\b') | (1U << '\t') | (1U << '\f') |
                            (1U << '\n') | (1U << '\r') | (1U << '\033') |
                            (1U << 0x0e) | (1U << 0x0f) | (1U << 0x1a) |
                            (1U << 0x1f));

    bits = 0;

    for (i = 0; i < size;) {
        c = buf[i++];
        if (c < 32) {
            if (!(magic & (1U << c)))
                bits |= HAS_BINARY;
        } else
        if (c < 0x80) {
            /* 0x7F should trigger HAS_BINARY? */
            continue;
        } else
        if (c < 0xC0) {
            bits |= TRAILING_BYTES;
        } else {
            bits |= HAS_UTF8;
            if (c < 0xE0) {
                trail = 1;
                min_code = 0x80;
                c &= 0x1f;
            } else
            if (c < 0xF0) {
                trail = 2;
                min_code = 0x800;
                c &= 0x0f;
            } else
            if (c < 0xF8) {
                trail = 3;
                min_code = 0x10000;
                c &= 0x07;
            } else
            if (c < 0xFC) {
                trail = 4;
                min_code = 0x00200000;
                c &= 0x03;
            } else {
                trail = 5;
                min_code = 0x04000000;
                c &= 0x03;
            }
            if (i + trail > size)
                break;
            for (j = 0; j < trail; j++) {
                char32_t c1 = buf[i + j] ^ 0x80;
                if (c1 > 0x3f)
                    break;
                c = (c << 6) + c1;
            }
            i += j;
            if (j < trail) {
                bits |= INVALID_SEQUENCE;
            } else {
                if (c < min_code)
                    bits |= OVERLONG_ENCODING;
                if (c >= 0xd800 && c <= 0xdfff) {
                    // XXX: should detect CESU-8 encoding
                    bits |= HAS_SURROGATE;
                }
                if (c == 0xfffe || c == 0xffff || c > 0x10ffff)
                    bits |= INVALID_CODE;
            }
        }
    }
    if ((bits & (HAS_UTF8 | HAS_BINARY | TRAILING_BYTES | INVALID_SEQUENCE)) == HAS_UTF8) {
        if (bits == HAS_UTF8) {
            /* strict UTF-8 encoding */
            return detect_eol_type(buf, size, &charset_utf8, eol_typep);
        } else {
            /* relax extended UTF-8 encoding */
            return detect_eol_type(buf, size, &charset_utf8x, eol_typep);
        }
    }

    /* Check for ZWNBSP as BOM: files starting with zero-width
     * no-break space as a byte-order mark (BOM) will be detected
     * as ucs2/UTF-16 or ucs4/UTF-32 encoded.
     */
    if (size >= 2 && buf[0] == 0xff && buf[1] == 0xfe) {
        if (size >= 4 && buf[2] == 0 && buf[3] == 0) {
            return detect_eol_type(buf, size, &charset_ucs4le, eol_typep);
        } else {
            return detect_eol_type(buf, size, &charset_ucs2le, eol_typep);
        }
    }

    if (size >= 2 && buf[0] == 0xfe && buf[1] == 0xff) {
        return detect_eol_type(buf, size, &charset_ucs2be, eol_typep);
    }

    if (size >= 4
    &&  buf[0] == 0 && buf[1] == 0 && buf[2] == 0xfe && buf[3] == 0xff) {
        return detect_eol_type(buf, size, &charset_ucs4be, eol_typep);
    }

#if 0
    {
        /* Need a more reliable generic ucs2/ucs4 detection */
        int maxc[4];

        maxc[0] = maxc[1] = maxc[2] = maxc[3] = 0;
        for (i = 0; i < size; i += 2) {
            if (buf[i] > maxc[i & 3])
                maxc[i & 3] = buf[i];
        }
        if (maxc[0] > 'a' && maxc[1] < 0x2f && maxc[2] > 'a' && maxc[3] < 0x2f) {
            detect_eol_type(buf, size, &charset_ucs2le, eol_typep);
            return &charset_ucs2le;
        }
        if (maxc[1] > 'a' && maxc[0] < 0x2f && maxc[3] > 'a' && maxc[2] < 0x2f) {
            detect_eol_type(buf, size, &charset_ucs2be, eol_typep);
            return &charset_ucs2be;
        }
    }
#else
    if (charset_ucs4le.probe_func(&charset_ucs4le, buf, size))
        return detect_eol_type(buf, size, &charset_ucs4le, eol_typep);
    else
    if (charset_ucs4be.probe_func(&charset_ucs4be, buf, size))
        return detect_eol_type(buf, size, &charset_ucs4be, eol_typep);
    else
    if (charset_ucs2le.probe_func(&charset_ucs2le, buf, size))
        return detect_eol_type(buf, size, &charset_ucs2le, eol_typep);
    else
    if (charset_ucs2be.probe_func(&charset_ucs2be, buf, size))
        return detect_eol_type(buf, size, &charset_ucs2be, eol_typep);
#endif

    /* Should detect iso-2220-jp upon \033$@ and \033$B, but jis
     * support is not selected in tiny build
     * XXX: should use charset probe functions.
     */

    if (bits & HAS_BINARY) {
        *eol_typep = EOL_UNIX;
        return &charset_raw;
    }

    detect_eol_type(buf, size, &charset_raw, eol_typep);

#ifndef CONFIG_TINY
    if (*eol_typep == EOL_MAC) {
        /* XXX: default MAC files to Mac_roman, should be selectable */
        /* XXX: should use probe functions */
        return &charset_mac_roman;
    }
#endif
    if (*eol_typep == EOL_DOS
    ||  (bits & (TRAILING_BYTES | INVALID_PREFIX | INVALID_SEQUENCE))) {
        /* XXX: default DOS files to Latin1, should be selectable */
        return &charset_8859_1;
    }
    /* XXX: should use a state variable for default charset */
    return &charset_utf8;
#endif
}

/********************************************************/
/* 8 bit charsets */

void decode_8bit_init(CharsetDecodeState *s)
{
    QECharset *charset = s->charset;
    unsigned short *table;
    int i, n;

    table = unconst(unsigned short *)s->table;     /* remove const qualifier */
    for (i = 0; i < charset->min_char; i++)
        *table++ = i;
    n = charset->max_char - charset->min_char + 1;
    for (i = 0; i < n; i++)
        *table++ = charset->private_table[i];
    for (i = charset->max_char + 1; i < 256; i++)
        *table++ = i;
}

char32_t decode_8bit(CharsetDecodeState *s)
{
    return s->table[*(s->p)++];
}

/* not very fast, but not critical yet */
u8 *encode_8bit(QECharset *charset, u8 *q, char32_t c)
{
    int i, n;
    const unsigned short *table;

    if (c < charset->min_char) {
        /* nothing to do */
    } else
    if (c > charset->max_char && c <= 0xff) {
        /* nothing to do */
    } else {
        n = charset->max_char - charset->min_char + 1;
        table = charset->private_table;
        for (i = 0; i < n; i++) {
            if (table[i] == c)
                goto found;
        }
        return NULL;
    found:
        c = charset->min_char + i;
    }
    *q++ = c;
    return q;
}

/* return the number of lines and column position for a buffer */
void charset_get_pos_8bit(CharsetDecodeState *s, const u8 *buf, int size,
                          int *line_ptr, int *col_ptr)
{
    const u8 *p, *p1, *lp;
    int nl, line, col;

    QASSERT(size >= 0);

    line = 0;
    lp = p = buf;
    p1 = p + size;
    nl = s->eol_char;

    if (s->eol_type == EOL_DOS && p < p1 && *p == '\n') {
        /* Skip \n at start of buffer.
         * Should check for pending skip state */
        p++;
        lp++;
    }

    for (;;) {
        p = memchr(p, nl, p1 - p);
        if (!p)
            break;
        p++;
        if (s->eol_type == EOL_DOS && p < p1 && *p == '\n')
            p++;
        lp = p;
        line++;
    }
    col = p1 - lp;
    *line_ptr = line;
    *col_ptr = col;
}

int charset_goto_line_8bit(CharsetDecodeState *s,
                           const u8 *buf, int size, int nlines)
{
    const u8 *p, *p1, *lp;
    int nl;

    lp = p = buf;
    p1 = p + size;
    nl = s->eol_char;

    if (s->eol_type == EOL_DOS && p < p1 && *p == '\n') {
        /* Skip \n at start of buffer.
         * Should check for pending skip state */
        p++;
        lp++;
    }

    while (nlines > 0) {
        p = memchr(p, nl, p1 - p);
        if (!p)
            break;
        p++;
        if (s->eol_type == EOL_DOS && p < p1 && *p == '\n')
            p++;
        lp = p;
        nlines--;
    }
    return lp - buf;
}

int charset_get_chars_8bit(CharsetDecodeState *s,
                           const u8 *buf, int size)
{
    int count = size;
    const u8 *buf_end, *buf_ptr;

    if (s->eol_type != EOL_DOS)
        return count;

    buf_ptr = buf;
    buf_end = buf_ptr + count;
    while (buf_ptr < buf_end) {
        if (*buf_ptr++ == '\n') {
            /* ignore \n in EOL_DOS scan, but count \r. (see above) */
            count--;
        }
    }
    return count;
}

int charset_goto_char_8bit(CharsetDecodeState *s,
                           const u8 *buf, int size, int pos)
{
    int nb_chars;
    const u8 *buf_ptr, *buf_end;

    if (s->eol_type != EOL_DOS)
        return min_offset(pos, size);

    nb_chars = 0;
    buf_ptr = buf;
    buf_end = buf_ptr + size;
    for (; buf_ptr < buf_end; buf_ptr++) {
        if (*buf_ptr == '\n') {
            /* ignore \n in EOL_DOS scan, but count \r. */
            continue;
        }
        if (nb_chars >= pos)
            break;
        nb_chars++;
    }
    return buf_ptr - buf;
}

/********************************************************/

void charset_init(struct QEmacsState *qs) {
    qe_register_charset(qs, &charset_raw);
    qe_register_charset(qs, &charset_8859_1);
    qe_register_charset(qs, &charset_vt100);
    qe_register_charset(qs, &charset_7bit);
    qe_register_charset(qs, &charset_utf8);
    qe_register_charset(qs, &charset_utf8x);
    qe_register_charset(qs, &charset_ucs2le);
    qe_register_charset(qs, &charset_ucs2be);
    qe_register_charset(qs, &charset_ucs4le);
    qe_register_charset(qs, &charset_ucs4be);
}
