MWCC/compiler_and_linker/FrontEnd/C/CPrepTokenizer.c

#include "compiler/CPrepTokenizer.h"
#include "compiler/CError.h"
#include "compiler/CInt64.h"
#include "compiler/CMachine.h"
#include "compiler/CParser.h"
#include "compiler/CPrep.h"
#include "compiler/CompilerTools.h"
#include "compiler/InlineAsmPPC.h"
#include "compiler/tokens.h"
#include "cos.h"

static Boolean prepnextstringchar_foundnl;

short prepskipnextchar(void) {
    short c;
    StringPtr start;
    StringPtr p;

    start = (StringPtr) pos;
    p = (StringPtr) pos;

loop:
    switch ((c = *p)) {
        case 0:
            if (tokenstacklevel > 0 || p >= (StringPtr) prep_file_end) {
                pos = (char *) p;
                nextcharpos = (char *) p;
                return c;
            } else {
                pos = (char *) p++;
                nextcharpos = (char *) p;
                return c;
            }
        case 4:
        case 5:
            p++;
            goto loop;
        case ' ':
        case '\t':
        case '\n':
        case '\v':
        case '\f':
            spaceskip = 1;
            p++;
            goto loop;
        case ';':
            pos = (char *) p++;
            if (tokenstacklevel <= 0 && in_assembler && copts.asmsemicolcomment) {
                spaceskip = 1;
                nextcharpos = (char *) CPrep_SkipNewComment(p);
                return CPrep_SkipNewCommentChar;
            }
            nextcharpos = (char *) p;
            return c;
        case '#':
            pos = (char *) p++;
            if (tokenstacklevel <= 0 && in_assembler && copts.asmpoundcomment) {
                spaceskip = 1;
                nextcharpos = (char *) CPrep_SkipNewComment(p);
                return CPrep_SkipNewCommentChar;
            }
            nextcharpos = (char *) p;
            return c;
        case '/':
            pos = (char *) p++;
            if (copts.multibyteaware) {
                while (p[0] == '\\' && !COS_IsMultiByte(start, p) && p[1] == '\r') {
                    foundnl();
                    if (p[2] == '\n')
                        p += 3;
                    else
                        p += 2;
                }
            } else {
                while (p[0] == '\\' && p[1] == '\r') {
                    foundnl();
                    if (p[2] == '\n')
                        p += 3;
                    else
                        p += 2;
                }
            }
            if (tokenstacklevel <= 0) {
                if (p[0] == '/' && (!copts.ANSI_strict || copts.cplusplus || copts.c9x)) {
                    spaceskip = 1;
                    nextcharpos = (char *) CPrep_SkipNewComment(p + 1);
                    return CPrep_SkipNewCommentChar;
                } else if (p[0] == '*') {
                    spaceskip = 1;
                    p++;
                    for (;;) {
                        if (!(c = *(p++))) {
                            if (tokenstacklevel > 0 || p >= (StringPtr) prep_file_end) {
                                CPrep_Error(CErrorStr103);
                                nextcharpos = (char *) p - 1;
                                return c;
                            }
                            p[-1] = ' ';
                        }
                        if (c == '*') {
                            if (copts.multibyteaware) {
                                while (p[0] == '\\' && !COS_IsMultiByte(start, p) && p[1] == '\r') {
                                    foundnl();
                                    if (p[2] == '\n')
                                        p += 3;
                                    else
                                        p += 2;
                                }
                            } else {
                                while (p[0] == '\\' && p[1] == '\r') {
                                    foundnl();
                                    if (p[2] == '\n')
                                        p += 3;
                                    else
                                        p += 2;
                                }
                            }
                            if (*p == '/') {
                                p++;
                                goto loop;
                            }
                        }
                        if (c == '\r')
                            foundnl();
                    }
                }
            }
            nextcharpos = (char *) p;
            return c;
        case '%':
            pos = (char *) p++;
            if (*p == ':' && copts.cplusplus) {
                nextcharpos = (char *) p + 1;
                return '#';
            }
            nextcharpos = (char *) p;
            return c;
        case '\\':
            pos = (char *) p++;
        mid_backslash:
            if (*p == '\r') {
                if (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1)) {
                    foundnl();
                    if (p[1] == '\n')
                        p += 2;
                    else
                        p += 1;
                    goto loop;
                }
            }
            nextcharpos = (char *) p;
            return c;
        case '?':
            pos = (char *) p++;
            if (copts.trigraphs && p[0] == '?') {
                switch (p[1]) {
                    case '=':
                        nextcharpos = (char *) p + 2;
                        return '#';
                    case '/':
                        c = '\\';
                        p += 2;
                        goto mid_backslash;
                    case '\'':
                        nextcharpos = (char *) p + 2;
                        return '^';
                    case '(':
                        nextcharpos = (char *) p + 2;
                        return '[';
                    case ')':
                        nextcharpos = (char *) p + 2;
                        return ']';
                    case '!':
                        nextcharpos = (char *) p + 2;
                        return '|';
                    case '<':
                        nextcharpos = (char *) p + 2;
                        return '{';
                    case '>':
                        nextcharpos = (char *) p + 2;
                        return '}';
                    case '-':
                        nextcharpos = (char *) p + 2;
                        return '~';
                }
            }
            nextcharpos = (char *) p;
            return c;
        default:
            pos = (char *) p++;
            nextcharpos = (char *) p;
            return c;
    }
}

short prepnextchar(void) {
    short c;
    StringPtr start;
    StringPtr p;

    start = (StringPtr) pos;
    p = (StringPtr) pos;

loop:
    switch ((c = *(p++))) {
        case 0:
            pos = (char *) p - 1;
            return c;
        case '/':
            if (copts.multibyteaware) {
                while (p[0] == '\\' && !COS_IsMultiByte(start, p) && p[1] == '\r') {
                    foundnl();
                    if (p[2] == '\n')
                        p += 3;
                    else
                        p += 2;
                }
            } else {
                while (p[0] == '\\' && p[1] == '\r') {
                    foundnl();
                    if (p[2] == '\n')
                        p += 3;
                    else
                        p += 2;
                }
            }
            if (p[0] == '/' && (!copts.ANSI_strict || copts.cplusplus)) {
                spaceskip = 1;
                pos = (char *) CPrep_SkipNewComment(p + 1);
                return CPrep_SkipNewCommentChar;
            } else if (p[0] == '*') {
                spaceskip = 1;
                p++;
                for (;;) {
                    if (!(c = *(p++))) {
                        if (tokenstacklevel > 0 || p >= (StringPtr) prep_file_end) {
                            pos = (char *) p - 1;
                            CPrep_Error(CErrorStr103);
                            return c;
                        }
                        p[-1] = ' ';
                    }
                    if (c == '*') {
                        if (copts.multibyteaware) {
                            while (p[0] == '\\' && !COS_IsMultiByte(start, p) && p[1] == '\r') {
                                foundnl();
                                if (p[2] == '\n')
                                    p += 3;
                                else
                                    p += 2;
                            }
                        } else {
                            while (p[0] == '\\' && p[1] == '\r') {
                                foundnl();
                                if (p[2] == '\n')
                                    p += 3;
                                else
                                    p += 2;
                            }
                        }
                        if (*p == '/')
                            break;
                    }
                    if (c == '\r')
                        foundnl();
                }
                pos = (char *) p + 1;
                return ' ';
            } else {
                pos = (char *) p;
                return c;
            }
        case '\\':
        backslash:
            if (*p == '\r') {
                if (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1)) {
                    foundnl();
                    if (p[1] == '\n')
                        p += 2;
                    else
                        p += 1;
                    goto loop;
                }
            }
            pos = (char *) p;
            return c;
        case '%':
            if (*p == ':' && copts.cplusplus) {
                pos = (char *) p + 1;
                return '#';
            }
            pos = (char *) p;
            return c;
        case '?':
            if (copts.trigraphs && p[0] == '?') {
                switch (p[1]) {
                    case '=':
                        pos = (char *) p + 2;
                        return '#';
                    case '/':
                        c = '\\';
                        p += 2;
                        goto backslash;
                    case '\'':
                        pos = (char *) p + 2;
                        return '^';
                    case '(':
                        pos = (char *) p + 2;
                        return '[';
                    case ')':
                        pos = (char *) p + 2;
                        return ']';
                    case '!':
                        pos = (char *) p + 2;
                        return '|';
                    case '<':
                        pos = (char *) p + 2;
                        return '{';
                    case '>':
                        pos = (char *) p + 2;
                        return '}';
                    case '-':
                        pos = (char *) p + 2;
                        return '~';
                }
            }
            pos = (char *) p;
            return c;
        default:
            pos = (char *) p;
            return c;
    }
}

short prepnextstringchar(StringPtr str, Boolean flag) {
    StringPtr p;
    short c;

    p = (StringPtr) pos;

loop:
    switch ((c = *(p++))) {
        case 0:
            pos = (char *) p - 1;
            return c;
        case '\\':
        backslash:
            if (p[0] == '\r' && (!copts.multibyteaware || !COS_IsMultiByte(str, p - 1))) {
                if (flag)
                    foundnl();
                else
                    prepnextstringchar_foundnl = 1;
                if (p[1] == '\n')
                    p += 2;
                else
                    p += 1;
                goto loop;
            }
            pos = (char *) p;
            return c;
        case '?':
            if (copts.trigraphs && p[0] == '?') {
                switch (p[1]) {
                    case '=':
                        pos = (char *) p + 2;
                        return '#';
                    case '/':
                        c = '\\';
                        p += 2;
                        goto backslash;
                    case '\'':
                        pos = (char *) p + 2;
                        return '^';
                    case '(':
                        pos = (char *) p + 2;
                        return '[';
                    case ')':
                        pos = (char *) p + 2;
                        return ']';
                    case '!':
                        pos = (char *) p + 2;
                        return '|';
                    case '<':
                        pos = (char *) p + 2;
                        return '{';
                    case '>':
                        pos = (char *) p + 2;
                        return '}';
                    case '-':
                        pos = (char *) p + 2;
                        return '~';
                }
            }
            pos = (char *) p;
            return c;
        default:
            pos = (char *) p;
            return c;
    }
}

void CPrep_MatchChar(char ch, Boolean flag) {
    char *p;
    char *start;
    char c;
    Boolean haveBackslash;

    p = start = pos;
    haveBackslash = 0;

loop:
    if (!(c = *(p++))) {
        if (tokenstacklevel > 0 || p >= prep_file_end)
            CPrep_Error(CErrorStr102);
        else if (ch == '"')
            CPrep_Error(CErrorStr101);
        else if (ch == '\'')
            CPrep_Error(CErrorStr100);
        CPrep_Error(CErrorStr117);
        pos = p - 1;
        return;
    }

    if (c == '\r') {
        if (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1)) {
            if (flag)
                foundnl();
            if (*p == '\n')
                p++;
            CPrep_Error(CErrorStr112);
            pos = p;
            return;
        }
        haveBackslash = 0;
        goto loop;
    }

    if (c == '?' && p[0] == '?' && copts.trigraphs) {
        switch (p[1]) {
            case '!':
            case '\'':
            case '(':
            case ')':
            case '-':
            case '<':
            case '=':
            case '>':
                haveBackslash = 0;
                p += 2;
                goto loop;
            case '/':
                p += 2;
                goto backslash;
        }
    }

    if (c == '\\' && !haveBackslash && (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1))) {
    backslash:
        if (*p == '\r') {
            if (flag)
                foundnl();
            if (*(++p) == '\n')
                p++;
            haveBackslash = 0;
        } else {
            haveBackslash = 1;
        }
        goto loop;
    }

    if (c == ch && !haveBackslash) {
        pos = p;
    } else {
        haveBackslash = 0;
        goto loop;
    }
}

char *CPrep_MatchChar2(char *start, char ch, Boolean flag) {
    char *p;
    char c;
    Boolean haveBackslash;

    p = start;
    haveBackslash = 0;

loop:
    if (!(c = *(p++))) {
        if (tokenstacklevel > 0 || p >= prep_file_end)
            CPrep_Error(CErrorStr102);
        else if (ch == '"')
            CPrep_Error(CErrorStr101);
        else if (ch == '\'')
            CPrep_Error(CErrorStr100);
        CPrep_Error(CErrorStr117);
        return p - 1;
    }

    if (c == '\r') {
        if (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1)) {
            if (flag)
                foundnl();
            if (*p == '\n')
                p++;
            CPrep_Error(CErrorStr112);
            return p;
        }
        haveBackslash = 0;
        goto loop;
    }

    if (c == '?' && p[0] == '?' && copts.trigraphs) {
        switch (p[1]) {
            case '!':
            case '\'':
            case '(':
            case ')':
            case '-':
            case '<':
            case '=':
            case '>':
                haveBackslash = 0;
                p += 2;
                goto loop;
            case '/':
                p += 2;
                goto backslash;
        }
    }

    if (c == '\\' && !haveBackslash && (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1))) {
        backslash:
        if (*p == '\r') {
            if (flag)
                foundnl();
            if (*(++p) == '\n')
                p++;
            haveBackslash = 0;
        } else {
            haveBackslash = 1;
        }
        goto loop;
    }

    if (c == ch && !haveBackslash) {
        return p;
    } else {
        haveBackslash = 0;
        goto loop;
    }
}

short prepcurchar(void) {
    char *save_pos;
    short t;

    save_pos = pos;
    t = prepnextchar();
    nextcharpos = pos;
    pos = save_pos;

    return t;
}

static short prepcurstringchar(StringPtr str) {
    char *save_pos;
    short t;

    save_pos = pos;
    prepnextstringchar_foundnl = 0;
    t = prepnextstringchar(str, 0);
    nextcharpos = pos;
    pos = save_pos;

    return t;
}

static void prepcurstringchar_skip(void) {
    pos = nextcharpos;
    if (prepnextstringchar_foundnl)
        foundnl();
}

char *ReadIdentifier(const char *p) {
    char buf[256];
    const char *start;
    unsigned int ch;
    int len;

    start = p;
    len = 0;
    for (;;) {
        ch = (unsigned char) *p;
        if (cprep_idarray[ch]) {
            if (len < 255)
                buf[len++] = *p;
            p++;
        } else {
            if (copts.multibyteaware) {
                if ((unsigned char) *p == '\\' && !COS_IsMultiByte(start, p) && p[1] == '\r') {
                    if (p[2] == '\n')
                        p += 3;
                    else
                        p += 2;
                    foundnl();
                } else {
                    break;
                }
            } else {
                if ((unsigned char) *p == '\\' && p[1] == '\r') {
                    if (p[2] == '\n')
                        p += 3;
                    else
                        p += 2;
                    foundnl();
                } else {
                    break;
                }
            }
        }
    }

    buf[len] = 0;
    tkidentifier = GetHashNameNodeExport(buf);
    return (char *) p;
}

static short intsuffix(short token, Boolean flag) {
    Boolean is_unsigned;
    Boolean is_long;
    Boolean is_longlong;
    short t;

    is_unsigned = is_long = is_longlong = 0;
    t = token;
    switch (t) {
        case 'u':
        case 'U':
            is_unsigned = 1;
            pos = nextcharpos;
            t = prepcurchar();
            if (t == 'l' || t == 'L') {
                is_long = 1;
                pos = nextcharpos;
                t = *pos;
                if (copts.longlong) {
                    pos = nextcharpos;
                    t = prepcurchar();
                    if (t == 'l' || t == 'L') {
                        is_longlong = 1;
                        pos = nextcharpos;
                        t = *pos;
                    }
                }
            }
            break;
        case 'l':
        case 'L':
            is_long = 1;
            pos = nextcharpos;
            t = prepcurchar();
            if (t == 'u' || t == 'U') {
                is_unsigned = 1;
                pos = nextcharpos;
                t = *pos;
            } else if (copts.longlong && (t == 'l' || t == 'L')) {
                is_longlong = 1;
                pos = nextcharpos;
                t = prepcurchar();
                if (t == 'u' || t == 'U') {
                    is_unsigned = 1;
                    pos = nextcharpos;
                    t = *pos;
                }
            }
            break;
        case 'i':
        case 'I':
            if (copts.cpp_extensions && !copts.ANSI_strict && copts.longlong) {
                pos = nextcharpos;
                t = prepcurchar();
                if (t == '6') {
                    pos = nextcharpos;
                    t = prepcurchar();
                    if (t == '4') {
                        is_long = is_longlong = 1;
                        pos = nextcharpos;
                        t = *pos;
                    }
                }
            }
            break;
        case 'b':
        case 'B':
        case 'f':
        case 'F':
            if (InlineAsm_gccmode)
                return IT_INT;
            break;
    }

    if ((t >= 'a' && t <= 'z') || (t >= 'A' && t <= 'Z') || (t == '_') || (t >= '0' && t <= '9'))
        CPrep_Error(CErrorStr105);

    if (!is_longlong && !is_long) {
        if (!is_unsigned && CInt64_IsInRange(tkintconst, stsignedint.size))
            return IT_INT;
        if ((flag || is_unsigned) && CInt64_IsInURange(tkintconst, stunsignedint.size))
            return IT_UINT;
    }
    if (!is_longlong) {
        if (!is_unsigned && CInt64_IsInRange(tkintconst, stsignedlong.size))
            return IT_LONG;
        if (CInt64_IsInURange(tkintconst, stunsignedlong.size))
            return IT_ULONG;
    }
    if (copts.longlong) {
        if (!is_unsigned && CInt64_IsInRange(tkintconst, stsignedlonglong.size))
            return IT_LONGLONG;
        if (CInt64_IsInURange(tkintconst, stsignedlonglong.size))
            return IT_ULONGLONG;
    }
    CError_Error(CErrorStr154);
    tkintconst = cint64_zero;
    return IT_INT;
}

static short floatsuffix(short token) {
    short whichtype;

    switch (token) {
        case 'f':
        case 'F':
            pos = nextcharpos;
            token = *pos;
            whichtype = IT_FLOAT;
            if (stfloat.size != stlongdouble.size)
                tkfloatconst = CMach_CalcFloatConvert((Type *) &stfloat, tkfloatconst);
            break;
        case 'l':
        case 'L':
            pos = nextcharpos;
            token = *pos;
            whichtype = IT_LONGDOUBLE;
            break;
        case 'd':
        case 'D':
            if (!copts.ANSI_strict) {
                pos = nextcharpos;
                token = *pos;
                whichtype = IT_DOUBLE;
                if (stdouble.size != stlongdouble.size)
                    tkfloatconst = CMach_CalcFloatConvert((Type *) &stdouble, tkfloatconst);
            }
            break;
        default:
            if (copts.float_constants) {
                whichtype = IT_FLOAT;
                if (stfloat.size != stlongdouble.size)
                    tkfloatconst = CMach_CalcFloatConvert((Type *) &stfloat, tkfloatconst);
            } else {
                whichtype = IT_DOUBLE;
                if (stdouble.size != stlongdouble.size)
                    tkfloatconst = CMach_CalcFloatConvert((Type *) &stdouble, tkfloatconst);
            }
    }

    if ((token >= 'a' && token <= 'z') || (token >= 'A' && token <= 'Z') || (token == '_') || (token >= '0' && token <= '9')) {
        CPrep_Error(CErrorStr105);
        whichtype = IT_DOUBLE;
    }

    return whichtype;
}

static short tohex(short token) {
    if (token >= '0' && token <= '9')
        return token - '0';
    if (token >= 'a' && token <= 'f')
        return token - 'a' + 10;
    if (token >= 'A' && token <= 'F')
        return token - 'A' + 10;
    return -1;
}

static SInt32 nextchar(StringPtr str) {
    short tmpchr;
    SInt32 chr_max;
    Boolean out_of_bounds;
    SInt32 chr;
    short i;
    short t;

    chr_max = 0xFF;
    if (widestring) chr_max = 0xFFFF;

    was_escchar = 0;
    t = prepnextstringchar(str, 1);

    if (t == '\\' && (!copts.multibyteaware || !COS_IsMultiByte(str, pos - 1))) {
        was_escchar = 1;
        switch ((t = prepnextstringchar(str, 1))) {
            case 'a': return 7;
            case 'b': return 8;
            case 't': return 9;
            case 'v': return 11;
            case 'f': return 12;
            case 'n':
                if (copts.mpwc_newline)
                    return 13;
                else
                    return 10;
            case 'r':
                if (copts.mpwc_newline)
                    return 10;
                else
                    return 13;
            case 'e': return 27;
            case 'x':
                chr = tohex(prepnextstringchar(str, 1));
                if (chr == -1) {
                    CPrep_Error(CErrorStr100);
                    return ' ';
                }
                out_of_bounds = 0;
                while (tohex((tmpchr = prepcurstringchar(str))) != -1) {
                    chr = (chr << 4) + tohex(tmpchr);
                    prepcurstringchar_skip();
                    if (chr & ~chr_max)
                        out_of_bounds = 1;
                }
                if (out_of_bounds)
                    CError_Error(CErrorStr100);
                break;
            default:
                if (t >= '0' && t <= '7') {
                    out_of_bounds = 0;
                    i = 1;
                    chr = t - '0';
                    while (i < 3 && (tmpchr = prepcurstringchar(str)) >= '0' && tmpchr <= '7') {
                        chr = (chr << 3) + tmpchr - '0';
                        prepcurstringchar_skip();
                        if (chr & ~chr_max)
                            out_of_bounds = 1;
                        i++;
                    }
                    if (out_of_bounds)
                        CError_Error(CErrorStr100);
                } else {
                    chr = t;
                }
        }
    } else {
        chr = t;
    }

    if (chr_max == 0xFF)
        return (char) (chr & chr_max);
    else
        return (chr & chr_max);
}

StringPtr CPrep_SkipNewComment(StringPtr str) {
    StringPtr p;

    p = str;
    while (1) {
        switch (*(p++)) {
            case 0:
                if (tokenstacklevel > 0 || p >= (StringPtr) prep_file_end) {
                    CPrep_SkipNewCommentChar = 0;
                    return p - 1;
                }
                p[-1] = ' ';
                continue;

            case '\r':
                CPrep_SkipNewCommentChar = '\r';
                return p;

            case '\\':
                if (copts.multibyteaware && COS_IsMultiByte(str, p - 1))
                    continue;

                if (*p == '\r') {
                    ++p;
                    foundnl();
                }

                if (*p == '\n')
                    p++;
                continue;

            case '"':
                if (InlineAsm_gccmode) {
                    CPrep_SkipNewCommentChar = '\r';
                    return p - 1;
                }
                continue;
        }
    }
}

Boolean skipendoflinematch(StringPtr str, short token) {
    StringPtr start;

    start = str - 1;
    pos = (char *) start;

    while (1) {
        switch (*(str++)) {
            case 0:
            case '\r':
                pos = (char *) str - 1;
                return 0;

            case '"':
                if (token == '"') {
                    pos = (char *) str;
                    return 1;
                }
                break;

            case '\'':
                if (token == '\'') {
                    pos = (char *) str;
                    return 1;
                }
                break;

            case '\\':
                if (copts.multibyteaware && COS_IsMultiByte(start, str - 1))
                    continue;

                switch (*(str++)) {
                    case 0:
                        pos = (char *) str - 1;
                        return 0;

                    case '\r':
                        foundnl();
                        if (*str == '\n')
                            str++;
                        break;
                }
                break;

            case '?':
                if (copts.trigraphs && str[0] == '?' && str[1] == '/' && str[2] == '\r') {
                    foundnl();
                    str += 3;
                }
                break;
        }
    }
}

void skipendofline(void) {
    StringPtr start;
    StringPtr p;
    short ch;

    start = (StringPtr) pos;
    p = (StringPtr) pos;

    while (1) {
        switch (*(p++)) {
            case 0:
            case '\r':
                pos = (char *) p - 1;
                return;

            case '"':
                if (!skipendoflinematch(p, '"'))
                    return;
                p = (StringPtr) pos;
                break;

            case '\'':
                if (!skipendoflinematch(p, '\''))
                    return;
                p = (StringPtr) pos;
                break;

            case '/':
                if (copts.multibyteaware) {
                    while (*p == '\\' && !COS_IsMultiByte(start, p) && p[1] == '\r') {
                        foundnl();
                        if (p[2] == '\n')
                            p += 3;
                        else
                            p += 2;
                    }
                } else {
                    while (*p == '\\' && p[1] == '\r') {
                        foundnl();
                        if (p[2] == '\n')
                            p += 3;
                        else
                            p += 2;
                    }
                }

                if (*p == '/' && (!copts.ANSI_strict || copts.cplusplus)) {
                    // EOL comment
                    pos = (char *) CPrep_SkipNewComment(p + 1);
                    if (CPrep_SkipNewCommentChar == '\r')
                        pos--;
                    return;
                }

                if (*p == '*') {
                    // Block comment
                    pos = (char *) p - 1;
                    spaceskip = 1;
                    p++;
                    while (1) {
                        if ((ch = *(p++)) == 0) {
                            if (tokenstacklevel > 0 || p >= (StringPtr) prep_file_end) {
                                pos = (char *) p - 1;
                                CPrep_Error(CErrorStr103);
                                return;
                            }
                            p[-1] = ' ';
                        }

                        if (ch == '*') {
                            if (copts.multibyteaware) {
                                while (*p == '\\' && !COS_IsMultiByte(start, p) && p[1] == '\r') {
                                    foundnl();
                                    if (p[2] == '\n')
                                        p += 3;
                                    else
                                        p += 2;
                                }
                            } else {
                                while (*p == '\\' && p[1] == '\r') {
                                    foundnl();
                                    if (p[2] == '\n')
                                        p += 3;
                                    else
                                        p += 2;
                                }
                            }

                            if (*p == '/') {
                                p++;
                                break;
                            }
                        }

                        if (ch == '\r')
                            foundnl();
                    }
                }
                break;

            case '\\':
                if (copts.multibyteaware && COS_IsMultiByte(start, p - 1))
                    continue;

                if (*p == '\r') {
                    foundnl();
                    if (p[1] == '\n')
                        p += 2;
                    else
                        p += 1;
                }
                break;

            case '?':
                if (copts.trigraphs && p[0] == '?' && p[1] == '/' && p[2] == '\r') {
                    foundnl();
                    p += 3;
                }
                break;
        }
    }
}

void CPrep_SkipAsmComment(void) {
    Boolean save_macrocheck = macrocheck;
    skipendofline();
    macrocheck = save_macrocheck;
}

static short tille(void) { return TK_ILLEGAL; }

static short tcret(void) {
    newline();
    if (cprep_eoltokens)
        return TK_EOL;
    else
        return 0;
}

static short tapos(short _t) {
    int t;
    int t2;
    int i;
    StringPtr start;
    UInt32 maxvalue;
    int bytecount;
    CInt64 newBits;
    CInt64 val;

    start = (StringPtr) pos;
    t = nextchar((StringPtr) pos);
    if ((t == '\'' || t == 0 || t == '\r') && !was_escchar) {
        CPrep_Error(CErrorStr100);
        tkintconst = cint64_zero;
        tksize = IT_INT;
        return TK_INTCONST;
    }

    t2 = nextchar(start);
    if (t2 == '\'' && !was_escchar) {
        if (widestring) {
            switch (stwchar.size) {
                case 1:
                    CInt64_SetULong(&tkintconst, (UInt8) t);
                    tksize = copts.cplusplus ? IT_SCHAR : IT_INT;
                    break;
                case 2:
                    CInt64_SetULong(&tkintconst, (UInt16) t);
                    tksize = IT_UINT;
                    break;
                case 4:
                    CInt64_SetULong(&tkintconst, (UInt32) t);
                    tksize = IT_UINT;
                    break;
                default:
                    CError_FATAL(1386);
            }
        } else {
            if (copts.unsignedchars)
                t &= 0xFF;
            CInt64_SetLong(&tkintconst, t);
            tksize = copts.cplusplus ? IT_SCHAR : IT_INT;
        }
    } else {
        if (widestring) {
            switch (stwchar.size) {
                case 1:
                    CInt64_SetULong(&val, 8);
                    maxvalue = 0xFF;
                    bytecount = 1;
                    break;
                case 2:
                    CInt64_SetULong(&val, 16);
                    maxvalue = 0xFFFF;
                    bytecount = 2;
                    break;
                case 4:
                    CInt64_SetULong(&val, 32);
                    maxvalue = 0xFFFFFFFF;
                    bytecount = 4;
                    break;
                default:
                    CError_FATAL(1424);
            }
        } else {
            CInt64_SetULong(&val, 8);
            maxvalue = 0xFF;
            bytecount = 1;
        }

        CInt64_SetULong(&tkintconst, t & maxvalue);
        i = bytecount;

        while (1) {
            if ((t2 == 0 || t2 == '\r') && !was_escchar) {
                CPrep_Error(CErrorStr100);
                break;
            }

            CInt64_SetULong(&newBits, t2 & maxvalue);
            tkintconst = CInt64_Or(CInt64_Shl(tkintconst, val), newBits);
            i += bytecount;

            t2 = nextchar(start);
            if (t2 == '\'' && !was_escchar)
                break;

            if (i >= stunsignedlonglong.size) {
                CPrep_Error(CErrorStr100);
                break;
            }
        }

        if (i <= stunsignedint.size && stunsignedint.size < stunsignedlong.size)
            tksize = IT_UINT;
        else if (i <= stunsignedlong.size)
            tksize = IT_ULONG;
        else
            tksize = IT_ULONGLONG;
    }

    return TK_INTCONST;
}

static short tquot(short t) {
    Type *type;
    StringPtr string_ptr;
    SInt32 size;
    short c;
    StringPtr save_pos;
    StringPtr start_pos;
    CInt64 ch64;

    start_pos = (StringPtr) pos;
    at_linestart = 0;
    if (cprep_nostring)
        return '"';

    string_ptr = (StringPtr) *stringmem;
    ispascalstring = 0;
    size = 0;
    if (widestring)
        type = TYPE(&stwchar);
    else
        type = TYPE(&stchar);

    if (prepcurstringchar(start_pos) == '\\') {
        save_pos = (StringPtr) pos;
        prepcurstringchar_skip();
        if (prepcurstringchar(start_pos) == 'p') {
            prepcurstringchar_skip();
            ispascalstring = 1;
            string_ptr += type->size;
        } else {
            pos = (char *) save_pos;
        }
    }

    while (1) {
        c = nextchar(start_pos);
        if ((c == '"' || c == '\r' || c == 0) && !was_escchar) {
            if (c == 0 && (tokenstacklevel > 0 || pos >= prep_file_end))
                CPrep_Error(CErrorStr102);
            else if (c != '"')
                CPrep_Error(CErrorStr101);
            break;
        }
        if ((size + type->size) >= maxstringsize) {
            SInt32 offset = string_ptr - (StringPtr) *stringmem;
            if (!COS_ResizeHandle(stringmem, maxstringsize += 256))
                CError_NoMem();
            string_ptr = (StringPtr) *stringmem + offset;
        }
        if (type->size != 1) {
            CInt64_SetLong(&ch64, c);
            CMach_InitIntMem(type, ch64, string_ptr);
            string_ptr += type->size;
            size += type->size;
        } else {
            *string_ptr = c;
            string_ptr++;
            size++;
        }
    }

    if (ispascalstring) {
        if (size > 255 && type->size == 1) {
            CPrep_Error(CErrorStr106);
            size = 255;
        }
        CInt64_SetLong(&ch64, size / type->size);
        CMach_InitIntMem(type, ch64, *stringmem);
        size += type->size;
    } else {
        if ((size + 1) >= maxstringsize) {
            if (!COS_ResizeHandle(stringmem, maxstringsize += 256))
                CError_NoMem();
        }
        CMach_InitIntMem(type, cint64_zero, *stringmem + size);
        size += type->size;
    }

    tksize = size;
    tkstring = galloc(size);
    memcpy(tkstring, *stringmem, tksize);

    if (widestring)
        c = TK_STRING_WIDE;
    else
        c = TK_STRING;
    return (short) c;
}

static short thash(void) {
    if (at_linestart) {
        preprocessor();
        return 0;
    }
    return '#';
}

static short tmult(void) {
    at_linestart = 0;
    if (prepcurchar() == '=') {
        pos = nextcharpos;
        return TK_MULT_ASSIGN;
    }
    return '*';
}

static short tcolo(void) {
    char t2;
    at_linestart = 0;
    if (copts.cplusplus) {
        t2 = prepcurchar();
        if (t2 == ':') {
            pos = nextcharpos;
            return TK_COLON_COLON;
        }
        if (t2 == '>' && !preprocessing_only) {
            pos = nextcharpos;
            return ']';
        }
    }
    return ':';
}

static short tless(void) {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_LESS_EQUAL;
    }
    if (t2 == '<') {
        pos = nextcharpos;
        if (prepcurchar() == '=') {
            pos = nextcharpos;
            return TK_SHL_ASSIGN;
        } else {
            return TK_SHL;
        }
    }
    if (t2 == '%' && copts.cplusplus && !preprocessing_only) {
        pos = nextcharpos;
        return '{';
    }
    if (t2 == ':' && copts.cplusplus && !preprocessing_only) {
        pos = nextcharpos;
        return '[';
    }
    return '<';
}

static short tequa(short) {
    at_linestart = 0;
    if (prepcurchar() == '=') {
        pos = nextcharpos;
        return TK_LOGICAL_EQ;
    }
    return '=';
}

static short tgrea(short) {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_GREATER_EQUAL;
    }
    if (t2 == '>') {
        pos = nextcharpos;
        if (prepcurchar() == '=') {
            pos = nextcharpos;
            return TK_SHR_ASSIGN;
        } else {
            return TK_SHR;
        }
    }
    return '>';
}

static short tatsg(short) {
    char save_idarray;
    char *start_pos = pos;

    at_linestart = 0;

    if (copts.objective_c || in_assembler) {
        save_idarray = cprep_idarray['@'];
        cprep_idarray['@'] = 1;
        pos = ReadIdentifier(start_pos - 1);
        cprep_idarray['@'] = save_idarray;

        if (copts.objective_c) {
            if (!strcmp("@interface", tkidentifier->name))
                return TK_AT_INTERFACE;
            if (!strcmp("@implementation", tkidentifier->name))
                return TK_AT_IMPLEMENTATION;
            if (!strcmp("@protocol", tkidentifier->name))
                return TK_AT_PROTOCOL;
            if (!strcmp("@end", tkidentifier->name))
                return TK_AT_END;
            if (!strcmp("@private", tkidentifier->name))
                return TK_AT_PRIVATE;
            if (!strcmp("@protected", tkidentifier->name))
                return TK_AT_PROTECTED;
            if (!strcmp("@public", tkidentifier->name))
                return TK_AT_PUBLIC;
            if (!strcmp("@class", tkidentifier->name))
                return TK_AT_CLASS;
            if (!strcmp("@selector", tkidentifier->name))
                return TK_AT_SELECTOR;
            if (!strcmp("@encode", tkidentifier->name))
                return TK_AT_ENCODE;
            if (!strcmp("@defs", tkidentifier->name))
                return TK_AT_DEFS;
        }

        if (in_assembler)
            return TK_IDENTIFIER;

        pos = start_pos;
    }

    return '@';
}

static short tperc(short) {
    char t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_MOD_ASSIGN;
    }
    if (t2 == '>' && copts.cplusplus && !preprocessing_only) {
        pos = nextcharpos;
        return '}';
    }
    return '%';
}

static short texcl(void) {
    at_linestart = 0;
    if (prepcurchar() == '=') {
        pos = nextcharpos;
        return TK_LOGICAL_NE;
    }
    return '!';
}

static short tplus(void) {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_ADD_ASSIGN;
    }
    if (t2 == '+') {
        pos = nextcharpos;
        return TK_INCREMENT;
    }
    return '+';
}

static short tminu(void) {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_SUB_ASSIGN;
    }
    if (t2 == '-') {
        pos = nextcharpos;
        return TK_DECREMENT;
    }
    if (t2 == '>') {
        pos = nextcharpos;
        if (copts.cplusplus && prepcurchar() == '*') {
            pos = nextcharpos;
            return TK_ARROW_STAR;
        }
        return TK_ARROW;
    }
    return '-';
}

static short torrr(void) {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_OR_ASSIGN;
    }
    if (t2 == '|') {
        pos = nextcharpos;
        return TK_LOGICAL_OR;
    }
    return '|';
}

static short tampe(void) {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_AND_ASSIGN;
    }
    if (t2 == '&') {
        pos = nextcharpos;
        return TK_LOGICAL_AND;
    }
    return '&';
}

static short tpowe(void) {
    at_linestart = 0;
    if (prepcurchar() == '=') {
        pos = nextcharpos;
        return TK_XOR_ASSIGN;
    }
    return '^';
}

static short tdivi(void) {
    at_linestart = 0;
    if (prepcurchar() == '=') {
        pos = nextcharpos;
        return TK_DIV_ASSIGN;
    }
    return '/';
}

// forward decl
static short tnumb(short t);

static short tzero(short t) {
    char *p;
    short ch;
    Boolean failed;
    Boolean floatFailed;

    if (InlineAsm_gccmode && InlineAsm_labelref)
        return tnumb(t);

    at_linestart = 0;

    p = pos - 1;
    ch = prepnextchar();
    if (ch == 'x' || ch == 'X') {
        p = CInt64_ScanHexString(&tkintconst, pos, &failed);
        if (pos == p)
            CError_Error(CErrorStr105);
        pos = p;
    } else if (!copts.ANSI_strict && (ch == 'b' || ch == 'B')) {
        p = CInt64_ScanBinString(&tkintconst, pos, &failed);
        if (pos == p)
            CError_Error(CErrorStr105);
        pos = p;
    } else {
        while (ch >= '0' && ch <= '9')
            ch = prepnextchar();

        switch (ch) {
            case '.':
            case 'E':
            case 'e':
                pos = CMach_FloatScan(p, &tkfloatconst, &floatFailed);
                if (floatFailed)
                    CPrep_Error(CErrorStr154);
                tksize = floatsuffix(prepcurchar());
                return TK_FLOATCONST;
        }

        pos = CInt64_ScanOctString(&tkintconst, p, &failed);
    }

    if (failed) {
        CPrep_Error(CErrorStr154);
        tkintconst = cint64_zero;
    }

    tksize = intsuffix(prepcurchar(), 1);
    return TK_INTCONST;
}

static short tpoin(void) {
    char *p;
    short ch;
    Boolean failed;

    at_linestart = 0;

    p = pos - 1;
    ch = prepnextchar();

    if (ch >= '0' && ch <= '9') {
        pos = CMach_FloatScan(p, &tkfloatconst, &failed);
        if (failed)
            CPrep_Error(CErrorStr154);
        tksize = floatsuffix(prepcurchar());
        return TK_FLOATCONST;
    }

    if (copts.cplusplus && ch == '*')
        return TK_DOT_STAR;

    if (ch == '.' && prepnextchar() == '.')
        return TK_ELLIPSIS;

    pos = p + 1;
    return '.';
}

static short tnumb(short t) {
    char *p;
    short ch;
    Boolean failed;
    Boolean floatFailed;

    at_linestart = 0;

    p = pos - 1;
    pos = CInt64_ScanDecString(&tkintconst, p, &failed);

    ch = prepcurchar();
    if (ch == '.' || ch == 'e' || ch == 'E') {
        pos = CMach_FloatScan(p, &tkfloatconst, &floatFailed);
        if (floatFailed)
            CPrep_Error(CErrorStr154);
        tksize = floatsuffix(prepcurchar());
        return TK_FLOATCONST;
    }

    if (failed) {
        CPrep_Error(CErrorStr154);
        tkintconst = cint64_zero;
    }

    tksize = intsuffix(prepcurchar(), 0);
    return TK_INTCONST;
}

static short tiden(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    else
        return TK_IDENTIFIER;
}

static short tchrL(void) {
    short ch;

    at_linestart = 0;

    if ((ch = prepcurchar()) == '\'') {
        widestring = 1;
        pos = nextcharpos;
        ch = tapos(ch);
        widestring = 0;

        tksize = (copts.wchar_type && copts.cplusplus) ? IT_WCHAR_T : IT_INT;
        return ch;
    }

    if (ch == '"') {
        widestring = 1;
        pos = nextcharpos;
        ch = tquot(ch);
        widestring = 0;
        return ch;
    }

    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;

    return TK_IDENTIFIER;
}

static short tchra(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("auto", tkidentifier->name))
        return TK_AUTO;
    if ((copts.cplusplus || !copts.only_std_keywords) && !strcmp("asm", tkidentifier->name))
        return TK_ASM;
    if (copts.cplusplus && !in_assembler && !preprocessing_only) {
        if (!strcmp("and", tkidentifier->name))
            return TK_LOGICAL_AND;
        if (!strcmp("and_eq", tkidentifier->name))
            return TK_AND_ASSIGN;
    }
    return TK_IDENTIFIER;
}

static short tchrb(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("break", tkidentifier->name))
        return TK_BREAK;
    if (copts.cplusplus) {
        if (copts.booltruefalse && !strcmp("bool", tkidentifier->name))
            return TK_BOOL;
        if (!in_assembler && !preprocessing_only) {
            if (!strcmp("bitand", tkidentifier->name))
                return '&';
            if (!strcmp("bitor", tkidentifier->name))
                return '|';
        }
    }
    if (copts.objective_c) {
        if (!strcmp("bycopy", tkidentifier->name))
            return TK_BYCOPY;
        if (!strcmp("byref", tkidentifier->name))
            return TK_BYREF;
    }
    return TK_IDENTIFIER;
}

static short tchrc(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("case", tkidentifier->name))
        return TK_CASE;
    if (!strcmp("char", tkidentifier->name))
        return TK_CHAR;
    if (!strcmp("const", tkidentifier->name))
        return TK_CONST;
    if (!strcmp("continue", tkidentifier->name))
        return TK_CONTINUE;
    if (copts.cplusplus) {
        if (!strcmp("const_cast", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(CErrorStr339);
            return TK_CONST_CAST;
        }
        if (!strcmp("catch", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(CErrorStr339);
            return TK_CATCH;
        }
        if (!strcmp("class", tkidentifier->name))
            return TK_CLASS;
        if (!in_assembler && !preprocessing_only) {
            if (!strcmp("compl", tkidentifier->name))
                return '~';
        }
    }
    return TK_IDENTIFIER;
}

static short tchrd(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("default", tkidentifier->name))
        return TK_DEFAULT;
    if (!strcmp("do", tkidentifier->name))
        return TK_DO;
    if (!strcmp("double", tkidentifier->name))
        return TK_DOUBLE;
    if (copts.cplusplus) {
        if (!strcmp("delete", tkidentifier->name))
            return TK_DELETE;
        if (!strcmp("dynamic_cast", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(CErrorStr339);
            return TK_DYNAMIC_CAST;
        }
    }
    return TK_IDENTIFIER;
}

static short tchre(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("else", tkidentifier->name))
        return TK_ELSE;
    if (!strcmp("enum", tkidentifier->name))
        return TK_ENUM;
    if (!strcmp("extern", tkidentifier->name))
        return TK_EXTERN;
    if (copts.cplusplus) {
        if (!strcmp("explicit", tkidentifier->name))
            return TK_EXPLICIT;
        if (!strcmp("export", tkidentifier->name) && !copts.ecplusplus)
            return TK_EXPORT;
    }
    return TK_IDENTIFIER;
}

static short tchrf(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("float", tkidentifier->name))
        return TK_FLOAT;
    if (!strcmp("for", tkidentifier->name))
        return TK_FOR;
    if (copts.cplusplus) {
        if (!strcmp("friend", tkidentifier->name))
            return TK_FRIEND;
        if (copts.booltruefalse && !strcmp("false", tkidentifier->name))
            return TK_FALSE;
    }
    return TK_IDENTIFIER;
}

static short tchrg(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("goto", tkidentifier->name))
        return TK_GOTO;
    return TK_IDENTIFIER;
}

static short tchri(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("if", tkidentifier->name))
        return TK_IF;
    if (!strcmp("int", tkidentifier->name))
        return TK_INT;
    if (copts.cplusplus || copts.c9x) {
        if (!strcmp("inline", tkidentifier->name))
            return TK_INLINE;
    } else if (!copts.only_std_keywords) {
        if (!strcmp("inline", tkidentifier->name))
            return TK_INLINE;
    }
    if (copts.objective_c) {
        if (!strcmp("in", tkidentifier->name))
            return TK_IN;
        if (!strcmp("inout", tkidentifier->name))
            return TK_INOUT;
    }
    return TK_IDENTIFIER;
}

static short tchrl(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("long", tkidentifier->name))
        return TK_LONG;
    return TK_IDENTIFIER;
}

static short tchrm(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (copts.cplusplus && !strcmp("mutable", tkidentifier->name)) {
        if (copts.ecplusplus)
            CPrep_Warning(CErrorStr339);
        return TK_MUTABLE;
    }
    return TK_IDENTIFIER;
}

static short tchrn(short t) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (copts.cplusplus && t == 'n') {
        if (!strcmp("new", tkidentifier->name))
            return TK_NEW;
        if (!strcmp("namespace", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(CErrorStr339);
            return TK_NAMESPACE;
        }
        if (!in_assembler && !preprocessing_only) {
            if (!strcmp("not", tkidentifier->name))
                return '!';
            if (!strcmp("not_eq", tkidentifier->name))
                return TK_LOGICAL_NE;
        }
    }
    return TK_IDENTIFIER;
}

static short tchro(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (copts.cplusplus) {
        if (!strcmp("operator", tkidentifier->name))
            return TK_OPERATOR;
        if (!in_assembler && !preprocessing_only) {
            if (!strcmp("or", tkidentifier->name))
                return TK_LOGICAL_OR;
            if (!strcmp("or_eq", tkidentifier->name))
                return TK_OR_ASSIGN;
        }
    }
    if (copts.objective_c) {
        if (!strcmp("out", tkidentifier->name))
            return TK_OUT;
        if (!strcmp("oneway", tkidentifier->name))
            return TK_ONEWAY;
    }
    return TK_IDENTIFIER;
}

static short tchrp(short t) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!copts.only_std_keywords && !strcmp("pascal", tkidentifier->name))
        return TK_PASCAL;
    if (copts.cplusplus && t == 'p') {
        if (!strcmp("private", tkidentifier->name))
            return TK_PRIVATE;
        if (!strcmp("protected", tkidentifier->name))
            return TK_PROTECTED;
        if (!strcmp("public", tkidentifier->name))
            return TK_PUBLIC;
    }
    return TK_IDENTIFIER;
}

static short tchrr(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("register", tkidentifier->name))
        return TK_REGISTER;
    if (!strcmp("return", tkidentifier->name))
        return TK_RETURN;
    if (copts.cplusplus && !strcmp("reinterpret_cast", tkidentifier->name)) {
        if (copts.ecplusplus)
            CPrep_Warning(CErrorStr339);
        return TK_REINTERPRET_CAST;
    }
    if (copts.c9x && !strcmp("restrict", tkidentifier->name))
        return TK_RESTRICT;
    return TK_IDENTIFIER;
}

static short tchrs(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("short", tkidentifier->name))
        return TK_SHORT;
    if (!strcmp("signed", tkidentifier->name))
        return TK_SIGNED;
    if (!strcmp("sizeof", tkidentifier->name))
        return TK_SIZEOF;
    if (!strcmp("static", tkidentifier->name))
        return TK_STATIC;
    if (!strcmp("struct", tkidentifier->name))
        return TK_STRUCT;
    if (!strcmp("switch", tkidentifier->name))
        return TK_SWITCH;
    if (copts.cplusplus && !strcmp("static_cast", tkidentifier->name)) {
        if (copts.ecplusplus)
            CPrep_Warning(CErrorStr339);
        return TK_STATIC_CAST;
    }
    return TK_IDENTIFIER;
}

static short tchrt(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("typedef", tkidentifier->name))
        return TK_TYPEDEF;
    if (copts.gcc_extensions && !strcmp("typeof", tkidentifier->name))
        return TK_UU_TYPEOF_UU;
    if (copts.cplusplus) {
        if (!strcmp("this", tkidentifier->name))
            return TK_THIS;
        if (copts.booltruefalse && !strcmp("true", tkidentifier->name))
            return TK_TRUE;
        if (!strcmp("template", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(CErrorStr339);
            return TK_TEMPLATE;
        }
        if (!strcmp("try", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(CErrorStr339);
            return TK_TRY;
        }
        if (!strcmp("throw", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(CErrorStr339);
            return TK_THROW;
        }
        if (!strcmp("typeid", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(CErrorStr339);
            return TK_TYPEID;
        }
        if (!strcmp("typename", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(CErrorStr339);
            return TK_TYPENAME;
        }
    }
    return TK_IDENTIFIER;
}

static short tchru(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("union", tkidentifier->name))
        return TK_UNION;
    if (!strcmp("unsigned", tkidentifier->name))
        return TK_UNSIGNED;
    if (copts.cplusplus && !strcmp("using", tkidentifier->name)) {
        if (copts.ecplusplus)
            CPrep_Warning(CErrorStr339);
        return TK_USING;
    }
    return TK_IDENTIFIER;
}

static short tchrv(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("void", tkidentifier->name))
        return TK_VOID;
    if (!strcmp("volatile", tkidentifier->name))
        return TK_VOLATILE;
    if (copts.cplusplus && !strcmp("virtual", tkidentifier->name))
        return TK_VIRTUAL;
    return TK_IDENTIFIER;
}

static short tchrw(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("while", tkidentifier->name))
        return TK_WHILE;
    if (copts.cplusplus && copts.wchar_type && !strcmp("wchar_t", tkidentifier->name))
        return TK_WCHAR_T;
    return TK_IDENTIFIER;
}

static short tchrx(short t) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;

    if (copts.cplusplus && t == 'x' && !in_assembler && !preprocessing_only) {
        if (!strcmp("xor", tkidentifier->name))
            return '^';
        if (!strcmp("xor_eq", tkidentifier->name))
            return TK_XOR_ASSIGN;
    }

    return TK_IDENTIFIER;
}

static short tchr_(void) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;

    if (!strcmp("__stdcall", tkidentifier->name))
        return TK_UU_STDCALL;
    if (!strcmp("__cdecl", tkidentifier->name))
        return TK_UU_CDECL;
    if (!strcmp("__fastcall", tkidentifier->name))
        return TK_UU_FASTCALL;
    if (!strcmp("__declspec", tkidentifier->name))
        return TK_UU_DECLSPEC;
    if (!strcmp("__asm", tkidentifier->name))
        return TK_ASM;
    if (!strcmp("__asm__", tkidentifier->name))
        return TK_ASM;
    if (!strcmp("__inline", tkidentifier->name))
        return TK_INLINE;
    if (!strcmp("__inline__", tkidentifier->name))
        return TK_INLINE;
    if (!strcmp("__restrict", tkidentifier->name))
        return TK_RESTRICT;
    if (!strcmp("__restrict__", tkidentifier->name))
        return TK_RESTRICT;
    if (!strcmp("__attribute__", tkidentifier->name))
        return TK_UU_ATTRIBUTE_UU;
    if (!strcmp("__far", tkidentifier->name))
        return TK_UU_FAR;
    if (!strcmp("__alignof__", tkidentifier->name))
        return TK_UU_ALIGNOF_UU;
    if (!strcmp("__typeof__", tkidentifier->name))
        return TK_UU_TYPEOF_UU;
    if (copts.c9x) {
        if (!strcmp("_Bool", tkidentifier->name))
            return TK_BOOL;
        if (!strcmp("_Complex", tkidentifier->name))
            return TK_U_COMPLEX;
        if (!strcmp("_Imaginary", tkidentifier->name))
            return TK_U_IMAGINARY;
    }
    if (copts.altivec_model && !strcmp("__vector", tkidentifier->name))
        return TK_UU_VECTOR;
    return TK_IDENTIFIER;
}

static short token(short t) {
    at_linestart = 0;
    return t;
}

static short tdoll(short t) {
    at_linestart = 0;
    return t;
}

static short tisid(void) {
    if (!tokenstacklevel)
        CError_Error(CErrorStr105);
    return 0;
}

static short tnull(short t) {
    if (tokenstacklevel > 0 || pos >= prep_file_end) {
        at_linestart = 0;
        return t;
    } else {
        pos[-1] = -64;
        return TK_ILLEGAL;
    }
}

static short t0x1a(void) {
    if (*pos == 0)
        return 0;
    else
        return TK_ILLEGAL;
}

typedef short (*TokenizePtr)(short);
#define F(func) ((TokenizePtr) (&func))

static TokenizePtr cprep_tokenize[256] = {
        F(tnull), F(tille), F(tille), F(tille), F(tisid), F(tisid), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tcret), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(t0x1a), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(texcl), F(tquot), F(thash), F(tdoll), F(tperc), F(tampe), F(tapos),
        F(token), F(token), F(tmult), F(tplus), F(token), F(tminu), F(tpoin), F(tdivi),
        F(tzero), F(tnumb), F(tnumb), F(tnumb), F(tnumb), F(tnumb), F(tnumb), F(tnumb),
        F(tnumb), F(tnumb), F(tcolo), F(token), F(tless), F(tequa), F(tgrea), F(token),
        F(tatsg), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden),
        F(tiden), F(tiden), F(tiden), F(tiden), F(tchrL), F(tiden), F(tchrn), F(tiden),
        F(tchrp), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden),
        F(tchrx), F(tiden), F(tiden), F(token), F(token), F(token), F(tpowe), F(tchr_),
        F(token), F(tchra), F(tchrb), F(tchrc), F(tchrd), F(tchre), F(tchrf), F(tchrg),
        F(tiden), F(tchri), F(tiden), F(tiden), F(tchrl), F(tchrm), F(tchrn), F(tchro),
        F(tchrp), F(tiden), F(tchrr), F(tchrs), F(tchrt), F(tchru), F(tchrv), F(tchrw),
        F(tchrx), F(tiden), F(tiden), F(token), F(torrr), F(token), F(token), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille)
};

short lookahead(void) {
    SInt32 state;
    short t;

    CPrep_TokenStreamGetState(&state);
    t = lex();
    CPrep_TokenStreamSetState(&state);

    return t;
}

short lookahead_noeol(void) {
    SInt32 state;
    short t;

    CPrep_TokenStreamGetState(&state);
    do {
        t = lex();
    } while (t == TK_EOL);
    CPrep_TokenStreamSetState(&state);

    return t;
}

typedef struct StringChunk {
    struct StringChunk *next;
    char *data;
    SInt32 offset;
    SInt32 size;
} StringChunk;

static void CPrep_StringConCat(Boolean flag) {
    StringPtr buffer;
    StringChunk *chunks;
    StringChunk *chunk;
    char *saveString;
    SInt32 saveSize;
    SInt32 offset;
    Boolean savePascalString;
    TStreamElement *elem;
    short ch;

    saveString = tkstring;
    saveSize = tksize;
    savePascalString = ispascalstring;

    cprep_strconcat = 1;

    chunks = NULL;
    offset = tksize;

    while (1) {
        ch = lookahead_noeol();
        if (ch != TK_STRING && ch != TK_STRING_WIDE) {
            ispascalstring = savePascalString;
            if (chunks) {
                tksize = offset;
                buffer = galloc(offset);
                tkstring = (char *) buffer;

                memcpy(buffer, saveString, saveSize);
                for (chunk = chunks; chunk; chunk = chunk->next)
                    memcpy(buffer + chunk->offset, chunk->data, chunk->size);

                if (savePascalString) {
                    if (offset > 256) {
                        CError_Error(CErrorStr106);
                        offset = 256;
                    }
                    buffer[0] = offset - 1;
                } else if (flag) {
                    CMach_InitIntMem(TYPE(&stwchar), cint64_zero, buffer + offset - stwchar.size);
                } else {
                    CMach_InitIntMem(TYPE(&stchar), cint64_zero, buffer + offset - stchar.size);
                }

                elem = ts_current - 1;
                elem->data.tkstring.data = (char *) buffer;
                elem->data.tkstring.size = offset;
                elem->subtype = ispascalstring;
                cprep_strconcat = 0;
            } else {
                tkstring = saveString;
                tksize = saveSize;
                cprep_strconcat = 0;
            }
            break;
        }

        do {
            ch = lex();
            elem = --ts_current;
            memmove(elem, elem + 1, sizeof(TStreamElement) * ts_preread_elements);
        } while (ch == TK_EOL);

        chunk = lalloc(sizeof(StringChunk));
        chunk->next = chunks;
        chunk->data = tkstring;
        chunk->size = tksize - 1;
        chunk->offset = offset - 1;
        chunks = chunk;

        if (ispascalstring)
            chunk->data++;
        if (savePascalString)
            chunk->offset++;

        if (ch == TK_STRING_WIDE)
            chunk->size = chunk->size + 1 - stwchar.size;
        if (flag)
            chunk->offset = chunk->offset + 1 - stwchar.size;

        offset += chunk->size;
    }
}

short lex(void) {
    short t;

    while (1) {
        if (ts_preread_elements <= 0)
            break;

        t = ts_current->tokentype;
        if (t < 0) {
            switch (t) {
                case TK_INTCONST:
                    tkintconst = ts_current->data.tkintconst;
                    tksize = ts_current->subtype;
                    break;
                case TK_FLOATCONST:
                    tkfloatconst = ts_current->data.tkfloatconst;
                    tksize = ts_current->subtype;
                    break;
                case TK_IDENTIFIER:
                    tkidentifier = ts_current->data.tkidentifier;
                    break;
                case TK_STRING:
                case TK_STRING_WIDE:
                    tkstring = ts_current->data.tkstring.data;
                    tksize = ts_current->data.tkstring.size;
                    ispascalstring = ts_current->subtype;
                    break;
                case TK_EOL:
                    nlflag = 1;
                    if (!cprep_eoltokens) {
                        ts_current++;
                        ts_preread_elements--;
                        continue;
                    }
                    break;
                default:
                    CError_FATAL(3007);
            }
        }

        ts_current++;
        ts_preread_elements--;
        return t;
    }

    while (1) {
        t = prepskipnextchar();

        ts_current->tokenfile = prep_file;
        if (tokenstacklevel > 0) {
            ts_current->tokenoffset = tokenstack[0].pos - prep_file_start;
            macropos = pos;
        } else {
            ts_current->tokenoffset = pos - prep_file_start;
        }
        ts_current->tokenline = linenumber;
        pos = nextcharpos;

        t = cprep_tokenize[t](t);
        if (t != 0) {
            if (ts_current >= ts_last)
                CPrep_TSBufferGrow(1024);
            ts_current->tokentype = t;

            if ((t = ts_current->tokentype) < 0) {
                switch (t) {
                    case TK_INTCONST:
                        ts_current->data.tkintconst = tkintconst;
                        ts_current->subtype = tksize;
                        break;
                    case TK_FLOATCONST:
                        ts_current->data.tkfloatconst = tkfloatconst;
                        ts_current->subtype = tksize;
                        break;
                    case TK_IDENTIFIER:
                        ts_current->data.tkidentifier = tkidentifier;
                        break;
                    case TK_STRING:
                    case TK_STRING_WIDE:
                        ts_current->data.tkstring.data = tkstring;
                        ts_current->data.tkstring.size = tksize;
                        ts_current->subtype = ispascalstring;
                        if (!cprep_strconcat) {
                            ts_current++;
                            CPrep_StringConCat(t == TK_STRING_WIDE);
                            return t;
                        }
                        break;
                    case TK_ILLEGAL:
                        ts_current++;
                        CError_Error(CErrorStr105);
                        ts_current--;
                        break;
                }
            }

            ts_current++;
            return t;
        }

        if (*pos == 0) {
            if (tokenstacklevel > 0) {
                poptokenseq();
            } else if (tokenstacklevel > 0 || pos >= prep_file_end) {
                if (filesp > 0)
                    popfile();
                else
                    return 0;
            } else {
                ts_current++;
                CError_ErrorTerm(CErrorStr105);
            }
        }
    }
}

short plex(void) {
    short t;

    while (1) {
        t = prepskipnextchar();
        pos = nextcharpos;
        if ((t = cprep_tokenize[t](t)))
            return t;

        if (!*pos) {
            if (tokenstacklevel > 0) {
                poptokenseq();
            } else if (filesp > 0) {
                popfile();
            } else {
                break;
            }
        }
    }

    return 0;
}

short lexidentifier(void) {
    short t;
    char *save_pos;

    t = prepskipnextchar();
    pos = nextcharpos;
    at_linestart = 0;
    if ((t >= 'a' && t <= 'z') || (t >= 'A' && t <= 'Z') || (t == '_')) {
        pos = ReadIdentifier(pos - 1);
        return TK_IDENTIFIER;
    } else if (t == '.') {
        save_pos = pos;
        if (prepnextchar() == '.' && prepnextchar() == '.')
            return TK_ELLIPSIS;
        else
            pos = save_pos;
    }
    return t;
}