MWCC/compiler_and_linker/FrontEnd/C/CPrepTokenizer.c

#include "compiler.h"
#include "compiler/tokens.h"
#include "compiler/CError.h"
#include "compiler/CInt64.h"
#include "cos.h"

static Boolean prepnextstringchar_foundnl;

#define UCD (unsigned char *)
#define CD (char *)

short prepskipnextchar() {
    short c;
    unsigned char *start;
    unsigned char *p;

    start = UCD pos;
    p = UCD pos;

loop:
    switch ((c = *p)) {
        case 0:
            if (tokenstacklevel > 0 || CD p >= prep_file_end) {
                pos = CD p;
                nextcharpos = CD p;
                return c;
            } else {
                pos = CD p++;
                nextcharpos = CD p;
                return c;
            }
        case 4:
        case 5:
            p++;
            goto loop;
        case ' ':
        case '\t':
        case '\n':
        case '\v':
        case '\f':
            spaceskip = 1;
            p++;
            goto loop;
        case ';':
            pos = CD p++;
            if (tokenstacklevel <= 0 && in_assembler && copts.asmsemicolcomment) {
                spaceskip = 1;
                nextcharpos = CPrep_SkipNewComment(CD p);
                return CPrep_SkipNewCommentChar;
            }
            nextcharpos = CD p;
            return c;
        case '#':
            pos = CD p++;
            if (tokenstacklevel <= 0 && in_assembler && copts.asmpoundcomment) {
                spaceskip = 1;
                nextcharpos = CPrep_SkipNewComment(CD p);
                return CPrep_SkipNewCommentChar;
            }
            nextcharpos = CD p;
            return c;
        case '/':
            pos = CD p++;
            if (copts.multibyteaware) {
                while (p[0] == '\\' && !COS_IsMultiByte(CD start, CD p) && p[1] == '\r') {
                    foundnl();
                    if (p[2] == '\n')
                        p += 3;
                    else
                        p += 2;
                }
            } else {
                while (p[0] == '\\' && p[1] == '\r') {
                    foundnl();
                    if (p[2] == '\n')
                        p += 3;
                    else
                        p += 2;
                }
            }
            if (tokenstacklevel <= 0) {
                if (p[0] == '/' && (!copts.ANSI_strict || copts.cplusplus || copts.c9x)) {
                    spaceskip = 1;
                    nextcharpos = CPrep_SkipNewComment(CD p + 1);
                    return CPrep_SkipNewCommentChar;
                } else if (p[0] == '*') {
                    spaceskip = 1;
                    p++;
                    for (;;) {
                        if (!(c = *(p++))) {
                            if (tokenstacklevel > 0 || CD p >= prep_file_end) {
                                CPrep_Error(103);
                                nextcharpos = CD p - 1;
                                return c;
                            }
                            p[-1] = ' ';
                        }
                        if (c == '*') {
                            if (copts.multibyteaware) {
                                while (p[0] == '\\' && !COS_IsMultiByte(CD start, CD p) && p[1] == '\r') {
                                    foundnl();
                                    if (p[2] == '\n')
                                        p += 3;
                                    else
                                        p += 2;
                                }
                            } else {
                                while (p[0] == '\\' && p[1] == '\r') {
                                    foundnl();
                                    if (p[2] == '\n')
                                        p += 3;
                                    else
                                        p += 2;
                                }
                            }
                            if (*p == '/') {
                                p++;
                                goto loop;
                            }
                        }
                        if (c == '\r')
                            foundnl();
                    }
                }
            }
            nextcharpos = CD p;
            return c;
        case '%':
            pos = CD p++;
            if (*p == ':' && copts.cplusplus) {
                nextcharpos = CD p + 1;
                return '#';
            }
            nextcharpos = CD p;
            return c;
        case '\\':
            pos = CD p++;
        mid_backslash:
            if (*p == '\r') {
                if (!copts.multibyteaware || !COS_IsMultiByte(CD start, CD p - 1)) {
                    foundnl();
                    if (p[1] == '\n')
                        p += 2;
                    else
                        p += 1;
                    goto loop;
                }
            }
            nextcharpos = CD p;
            return c;
        case '?':
            pos = CD p++;
            if (copts.trigraphs && p[0] == '?') {
                switch (p[1]) {
                    case '=':
                        nextcharpos = CD p + 2;
                        return '#';
                    case '/':
                        c = '\\';
                        p += 2;
                        goto mid_backslash;
                    case '\'':
                        nextcharpos = CD p + 2;
                        return '^';
                    case '(':
                        nextcharpos = CD p + 2;
                        return '[';
                    case ')':
                        nextcharpos = CD p + 2;
                        return ']';
                    case '!':
                        nextcharpos = CD p + 2;
                        return '|';
                    case '<':
                        nextcharpos = CD p + 2;
                        return '{';
                    case '>':
                        nextcharpos = CD p + 2;
                        return '}';
                    case '-':
                        nextcharpos = CD p + 2;
                        return '~';
                }
            }
            nextcharpos = CD p;
            return c;
        default:
            pos = CD p++;
            nextcharpos = CD p;
            return c;
    }
}

short prepnextchar() {
    short c;
    unsigned char *start;
    unsigned char *p;

    start = UCD pos;
    p = UCD pos;

loop:
    switch ((c = *(p++))) {
        case 0:
            pos = CD p - 1;
            return c;
        case '/':
            if (copts.multibyteaware) {
                while (p[0] == '\\' && !COS_IsMultiByte(CD start, CD p) && p[1] == '\r') {
                    foundnl();
                    if (p[2] == '\n')
                        p += 3;
                    else
                        p += 2;
                }
            } else {
                while (p[0] == '\\' && p[1] == '\r') {
                    foundnl();
                    if (p[2] == '\n')
                        p += 3;
                    else
                        p += 2;
                }
            }
            if (p[0] == '/' && (!copts.ANSI_strict || copts.cplusplus)) {
                spaceskip = 1;
                pos = CPrep_SkipNewComment(CD p + 1);
                return CPrep_SkipNewCommentChar;
            } else if (p[0] == '*') {
                spaceskip = 1;
                p++;
                for (;;) {
                    if (!(c = *(p++))) {
                        if (tokenstacklevel > 0 || CD p >= prep_file_end) {
                            pos = CD p - 1;
                            CPrep_Error(103);
                            return c;
                        }
                        p[-1] = ' ';
                    }
                    if (c == '*') {
                        if (copts.multibyteaware) {
                            while (p[0] == '\\' && !COS_IsMultiByte(CD start, CD p) && p[1] == '\r') {
                                foundnl();
                                if (p[2] == '\n')
                                    p += 3;
                                else
                                    p += 2;
                            }
                        } else {
                            while (p[0] == '\\' && p[1] == '\r') {
                                foundnl();
                                if (p[2] == '\n')
                                    p += 3;
                                else
                                    p += 2;
                            }
                        }
                        if (*p == '/')
                            break;
                    }
                    if (c == '\r')
                        foundnl();
                }
                pos = CD p + 1;
                return ' ';
            } else {
                pos = CD p;
                return c;
            }
        case '\\':
        backslash:
            if (*p == '\r') {
                if (!copts.multibyteaware || !COS_IsMultiByte(CD start, CD p - 1)) {
                    foundnl();
                    if (p[1] == '\n')
                        p += 2;
                    else
                        p += 1;
                    goto loop;
                }
            }
            pos = CD p;
            return c;
        case '%':
            if (*p == ':' && copts.cplusplus) {
                pos = CD p + 1;
                return '#';
            }
            pos = CD p;
            return c;
        case '?':
            if (copts.trigraphs && p[0] == '?') {
                switch (p[1]) {
                    case '=':
                        pos = CD p + 2;
                        return '#';
                    case '/':
                        c = '\\';
                        p += 2;
                        goto backslash;
                    case '\'':
                        pos = CD p + 2;
                        return '^';
                    case '(':
                        pos = CD p + 2;
                        return '[';
                    case ')':
                        pos = CD p + 2;
                        return ']';
                    case '!':
                        pos = CD p + 2;
                        return '|';
                    case '<':
                        pos = CD p + 2;
                        return '{';
                    case '>':
                        pos = CD p + 2;
                        return '}';
                    case '-':
                        pos = CD p + 2;
                        return '~';
                }
            }
            pos = CD p;
            return c;
        default:
            pos = CD p;
            return c;
    }
}

short prepnextstringchar(char *str, Boolean flag) {
    unsigned char *p;
    short c;

    p = UCD pos;

loop:
    switch ((c = *(p++))) {
        case 0:
            pos = CD p - 1;
            return c;
        case '\\':
        backslash:
            if (p[0] == '\r' && (!copts.multibyteaware || !COS_IsMultiByte(str, CD p - 1))) {
                if (flag)
                    foundnl();
                else
                    prepnextstringchar_foundnl = 1;
                if (p[1] == '\n')
                    p += 2;
                else
                    p += 1;
                goto loop;
            }
            pos = CD p;
            return c;
        case '?':
            if (copts.trigraphs && p[0] == '?') {
                switch (p[1]) {
                    case '=':
                        pos = CD p + 2;
                        return '#';
                    case '/':
                        c = '\\';
                        p += 2;
                        goto backslash;
                    case '\'':
                        pos = CD p + 2;
                        return '^';
                    case '(':
                        pos = CD p + 2;
                        return '[';
                    case ')':
                        pos = CD p + 2;
                        return ']';
                    case '!':
                        pos = CD p + 2;
                        return '|';
                    case '<':
                        pos = CD p + 2;
                        return '{';
                    case '>':
                        pos = CD p + 2;
                        return '}';
                    case '-':
                        pos = CD p + 2;
                        return '~';
                }
            }
            pos = CD p;
            return c;
        default:
            pos = CD p;
            return c;
    }
}

void CPrep_MatchChar(char ch, Boolean flag) {
    char *p;
    char *start;
    char c;
    Boolean r26;

    p = start = pos;
    r26 = 0;

loop:
    if (!(c = *(p++))) {
        if (tokenstacklevel > 0 || p >= prep_file_end)
            CPrep_Error(102);
        else if (ch == '"')
            CPrep_Error(101);
        else if (ch == '\'')
            CPrep_Error(100);
        CPrep_Error(117);
        pos = p - 1;
        return;
    }

    if (c == '\r') {
        if (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1)) {
            if (flag)
                foundnl();
            if (*p == '\n')
                p++;
            CPrep_Error(112);
            pos = p;
            return;
        }
        r26 = 0;
        goto loop;
    }

    if (c == '?' && p[0] == '?' && copts.trigraphs) {
        switch (p[1]) {
            case '!':
            case '\'':
            case '(':
            case ')':
            case '-':
            case '<':
            case '=':
            case '>':
                r26 = 0;
                p += 2;
                goto loop;
            case '/':
                p += 2;
                goto backslash;
        }
    }

    if (c == '\\' && !r26 && (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1))) {
    backslash:
        if (*p == '\r') {
            if (flag)
                foundnl();
            if (*(++p) == '\n')
                p++;
            r26 = 0;
        } else {
            r26 = 1;
        }
        goto loop;
    }

    if (c == ch && !r26) {
        pos = p;
    } else {
        r26 = 0;
        goto loop;
    }
}

char *CPrep_MatchChar2(char *start, char ch, Boolean flag) {
    char *p;
    char c;
    Boolean r28;

    p = start;
    r28 = 0;

loop:
    if (!(c = *(p++))) {
        if (tokenstacklevel > 0 || p >= prep_file_end)
            CPrep_Error(102);
        else if (ch == '"')
            CPrep_Error(101);
        else if (ch == '\'')
            CPrep_Error(100);
        CPrep_Error(117);
        return p - 1;
    }

    if (c == '\r') {
        if (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1)) {
            if (flag)
                foundnl();
            if (*p == '\n')
                p++;
            CPrep_Error(112);
            return p;
        }
        r28 = 0;
        goto loop;
    }

    if (c == '?' && p[0] == '?' && copts.trigraphs) {
        switch (p[1]) {
            case '!':
            case '\'':
            case '(':
            case ')':
            case '-':
            case '<':
            case '=':
            case '>':
                r28 = 0;
                p += 2;
                goto loop;
            case '/':
                p += 2;
                goto backslash;
        }
    }

    if (c == '\\' && !r28 && (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1))) {
        backslash:
        if (*p == '\r') {
            if (flag)
                foundnl();
            if (*(++p) == '\n')
                p++;
            r28 = 0;
        } else {
            r28 = 1;
        }
        goto loop;
    }

    if (c == ch && !r28) {
        return p;
    } else {
        r28 = 0;
        goto loop;
    }
}

short prepcurchar() {
    char *save_pos;
    short t;

    save_pos = pos;
    t = prepnextchar();
    nextcharpos = pos;
    pos = save_pos;

    return t;
}

static short prepcurstringchar(char *str) {
    char *save_pos;
    short t;

    save_pos = pos;
    prepnextstringchar_foundnl = 0;
    t = prepnextstringchar(str, 0);
    nextcharpos = pos;
    pos = save_pos;

    return t;
}

static void prepcurstringchar_skip() {
    pos = nextcharpos;
    if (prepnextstringchar_foundnl)
        foundnl();
}

char *ReadIdentifier(char *p) {
    char buf[256];
    char *start;
    int len;

    start = p;
    len = 0;
loop:
    for (;;) {
        if (!cprep_idarray[(unsigned char) *p])
            break;
        if (len < 255)
            buf[len++] = *p;
        p++;
    }

    if (copts.multibyteaware) {
        if ((unsigned char) *p == '\\' && !COS_IsMultiByte(start, p) && p[1] == '\r') {
            if (p[2] == '\n')
                p += 3;
            else
                p += 2;
            foundnl();
            goto loop;
        }
    } else {
        if ((unsigned char) *p == '\\' && p[1] == '\r') {
            if (p[2] == '\n')
                p += 3;
            else
                p += 2;
            foundnl();
            goto loop;
        }
    }

    buf[len] = 0;
    tkidentifier = GetHashNameNodeExport(buf);
    return p;
}

static short intsuffix(short token, Boolean flag) {
    Boolean is_unsigned;
    Boolean is_long;
    Boolean is_longlong;
    short t;

    is_unsigned = is_long = is_longlong = 0;
    t = token;
    switch (t) {
        case 'u':
        case 'U':
            is_unsigned = 1;
            pos = nextcharpos;
            t = prepcurchar();
            if (t == 'l' || t == 'L') {
                is_long = 1;
                pos = nextcharpos;
                t = *pos;
                if (copts.longlong) {
                    pos = nextcharpos;
                    t = prepcurchar();
                    if (t == 'l' || t == 'L') {
                        is_longlong = 1;
                        pos = nextcharpos;
                        t = *pos;
                    }
                }
            }
            break;
        case 'l':
        case 'L':
            is_long = 1;
            pos = nextcharpos;
            t = prepcurchar();
            if (t == 'u' || t == 'U') {
                is_unsigned = 1;
                pos = nextcharpos;
                t = *pos;
            } else if (copts.longlong && (t == 'l' || t == 'L')) {
                is_longlong = 1;
                pos = nextcharpos;
                t = prepcurchar();
                if (t == 'u' || t == 'U') {
                    is_unsigned = 1;
                    pos = nextcharpos;
                    t = *pos;
                }
            }
            break;
        case 'i':
        case 'I':
            if (copts.cpp_extensions && !copts.ANSI_strict && copts.longlong) {
                pos = nextcharpos;
                t = prepcurchar();
                if (t == '6') {
                    pos = nextcharpos;
                    t = prepcurchar();
                    if (t == '4') {
                        is_long = is_longlong = 1;
                        pos = nextcharpos;
                        t = *pos;
                    }
                }
            }
            break;
        case 'b':
        case 'B':
        case 'f':
        case 'F':
            if (InlineAsm_gccmode)
                return IT_INT;
            break;
    }

    if ((t >= 'a' && t <= 'z') || (t >= 'A' && t <= 'Z') || (t == '_') || (t >= '0' && t <= '9'))
        CPrep_Error(105);

    if (!is_longlong && !is_long) {
        if (!is_unsigned && CInt64_IsInRange(tkintconst, stsignedint.size))
            return IT_INT;
        if ((flag || is_unsigned) && CInt64_IsInURange(tkintconst, stunsignedint.size))
            return IT_UINT;
    }
    if (!is_longlong) {
        if (!is_unsigned && CInt64_IsInRange(tkintconst, stsignedlong.size))
            return IT_LONG;
        if (CInt64_IsInURange(tkintconst, stunsignedlong.size))
            return IT_ULONG;
    }
    if (copts.longlong) {
        if (!is_unsigned && CInt64_IsInRange(tkintconst, stsignedlonglong.size))
            return IT_LONGLONG;
        if (CInt64_IsInURange(tkintconst, stsignedlonglong.size))
            return IT_ULONGLONG;
    }
    CError_Error(154);
    tkintconst = cint64_zero;
    return IT_INT;
}

static short floatsuffix(short token) {
    short whichtype;

    switch (token) {
        case 'f':
        case 'F':
            pos = nextcharpos;
            token = *pos;
            whichtype = IT_FLOAT;
            if (stfloat.size != stlongdouble.size)
                tkfloatconst = CMach_CalcFloatConvert((Type *) &stfloat, tkfloatconst);
            break;
        case 'l':
        case 'L':
            pos = nextcharpos;
            token = *pos;
            whichtype = IT_LONGDOUBLE;
            break;
        case 'd':
        case 'D':
            if (!copts.ANSI_strict) {
                pos = nextcharpos;
                token = *pos;
                whichtype = IT_DOUBLE;
                if (stdouble.size != stlongdouble.size)
                    tkfloatconst = CMach_CalcFloatConvert((Type *) &stdouble, tkfloatconst);
            }
            break;
        default:
            if (copts.float_constants) {
                whichtype = IT_FLOAT;
                if (stfloat.size != stlongdouble.size)
                    tkfloatconst = CMach_CalcFloatConvert((Type *) &stfloat, tkfloatconst);
            } else {
                whichtype = IT_DOUBLE;
                if (stdouble.size != stlongdouble.size)
                    tkfloatconst = CMach_CalcFloatConvert((Type *) &stdouble, tkfloatconst);
            }
    }

    if ((token >= 'a' && token <= 'z') || (token >= 'A' && token <= 'Z') || (token == '_') || (token >= '0' && token <= '9')) {
        CPrep_Error(105);
        whichtype = IT_DOUBLE;
    }

    return whichtype;
}

static short tohex(short token) {
    if (token >= '0' && token <= '9')
        return token - '0';
    if (token >= 'a' && token <= 'f')
        return token - 'a' + 10;
    if (token >= 'A' && token <= 'F')
        return token - 'A' + 10;
    return -1;
}

static SInt32 nextchar(char *str) {
    // minor register allocation issues
    SInt32 chr_max;
    short tmpchr;
    SInt32 chr;
    Boolean out_of_bounds;
    short i;
    short t;

    chr_max = 0xFF;
    if (widestring) chr_max = 0xFFFF;

    was_escchar = 0;
    t = prepnextstringchar(str, 1);

    if (t == '\\' && (!copts.multibyteaware || !COS_IsMultiByte(str, pos - 1))) {
        was_escchar = 1;
        t = prepnextstringchar(str, 1);
        switch (t) {
            case 'a': return 7;
            case 'b': return 8;
            case 't': return 9;
            case 'v': return 11;
            case 'f': return 12;
            case 'n':
                if (copts.mpwc_newline)
                    return 13;
                else
                    return 10;
            case 'r':
                if (copts.mpwc_newline)
                    return 10;
                else
                    return 13;
            case 'e': return 27;
            case 'x':
                chr = tohex(prepnextstringchar(str, 1));
                if (chr == -1) {
                    CPrep_Error(100);
                    return ' ';
                }
                out_of_bounds = 0;
                while (tohex(tmpchr = prepcurstringchar(str)) != -1) {
                    chr = (chr << 4) + tohex(tmpchr);
                    prepcurstringchar_skip();
                    if (chr & ~chr_max)
                        out_of_bounds = 1;
                }
                if (out_of_bounds)
                    CError_Error(100);
                break;
            default:
                if (t >= '0' && t <= '7') {
                    out_of_bounds = 0;
                    i = 1;
                    chr = t - '0';
                    while (i < 3 && (tmpchr = prepcurstringchar(str)) >= '0' && tmpchr <= '9') {
                        chr = (chr << 3) + (tmpchr - '0');
                        prepcurstringchar_skip();
                        if (chr & ~chr_max)
                            out_of_bounds = 1;
                        i++;
                    }
                    if (out_of_bounds)
                        CError_Error(100);
                } else {
                    chr = t;
                }
        }
    } else {
        chr = t;
    }

    if (chr_max == 0xFF)
        return (char) (chr & chr_max);
    else
        return (chr & chr_max);
}

char *CPrep_SkipNewComment(char *str) {
}

Boolean skipendoflinematch(char *str, short token) {
}

void skipendofline() {
}

void CPrep_SkipAsmComment() {
    Boolean save_macrocheck = macrocheck;
    skipendofline();
    macrocheck = save_macrocheck;
}

static short tille() { return TK_NEG6; }

static short tcret() {
    newline();
    if (cprep_eoltokens)
        return TK_NEG7;
    else
        return 0;
}

static short tapos() {
    // has register allocation issues
    UInt32 maxvalue;
    int bytecount;
    CInt64 ci58;
    CInt64 ci50;
    CInt64 ci48;
    SInt32 t;
    SInt32 t2;
    int i;
    char *start;

    start = pos;
    t = nextchar(pos);
    if ((t == '\'' || t == 0 || t == '\r') && !was_escchar) {
        CPrep_Error(100);
        tkintconst = cint64_zero;
        tksize = IT_INT;
        return TK_INTCONST;
    }

    t2 = nextchar(start);
    if (t2 == '\'' && !was_escchar) {
        if (widestring) {
            switch (stwchar.size) {
                case 1:
                    CInt64_SetULong(&tkintconst, (UInt8) t);
                    tksize = copts.cplusplus ? IT_SCHAR : IT_INT;
                    break;
                case 2:
                    CInt64_SetULong(&tkintconst, (UInt16) t);
                    tksize = IT_UINT;
                    break;
                case 4:
                    CInt64_SetULong(&tkintconst, (UInt32) t);
                    tksize = IT_UINT;
                    break;
                default:
#line 1386
                    CError_FATAL();
            }
        } else {
            if (copts.unsignedchars)
                t &= 0xFF;
            CInt64_SetLong(&tkintconst, t);
            tksize = copts.cplusplus ? IT_SCHAR : IT_INT;
        }
    } else {
        if (widestring) {
            switch (stwchar.size) {
                case 1:
                    CInt64_SetULong(&ci50, 8);
                    maxvalue = 0xFF;
                    bytecount = 1;
                    break;
                case 2:
                    CInt64_SetULong(&ci50, 16);
                    maxvalue = 0xFFFF;
                    bytecount = 2;
                    break;
                case 4:
                    CInt64_SetULong(&ci50, 32);
                    maxvalue = 0xFFFFFFFF;
                    bytecount = 4;
                    break;
                default:
#line 1424
                    CError_FATAL();
            }
        } else {
            CInt64_SetULong(&ci50, 8);
            maxvalue = 0xFF;
            bytecount = 1;
        }

        CInt64_SetULong(&tkintconst, t & maxvalue);
        i = bytecount;
        do {
            if ((t2 == 0 || t2 == '\r') && !was_escchar) {
                CPrep_Error(100);
                goto after_char_const;
            }
            CInt64_SetULong(&ci48, t2 & maxvalue);
            ci58 = CInt64_Shl(tkintconst, ci50);
            tkintconst = CInt64_Or(ci58, ci48);
            i += bytecount;
            t2 = nextchar(start);
            if (t2 == '\'' && !was_escchar)
                goto after_char_const;
        } while (i < stunsignedlonglong.size);
        CPrep_Error(100);
    after_char_const:
        if (i <= stunsignedint.size && i < stunsignedlong.size)
            tksize = IT_UINT;
        else if (i <= stunsignedlong.size)
            tksize = IT_ULONG;
        else
            tksize = IT_ULONGLONG;
    }

    return TK_INTCONST;
}

static short tquot() {
    // has register allocation issues
    TypeIntegral *type; // r22
    char *string_ptr; // r21
    SInt32 r20; // r20
    char *save_pos; // r19
    short c; // r19
    char *start_pos; // r18
    CInt64 ch64;

    start_pos = pos;
    at_linestart = 0;
    if (cprep_nostring)
        return '"';

    string_ptr = *stringmem;
    ispascalstring = 0;
    r20 = 0;
    if (widestring)
        type = &stwchar;
    else
        type = &stchar;

    if (prepcurstringchar(start_pos) == '\\') {
        save_pos = pos;
        prepcurstringchar_skip();
        if (prepcurstringchar(start_pos) == 'p') {
            prepcurstringchar_skip();
            ispascalstring = 1;
            string_ptr += type->size;
        } else {
            pos = save_pos;
        }
    }

    do {
        c = nextchar(start_pos);
        if ((c == '"' || c == '\r' || c == 0) && !was_escchar) {
            if (c == 0 && (tokenstacklevel > 0 || pos >= prep_file_end))
                CPrep_Error(102);
            else if (c != '"')
                CPrep_Error(101);
            break;
        }
        if ((r20 + type->size) >= maxstringsize) {
            string_ptr = (char *) (string_ptr - *stringmem);
            if (!COS_ResizeHandle(stringmem, maxstringsize += 256))
                CError_NoMem();
            string_ptr = *stringmem + (SInt32) string_ptr;
        }
        if (type->size != 1) {
            CInt64_SetLong(&ch64, c);
            CMach_InitIntMem((Type *) type, ch64, string_ptr);
            string_ptr += type->size;
            r20 += type->size;
        } else {
            *string_ptr = c;
            string_ptr++;
            r20++;
        }
    } while (1);

    if (ispascalstring) {
        if (r20 > 255 && type->size == 1) {
            CPrep_Error(106);
            r20 = 255;
        }
        CInt64_SetLong(&ch64, r20 / type->size);
        CMach_InitIntMem((Type *) type, ch64, *stringmem);
        r20 += type->size;
    } else {
        if ((r20 + 1) >= maxstringsize) {
            if (!COS_ResizeHandle(stringmem, maxstringsize += 256))
                CError_NoMem();
            CMach_InitIntMem((Type *) type, cint64_zero, *stringmem + r20);
            r20 += type->size;
        }
    }

    tksize = r20;
    tkstring = galloc(r20);
    memcpy(tkstring, *stringmem, tksize);
    if (widestring)
        return TK_STRING_WIDE;
    else
        return TK_STRING;
}

static short thash() {
    if (at_linestart) {
        preprocessor();
        return 0;
    }
    return '#';
}

static short tmult() {
    at_linestart = 0;
    if (prepcurchar() == '=') {
        pos = nextcharpos;
        return TK_MULT_ASSIGN;
    }
    return '*';
}

static short tcolo() {
    char t2;
    at_linestart = 0;
    if (copts.cplusplus) {
        t2 = prepcurchar();
        if (t2 == ':') {
            pos = nextcharpos;
            return TK_COLON_COLON;
        }
        if (t2 == '>' && !preprocessing_only) {
            pos = nextcharpos;
            return ']';
        }
    }
    return ':';
}

static short tless() {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_LESS_EQUAL;
    }
    if (t2 == '<') {
        pos = nextcharpos;
        if (prepcurchar() == '=') {
            pos = nextcharpos;
            return TK_SHL_ASSIGN;
        } else {
            return TK_SHL;
        }
    }
    if (t2 == '%' && copts.cplusplus && !preprocessing_only) {
        pos = nextcharpos;
        return '{';
    }
    if (t2 == ':' && copts.cplusplus && !preprocessing_only) {
        pos = nextcharpos;
        return '[';
    }
    return '<';
}

static short tequa(short) {
    at_linestart = 0;
    if (prepcurchar() == '=') {
        pos = nextcharpos;
        return TK_LOGICAL_EQ;
    }
    return '=';
}

static short tgrea(short) {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_GREATER_EQUAL;
    }
    if (t2 == '>') {
        pos = nextcharpos;
        if (prepcurchar() == '=') {
            pos = nextcharpos;
            return TK_SHR_ASSIGN;
        } else {
            return TK_SHR;
        }
    }
    return '>';
}

static short tatsg(short) {
    char *start_pos;
    Boolean save_idarray;

    start_pos = pos;
    at_linestart = 0;

    if (copts.objective_c || in_assembler) {
        save_idarray = cprep_idarray['@'];
        cprep_idarray['@'] = 1;
        pos = ReadIdentifier(start_pos - 1);
        cprep_idarray['@'] = save_idarray;

        if (copts.objective_c) {
            if (!strcmp("@interface", tkidentifier->name))
                return TK_AT_INTERFACE;
            if (!strcmp("@implementation", tkidentifier->name))
                return TK_AT_IMPLEMENTATION;
            if (!strcmp("@protocol", tkidentifier->name))
                return TK_AT_PROTOCOL;
            if (!strcmp("@end", tkidentifier->name))
                return TK_AT_END;
            if (!strcmp("@private", tkidentifier->name))
                return TK_AT_PRIVATE;
            if (!strcmp("@protected", tkidentifier->name))
                return TK_AT_PROTECTED;
            if (!strcmp("@public", tkidentifier->name))
                return TK_AT_PUBLIC;
            if (!strcmp("@class", tkidentifier->name))
                return TK_AT_CLASS;
            if (!strcmp("@selector", tkidentifier->name))
                return TK_AT_SELECTOR;
            if (!strcmp("@encode", tkidentifier->name))
                return TK_AT_ENCODE;
            if (!strcmp("@defs", tkidentifier->name))
                return TK_AT_DEFS;
        }

        if (in_assembler)
            return TK_IDENTIFIER;

        pos = start_pos;
    }

    return '@';
}

static short tperc(short) {
    char t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_MOD_ASSIGN;
    }
    if (t2 == '>' && copts.cplusplus && !preprocessing_only) {
        pos = nextcharpos;
        return '}';
    }
    return '%';
}

static short texcl() {
    at_linestart = 0;
    if (prepcurchar() == '=') {
        pos = nextcharpos;
        return TK_LOGICAL_NE;
    }
    return '!';
}

static short tplus() {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_ADD_ASSIGN;
    }
    if (t2 == '+') {
        pos = nextcharpos;
        return TK_INCREMENT;
    }
    return '+';
}

static short tminu() {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_SUB_ASSIGN;
    }
    if (t2 == '-') {
        pos = nextcharpos;
        return TK_DECREMENT;
    }
    if (t2 == '>') {
        pos = nextcharpos;
        if (copts.cplusplus && prepcurchar() == '*') {
            pos = nextcharpos;
            return TK_ARROW_STAR;
        }
        return TK_ARROW;
    }
    return '-';
}

static short torrr() {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_OR_ASSIGN;
    }
    if (t2 == '|') {
        pos = nextcharpos;
        return TK_LOGICAL_OR;
    }
    return '|';
}

static short tampe() {
    short t2;
    at_linestart = 0;
    t2 = prepcurchar();
    if (t2 == '=') {
        pos = nextcharpos;
        return TK_AND_ASSIGN;
    }
    if (t2 == '&') {
        pos = nextcharpos;
        return TK_LOGICAL_AND;
    }
    return '&';
}

static short tpowe() {
    at_linestart = 0;
    if (prepcurchar() == '=') {
        pos = nextcharpos;
        return TK_XOR_ASSIGN;
    }
    return '^';
}

static short tdivi() {
    at_linestart = 0;
    if (prepcurchar() == '=') {
        pos = nextcharpos;
        return TK_DIV_ASSIGN;
    }
    return '/';
}

static short tzero() {}
static short tpoin() {}
static short tnumb() {}

static short tiden() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    else
        return TK_IDENTIFIER;
}

static short tchrL() {}

static short tchra() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("auto", tkidentifier->name))
        return TK_AUTO;
    if ((copts.cplusplus || !copts.only_std_keywords) && !strcmp("asm", tkidentifier->name))
        return TK_ASM;
    if (copts.cplusplus && !in_assembler && !preprocessing_only) {
        if (!strcmp("and", tkidentifier->name))
            return TK_LOGICAL_AND;
        if (!strcmp("and_eq", tkidentifier->name))
            return TK_AND_ASSIGN;
    }
    return TK_IDENTIFIER;
}

static short tchrb() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("break", tkidentifier->name))
        return TK_BREAK;
    if (copts.cplusplus) {
        if (copts.booltruefalse && !strcmp("bool", tkidentifier->name))
            return TK_BOOL;
        if (!in_assembler && !preprocessing_only) {
            if (!strcmp("bitand", tkidentifier->name))
                return '&';
            if (!strcmp("bitor", tkidentifier->name))
                return '|';
        }
    }
    if (copts.objective_c) {
        if (!strcmp("bycopy", tkidentifier->name))
            return TK_BYCOPY;
        if (!strcmp("byref", tkidentifier->name))
            return TK_BYREF;
    }
    return TK_IDENTIFIER;
}

static short tchrc() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("case", tkidentifier->name))
        return TK_CASE;
    if (!strcmp("char", tkidentifier->name))
        return TK_CHAR;
    if (!strcmp("const", tkidentifier->name))
        return TK_CONST;
    if (!strcmp("continue", tkidentifier->name))
        return TK_CONTINUE;
    if (copts.cplusplus) {
        if (!strcmp("const_cast", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(339);
            return TK_CONST_CAST;
        }
        if (!strcmp("catch", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(339);
            return TK_CATCH;
        }
        if (!strcmp("class", tkidentifier->name))
            return TK_CLASS;
        if (!in_assembler && !preprocessing_only) {
            if (!strcmp("compl", tkidentifier->name))
                return '~';
        }
    }
    return TK_IDENTIFIER;
}

static short tchrd() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("default", tkidentifier->name))
        return TK_DEFAULT;
    if (!strcmp("do", tkidentifier->name))
        return TK_DO;
    if (!strcmp("double", tkidentifier->name))
        return TK_DOUBLE;
    if (copts.cplusplus) {
        if (!strcmp("delete", tkidentifier->name))
            return TK_DELETE;
        if (!strcmp("dynamic_cast", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(339);
            return TK_DYNAMIC_CAST;
        }
    }
    return TK_IDENTIFIER;
}

static short tchre() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("else", tkidentifier->name))
        return TK_ELSE;
    if (!strcmp("enum", tkidentifier->name))
        return TK_ENUM;
    if (!strcmp("extern", tkidentifier->name))
        return TK_EXTERN;
    if (copts.cplusplus) {
        if (!strcmp("explicit", tkidentifier->name))
            return TK_EXPLICIT;
        if (!strcmp("export", tkidentifier->name) && !copts.ecplusplus)
            return TK_EXPORT;
    }
    return TK_IDENTIFIER;
}

static short tchrf() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("float", tkidentifier->name))
        return TK_FLOAT;
    if (!strcmp("for", tkidentifier->name))
        return TK_FOR;
    if (copts.cplusplus) {
        if (!strcmp("friend", tkidentifier->name))
            return TK_FRIEND;
        if (copts.booltruefalse && !strcmp("false", tkidentifier->name))
            return TK_FALSE;
    }
    return TK_IDENTIFIER;
}

static short tchrg() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("goto", tkidentifier->name))
        return TK_GOTO;
    return TK_IDENTIFIER;
}

static short tchri() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("if", tkidentifier->name))
        return TK_IF;
    if (!strcmp("int", tkidentifier->name))
        return TK_INT;
    if (copts.cplusplus || copts.c9x) {
        if (!strcmp("inline", tkidentifier->name))
            return TK_INLINE;
    } else if (!copts.only_std_keywords) {
        if (!strcmp("inline", tkidentifier->name))
            return TK_INLINE;
    }
    if (copts.objective_c) {
        if (!strcmp("in", tkidentifier->name))
            return TK_IN;
        if (!strcmp("inout", tkidentifier->name))
            return TK_INOUT;
    }
    return TK_IDENTIFIER;
}

static short tchrl() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("long", tkidentifier->name))
        return TK_LONG;
    return TK_IDENTIFIER;
}

static short tchrm() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (copts.cplusplus && !strcmp("mutable", tkidentifier->name)) {
        if (copts.ecplusplus)
            CPrep_Warning(339);
        return TK_MUTABLE;
    }
    return TK_IDENTIFIER;
}

static short tchrn(short t) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (copts.cplusplus && t == 'n') {
        if (!strcmp("new", tkidentifier->name))
            return TK_NEW;
        if (!strcmp("namespace", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(339);
            return TK_NAMESPACE;
        }
        if (!in_assembler && !preprocessing_only) {
            if (!strcmp("not", tkidentifier->name))
                return '!';
            if (!strcmp("not_eq", tkidentifier->name))
                return TK_LOGICAL_NE;
        }
    }
    return TK_IDENTIFIER;
}

static short tchro() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (copts.cplusplus) {
        if (!strcmp("operator", tkidentifier->name))
            return TK_OPERATOR;
        if (!in_assembler && !preprocessing_only) {
            if (!strcmp("or", tkidentifier->name))
                return TK_LOGICAL_OR;
            if (!strcmp("or_eq", tkidentifier->name))
                return TK_OR_ASSIGN;
        }
    }
    if (copts.objective_c) {
        if (!strcmp("out", tkidentifier->name))
            return TK_OUT;
        if (!strcmp("oneway", tkidentifier->name))
            return TK_ONEWAY;
    }
    return TK_IDENTIFIER;
}

static short tchrp(short t) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!copts.only_std_keywords && !strcmp("pascal", tkidentifier->name))
        return TK_PASCAL;
    if (copts.cplusplus && t == 'p') {
        if (!strcmp("private", tkidentifier->name))
            return TK_PRIVATE;
        if (!strcmp("protected", tkidentifier->name))
            return TK_PROTECTED;
        if (!strcmp("public", tkidentifier->name))
            return TK_PUBLIC;
    }
    return TK_IDENTIFIER;
}

static short tchrr() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("register", tkidentifier->name))
        return TK_REGISTER;
    if (!strcmp("return", tkidentifier->name))
        return TK_RETURN;
    if (copts.cplusplus && !strcmp("reinterpret_cast", tkidentifier->name)) {
        if (copts.ecplusplus)
            CPrep_Warning(339);
        return TK_REINTERPRET_CAST;
    }
    if (copts.c9x && !strcmp("restrict", tkidentifier->name))
        return TK_RESTRICT;
    return TK_IDENTIFIER;
}

static short tchrs() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("short", tkidentifier->name))
        return TK_SHORT;
    if (!strcmp("signed", tkidentifier->name))
        return TK_SIGNED;
    if (!strcmp("sizeof", tkidentifier->name))
        return TK_SIZEOF;
    if (!strcmp("static", tkidentifier->name))
        return TK_STATIC;
    if (!strcmp("struct", tkidentifier->name))
        return TK_STRUCT;
    if (!strcmp("switch", tkidentifier->name))
        return TK_SWITCH;
    if (copts.cplusplus && !strcmp("static_cast", tkidentifier->name)) {
        if (copts.ecplusplus)
            CPrep_Warning(339);
        return TK_STATIC_CAST;
    }
    return TK_IDENTIFIER;
}

static short tchrt() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("typedef", tkidentifier->name))
        return TK_TYPEDEF;
    if (copts.gcc_extensions && !strcmp("typeof", tkidentifier->name))
        return TK_UU_TYPEOF_UU;
    if (copts.cplusplus) {
        if (!strcmp("this", tkidentifier->name))
            return TK_THIS;
        if (copts.booltruefalse && !strcmp("true", tkidentifier->name))
            return TK_TRUE;
        if (!strcmp("template", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(339);
            return TK_TEMPLATE;
        }
        if (!strcmp("try", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(339);
            return TK_TRY;
        }
        if (!strcmp("throw", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(339);
            return TK_THROW;
        }
        if (!strcmp("typeid", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(339);
            return TK_TYPEID;
        }
        if (!strcmp("typename", tkidentifier->name)) {
            if (copts.ecplusplus)
                CPrep_Warning(339);
            return TK_TYPENAME;
        }
    }
    return TK_IDENTIFIER;
}

static short tchru() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("union", tkidentifier->name))
        return TK_UNION;
    if (!strcmp("unsigned", tkidentifier->name))
        return TK_UNSIGNED;
    if (copts.cplusplus && !strcmp("using", tkidentifier->name)) {
        if (copts.ecplusplus)
            CPrep_Warning(339);
        return TK_USING;
    }
    return TK_IDENTIFIER;
}

static short tchrv() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("void", tkidentifier->name))
        return TK_VOID;
    if (!strcmp("volatile", tkidentifier->name))
        return TK_VOLATILE;
    if (copts.cplusplus && !strcmp("virtual", tkidentifier->name))
        return TK_VIRTUAL;
    return TK_IDENTIFIER;
}

static short tchrw() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;
    if (!strcmp("while", tkidentifier->name))
        return TK_WHILE;
    if (copts.cplusplus && copts.wchar_type && !strcmp("wchar_t", tkidentifier->name))
        return TK_WCHAR_T;
    return TK_IDENTIFIER;
}

static short tchrx(short t) {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;

    if (copts.cplusplus && t == 'x' && !in_assembler && !preprocessing_only) {
        if (!strcmp("xor", tkidentifier->name))
            return '^';
        if (!strcmp("xor_eq", tkidentifier->name))
            return TK_XOR_ASSIGN;
    }

    return TK_IDENTIFIER;
}

static short tchr_() {
    at_linestart = 0;
    pos = ReadIdentifier(pos - 1);
    if (macrocheck && macrotest())
        return 0;

    if (!strcmp("__stdcall", tkidentifier->name))
        return TK_UU_STDCALL;
    if (!strcmp("__cdecl", tkidentifier->name))
        return TK_UU_CDECL;
    if (!strcmp("__fastcall", tkidentifier->name))
        return TK_UU_FASTCALL;
    if (!strcmp("__declspec", tkidentifier->name))
        return TK_UU_DECLSPEC;
    if (!strcmp("__asm", tkidentifier->name))
        return TK_ASM;
    if (!strcmp("__asm__", tkidentifier->name))
        return TK_ASM;
    if (!strcmp("__inline", tkidentifier->name))
        return TK_INLINE;
    if (!strcmp("__inline__", tkidentifier->name))
        return TK_INLINE;
    if (!strcmp("__restrict", tkidentifier->name))
        return TK_RESTRICT;
    if (!strcmp("__restrict__", tkidentifier->name))
        return TK_RESTRICT;
    if (!strcmp("__attribute__", tkidentifier->name))
        return TK_UU_ATTRIBUTE_UU;
    if (!strcmp("__far", tkidentifier->name))
        return TK_UU_FAR;
    if (!strcmp("__alignof__", tkidentifier->name))
        return TK_UU_ALIGNOF_UU;
    if (!strcmp("__typeof__", tkidentifier->name))
        return TK_UU_TYPEOF_UU;
    if (copts.c9x) {
        if (!strcmp("_Bool", tkidentifier->name))
            return TK_BOOL;
        if (!strcmp("_Complex", tkidentifier->name))
            return TK_U_COMPLEX;
        if (!strcmp("_Imaginary", tkidentifier->name))
            return TK_U_IMAGINARY;
    }
    if (copts.altivec_model && !strcmp("__vector", tkidentifier->name))
        return TK_UU_VECTOR;
    return TK_IDENTIFIER;
}

static short token(short t) {
    at_linestart = 0;
    return t;
}

static short tdoll(short t) {
    at_linestart = 0;
    return t;
}

static short tisid() {
    if (!tokenstacklevel)
        CError_Error(105);
    return 0;
}

static short tnull(short t) {
    if (tokenstacklevel > 0 || pos >= prep_file_end) {
        at_linestart = 0;
        return t;
    } else {
        pos[-1] = -64;
        return TK_NEG6;
    }
}

static short t0x1a() {
    if (*pos == 0)
        return 0;
    else
        return TK_NEG6;
}

typedef short (*TokenizePtr)(short);
#define F(func) ((TokenizePtr) (&func))

static TokenizePtr cprep_tokenize[256] = {
        F(tnull), F(tille), F(tille), F(tille), F(tisid), F(tisid), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tcret), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(t0x1a), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(texcl), F(tquot), F(thash), F(tdoll), F(tperc), F(tampe), F(tapos),
        F(token), F(token), F(tmult), F(tplus), F(token), F(tminu), F(tpoin), F(tdivi),
        F(tzero), F(tnumb), F(tnumb), F(tnumb), F(tnumb), F(tnumb), F(tnumb), F(tnumb),
        F(tnumb), F(tnumb), F(tcolo), F(token), F(tless), F(tequa), F(tgrea), F(token),
        F(tatsg), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden),
        F(tiden), F(tiden), F(tiden), F(tiden), F(tchrL), F(tiden), F(tchrn), F(tiden),
        F(tchrp), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden),
        F(tchrx), F(tiden), F(tiden), F(token), F(token), F(token), F(tpowe), F(tchr_),
        F(token), F(tchra), F(tchrb), F(tchrc), F(tchrd), F(tchre), F(tchrf), F(tchrg),
        F(tiden), F(tchri), F(tiden), F(tiden), F(tchrl), F(tchrm), F(tchrn), F(tchro),
        F(tchrp), F(tiden), F(tchrr), F(tchrs), F(tchrt), F(tchru), F(tchrv), F(tchrw),
        F(tchrx), F(tiden), F(tiden), F(token), F(torrr), F(token), F(token), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille)
};

short lookahead() {
    SInt32 state;
    short t;

    CPrep_TokenStreamGetState(&state);
    t = lex();
    CPrep_TokenStreamSetState(&state);

    return t;
}

short lookahead_noeol() {
    SInt32 state;
    short t;

    CPrep_TokenStreamGetState(&state);
    do {
        t = lex();
    } while (t == TK_NEG7);
    CPrep_TokenStreamSetState(&state);

    return t;
}

static void CPrep_StringConCat(Boolean flag) {
}

short lex() {
}

short plex() {
    // does not match - global refs in wrong order
    short t;

    while (1) {
        t = prepskipnextchar();
        pos = nextcharpos;
        if ((t = cprep_tokenize[t](t)))
            return t;

        if (!*pos) {
            if (tokenstacklevel > 0) {
                poptokenseq();
            } else if (filesp > 0) {
                popfile();
            } else {
                break;
            }
        }
    }

    return 0;
}

short lexidentifier() {
    short t;
    char *save_pos;

    t = prepskipnextchar();
    pos = nextcharpos;
    at_linestart = 0;
    if ((t >= 'a' && t <= 'z') || (t >= 'A' && t <= 'Z') || (t == '_')) {
        pos = ReadIdentifier(pos - 1);
        return TK_IDENTIFIER;
    } else if (t == '.') {
        save_pos = pos;
        if (prepnextchar() == '.' && prepnextchar() == '.')
            return TK_ELLIPSIS;
        else
            pos = save_pos;
    }
    return t;
}