textconv.c

#include <ctype.h>
#include <errno.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "hashtable.h"
#include "utf8.h"

#define ARRAY_COUNT(arr) (sizeof(arr) / sizeof(arr[0]))

#define INVALID_CHAR 0xFFFFFFFF

struct CharmapEntry
{
    uint32_t unicode[3];
    int length; // length of the unicode array. TODO: use dynamic memory allocation
    int bytesCount;
    uint8_t bytes[4]; // bytes to convert unicode array to, (e.g. 'A' = 0x0A)
};

static struct HashTable *charmap;

static void fatal_error(const char *msgfmt, ...)
{
    va_list args;

    fputs("error: ", stderr);

    va_start(args, msgfmt);
    vfprintf(stderr, msgfmt, args);
    va_end(args);

    fputc('\n', stderr);

    exit(1);
}

static void parse_error(const char *filename, int lineNum, const char *msgfmt, ...)
{
    va_list args;

    fprintf(stderr, "%s: line %i: ", filename, lineNum);

    va_start(args, msgfmt);
    vfprintf(stderr, msgfmt, args);
    va_end(args);

    fputc('\n', stderr);

    exit(1);
}

// Reads the whole file and returns a null-terminated buffer with its contents
void *read_text_file(const char *filename)
{
    if (strcmp(filename, "-") != 0)
    {
        FILE *file = fopen(filename, "rb");
        uint8_t *buffer;
        size_t size;

        if (file == NULL)
            fatal_error("failed to open file '%s' for reading: %s", filename, strerror(errno));

        // get size
        fseek(file, 0, SEEK_END);
        size = ftell(file);

        // allocate buffer
        buffer = malloc(size + 1);
        if (buffer == NULL)
            fatal_error("could not allocate buffer of size %u", (uint32_t)(size + 1));

        // read file
        fseek(file, 0, SEEK_SET);
        if (fread(buffer, size, 1, file) != 1)
            fatal_error("error reading from file '%s': %s", filename, strerror(errno));

        // null-terminate the buffer
        buffer[size] = 0;

        fclose(file);

        return buffer;
    }
    else
    {
        size_t size = 0;
        size_t capacity = 1024;
        uint8_t *buffer = malloc(capacity + 1);

        if (buffer == NULL)
            fatal_error("could not allocate buffer of size %u", (uint32_t)(capacity + 1));

        for (;;)
        {
            size += fread(buffer + size, 1, capacity - size, stdin);
            if (size == capacity)
            {
                capacity *= 2;
                buffer = realloc(buffer, capacity + 1);
                if (buffer == NULL)
                    fatal_error("could not allocate buffer of size %u", (uint32_t)(capacity + 1));
            }
            else if (feof(stdin))
            {
                break;
            }
            else
            {
                fatal_error("error reading from stdin: %s", strerror(errno));
            }
        }

        // null-terminate the buffer
        buffer[size] = 0;
        return buffer;
    }
}

static char *skip_whitespace(char *str)
{
    while (isspace(*str))
        str++;
    return str;
}

// null terminates the current line and returns a pointer to the next line
static char *line_split(char *str)
{
    while (*str != '\n')
    {
        if (*str == 0)
            return str;  // end of string
        str++;
    }
    *str = 0;  // terminate line
    return str + 1;
}

static char *parse_number(const char *str, unsigned int *num)
{
    char *endptr;
    unsigned int n = strtol(str, &endptr, 0);

    *num = n;
    if (endptr > str)
        return endptr;
    else
        return NULL;
}

static int is_identifier_char(char c)
{
    return isalnum(c) || c == '_';
}

static uint32_t get_escape_char(int c)
{
    const uint8_t escapeTable[] =
    {
        ['0'] = '\0',
        ['a'] = '\a',
        ['b'] = '\b',
        ['f'] = '\f',
        ['n'] = '\n',
        ['r'] = '\r',
        ['t'] = '\t',
        ['v'] = '\v',
        ['\\'] = '\\',
        ['\''] = '\'',
        ['"'] = '"',
    };

    if ((unsigned int)c < ARRAY_COUNT(escapeTable) && (escapeTable[c] != 0 || c == '0'))
        return escapeTable[c];
    else
        return INVALID_CHAR;
}

static void read_charmap(const char *filename)
{
    char *filedata = read_text_file(filename);
    char *line = filedata;
    int lineNum = 1;

    while (line[0] != 0)
    {
        char *nextLine = line_split(line);

        struct CharmapEntry entry;
        struct CharmapEntry *existing;

        line = skip_whitespace(line);
        if (line[0] != 0 && !(line[0] == '/' && line[1] == '/'))  // ignore empty lines and comments
        {
            int len = 0;
            /* Read Character */

            // opening quote
            if (*line != '\'')
                parse_error(filename, lineNum, "expected '");
            line++;

            // perform analysis of charmap entry, we are in the quote
            while(1)
            {
                if(*line == '\'')
                {
                    line++;
                    break;
                }
                else if(len == ARRAY_COUNT(entry.unicode))
                {
                    // TODO: Use dynamic memory allocation so this is unnecessary.
                    parse_error(filename, lineNum, "string limit exceeded");
                }
                else if (*line == '\\')
                {
                    line++; // advance to get the character being escaped
                    if (*line == '\r')
                        line++;
                    if (*line == '\n')
                    {
                        // Backslash at end of line is ignored
                        continue;
                    }
                    entry.unicode[len] = get_escape_char(*line);
                    if (entry.unicode[len] == INVALID_CHAR)
                        parse_error(filename, lineNum, "unknown escape sequence \\%c", *line);
                    line++; // increment again to get past the escape sequence.
                }
                else
                {
                    line = utf8_decode(line, &entry.unicode[len]);
                    if (line == NULL)
                        parse_error(filename, lineNum, "invalid UTF8");
                }
                len++;
            }
            entry.length = len;

            // equals sign
            line = skip_whitespace(line);
            if (*line != '=')
                parse_error(filename, lineNum, "expected = after character \\%c", *line);
            line++;

            entry.bytesCount = 0;

            // value
            while (1)
            {
                uint32_t value;

                if (entry.bytesCount >= 4)
                    parse_error(filename, lineNum, "more than 4 values specified");

                line = skip_whitespace(line);

                line = parse_number(line, &value);
                if (line == NULL)
                    parse_error(filename, lineNum, "expected number after =");
                if (value > 0xFF)
                    parse_error(filename, lineNum, "0x%X is larger than 1 byte", value);

                entry.bytes[entry.bytesCount] = value;
                entry.bytesCount++;

                line = skip_whitespace(line);
                if (*line == 0)
                    break;
                if (*line != ',')
                    parse_error(filename, lineNum, "junk at end of line");
                line++;
            }

            existing = hashtable_query(charmap, &entry);

            if (existing != NULL) {
                const char *fmt = "0x%02X, ";
                int fmtlen = 6;

                char str[32];
                int i;

                for (i = 0; i < existing->bytesCount; i++) {
                    sprintf(&str[fmtlen * i], fmt, existing->bytes[i]);
                }

                str[fmtlen * i - 2] = '\0';

                parse_error(filename, lineNum, "entry for character already exists (%s)", str);
            } else {
                hashtable_insert(charmap, &entry);
            }
        }

        line = nextLine;
        lineNum++;
    }

    free(filedata);
}

static int count_line_num(const char *start, const char *pos)
{
    const char *c;
    int lineNum = 1;

    for (c = start; c < pos; c++)
    {
        if (*c == '\n')
            lineNum++;
    }
    return lineNum;
}

static char *convert_string(char *pos, FILE *fout, const char *inputFileName, char *start, int uncompressed, int cnOneByte)
{
    const struct CharmapEntry terminatorInput = {.unicode = {'\0'}, .length = 1};
    struct CharmapEntry *terminator;
    int hasString = 0;
    int i;

    while (1)
    {
        pos = skip_whitespace(pos);
        if (*pos == ')')
        {
            if (hasString)
                break;
            else
                parse_error(inputFileName, count_line_num(start, pos), "expected quoted string after '_('");
        }
        else if (*pos != '"')
            parse_error(inputFileName, count_line_num(start, pos), "unexpected character '%c'", *pos);
        pos++;

        hasString = 1;

        // convert quoted string
        while (*pos != '"')
        {
            struct CharmapEntry input;
            struct CharmapEntry *last_valid_entry = NULL;
            struct CharmapEntry *entry;
            uint32_t c;
            int length = 0;
            char* last_valid_pos = NULL;
            // safely erase the unicode area before use
            memset(input.unicode, 0, sizeof (input.unicode));
            input.length = 0;

            // Find a charmap entry of longest length possible starting from this position
            while (*pos != '"')
            {
                if ((uncompressed && length == 1) || length == ARRAY_COUNT(entry->unicode))
                {
                    // Stop searching after length 3; we only support strings of lengths up
                    // to that right now. Unless uncompressed is set, in which we ignore multi
                    // texts by discarding entries longer than 1.
                    break;
                }

                if (*pos == 0)
                    parse_error(inputFileName, count_line_num(start, pos), "EOF in string literal");
                if (*pos == '\\')
                {
                    pos++;
                    c = get_escape_char(*pos);
                    if (c == INVALID_CHAR)
                        parse_error(inputFileName, count_line_num(start, pos), "unknown escape sequence \\%c", *pos);
                    input.unicode[length] = c;
                    pos++;
                }
                else
                {
                    pos = utf8_decode(pos, &input.unicode[length]);
                    if (pos == NULL)
                        parse_error(inputFileName, count_line_num(start, pos), "invalid unicode encountered in file");
                }
                length++;
                input.length = length;

                entry = hashtable_query(charmap, &input);
                if (entry != NULL)
                {
                    last_valid_entry = entry;
                    last_valid_pos = pos;
                }
            }

            entry = last_valid_entry;
            pos = last_valid_pos;
            if (entry == NULL)
                parse_error(inputFileName, count_line_num(start, pos), "no charmap entry for U+%X", input.unicode[0]);

            for (i = 0; i < entry->bytesCount; i++) {
                if (entry->bytesCount > 1 && cnOneByte && i % 2 == 0) {
                    continue;
                }
                fprintf(fout, "0x%02X,", entry->bytes[i]);
            }
        }
        pos++;  // skip over closing '"'
    }
    pos++;  // skip over closing ')'
    // use terminator \0 from charmap if provided, otherwise default 0xFF
    terminator = hashtable_query(charmap, &terminatorInput);
    if (terminator == NULL)
        fputs("0xFF", fout);
    else
    {
        for (i = 0; i < (cnOneByte ? 1 : terminator->bytesCount); i++)
            fprintf(fout, "0x%02X,", terminator->bytes[i]);
    }
    return pos;
}

static void convert_file(const char *infilename, const char *outfilename)
{
    char *in = read_text_file(infilename);
    FILE *fout = strcmp(outfilename, "-") != 0 ? fopen(outfilename, "wb") : stdout;

    if (fout == NULL)
        fatal_error("failed to open file '%s' for writing: %s", strerror(errno));

    char *start = in;
    char *end = in;
    char *pos = in;

    while (1)
    {
        if (*pos == 0)  // end of file
            goto eof;

        // check for comment
        if (*pos == '/')
        {
            pos++;
            // skip over // comment
            if (*pos == '/')
            {
                pos++;
                // skip over next newline
                while (*pos != '\n')
                {
                    if (*pos == 0)
                        goto eof;
                    pos++;
                }
                pos++;
            }
            // skip over /* */ comment
            else if (*pos == '*')
            {
                pos++;
                while (*pos != '*' && pos[1] != '/')
                {
                    if (*pos == 0)
                        goto eof;
                    pos++;
                }
                pos += 2;
            }
        }
        // skip over normal string literal
        else if (*pos == '"')
        {
            pos++;
            while (*pos != '"')
            {
                if (*pos == 0)
                    goto eof;
                if (*pos == '\\')
                    pos++;
                pos++;
            }
            pos++;
        }
        // check for _( sequence
        else if ((*pos == '_') && (pos == in || !is_identifier_char(pos[-1])))
        {
            int uncompressed = 0;
            int cnOneByte = 0;
            end = pos;
            pos++;
            if (*pos == '_') // an extra _ signifies uncompressed strings. Enable uncompressed flag
            {
                pos++;
                uncompressed = 1;
            }
            if (*pos == '%') // an extra % signifies a one-byte long characters on iQue instead of two-byte
            {
                pos++;
                cnOneByte = 1;
            }
            if (*pos == '(')
            {
                pos++;
                fwrite(start, end - start, 1, fout);
                pos = convert_string(pos, fout, infilename, in, uncompressed, cnOneByte);
                start = pos;
            }
        }
        else
        {
            pos++;
        }
    }

  eof:
    fwrite(start, pos - start, 1, fout);
    if (strcmp(outfilename, "-") != 0)
        fclose(fout);
    free(in);
}

static unsigned int charmap_hash(const void *value)
{
    const struct CharmapEntry* entry = value;
    unsigned int ret = 0;
    for (int i = 0; i < entry->length; i++)
        ret = ret * 17 + entry->unicode[i];
    return ret;
}

static int charmap_cmp(const void *a, const void *b)
{
    const struct CharmapEntry *ea = a;
    const struct CharmapEntry *eb = b;
    if (ea->length != eb->length)
        return 0;
    for(int i = 0; i < ea->length; i++)
        if(ea->unicode[i] != eb->unicode[i])
            return 0;
    return 1;
}

static void usage(const char *execName)
{
    fprintf(stderr, "Usage: %s CHARMAP INPUT OUTPUT\n", execName);
}

int main(int argc, char **argv)
{
    if (argc != 4)
    {
        usage(argv[0]);
        return 1;
    }

    charmap = hashtable_new(charmap_hash, charmap_cmp, 256, sizeof(struct CharmapEntry));

    read_charmap(argv[1]);
    convert_file(argv[2], argv[3]);

    hashtable_free(charmap);

    return 0;
}