Evaluate C define expressions using postfix evaluation, and rename 'asm' to 'parseutil'

asm.cpp

@@ -1,59 +0,0 @@
-#include "asm.h"
-
-Asm::Asm()
-{
-}
-
-void Asm::strip_comment(QString *line) {
-    bool in_string = false;
-    for (int i = 0; i < line->length(); i++) {
-        if (line->at(i) == '"') {
-            in_string = !in_string;
-        } else if (line->at(i) == '@') {
-            if (!in_string) {
-                line->truncate(i);
-                break;
-            }
-        }
-    }
-}
-
-QList<QStringList>* Asm::parse(QString text) {
-    QList<QStringList> *parsed = new QList<QStringList>;
-    QStringList lines = text.split('\n');
-    for (QString line : lines) {
-        QString label;
-        //QString macro;
-        //QStringList *params;
-        strip_comment(&line);
-        if (line.trimmed().isEmpty()) {
-        } else if (line.contains(':')) {
-            label = line.left(line.indexOf(':'));
-            QStringList *list = new QStringList;
-            list->append(".label"); // This is not a real keyword. It's used only to make the output more regular.
-            list->append(label);
-            parsed->append(*list);
-            // There should not be anything else on the line.
-            // gas will raise a syntax error if there is.
-        } else {
-            line = line.trimmed();
-            //parsed->append(line.split(QRegExp("\\s*,\\s*")));
-            QString macro;
-            QStringList params;
-            int index = line.indexOf(QRegExp("\\s+"));
-            macro = line.left(index);
-            params = line.right(line.length() - index).trimmed().split(QRegExp("\\s*,\\s*"));
-            params.prepend(macro);
-            parsed->append(params);
-        }
-        //if (macro != NULL) {
-        //    if (macros->contains(macro)) {
-        //        void* function = macros->value(macro);
-        //        if (function != NULL) {
-        //            std::function function(params);
-        //        }
-        //    }
-        //}
-    }
-    return parsed;
-}

asm.h

@@ -1,15 +0,0 @@
-#ifndef ASM_H
-#define ASM_H
-
-#include <QString>
-#include <QList>
-
-class Asm
-{
-public:
-    Asm();
-    void strip_comment(QString*);
-    QList<QStringList>* parse(QString);
-};
-
-#endif // ASM_H

parseutil.cpp

@@ -0,0 +1,199 @@
+#include "parseutil.h"
+
+#include <QDebug>
+#include <QRegularExpression>
+#include <QStack>
+
+ParseUtil::ParseUtil()
+{
+}
+
+void ParseUtil::strip_comment(QString *line) {
+    bool in_string = false;
+    for (int i = 0; i < line->length(); i++) {
+        if (line->at(i) == '"') {
+            in_string = !in_string;
+        } else if (line->at(i) == '@') {
+            if (!in_string) {
+                line->truncate(i);
+                break;
+            }
+        }
+    }
+}
+
+QList<QStringList>* ParseUtil::parseAsm(QString text) {
+    QList<QStringList> *parsed = new QList<QStringList>;
+    QStringList lines = text.split('\n');
+    for (QString line : lines) {
+        QString label;
+        //QString macro;
+        //QStringList *params;
+        strip_comment(&line);
+        if (line.trimmed().isEmpty()) {
+        } else if (line.contains(':')) {
+            label = line.left(line.indexOf(':'));
+            QStringList *list = new QStringList;
+            list->append(".label"); // This is not a real keyword. It's used only to make the output more regular.
+            list->append(label);
+            parsed->append(*list);
+            // There should not be anything else on the line.
+            // gas will raise a syntax error if there is.
+        } else {
+            line = line.trimmed();
+            //parsed->append(line.split(QRegExp("\\s*,\\s*")));
+            QString macro;
+            QStringList params;
+            int index = line.indexOf(QRegExp("\\s+"));
+            macro = line.left(index);
+            params = line.right(line.length() - index).trimmed().split(QRegExp("\\s*,\\s*"));
+            params.prepend(macro);
+            parsed->append(params);
+        }
+        //if (macro != NULL) {
+        //    if (macros->contains(macro)) {
+        //        void* function = macros->value(macro);
+        //        if (function != NULL) {
+        //            std::function function(params);
+        //        }
+        //    }
+        //}
+    }
+    return parsed;
+}
+
+int ParseUtil::evaluateDefine(QString define, QMap<QString, int>* knownDefines) {
+    QList<Token> tokens = tokenizeExpression(define, knownDefines);
+    QList<Token> postfixExpression = generatePostfix(tokens);
+    return evaluatePostfix(postfixExpression);
+}
+
+QList<Token> ParseUtil::tokenizeExpression(QString expression, QMap<QString, int>* knownIdentifiers) {
+    QList<Token> tokens;
+
+    QStringList tokenTypes = (QStringList() << "hex" << "decimal" << "identifier" << "operator" << "leftparen" << "rightparen");
+    QRegularExpression re("^(?<hex>0x[0-9a-fA-F]+)|(?<decimal>[0-9]+)|(?<identifier>[a-zA-Z_0-9]+)|(?<operator>[+\\-*\\/<>|^%]+)|(?<leftparen>\\()|(?<rightparen>\\))");
+
+    expression = expression.trimmed();
+    while (!expression.isEmpty()) {
+        QRegularExpressionMatch match = re.match(expression);
+        if (!match.hasMatch()) {
+            qDebug() << "Failed to tokenize expression: " << expression;
+            break;
+        }
+        for (QString tokenType : tokenTypes) {
+            QString token = match.captured(tokenType);
+            if (!token.isEmpty()) {
+                if (tokenType == "identifier") {
+                    if (knownIdentifiers->contains(token)) {
+                        QString actualToken = QString("%1").arg(knownIdentifiers->value(token));
+                        expression = expression.replace(0, token.length(), actualToken);
+                        token = actualToken;
+                        tokenType = "decimal";
+                    } else {
+                        qDebug() << "Unknown identifier found in expression: " << token;
+                    }
+                }
+
+                tokens.append(Token(token, tokenType));
+                expression = expression.remove(0, token.length()).trimmed();
+                break;
+            }
+        }
+    }
+    return tokens;
+}
+
+QMap<QString, int> Token::precedenceMap = QMap<QString, int>(
+{
+    {"*", 3},
+    {"/", 3},
+    {"+", 4},
+    {"-", 4},
+    {"<<", 5},
+    {">>", 5},
+    {"&", 8},
+    {"^", 9},
+    {"|", 10}
+});
+
+// Shunting-yard algorithm for generating postfix notation.
+// https://en.wikipedia.org/wiki/Shunting-yard_algorithm
+QList<Token> ParseUtil::generatePostfix(QList<Token> tokens) {
+    QList<Token> output;
+    QStack<Token> operatorStack;
+    for (Token token : tokens) {
+        if (token.type == TokenType::Number) {
+            output.append(token);
+        } else if (token.value == "(") {
+            operatorStack.push(token);
+        } else if (token.value == ")") {
+            while (!operatorStack.empty() && operatorStack.top().value != "(") {
+                output.append(operatorStack.pop());
+            }
+            if (!operatorStack.empty()) {
+                // pop the left parenthesis token
+                operatorStack.pop();
+            } else {
+                qDebug() << "Mismatched parentheses detected in expression!";
+            }
+        } else {
+            // token is an operator
+            while (!operatorStack.isEmpty()
+                   && operatorStack.top().operatorPrecedence <= token.operatorPrecedence
+                   && operatorStack.top().value != "(") {
+                output.append(operatorStack.pop());
+            }
+            operatorStack.push(token);
+        }
+    }
+
+    while (!operatorStack.isEmpty()) {
+        if (operatorStack.top().value == "(" || operatorStack.top().value == ")") {
+            qDebug() << "Mismatched parentheses detected in expression!";
+        } else {
+            output.append(operatorStack.pop());
+        }
+    }
+
+    return output;
+}
+
+// Evaluate postfix expression.
+// https://en.wikipedia.org/wiki/Reverse_Polish_notation#Postfix_evaluation_algorithm
+int ParseUtil::evaluatePostfix(QList<Token> postfix) {
+    QStack<Token> stack;
+    for (Token token : postfix) {
+        if (token.type == TokenType::Operator) {
+            int op2 = stack.pop().value.toInt(nullptr, 0);
+            int op1 = stack.pop().value.toInt(nullptr, 0);
+            int result = 0;
+            if (token.value == "*") {
+                result = op1 * op2;
+            } else if (token.value == "/") {
+                result = op1 / op2;
+            } else if (token.value == "+") {
+                result = op1 + op2;
+            } else if (token.value == "-") {
+                result = op1 - op2;
+            } else if (token.value == "<<") {
+                result = op1 << op2;
+            } else if (token.value == ">>") {
+                result = op1 >> op2;
+            } else if (token.value == "&") {
+                result = op1 & op2;
+            } else if (token.value == "^") {
+                result = op1 ^ op2;
+            } else if (token.value == "|") {
+                result = op1 | op2;
+            } else {
+                qDebug() << "Unsupported postfix operator: " << token.value;
+            }
+            stack.push(Token(QString("%1").arg(result), "decimal"));
+        } else {
+            stack.push(token);
+        }
+    }
+
+    return stack.pop().value.toInt(nullptr, 0);
+}

parseutil.h

@@ -0,0 +1,43 @@
+#ifndef PARSEUTIL_H
+#define PARSEUTIL_H
+
+#include <QString>
+#include <QList>
+
+enum TokenType {
+    Number,
+    Operator,
+};
+
+class Token {
+public:
+    Token(QString value = "", QString type = "") {
+        this->value = value;
+        this->type = TokenType::Operator;
+        if (type == "decimal" || type == "hex") {
+            this->type = TokenType::Number;
+            this->operatorPrecedence = -1;
+        } else if (type == "operator") {
+            this->operatorPrecedence = precedenceMap[value];
+        }
+    }
+    static QMap<QString, int> precedenceMap;
+    QString value;
+    TokenType type;
+    int operatorPrecedence; // only relevant for operator tokens
+};
+
+class ParseUtil
+{
+public:
+    ParseUtil();
+    void strip_comment(QString*);
+    QList<QStringList>* parseAsm(QString);
+    int evaluateDefine(QString, QMap<QString, int>*);
+private:
+    QList<Token> tokenizeExpression(QString expression, QMap<QString, int>* knownIdentifiers);
+    QList<Token> generatePostfix(QList<Token> tokens);
+    int evaluatePostfix(QList<Token> postfix);
+};
+
+#endif // PARSEUTIL_H

pretmap.pro

@@ -15,7 +15,6 @@ TEMPLATE = app
 SOURCES += main.cpp\
         mainwindow.cpp \
     project.cpp \
-    asm.cpp \
     map.cpp \
     blockdata.cpp \
     block.cpp \
@@ -25,11 +24,11 @@ SOURCES += main.cpp\
     event.cpp \
     editor.cpp \
     objectpropertiesframe.cpp \
-    graphicsview.cpp
+    graphicsview.cpp \
+    parseutil.cpp
 
 HEADERS  += mainwindow.h \
     project.h \
-    asm.h \
     map.h \
     blockdata.h \
     block.h \
@@ -39,7 +38,8 @@ HEADERS  += mainwindow.h \
     event.h \
     editor.h \
     objectpropertiesframe.h \
-    graphicsview.h
+    graphicsview.h \
+    parseutil.h
 
 FORMS    += mainwindow.ui \
     objectpropertiesframe.ui

project.cpp

@@ -1,4 +1,4 @@
-#include "asm.h"
+#include "parseutil.h"
 #include "project.h"
 #include "tile.h"
 #include "tileset.h"
@@ -71,7 +71,7 @@ void Project::loadMapConnections(Map *map) {
         QString path = root + QString("/data/maps/%1/connections.inc").arg(map->name);
         QString text = readTextFile(path);
         if (!text.isNull()) {
-            QList<QStringList> *commands = parse(text);
+            QList<QStringList> *commands = parseAsm(text);
             QStringList *list = getLabelValues(commands, map->connections_label);
 
             //// Avoid using this value. It ought to be generated instead.
@@ -150,13 +150,13 @@ void Project::readMapHeader(Map* map) {
     }
 
     QString label = map->name;
-    Asm *parser = new Asm;
+    ParseUtil *parser = new ParseUtil;
 
     QString header_text = readTextFile(root + "/data/maps/" + label + "/header.inc");
     if (header_text.isNull()) {
         return;
     }
-    QStringList *header = getLabelValues(parser->parse(header_text), label);
+    QStringList *header = getLabelValues(parser->parseAsm(header_text), label);
     map->attributes_label = header->value(0);
     map->events_label = header->value(1);
     map->scripts_label = header->value(2);
@@ -212,7 +212,7 @@ void Project::saveMapHeader(Map *map) {
 void Project::readMapAttributesTable() {
     int curMapIndex = 1;
     QString attributesText = readTextFile(getMapAttributesTableFilepath());
-    QList<QStringList>* values = parse(attributesText);
+    QList<QStringList>* values = parseAsm(attributesText);
     bool inAttributePointers = false;
     for (int i = 0; i < values->length(); i++) {
         QStringList params = values->value(i);
@@ -265,13 +265,13 @@ void Project::readMapAttributes(Map* map) {
         return;
     }
 
-    Asm *parser = new Asm;
+    ParseUtil *parser = new ParseUtil;
 
     QString assets_text = readTextFile(getMapAssetsFilepath());
     if (assets_text.isNull()) {
         return;
     }
-    QStringList *attributes = getLabelValues(parser->parse(assets_text), map->attributes_label);
+    QStringList *attributes = getLabelValues(parser->parseAsm(assets_text), map->attributes_label);
     map->width = attributes->value(0);
     map->height = attributes->value(1);
     map->border_label = attributes->value(2);
@@ -283,13 +283,13 @@ void Project::readMapAttributes(Map* map) {
 void Project::readAllMapAttributes() {
     mapAttributes.clear();
 
-    Asm *parser = new Asm;
+    ParseUtil *parser = new ParseUtil;
     QString assets_text = readTextFile(getMapAssetsFilepath());
     if (assets_text.isNull()) {
         return;
     }
 
-    QList<QStringList> *commands = parser->parse(assets_text);
+    QList<QStringList> *commands = parser->parseAsm(assets_text);
 
     // Assume the _assets.inc file is grouped consistently in the order of:
     // 1. <map_name>_MapBorder
@@ -539,10 +539,10 @@ void Project::getTilesets(Map* map) {
 }
 
 Tileset* Project::loadTileset(QString label) {
-    Asm *parser = new Asm;
+    ParseUtil *parser = new ParseUtil;
 
     QString headers_text = readTextFile(root + "/data/tilesets/headers.inc");
-    QStringList *values = getLabelValues(parser->parse(headers_text), label);
+    QStringList *values = getLabelValues(parser->parseAsm(headers_text), label);
     Tileset *tileset = new Tileset;
     tileset->name = label;
     tileset->is_compressed = values->value(0);
@@ -562,7 +562,7 @@ Tileset* Project::loadTileset(QString label) {
 
 QString Project::getBlockdataPath(Map* map) {
     QString text = readTextFile(getMapAssetsFilepath());
-    QStringList *values = getLabelValues(parse(text), map->blockdata_label);
+    QStringList *values = getLabelValues(parseAsm(text), map->blockdata_label);
     QString path;
     if (!values->isEmpty()) {
         path = root + "/" + values->value(0).section('"', 1, 1);
@@ -574,7 +574,7 @@ QString Project::getBlockdataPath(Map* map) {
 
 QString Project::getMapBorderPath(Map *map) {
     QString text = readTextFile(getMapAssetsFilepath());
-    QStringList *values = getLabelValues(parse(text), map->border_label);
+    QStringList *values = getLabelValues(parseAsm(text), map->border_label);
     QString path;
     if (!values->isEmpty()) {
         path = root + "/" + values->value(0).section('"', 1, 1);
@@ -705,7 +705,7 @@ void Project::saveAllDataStructures() {
 }
 
 void Project::loadTilesetAssets(Tileset* tileset) {
-    Asm* parser = new Asm;
+    ParseUtil* parser = new ParseUtil;
     QString category = (tileset->is_secondary == "TRUE") ? "secondary" : "primary";
     if (tileset->name.isNull()) {
         return;
@@ -713,7 +713,7 @@ void Project::loadTilesetAssets(Tileset* tileset) {
     QString dir_path = root + "/data/tilesets/" + category + "/" + tileset->name.replace("gTileset_", "").toLower();
 
     QString graphics_text = readTextFile(root + "/data/tilesets/graphics.inc");
-    QList<QStringList> *graphics = parser->parse(graphics_text);
+    QList<QStringList> *graphics = parser->parseAsm(graphics_text);
     QStringList *tiles_values = getLabelValues(graphics, tileset->tiles_label);
     QStringList *palettes_values = getLabelValues(graphics, tileset->palettes_label);
 
@@ -743,7 +743,7 @@ void Project::loadTilesetAssets(Tileset* tileset) {
     QString metatiles_path;
     QString metatile_attrs_path;
     QString metatiles_text = readTextFile(root + "/data/tilesets/metatiles.inc");
-    QList<QStringList> *metatiles_macros = parser->parse(metatiles_text);
+    QList<QStringList> *metatiles_macros = parser->parseAsm(metatiles_text);
     QStringList *metatiles_values = getLabelValues(metatiles_macros, tileset->metatiles_label);
     if (!metatiles_values->isEmpty()) {
         metatiles_path = root + "/" + metatiles_values->value(0).section('"', 1, 1);
@@ -918,8 +918,8 @@ void Project::readMapGroups() {
     if (text.isNull()) {
         return;
     }
-    Asm *parser = new Asm;
-    QList<QStringList> *commands = parser->parse(text);
+    ParseUtil *parser = new ParseUtil;
+    QList<QStringList> *commands = parser->parseAsm(text);
 
     bool in_group_pointers = false;
     QStringList *groups = new QStringList;
@@ -1015,9 +1015,9 @@ QString Project::getNewMapName() {
     return newMapName;
 }
 
-QList<QStringList>* Project::parse(QString text) {
-    Asm *parser = new Asm;
-    return parser->parse(text);
+QList<QStringList>* Project::parseAsm(QString text) {
+    ParseUtil *parser = new ParseUtil;
+    return parser->parseAsm(text);
 }
 
 QStringList Project::getLocations() {
@@ -1074,7 +1074,7 @@ void Project::readItemNames() {
 void Project::readFlagNames() {
     QString filepath = root + "/include/constants/flags.h";
     QStringList prefixes = (QStringList() << "FLAG_");
-    readCDefinesSorted(filepath, prefixes, flagNames, "SYSTEM_FLAGS", 0x800);
+    readCDefinesSorted(filepath, prefixes, flagNames);
 }
 
 void Project::readVarNames() {
@@ -1084,13 +1084,9 @@ void Project::readVarNames() {
 }
 
 void Project::readCDefinesSorted(QString filepath, QStringList prefixes, QStringList* definesToSet) {
-    return readCDefinesSorted(filepath, prefixes, definesToSet, "", 0);
-}
-
-void Project::readCDefinesSorted(QString filepath, QStringList prefixes, QStringList* definesToSet, QString hardcodedDefine, int hardcodedDefineValue) {
     QString text = readTextFile(filepath);
     if (!text.isNull()) {
-        QMap<QString, int> defines = readCDefines(text, prefixes, hardcodedDefine, hardcodedDefineValue);
+        QMap<QString, int> defines = readCDefines(text, prefixes);
 
         // The defines should to be sorted by their underlying value, not alphabetically.
         // Reverse the map and read out the resulting keys in order.
@@ -1318,13 +1314,13 @@ void Project::readMapEvents(Map *map) {
         return;
     }
 
-    QStringList *labels = getLabelValues(parse(text), map->events_label);
+    QStringList *labels = getLabelValues(parseAsm(text), map->events_label);
     map->object_events_label = labels->value(0);
     map->warps_label = labels->value(1);
     map->coord_events_label = labels->value(2);
     map->bg_events_label = labels->value(3);
 
-    QList<QStringList> *object_events = getLabelMacros(parse(text), map->object_events_label);
+    QList<QStringList> *object_events = getLabelMacros(parseAsm(text), map->object_events_label);
     map->events["object"].clear();
     for (QStringList command : *object_events) {
         if (command.value(0) == "object_event") {
@@ -1368,7 +1364,7 @@ void Project::readMapEvents(Map *map) {
         }
     }
 
-    QList<QStringList> *warps = getLabelMacros(parse(text), map->warps_label);
+    QList<QStringList> *warps = getLabelMacros(parseAsm(text), map->warps_label);
     map->events["warp"].clear();
     for (QStringList command : *warps) {
         if (command.value(0) == "warp_def") {
@@ -1392,7 +1388,7 @@ void Project::readMapEvents(Map *map) {
         }
     }
 
-    QList<QStringList> *coords = getLabelMacros(parse(text), map->coord_events_label);
+    QList<QStringList> *coords = getLabelMacros(parseAsm(text), map->coord_events_label);
     map->events["trap"].clear();
     map->events["trap_weather"].clear();
     for (QStringList command : *coords) {
@@ -1430,7 +1426,7 @@ void Project::readMapEvents(Map *map) {
         }
     }
 
-    QList<QStringList> *bgs = getLabelMacros(parse(text), map->bg_events_label);
+    QList<QStringList> *bgs = getLabelMacros(parseAsm(text), map->bg_events_label);
     map->events["sign"].clear();
     map->events["event_hidden_item"].clear();
     map->events["event_secret_base"].clear();
@@ -1535,41 +1531,23 @@ QString Project::readCIncbin(QString text, QString label) {
 }
 
 QMap<QString, int> Project::readCDefines(QString text, QStringList prefixes) {
-    return readCDefines(text, prefixes, "", 0);
-}
-
-QMap<QString, int> Project::readCDefines(QString text, QStringList prefixes, QString hardcodedDefine, int hardcodedDefineValue) {
-    QMap<QString, int> defines;
-
-    QString combinedPrefixes = "(" + prefixes.join('|') + ")";
-    QString regex;
-    if (hardcodedDefine.isEmpty()) {
-        regex = QString("#define\\s+(?<defineName>(%1)\\w+)\\s+(?<defineValue>\\w+)").arg(combinedPrefixes);
-    } else {
-        regex = QString("#define\\s+(?<defineName>(%1)\\w+)\\s+\\(*(?<hardcodedDefineName>\\\s*%2\\s+\\+\\s+)*(?<defineValue>\\w+\)\\)*").arg(combinedPrefixes, hardcodedDefine);
-    }
-
-    QRegularExpression re(regex);
+    ParseUtil parser;
+    QMap<QString, int> allDefines;
+    QMap<QString, int> filteredDefines;
+    QRegularExpression re("#define\\s+(?<defineName>\\w+)[^\\S\\n]+(?<defineValue>.+)");
     QRegularExpressionMatchIterator iter = re.globalMatch(text);
     while (iter.hasNext()) {
         QRegularExpressionMatch match = iter.next();
         QString name = match.captured("defineName");
-        QString hardcodedDefineName = match.captured("hardcodedDefineName");
-        QString value = match.captured("defineValue");
-        bool valid;
-        int parsedValue = value.startsWith("0x") ? value.toInt(&valid, 16) : value.toInt(&valid, 10);
-        if (valid) {
-            int actualValue = parsedValue;
-            if (!hardcodedDefine.isEmpty() && !hardcodedDefineName.isEmpty()) {
-                actualValue += hardcodedDefineValue;
+        QString expression = match.captured("defineValue");
+        expression.replace(QRegularExpression("//.*"), "");
+        int value = parser.evaluateDefine(expression, &allDefines);
+        allDefines.insert(name, value);
+        for (QString prefix : prefixes) {
+            if (name.startsWith(prefix)) {
+                filteredDefines.insert(name, value);
             }
-            defines.insert(name, actualValue);
-        } else if (defines.contains(value)) {
-            defines.insert(name, defines.value(value));
-        } else {
-            qDebug() << QString("Failed to parse define '%1' value '%2' as base 10 or hexadecimal value").arg(name, value);
         }
     }
-
-    return defines;
+    return filteredDefines;
 }

project.h

@@ -67,7 +67,7 @@ public:
     void saveMapGroupsTable();
     void saveMapConstantsHeader();
 
-    QList<QStringList>* parse(QString text);
+    QList<QStringList>* parseAsm(QString text);
     QStringList getSongNames();
     QStringList getLocations();
     QStringList getVisibilities();
@@ -93,7 +93,6 @@ public:
     QStringList readCArray(QString text, QString label);
     QString readCIncbin(QString text, QString label);
     QMap<QString, int> readCDefines(QString text, QStringList prefixes);
-    QMap<QString, int> readCDefines(QString text, QStringList prefixes, QString hardcodedDefine, int hardcodedDefineValue);
 private:
     QString getMapAttributesTableFilepath();
     QString getMapAssetsFilepath();