From 09f3d4436729cecb81a8e8b1e49009415282e6a2 Mon Sep 17 00:00:00 2001 From: Gabe Ambrosio Date: Sat, 21 Oct 2023 10:30:57 -0500 Subject: [PATCH 01/78] Add very basic syntax highlighting that needs to be fixed a lot --- .../include/editor/panels/CodeEditorPanel.h | 5 +- .../editor/panels/CodeEditorPanelManager.h | 5 + .../include/parsers/SyntaxHighlighter.h | 10 +- .../src/editor/panels/CodeEditorPanel.cpp | 95 +- .../editor/panels/CodeEditorPanelManager.cpp | 20 + Animations/src/parsers/Grammar.cpp | 47 +- Animations/src/parsers/SyntaxHighlighter.cpp | 34 + assets/customGrammars/lua.grammar.json | 1096 ++++++++++++++++ assets/customGrammars/og_lua.grammar.json | 1152 +++++++++++++++++ 9 files changed, 2439 insertions(+), 25 deletions(-) create mode 100644 assets/customGrammars/lua.grammar.json create mode 100644 assets/customGrammars/og_lua.grammar.json diff --git a/Animations/include/editor/panels/CodeEditorPanel.h b/Animations/include/editor/panels/CodeEditorPanel.h index 0c366d20..4657ca67 100644 --- a/Animations/include/editor/panels/CodeEditorPanel.h +++ b/Animations/include/editor/panels/CodeEditorPanel.h @@ -1,4 +1,5 @@ #include "core.h" +#include "parsers/SyntaxHighlighter.h" #include @@ -35,6 +36,8 @@ namespace MathAnim // A map that contains this files byte->codepoint mapping uint32 byteMap[1 << 8]; + + CodeHighlights syntaxHighlightTree; }; namespace CodeEditorPanel @@ -54,6 +57,6 @@ namespace MathAnim bool removeTextWithDelete(CodeEditorPanelData& panel, int32 textToRemoveStart, int32 textToRemoveLength); void translateStringToLocalByteMapping(CodeEditorPanelData& panel, uint8* utf8String, size_t stringNumBytes, uint8** outStr, size_t* outStrLength, uint32* numberLines = nullptr); - void translateLocalByteMappingToString(CodeEditorPanelData const& panel, uint8* byteMappedString, size_t byteMappedStringNumBytes, uint8** outUtf8String, size_t* outUtf8StringNumBytes); + void translateLocalByteMappingToString(CodeEditorPanelData const& panel, uint8* byteMappedString, size_t byteMappedStringNumBytes, uint8** outUtf8String, size_t* outUtf8StringNumBytes, bool includeCarriageReturnsForWindows = true); } } \ No newline at end of file diff --git a/Animations/include/editor/panels/CodeEditorPanelManager.h b/Animations/include/editor/panels/CodeEditorPanelManager.h index 356e7c02..835b7405 100644 --- a/Animations/include/editor/panels/CodeEditorPanelManager.h +++ b/Animations/include/editor/panels/CodeEditorPanelManager.h @@ -4,6 +4,8 @@ namespace MathAnim { struct AnimationManagerData; struct SizedFont; + class SyntaxHighlighter; + struct SyntaxTheme; namespace CodeEditorPanelManager { @@ -24,5 +26,8 @@ namespace MathAnim uint8 addCharToFont(uint32 codepoint); void imguiStats(); + + SyntaxHighlighter const& getHighlighter(); + SyntaxTheme const& getTheme(); } } \ No newline at end of file diff --git a/Animations/include/parsers/SyntaxHighlighter.h b/Animations/include/parsers/SyntaxHighlighter.h index 6b57ad5e..6e484804 100644 --- a/Animations/include/parsers/SyntaxHighlighter.h +++ b/Animations/include/parsers/SyntaxHighlighter.h @@ -14,6 +14,7 @@ namespace MathAnim Cpp, Glsl, Javascript, + Custom, Length }; @@ -21,14 +22,16 @@ namespace MathAnim "None", "C++", "Glsl", - "JavaScript" + "JavaScript", + "Undefined" ); constexpr auto _highlighterLanguageFilenames = fixedSizeArray( "None", "assets/grammars/cpp.tmLanguage.json", "assets/grammars/glsl.tmLanguage.json", - "assets/grammars/javascript.json" + "assets/grammars/javascript.json", + "Undefined" ); enum class HighlighterTheme : uint8 @@ -99,6 +102,9 @@ namespace MathAnim { void init(); + void importGrammar(const char* filename); + const SyntaxHighlighter* getImportedHighlighter(const char* filename); + const SyntaxHighlighter* getHighlighter(HighlighterLanguage language); const SyntaxTheme* getTheme(HighlighterTheme theme); diff --git a/Animations/src/editor/panels/CodeEditorPanel.cpp b/Animations/src/editor/panels/CodeEditorPanel.cpp index a46c38d2..1e5d8e5d 100644 --- a/Animations/src/editor/panels/CodeEditorPanel.cpp +++ b/Animations/src/editor/panels/CodeEditorPanel.cpp @@ -7,6 +7,9 @@ #include "core/Window.h" #include "renderer/Fonts.h" #include "editor/TextEditUndo.h" +#include "parsers/SyntaxTheme.h" + +#include using namespace CppUtils; @@ -36,8 +39,8 @@ namespace MathAnim static void renderTextCursor(CodeEditorPanelData& panel, ImVec2 const& textCursorDrawPosition, SizedFont const* const font); static ImVec2 renderNextLinePrefix(CodeEditorPanelData& panel, uint32 lineNumber, SizedFont const* const font); static bool mouseInTextEditArea(CodeEditorPanelData const& panel); - static ImVec2 addStringToDrawList(ImDrawList* drawList, SizedFont const* const font, std::string const& str, ImVec2 const& drawPos); - static ImVec2 addCharToDrawList(ImDrawList* drawList, SizedFont const* const font, uint32 charToAdd, ImVec2 const& drawPos); + static ImVec2 addStringToDrawList(ImDrawList* drawList, SizedFont const* const font, std::string const& str, ImVec2 const& drawPos, ImColor const& color); + static ImVec2 addCharToDrawList(ImDrawList* drawList, SizedFont const* const font, uint32 charToAdd, ImVec2 const& drawPos, ImColor const& color); static bool removeSelectedTextWithBackspace(CodeEditorPanelData& panel, bool addBackspaceToUndoHistory = true); static bool removeSelectedTextWithDelete(CodeEditorPanelData& panel); static bool removeText(CodeEditorPanelData& panel, int32 textToRemoveOffset, int32 textToRemoveNumBytes); @@ -184,9 +187,11 @@ namespace MathAnim RawMemory memory; memory.init(fileSize); - fread(memory.data, fileSize, 1, fp); + fread(memory.data, fileSize + 1, 1, fp); fclose(fp); + memory.data[fileSize + 1] = '\0'; + // Preprocess file into usable buffer res->visibleCharacterBuffer = nullptr; res->visibleCharacterBufferSize = 0; @@ -199,6 +204,29 @@ namespace MathAnim // +1 for the extra line for EOF res->totalNumberLines++; + // TODO: This is very gross, instead of copying a million times, we should get rid of the local byte + // mapping garbage since it's not working well. + uint8* noCarriageReturnStr = nullptr; + size_t noCarriageReturnStrLength = 0; + translateLocalByteMappingToString( + *res, + res->visibleCharacterBuffer, + res->visibleCharacterBufferSize, + &noCarriageReturnStr, + &noCarriageReturnStrLength, + false + ); + + // Parse the syntax + // TODO: This will not work with UTF8, fix it. + res->syntaxHighlightTree = CodeEditorPanelManager::getHighlighter().parse( + std::string((const char*)noCarriageReturnStr, noCarriageReturnStrLength), + CodeEditorPanelManager::getTheme(), + true + ); + + g_memory_free(noCarriageReturnStr); + res->undoSystem = UndoSystem::createTextEditorUndoSystem(res, MAX_UNDO_HISTORY); g_memory_free(memory.data); @@ -540,15 +568,38 @@ namespace MathAnim ImVec2 textHighlightRectStart = ImVec2(); int32 closestByteToMouseCursor = (int32)panel.lineNumberByteStart; + auto highlightIter = panel.syntaxHighlightTree.segments.begin(); + auto syntaxTheme = CodeEditorPanelManager::getTheme(); + for (size_t cursor = panel.lineNumberByteStart; cursor < panel.visibleCharacterBufferSize; cursor++) { + ImColor highlightedColor = syntaxTheme.defaultForeground.color; + + // Figure out what color this character should be + if (highlightIter != panel.syntaxHighlightTree.segments.end() && cursor >= highlightIter->endPos) + { + highlightIter++; + } + + if (highlightIter != panel.syntaxHighlightTree.segments.end() && + cursor >= highlightIter->startPos && cursor < highlightIter->endPos) + { + highlightedColor = ImColor( + highlightIter->color.r, + highlightIter->color.g, + highlightIter->color.b, + highlightIter->color.a + ); + } + uint32 currentCodepoint = panel.byteMap[panel.visibleCharacterBuffer[cursor]]; ImVec2 letterBoundsStart = currentLetterDrawPos; ImVec2 letterBoundsSize = addCharToDrawList( drawList, codeFont, currentCodepoint, - letterBoundsStart + letterBoundsStart, + highlightedColor ); if (cursor == panel.cursorBytePosition) @@ -868,7 +919,7 @@ namespace MathAnim } // Translate UTF8 string to local byte mapping - auto maybeParseInfo = Parser::makeParseInfo((char*)utf8String, stringNumBytes); + auto maybeParseInfo = ::Parser::makeParseInfo((char*)utf8String, stringNumBytes); if (!maybeParseInfo.hasValue()) { g_logger_error("Could not add UTF-8 string '{}' to editor. Had error '{}'.", utf8String, maybeParseInfo.error()); @@ -881,7 +932,7 @@ namespace MathAnim while (parseInfo.cursor < parseInfo.numBytes) { uint8 numBytesParsed = 1; - auto codepoint = Parser::parseCharacter(parseInfo, &numBytesParsed); + auto codepoint = ::Parser::parseCharacter(parseInfo, &numBytesParsed); if (!codepoint.hasValue()) { g_logger_error("File has malformed unicode. Skipping bad unicode data."); @@ -913,7 +964,7 @@ namespace MathAnim *outStr = (uint8*)g_memory_realloc((void*)(*outStr), (*outStrLength) * sizeof(uint8)); } - void translateLocalByteMappingToString(CodeEditorPanelData const& panel, uint8* byteMappedString, size_t byteMappedStringNumBytes, uint8** outUtf8String, size_t* outUtf8StringNumBytes) + void translateLocalByteMappingToString(CodeEditorPanelData const& panel, uint8* byteMappedString, size_t byteMappedStringNumBytes, uint8** outUtf8String, size_t* outUtf8StringNumBytes, bool includeCarriageReturnsForWindows) { // Translate to valid UTF-8 RawMemory memory{}; @@ -926,16 +977,19 @@ namespace MathAnim uint8 utf8Bytes[4] = {}; uint8 numBytes = codepointToUtf8Str(utf8Bytes, codepoint); -#ifdef _WIN32 - g_logger_assert(codepoint != (uint32)'\r', "We should never get carriage returns in our edit buffers"); - - // If we're on windows, translate newlines to carriage return + newlines when saving the files again - if (codepoint == (uint32)'\n') + #ifdef _WIN32 + if (includeCarriageReturnsForWindows) { - uint8 carriageReturn = '\r'; - memory.write(&carriageReturn); + g_logger_assert(codepoint != (uint32)'\r', "We should never get carriage returns in our edit buffers"); + + // If we're on windows, translate newlines to carriage return + newlines when saving the files again + if (codepoint == (uint32)'\n') + { + uint8 carriageReturn = '\r'; + memory.write(&carriageReturn); + } } -#endif + #endif memory.writeDangerous(utf8Bytes, numBytes * sizeof(uint8)); } @@ -1114,7 +1168,7 @@ namespace MathAnim ImVec2 lineStart = getTopLeftOfLine(panel, lineNumber, font); ImVec2 numberStart = lineStart + ImVec2(leftGutterWidth - textSize.x, 0.0f); - addStringToDrawList(drawList, font, numberText, numberStart); + addStringToDrawList(drawList, font, numberText, numberStart, ImColor(255, 255, 255, 255)); return lineStart + ImVec2(leftGutterWidth + style.FramePadding.x, 0.0f); } @@ -1126,7 +1180,7 @@ namespace MathAnim return mouseIntersectsRect(textAreaStart, textAreaEnd); } - static ImVec2 addStringToDrawList(ImDrawList* drawList, SizedFont const* const sizedFont, std::string const& str, ImVec2 const& drawPos) + static ImVec2 addStringToDrawList(ImDrawList* drawList, SizedFont const* const sizedFont, std::string const& str, ImVec2 const& drawPos, ImColor const& color) { Font const* const font = sizedFont->unsizedFont; ImVec2 cursorPos = drawPos; @@ -1139,14 +1193,14 @@ namespace MathAnim continue; } - ImVec2 charSize = addCharToDrawList(drawList, sizedFont, (uint32)str[i], cursorPos); + ImVec2 charSize = addCharToDrawList(drawList, sizedFont, (uint32)str[i], cursorPos, color); cursorPos = cursorPos + ImVec2(charSize.x, 0.0f); } return cursorPos - drawPos; } - static ImVec2 addCharToDrawList(ImDrawList* drawList, SizedFont const* const sizedFont, uint32 charToAdd, ImVec2 const& drawPos) + static ImVec2 addCharToDrawList(ImDrawList* drawList, SizedFont const* const sizedFont, uint32 charToAdd, ImVec2 const& drawPos, ImColor const& color) { Font const* const font = sizedFont->unsizedFont; uint32 texId = sizedFont->texture.graphicsId; @@ -1187,7 +1241,8 @@ namespace MathAnim uvMin, uvMin + ImVec2(uvSize.x, 0.0f), uvMin + uvSize, - uvMin + ImVec2(0.0f, uvSize.y) + uvMin + ImVec2(0.0f, uvSize.y), + color ); } diff --git a/Animations/src/editor/panels/CodeEditorPanelManager.cpp b/Animations/src/editor/panels/CodeEditorPanelManager.cpp index 9966f3ee..8033b707 100644 --- a/Animations/src/editor/panels/CodeEditorPanelManager.cpp +++ b/Animations/src/editor/panels/CodeEditorPanelManager.cpp @@ -4,6 +4,7 @@ #include "animation/AnimationManager.h" #include "core/Input.h" #include "renderer/Fonts.h" +#include "parsers/SyntaxHighlighter.h" namespace MathAnim { @@ -27,6 +28,9 @@ namespace MathAnim static std::string nextFileToAdd = ""; static int fileToMakeActive = -1; static int lineNumberToGoTo = -1; + static const char* luaGrammarJsonFile = "./assets/customGrammars/lua.grammar.json"; + static SyntaxHighlighter const* luaGrammar = nullptr; + static SyntaxTheme const* syntaxTheme = nullptr; static const char* codeFontRegularFile = "./assets/fonts/fira/FiraCode-SemiBold.ttf"; static SizedFont* codeFont = nullptr; @@ -37,6 +41,10 @@ namespace MathAnim void init() { codeFont = Fonts::loadSizedFont(codeFontRegularFile, fontSizePx, CharRange::Ascii, false); + + Highlighters::importGrammar(luaGrammarJsonFile); + luaGrammar = Highlighters::getImportedHighlighter(luaGrammarJsonFile); + syntaxTheme = Highlighters::getTheme(HighlighterTheme::MonokaiNight); for (uint32 i = CharRange::Ascii.firstCharCode; i <= CharRange::Ascii.lastCharCode; i++) { @@ -244,6 +252,18 @@ namespace MathAnim } } + SyntaxHighlighter const& getHighlighter() + { + g_logger_assert(luaGrammar != nullptr, "This shouldn't happen."); + return *luaGrammar; + } + + SyntaxTheme const& getTheme() + { + g_logger_assert(syntaxTheme != nullptr, "This shouldn't happen."); + return *syntaxTheme; + } + // ----------- Internal functinons ----------- } } \ No newline at end of file diff --git a/Animations/src/parsers/Grammar.cpp b/Animations/src/parsers/Grammar.cpp index 5350029a..f834b406 100644 --- a/Animations/src/parsers/Grammar.cpp +++ b/Animations/src/parsers/Grammar.cpp @@ -678,6 +678,17 @@ namespace MathAnim return ancestorScopes; } + static bool checkBufferUnderflow(size_t sizeLeft, size_t numBytesToRemove) + { + if (sizeLeft >= numBytesToRemove) + { + return false; + } + + g_logger_error("We have a buffer underflow. Please pass a larger buffer to the tree."); + return true; + } + std::string SourceGrammarTree::getStringifiedTree(size_t bufferSize) const { char* buffer = (char*)g_memory_allocate(bufferSize * sizeof(char)); @@ -697,6 +708,11 @@ namespace MathAnim { int numBytesWritten = snprintf(bufferPtr, bufferSizeLeft, " "); bufferPtr += numBytesWritten; + if (checkBufferUnderflow(bufferSizeLeft, numBytesWritten)) + { + // Break out of all loops + goto end; + } bufferSizeLeft -= numBytesWritten; } @@ -704,6 +720,11 @@ namespace MathAnim { int numBytesWritten = snprintf(bufferPtr, bufferSizeLeft, "'ATOM': "); bufferPtr += numBytesWritten; + if (checkBufferUnderflow(bufferSizeLeft, numBytesWritten)) + { + // Break out of all loops + goto end; + } bufferSizeLeft -= numBytesWritten; } else @@ -713,12 +734,22 @@ namespace MathAnim const std::optional& scope = tree[i].scope; int numBytesWritten = snprintf(bufferPtr, bufferSizeLeft, "'%s': ", scope->getFriendlyName().c_str()); bufferPtr += numBytesWritten; + if (checkBufferUnderflow(bufferSizeLeft, numBytesWritten)) + { + // Break out of all loops + goto end; + } bufferSizeLeft -= numBytesWritten; } else { int numBytesWritten = snprintf(bufferPtr, bufferSizeLeft, "'NULL_SCOPE': "); bufferPtr += numBytesWritten; + if (checkBufferUnderflow(bufferSizeLeft, numBytesWritten)) + { + // Break out of all loops + goto end; + } bufferSizeLeft -= numBytesWritten; } } @@ -734,6 +765,11 @@ namespace MathAnim + ">"; int numBytesWritten = snprintf(bufferPtr, bufferSizeLeft, "'%s'\n", offsetVal.c_str()); bufferPtr += numBytesWritten; + if (checkBufferUnderflow(bufferSizeLeft, numBytesWritten)) + { + // Break out of all loops + goto end; + } bufferSizeLeft -= numBytesWritten; } else @@ -759,6 +795,11 @@ namespace MathAnim } int numBytesWritten = snprintf(bufferPtr, bufferSizeLeft, "'%s'\n", val.c_str()); bufferPtr += numBytesWritten; + if (checkBufferUnderflow(bufferSizeLeft, numBytesWritten)) + { + // Break out of all loops + goto end; + } bufferSizeLeft -= numBytesWritten; } } @@ -778,16 +819,18 @@ namespace MathAnim } } + end: if ((size_t)(bufferPtr - buffer) < bufferSize) { bufferPtr[0] = '\0'; } else { + // We had a buffer overrun, truncate the string buffer[bufferSize - 1] = '\0'; } - std::string res = std::string(buffer); + std::string res = std::string((const char*)buffer); g_memory_free(buffer); return res; @@ -1622,7 +1665,7 @@ namespace MathAnim } else { - g_logger_error("Capture group in Onigiruma expression did not have a scoped name or a pattern array."); + g_logger_error("Capture group in Oniguruma expression did not have a scoped name or a pattern array."); } } } diff --git a/Animations/src/parsers/SyntaxHighlighter.cpp b/Animations/src/parsers/SyntaxHighlighter.cpp index f806d7a8..903a8113 100644 --- a/Animations/src/parsers/SyntaxHighlighter.cpp +++ b/Animations/src/parsers/SyntaxHighlighter.cpp @@ -112,6 +112,7 @@ namespace MathAnim { static std::unordered_map grammars = {}; static std::unordered_map themes = {}; + static std::unordered_map importedGrammars = {}; void init() { @@ -135,6 +136,32 @@ namespace MathAnim g_logger_info("Successfully imported default languages and themes for syntax highlighters."); } + void importGrammar(const char* filename) + { + std::filesystem::path absPath = std::filesystem::absolute(std::filesystem::path(filename)); + if (auto iter = importedGrammars.find(absPath); iter != importedGrammars.end()) + { + return; + } + + if (Platform::fileExists(absPath.string().c_str())) + { + SyntaxHighlighter* highlighter = g_memory_new SyntaxHighlighter(absPath); + importedGrammars[absPath] = highlighter; + } + } + + const SyntaxHighlighter* getImportedHighlighter(const char* filename) + { + std::filesystem::path absPath = std::filesystem::absolute(std::filesystem::path(filename)); + if (auto iter = importedGrammars.find(absPath); iter != importedGrammars.end()) + { + return iter->second; + } + + return nullptr; + } + const SyntaxHighlighter* getHighlighter(HighlighterLanguage language) { auto iter = grammars.find(language); @@ -159,6 +186,13 @@ namespace MathAnim void free() { + for (auto [k, v] : importedGrammars) + { + v->free(); + g_memory_delete(v); + } + importedGrammars.clear(); + for (auto [k, v] : grammars) { v->free(); diff --git a/assets/customGrammars/lua.grammar.json b/assets/customGrammars/lua.grammar.json new file mode 100644 index 00000000..d19ae9de --- /dev/null +++ b/assets/customGrammars/lua.grammar.json @@ -0,0 +1,1096 @@ +{ + "information_for_contributors": [ + "This file has been converted from https://github.com/textmate/lua.tmbundle/blob/master/Syntaxes/Lua.plist", + "If you want to provide a fix or improvement, please create a pull request against the original repository.", + "Once accepted there, we are happy to receive an update request." + ], + "version": "https://github.com/textmate/lua.tmbundle/commit/42da2c6ff5d86c068f72520f856190f413911a80", + "name": "Lua", + "scopeName": "source.lua", + "comment": "Lua Syntax: version 0.8", + "patterns": [ + { + "begin": "\\b(?:(local)\\s+)?(function)\\b", + "beginCaptures": { + "1": { + "name": "keyword.local.lua" + }, + "2": { + "name": "keyword.control.lua" + } + }, + "end": "(?<=[\\)\\-{}\\[\\]\"'])", + "name": "meta.function.lua", + "patterns": [ + { + "include": "#comment" + }, + { + "begin": "(\\()", + "beginCaptures": { + "1": { + "name": "punctuation.definition.parameters.begin.lua" + } + }, + "end": "(\\))|(?=[\\-\\.{}\\[\\]\"'])", + "endCaptures": { + "1": { + "name": "punctuation.definition.parameters.finish.lua" + } + }, + "name": "meta.parameter.lua", + "patterns": [ + { + "include": "#comment" + }, + { + "match": "[a-zA-Z_][a-zA-Z0-9_]*", + "name": "variable.parameter.function.lua" + }, + { + "match": ",", + "name": "punctuation.separator.arguments.lua" + }, + { + "begin": ":", + "beginCaptures": { + "0": { + "name": "punctuation.separator.arguments.lua" + } + }, + "end": "(?=[\\),])", + "patterns": [ + { + "include": "#luadoc.type" + } + ] + } + ] + }, + { + "match": "\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b\\s*(?=:)", + "name": "entity.name.function.lua" + }, + { + "match": "\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b", + "name": "entity.name.function.lua" + } + ] + }, + { + "match": "(?", + "captures": { + "1": { + "name": "string.tag.lua" + } + } + }, + { + "begin": "(^ ->[0-9]*\\.)(\\s)\\b([a-zA-Z_][a-zA-Z0-9_\\.\\s,\\<\\|\\>]*)", + "beginCaptures": { + "0": { + "name": "support.type.lua" + }, + "1": { + "name": "constant.numeric.integer.lua" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "match": "-- (.)*", + "name": "comment.block.lua" + } + ] + }, + { + "match": "\\b(break|do|else|for|if|elseif|goto|return|then|repeat|while|until|end|function|local|in|and|or|not)\\b", + "name": "keyword.control.lua" + }, + { + "match": "(?=?|(?|\\<", + "name": "keyword.operator.lua" + } + ] + }, + { + "begin": "(?<=---\\s*)@see", + "beginCaptures": { + "0": { + "name": "storage.type.annotation.lua" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "match": "\\b([a-zA-Z_\\*][a-zA-Z0-9_\\.\\*]*)", + "name": "support.class.lua" + }, + { + "match": "#", + "name": "keyword.operator.lua" + } + ] + }, + { + "begin": "(?<=---\\s*)@diagnostic", + "beginCaptures": { + "0": { + "name": "storage.type.annotation.lua" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "begin": "([a-zA-Z_\\-0-9]+)[ \\t]*(:)?", + "beginCaptures": { + "1": { + "name": "keyword.other.unit" + }, + "2": { + "name": "keyword.operator.unit" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "match": "\\b([a-zA-Z_\\*][a-zA-Z0-9_\\-]*)", + "name": "support.class.lua" + }, + { + "match": ",", + "name": "keyword.operator.lua" + } + ] + } + ] + }, + { + "begin": "(?<=---)\\|\\s*[\\>\\+]?", + "beginCaptures": { + "0": { + "name": "storage.type.annotation.lua" + } + }, + "end": "(?=[\\n#])", + "patterns": [ + { + "include": "#string" + } + ] + } + ] + }, + "luadoc.type": { + "patterns": [ + { + "begin": "\\bfun\\b", + "beginCaptures": { + "0": { + "name": "keyword.control.lua" + } + }, + "end": "(?=\\s)", + "patterns": [ + { + "begin": "@", + "end": "(?=\\n)", + "patterns": [ + { + "name": "constant.language.lua" + } + ] + }, + { + "match": "[\\(\\),:\\?][ \\t]*", + "name": "keyword.operator.lua" + }, + { + "match": "(([a-zA-Z_][a-zA-Z0-9_\\.\\*\\[\\]\\<\\>\\,]*))(?\\,]*))(?\\,]*))(?|\\<", + "name": "keyword.operator.lua" + } + ], + "name": "comment.block.lua" + }, + { + "begin": "^@see", + "beginCaptures": { + "0": { + "name": "storage.type.annotation.lua" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "match": "\\b([a-zA-Z_\\*][a-zA-Z0-9_\\.\\*]*)", + "name": "support.class.lua" + }, + { + "match": "#", + "name": "keyword.operator.lua" + } + ], + "name": "comment.block.lua" + }, + { + "begin": "^@diagnostic", + "beginCaptures": { + "0": { + "name": "storage.type.annotation.lua" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "begin": "([a-zA-Z_\\-0-9]+)[ \\t]*(:)?", + "beginCaptures": { + "1": { + "name": "keyword.other.unit" + }, + "2": { + "name": "keyword.operator.unit" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "match": "\\b([a-zA-Z_\\*][a-zA-Z0-9_\\-]*)", + "name": "support.class.lua" + }, + { + "match": ",", + "name": "keyword.operator.lua" + } + ] + } + ], + "name": "comment.block.lua" + } + ] + } + } +} \ No newline at end of file diff --git a/assets/customGrammars/og_lua.grammar.json b/assets/customGrammars/og_lua.grammar.json new file mode 100644 index 00000000..b46723c6 --- /dev/null +++ b/assets/customGrammars/og_lua.grammar.json @@ -0,0 +1,1152 @@ +{ + "information_for_contributors": [ + "This file has been converted from https://github.com/textmate/lua.tmbundle/blob/master/Syntaxes/Lua.plist", + "If you want to provide a fix or improvement, please create a pull request against the original repository.", + "Once accepted there, we are happy to receive an update request." + ], + "version": "https://github.com/textmate/lua.tmbundle/commit/42da2c6ff5d86c068f72520f856190f413911a80", + "name": "Lua", + "scopeName": "source.lua", + "comment": "Lua Syntax: version 0.8", + "patterns": [ + { + "begin": "\\b(?:(local)\\s+)?(function)\\b", + "beginCaptures": { + "1": { + "name": "keyword.local.lua" + }, + "2": { + "name": "keyword.control.lua" + } + }, + "end": "(?<=[\\)\\-{}\\[\\]\"'])", + "name": "meta.function.lua", + "patterns": [ + { + "include": "#comment" + }, + { + "begin": "(\\()", + "beginCaptures": { + "1": { + "name": "punctuation.definition.parameters.begin.lua" + } + }, + "end": "(\\))|(?=[\\-\\.{}\\[\\]\"'])", + "endCaptures": { + "1": { + "name": "punctuation.definition.parameters.finish.lua" + } + }, + "name": "meta.parameter.lua", + "patterns": [ + { + "include": "#comment" + }, + { + "match": "[a-zA-Z_][a-zA-Z0-9_]*", + "name": "variable.parameter.function.lua" + }, + { + "match": ",", + "name": "punctuation.separator.arguments.lua" + }, + { + "begin": ":", + "beginCaptures": { + "0": { + "name": "punctuation.separator.arguments.lua" + } + }, + "end": "(?=[\\),])", + "patterns": [ + { + "include": "#luadoc.type" + } + ] + } + ] + }, + { + "match": "\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b\\s*(?=:)", + "name": "entity.name.function.lua" + }, + { + "match": "\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b", + "name": "entity.name.function.lua" + } + ] + }, + { + "match": "(?", + "captures": { + "1": { + "name": "string.tag.lua" + } + } + }, + { + "begin": "(^ ->[0-9]*\\.)(\\s)\\b([a-zA-Z_][a-zA-Z0-9_\\.\\s,\\<\\|\\>]*)", + "beginCaptures": { + "0": { + "name": "support.type.lua" + }, + "1": { + "name": "constant.numeric.integer.lua" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "match": "-- (.)*", + "name": "comment.block.lua" + } + ] + }, + { + "match": "\\b(break|do|else|for|if|elseif|goto|return|then|repeat|while|until|end|function|local|in|and|or|not)\\b", + "name": "keyword.control.lua" + }, + { + "match": "(?=?|(?|\\<", + "name": "keyword.operator.lua" + } + ] + }, + { + "begin": "(?<=---\\s*)@see", + "beginCaptures": { + "0": { + "name": "storage.type.annotation.lua" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "match": "\\b([a-zA-Z_\\*][a-zA-Z0-9_\\.\\*]*)", + "name": "support.class.lua" + }, + { + "match": "#", + "name": "keyword.operator.lua" + } + ] + }, + { + "begin": "(?<=---\\s*)@diagnostic", + "beginCaptures": { + "0": { + "name": "storage.type.annotation.lua" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "begin": "([a-zA-Z_\\-0-9]+)[ \\t]*(:)?", + "beginCaptures": { + "1": { + "name": "keyword.other.unit" + }, + "2": { + "name": "keyword.operator.unit" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "match": "\\b([a-zA-Z_\\*][a-zA-Z0-9_\\-]*)", + "name": "support.class.lua" + }, + { + "match": ",", + "name": "keyword.operator.lua" + } + ] + } + ] + }, + { + "begin": "(?<=---)\\|\\s*[\\>\\+]?", + "beginCaptures": { + "0": { + "name": "storage.type.annotation.lua" + } + }, + "end": "(?=[\\n#])", + "patterns": [ + { + "include": "#string" + } + ] + } + ] + }, + "luadoc.type": { + "patterns": [ + { + "begin": "\\bfun\\b", + "beginCaptures": { + "0": { + "name": "keyword.control.lua" + } + }, + "end": "(?=\\s)", + "patterns": [ + { + "begin": "@", + "end": "(?=\\n)", + "patterns": [ + { + "name": "constant.language.lua" + } + ] + }, + { + "match": "[\\(\\),:\\?][ \\t]*", + "name": "keyword.operator.lua" + }, + { + "match": "(([a-zA-Z_][a-zA-Z0-9_\\.\\*\\[\\]\\<\\>\\,]*))(?\\,]*))(?\\,]*))(?|\\<", + "name": "keyword.operator.lua" + } + ], + "name": "comment.block.lua" + }, + { + "begin": "^@see", + "beginCaptures": { + "0": { + "name": "storage.type.annotation.lua" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "match": "\\b([a-zA-Z_\\*][a-zA-Z0-9_\\.\\*]*)", + "name": "support.class.lua" + }, + { + "match": "#", + "name": "keyword.operator.lua" + } + ], + "name": "comment.block.lua" + }, + { + "begin": "^@diagnostic", + "beginCaptures": { + "0": { + "name": "storage.type.annotation.lua" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "begin": "([a-zA-Z_\\-0-9]+)[ \\t]*(:)?", + "beginCaptures": { + "1": { + "name": "keyword.other.unit" + }, + "2": { + "name": "keyword.operator.unit" + } + }, + "end": "(?=\\n)", + "patterns": [ + { + "match": "\\b([a-zA-Z_\\*][a-zA-Z0-9_\\-]*)", + "name": "support.class.lua" + }, + { + "match": ",", + "name": "keyword.operator.lua" + } + ] + } + ], + "name": "comment.block.lua" + } + ] + } + } +} \ No newline at end of file From 9906b4a56355cfb03fe34b62b2a58d2e8f2d5a2f Mon Sep 17 00:00:00 2001 From: Gabe Ambrosio Date: Sat, 21 Oct 2023 10:52:58 -0500 Subject: [PATCH 02/78] Get basic themes working --- .../include/editor/panels/CodeEditorPanel.h | 2 + .../editor/panels/CodeEditorPanelManager.h | 4 ++ .../src/editor/panels/CodeEditorPanel.cpp | 66 ++++++++++--------- .../editor/panels/CodeEditorPanelManager.cpp | 9 +++ Animations/src/editor/panels/MenuBar.cpp | 18 +++++ 5 files changed, 68 insertions(+), 31 deletions(-) diff --git a/Animations/include/editor/panels/CodeEditorPanel.h b/Animations/include/editor/panels/CodeEditorPanel.h index 4657ca67..19eebb55 100644 --- a/Animations/include/editor/panels/CodeEditorPanel.h +++ b/Animations/include/editor/panels/CodeEditorPanel.h @@ -46,6 +46,8 @@ namespace MathAnim void saveFile(CodeEditorPanelData const& panel); void free(CodeEditorPanelData* panel); + void reparseSyntax(CodeEditorPanelData& panel); + bool update(CodeEditorPanelData& panel); void setCursorToLine(CodeEditorPanelData& panel, uint32 lineNumber); diff --git a/Animations/include/editor/panels/CodeEditorPanelManager.h b/Animations/include/editor/panels/CodeEditorPanelManager.h index 835b7405..6dec6d71 100644 --- a/Animations/include/editor/panels/CodeEditorPanelManager.h +++ b/Animations/include/editor/panels/CodeEditorPanelManager.h @@ -1,5 +1,7 @@ #include "core.h" +#include "parsers/SyntaxHighlighter.h" + namespace MathAnim { struct AnimationManagerData; @@ -29,5 +31,7 @@ namespace MathAnim SyntaxHighlighter const& getHighlighter(); SyntaxTheme const& getTheme(); + + void setTheme(HighlighterTheme theme); } } \ No newline at end of file diff --git a/Animations/src/editor/panels/CodeEditorPanel.cpp b/Animations/src/editor/panels/CodeEditorPanel.cpp index 1e5d8e5d..f9d427b0 100644 --- a/Animations/src/editor/panels/CodeEditorPanel.cpp +++ b/Animations/src/editor/panels/CodeEditorPanel.cpp @@ -50,6 +50,11 @@ namespace MathAnim static uint8 codepointToUtf8Str(uint8* const buffer, uint32 codepoint); // Simple calculation functions + static inline ImColor getColor(Vec4 const& color) + { + return ImColor(color.r, color.g, color.b, color.a); + } + static inline float getLineHeight(SizedFont const* const font) { return font->unsizedFont->lineHeight * font->fontSizePixels; @@ -204,28 +209,7 @@ namespace MathAnim // +1 for the extra line for EOF res->totalNumberLines++; - // TODO: This is very gross, instead of copying a million times, we should get rid of the local byte - // mapping garbage since it's not working well. - uint8* noCarriageReturnStr = nullptr; - size_t noCarriageReturnStrLength = 0; - translateLocalByteMappingToString( - *res, - res->visibleCharacterBuffer, - res->visibleCharacterBufferSize, - &noCarriageReturnStr, - &noCarriageReturnStrLength, - false - ); - - // Parse the syntax - // TODO: This will not work with UTF8, fix it. - res->syntaxHighlightTree = CodeEditorPanelManager::getHighlighter().parse( - std::string((const char*)noCarriageReturnStr, noCarriageReturnStrLength), - CodeEditorPanelManager::getTheme(), - true - ); - - g_memory_free(noCarriageReturnStr); + reparseSyntax(*res); res->undoSystem = UndoSystem::createTextEditorUndoSystem(res, MAX_UNDO_HISTORY); @@ -256,6 +240,32 @@ namespace MathAnim g_memory_delete(panel); } + void reparseSyntax(CodeEditorPanelData& panel) + { + // TODO: This is very gross, instead of copying a million times, we should get rid of the local byte + // mapping garbage since it's not working well. + uint8* noCarriageReturnStr = nullptr; + size_t noCarriageReturnStrLength = 0; + translateLocalByteMappingToString( + panel, + panel.visibleCharacterBuffer, + panel.visibleCharacterBufferSize, + &noCarriageReturnStr, + &noCarriageReturnStrLength, + false + ); + + // Parse the syntax + // TODO: This will not work with UTF8, fix it. + panel.syntaxHighlightTree = CodeEditorPanelManager::getHighlighter().parse( + std::string((const char*)noCarriageReturnStr, noCarriageReturnStrLength), + CodeEditorPanelManager::getTheme(), + true + ); + + g_memory_free(noCarriageReturnStr); + } + bool update(CodeEditorPanelData& panel) { bool fileHasBeenEdited = false; @@ -508,6 +518,7 @@ namespace MathAnim } // ---- Handle Rendering/mouse clicking ---- + auto syntaxTheme = CodeEditorPanelManager::getTheme(); panel.drawStart = ImGui::GetCursorScreenPos(); panel.drawEnd = panel.drawStart + ImGui::GetContentRegionAvail(); @@ -515,7 +526,7 @@ namespace MathAnim ImDrawList* drawList = ImGui::GetWindowDrawList(); drawList->PushClipRect(panel.drawStart, panel.drawEnd, true); - drawList->AddRectFilled(panel.drawStart, panel.drawEnd, backgroundColor); + drawList->AddRectFilled(panel.drawStart, panel.drawEnd, getColor(syntaxTheme.defaultBackground.color)); uint32 currentLine = panel.lineNumberStart; ImVec2 currentLetterDrawPos = renderNextLinePrefix(panel, currentLine, codeFont); @@ -567,9 +578,7 @@ namespace MathAnim bool passedFirstCharacter = false; ImVec2 textHighlightRectStart = ImVec2(); int32 closestByteToMouseCursor = (int32)panel.lineNumberByteStart; - auto highlightIter = panel.syntaxHighlightTree.segments.begin(); - auto syntaxTheme = CodeEditorPanelManager::getTheme(); for (size_t cursor = panel.lineNumberByteStart; cursor < panel.visibleCharacterBufferSize; cursor++) { @@ -584,12 +593,7 @@ namespace MathAnim if (highlightIter != panel.syntaxHighlightTree.segments.end() && cursor >= highlightIter->startPos && cursor < highlightIter->endPos) { - highlightedColor = ImColor( - highlightIter->color.r, - highlightIter->color.g, - highlightIter->color.b, - highlightIter->color.a - ); + highlightedColor = getColor(highlightIter->color); } uint32 currentCodepoint = panel.byteMap[panel.visibleCharacterBuffer[cursor]]; diff --git a/Animations/src/editor/panels/CodeEditorPanelManager.cpp b/Animations/src/editor/panels/CodeEditorPanelManager.cpp index 8033b707..e331269b 100644 --- a/Animations/src/editor/panels/CodeEditorPanelManager.cpp +++ b/Animations/src/editor/panels/CodeEditorPanelManager.cpp @@ -264,6 +264,15 @@ namespace MathAnim return *syntaxTheme; } + void setTheme(HighlighterTheme theme) + { + syntaxTheme = Highlighters::getTheme(theme); + for (auto editor = openEditors.begin(); editor != openEditors.end(); editor++) + { + CodeEditorPanel::reparseSyntax(*editor->panel); + } + } + // ----------- Internal functinons ----------- } } \ No newline at end of file diff --git a/Animations/src/editor/panels/MenuBar.cpp b/Animations/src/editor/panels/MenuBar.cpp index 1ed6b179..eb509a7d 100644 --- a/Animations/src/editor/panels/MenuBar.cpp +++ b/Animations/src/editor/panels/MenuBar.cpp @@ -1,10 +1,12 @@ #include "editor/panels/MenuBar.h" +#include "editor/panels/CodeEditorPanelManager.h" #include "editor/imgui/ImGuiLayer.h" #include "editor/EditorLayout.h" #include "editor/UndoSystem.h" #include "core/Application.h" #include "core/Profiling.h" #include "renderer/Colors.h" +#include "parsers/SyntaxHighlighter.h" #include @@ -67,6 +69,22 @@ namespace MathAnim UndoSystem::redo(Application::getUndoSystem()); } + ImGui::Separator(); + + if (ImGui::BeginMenu("Code Themes")) + { + for (uint8 i = 1; i < (uint8)_highlighterThemeNames.size(); i++) + { + const char* themeName = _highlighterThemeNames[i]; + if (ImGui::MenuItem(themeName)) + { + CodeEditorPanelManager::setTheme((HighlighterTheme)i); + } + } + + ImGui::EndMenu(); + } + ImGui::EndMenu(); } From aad989152c0db360b1baff9f59053d08e8e3bf7b Mon Sep 17 00:00:00 2001 From: Gabe Ambrosio Date: Sat, 21 Oct 2023 11:53:26 -0500 Subject: [PATCH 03/78] Add regex tester for some help --- .../include/editor/panels/RegexTester.h | 15 ++ Animations/src/editor/EditorGui.cpp | 2 + Animations/src/editor/panels/MenuBar.cpp | 11 + Animations/src/editor/panels/RegexTester.cpp | 190 ++++++++++++++++++ assets/customGrammars/lua.grammar.json | 15 ++ 5 files changed, 233 insertions(+) create mode 100644 Animations/include/editor/panels/RegexTester.h create mode 100644 Animations/src/editor/panels/RegexTester.cpp diff --git a/Animations/include/editor/panels/RegexTester.h b/Animations/include/editor/panels/RegexTester.h new file mode 100644 index 00000000..65ba9a16 --- /dev/null +++ b/Animations/include/editor/panels/RegexTester.h @@ -0,0 +1,15 @@ +#ifndef MATH_ANIM_REGEX_TESTER_H +#define MATH_ANIM_REGEX_TESTER_H +#include "core.h" + +namespace MathAnim +{ + namespace RegexTester + { + void update(); + + void showWindow(); + } +} + +#endif \ No newline at end of file diff --git a/Animations/src/editor/EditorGui.cpp b/Animations/src/editor/EditorGui.cpp index 8f0de50f..c20a58ce 100644 --- a/Animations/src/editor/EditorGui.cpp +++ b/Animations/src/editor/EditorGui.cpp @@ -7,6 +7,7 @@ #include "editor/panels/ErrorPopups.h" #include "editor/panels/ExportPanel.h" #include "editor/panels/InspectorPanel.h" +#include "editor/panels/RegexTester.h" #include "editor/panels/SceneHierarchyPanel.h" #include "editor/timeline/Timeline.h" #include "editor/imgui/ImGuiLayer.h" @@ -149,6 +150,7 @@ namespace MathAnim InspectorPanel::update(am); CodeEditorPanelManager::update(am, editorViewportDockId); ErrorPopups::update(am); + RegexTester::update(); // TODO: Do this in a central file checkHotKeys(am); diff --git a/Animations/src/editor/panels/MenuBar.cpp b/Animations/src/editor/panels/MenuBar.cpp index eb509a7d..2c1714a5 100644 --- a/Animations/src/editor/panels/MenuBar.cpp +++ b/Animations/src/editor/panels/MenuBar.cpp @@ -1,5 +1,6 @@ #include "editor/panels/MenuBar.h" #include "editor/panels/CodeEditorPanelManager.h" +#include "editor/panels/RegexTester.h" #include "editor/imgui/ImGuiLayer.h" #include "editor/EditorLayout.h" #include "editor/UndoSystem.h" @@ -90,6 +91,16 @@ namespace MathAnim if (ImGui::BeginMenu("View")) { + if (ImGui::BeginMenu("Windows")) + { + if (ImGui::MenuItem("Regex Tester")) + { + RegexTester::showWindow(); + } + + ImGui::EndMenu(); + } + if (ImGui::BeginMenu("Layouts")) { const std::vector& defaultLayouts = EditorLayout::getDefaultLayouts(); diff --git a/Animations/src/editor/panels/RegexTester.cpp b/Animations/src/editor/panels/RegexTester.cpp new file mode 100644 index 00000000..eb87e605 --- /dev/null +++ b/Animations/src/editor/panels/RegexTester.cpp @@ -0,0 +1,190 @@ +#include "editor/panels/RegexTester.h" + +namespace MathAnim +{ + struct Match + { + std::string text; + int start; + int end; + }; + + namespace RegexTester + { + static constexpr size_t bufferSize = 1024; + static char regexToTestBuffer[bufferSize] = {}; + static char stringToTestAgainst[bufferSize] = {}; + static bool multiline = false; + static bool shouldShowWindow = false; + + static std::vector matches = {}; + + void update() + { + if (!shouldShowWindow) + { + return; + } + + ImGui::Begin("Regex Tester", &shouldShowWindow); + + if (ImGui::Checkbox(": Is Multiline", &multiline)) + { + matches = {}; + } + + if (ImGui::InputTextMultiline(": Regex", regexToTestBuffer, bufferSize)) + { + matches = {}; + } + + if (ImGui::InputTextMultiline(": String to Test", stringToTestAgainst, bufferSize)) + { + matches = {}; + } + + if (ImGui::Button("Test")) + { + const char* pattern = regexToTestBuffer; + const char* patternEnd = regexToTestBuffer + strlen(regexToTestBuffer); + + int options = ONIG_OPTION_NONE; + options |= ONIG_OPTION_CAPTURE_GROUP; + if (multiline) + { + options |= ONIG_OPTION_MULTILINE; + } + + // Enable capture history for Oniguruma + OnigSyntaxType syn; + onig_copy_syntax(&syn, ONIG_SYNTAX_DEFAULT); + onig_set_syntax_op2(&syn, + onig_get_syntax_op2(&syn) | ONIG_SYN_OP2_ATMARK_CAPTURE_HISTORY); + + OnigRegex reg; + OnigErrorInfo error; + int parseRes = onig_new( + ®, + (uint8*)pattern, + (uint8*)patternEnd, + options, + ONIG_ENCODING_ASCII, + &syn, + &error + ); + + if (parseRes != ONIG_NORMAL) + { + char s[ONIG_MAX_ERROR_MESSAGE_LEN]; + onig_error_code_to_str((UChar*)s, parseRes, &error); + g_logger_error("Oniguruma Error: '{}'", &s[0]); + + Match match = {}; + match.text = std::string("Oniguruma Error: ") + s; + matches.emplace_back(match); + } + else + { + // Find and add all matches to list + const char* targetStr = stringToTestAgainst; + const char* targetStrEnd = targetStr + strlen(targetStr); + + const char* searchEnd = targetStr + strlen(targetStr); + + int searchRes = ONIG_MISMATCH; + auto region = onig_region_new(); + + while (targetStr < targetStrEnd) + { + const char* searchStart = targetStr; + + onig_region_clear(region); + searchRes = onig_search( + reg, + (uint8*)targetStr, + (uint8*)targetStrEnd, + (uint8*)searchStart, + (uint8*)searchEnd, + region, + ONIG_OPTION_NONE + ); + + if (searchRes >= 0) + { + for (int i = 0; i < region->num_regs; i++) + { + if (region->beg[0] >= 0 && region->end[0] >= region->beg[0]) + { + int matchStart = region->beg[0]; + int matchEnd = region->end[0]; + + Match match = {}; + match.text = std::string(targetStr + matchStart, matchEnd - matchStart); + match.start = matchStart + (int)(targetStr - stringToTestAgainst); + match.end = matchEnd + (int)(targetStr - stringToTestAgainst); + matches.emplace_back(match); + + targetStr += matchEnd; + if (matchEnd == matchStart) + { + targetStr++; + } + } + } + } + else if (searchRes != ONIG_MISMATCH) + { + // Error + char s[ONIG_MAX_ERROR_MESSAGE_LEN]; + onig_error_code_to_str((UChar*)s, searchRes); + g_logger_error("Oniguruma Error: '{}'", s); + + Match match = {}; + match.text = std::string("Oniguruma Error: ") + s; + matches.push_back(match); + break; + } + else + { + if (!multiline) + { + // No matches found on this line + // Move to the next line to find the next match + while (targetStr < targetStrEnd) + { + if (*targetStr == '\n') + { + targetStr++; + break; + } + targetStr++; + } + } + else + { + // If no multiline match was found, there are no matches left + break; + } + } + } + + onig_region_free(region, 1 /* 1:free self, 0:free contents only */); + } + + onig_free(reg); + } + + for (auto& match : matches) + { + ImGui::Text("Match<%d:%d>: '%s'", match.start, match.end, match.text.c_str()); + } + + ImGui::End(); + } + + void showWindow() + { + shouldShowWindow = true; + } + } +} \ No newline at end of file diff --git a/assets/customGrammars/lua.grammar.json b/assets/customGrammars/lua.grammar.json index d19ae9de..f4924271 100644 --- a/assets/customGrammars/lua.grammar.json +++ b/assets/customGrammars/lua.grammar.json @@ -341,6 +341,21 @@ } }, "patterns": [ + { + "begin": "--\\[(=*)\\[", + "beginCaptures": { + "0": { + "name": "punctuation.definition.comment.begin.lua" + } + }, + "end": "\\]\\1\\]", + "endCaptures": { + "0": { + "name": "punctuation.definition.comment.end.lua" + } + }, + "name": "comment.block.lua" + }, { "begin": "----", "beginCaptures": { From 20e8a44568abb0b4c7ec05c45455f52ba987d2b7 Mon Sep 17 00:00:00 2001 From: Gabe Ambrosio Date: Sat, 21 Oct 2023 15:14:42 -0500 Subject: [PATCH 04/78] Fix regex tester --- Animations/src/editor/panels/RegexTester.cpp | 24 +++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/Animations/src/editor/panels/RegexTester.cpp b/Animations/src/editor/panels/RegexTester.cpp index eb87e605..f493e5b8 100644 --- a/Animations/src/editor/panels/RegexTester.cpp +++ b/Animations/src/editor/panels/RegexTester.cpp @@ -111,12 +111,13 @@ namespace MathAnim if (searchRes >= 0) { + int maxEnd = 0; for (int i = 0; i < region->num_regs; i++) { - if (region->beg[0] >= 0 && region->end[0] >= region->beg[0]) + if (region->beg[i] >= 0 && region->end[i] >= region->beg[i]) { - int matchStart = region->beg[0]; - int matchEnd = region->end[0]; + int matchStart = region->beg[i]; + int matchEnd = region->end[i]; Match match = {}; match.text = std::string(targetStr + matchStart, matchEnd - matchStart); @@ -124,13 +125,20 @@ namespace MathAnim match.end = matchEnd + (int)(targetStr - stringToTestAgainst); matches.emplace_back(match); - targetStr += matchEnd; - if (matchEnd == matchStart) - { - targetStr++; - } + maxEnd = glm::max(matchEnd, maxEnd); } } + + targetStr += maxEnd; + if (maxEnd == 0) + { + targetStr++; + } + + if (multiline) + { + break; + } } else if (searchRes != ONIG_MISMATCH) { From 2ed287ab7dcb34892851b76faebd877f682d5cba Mon Sep 17 00:00:00 2001 From: Gabe Ambrosio Date: Sat, 21 Oct 2023 16:06:49 -0500 Subject: [PATCH 05/78] Add support for backreferences in end block captures --- Animations/include/parsers/Grammar.h | 17 +++- Animations/src/parsers/Grammar.cpp | 129 +++++++++++++++++++++++-- assets/customGrammars/lua.grammar.json | 41 ++++++++ 3 files changed, 178 insertions(+), 9 deletions(-) diff --git a/Animations/include/parsers/Grammar.h b/Animations/include/parsers/Grammar.h index 5b1e5df9..a60e8e62 100644 --- a/Animations/include/parsers/Grammar.h +++ b/Animations/include/parsers/Grammar.h @@ -51,6 +51,21 @@ namespace MathAnim static CaptureList from(const nlohmann::json& j, Grammar* self); }; + struct DynamicRegexCapture + { + size_t captureIndex; + size_t strReplaceStart; + size_t strReplaceEnd; + }; + + struct DynamicRegex + { + bool isDynamic; + OnigRegex simpleRegex; + std::string regexText; + std::vector backrefs; + }; + struct SimpleSyntaxPattern { std::optional scope; @@ -66,7 +81,7 @@ namespace MathAnim { std::optional scope; OnigRegex begin; - OnigRegex end; + DynamicRegex end; std::optional beginCaptures; std::optional endCaptures; std::optional patterns; diff --git a/Animations/src/parsers/Grammar.cpp b/Animations/src/parsers/Grammar.cpp index f834b406..363457f9 100644 --- a/Animations/src/parsers/Grammar.cpp +++ b/Animations/src/parsers/Grammar.cpp @@ -94,7 +94,7 @@ namespace MathAnim { std::optional match = getFirstMatch(str, anchor, start, end, this->regMatch, region, this->scope); if (match.has_value() && match->start < end && match->start >= start && match->end <= end) - { + { // `region` now contains any potential captures, so we'll search for any captures now std::vector subMatches = getCaptures(str, repo, region, this->captures, self); @@ -143,8 +143,42 @@ namespace MathAnim // If beginBlockMatch is valid, then we'll use the result stored in `region` to find any captures std::vector beginMatches = getCaptures(str, repo, region, this->beginCaptures, self); + OnigRegex endPattern = this->end.simpleRegex; + if (this->end.isDynamic) + { + // Generate an on the fly pattern + std::string regexPatternToTest = this->end.regexText; + + // Substitute backrefs in the string + int offsetToAdd = 0; + for (auto& backref : this->end.backrefs) + { + g_logger_assert(backref.captureIndex < region->num_regs, "An invalid regex was created in the grammar. This was the regex: '{}'", this->end.regexText); + + int replacementStringBegin = region->beg[backref.captureIndex]; + int replacementStringEnd = region->end[backref.captureIndex]; + + g_logger_assert(replacementStringEnd > replacementStringBegin, "An invalid backreference was captured for pattern: '{}'", this->end.regexText); + + int replaceBeginOffset = (int)backref.strReplaceStart + offsetToAdd; + int replaceEndOffset = (int)backref.strReplaceEnd + offsetToAdd + 1; + + std::string beginRegex = regexPatternToTest.substr(0, replaceBeginOffset); + std::string endRegex = regexPatternToTest.substr(replaceEndOffset); + std::string replacement = str.substr(replacementStringBegin, (replacementStringEnd - replacementStringBegin)); + regexPatternToTest = beginRegex + replacement + endRegex; + + offsetToAdd += (replacementStringEnd - replacementStringBegin) - (replaceEndOffset - replaceBeginOffset); + } + + // After substituting backrefs generate an on-the-fly end pattern to match + endPattern = onigFromString(regexPatternToTest, true); + + g_logger_assert(endPattern != nullptr, "Failed to generate dynamic regex pattern with backreferences for pattern: '{}'", this->end.regexText); + } + // This match can go to the end of the string - std::optional endBlockMatch = getFirstMatch(str, beginBlockMatch->end, beginBlockMatch->end, str.length(), this->end, region, std::nullopt); + std::optional endBlockMatch = getFirstMatch(str, beginBlockMatch->end, beginBlockMatch->end, str.length(), endPattern, region, std::nullopt); std::vector endMatches = {}; if (!endBlockMatch.has_value()) { @@ -231,7 +265,7 @@ namespace MathAnim // a new end that satisfies this match if (inBetweenStart > endBlockMatch->start) { - endBlockMatch = getFirstMatch(str, inBetweenStart, inBetweenStart, str.length(), this->end, region, std::nullopt); + endBlockMatch = getFirstMatch(str, inBetweenStart, inBetweenStart, str.length(), endPattern, region, std::nullopt); if (!endBlockMatch.has_value()) { GrammarMatch eof; @@ -269,6 +303,12 @@ namespace MathAnim } } + // Free dynamic regex if necessary + if (this->end.isDynamic) + { + onig_free(endPattern); + } + res.subMatches.insert(res.subMatches.end(), endMatches.begin(), endMatches.end()); res.start = beginBlockMatch->start; @@ -294,9 +334,9 @@ namespace MathAnim onig_free(begin); } - if (end) + if (end.simpleRegex) { - onig_free(end); + onig_free(end.simpleRegex); } if (beginCaptures.has_value()) @@ -315,7 +355,7 @@ namespace MathAnim } begin = nullptr; - end = nullptr; + end.simpleRegex = nullptr; } bool PatternArray::match(const std::string& str, size_t anchor, size_t start, size_t end, const PatternRepository& repo, OnigRegion* region, std::vector* outMatches, Grammar const* self) const @@ -683,7 +723,7 @@ namespace MathAnim if (sizeLeft >= numBytesToRemove) { return false; - } + } g_logger_error("We have a buffer underflow. Please pass a larger buffer to the tree."); return true; @@ -1170,7 +1210,80 @@ namespace MathAnim ComplexSyntaxPattern c = {}; c.begin = onigFromString(json["begin"], false); - c.end = onigFromString(json["end"], true); + + if (!c.begin) + { + return res; + } + + // Check if this end pattern has a dynamic backreferences and pre-process it if it does + std::string const& endPattern = json["end"]; + { + int numBeginCaptures = onig_number_of_captures(c.begin); + DynamicRegexCapture capture = {}; + int digitStart = 0; + bool parsingDigit = false; + for (int i = 0; i < (int)endPattern.size(); i++) + { + if (i < (int)endPattern.size() - 1 && endPattern[i] == '\\' && Parser::isDigit(endPattern[i + 1])) + { + c.end.isDynamic = true; + digitStart = i + 1; + parsingDigit = true; + } + else if (parsingDigit && !Parser::isDigit(endPattern[i])) + { + std::string substr = endPattern.substr(digitStart, i - digitStart); + capture.captureIndex = std::atoi(substr.c_str()); + + if (capture.captureIndex < 0 || capture.captureIndex > numBeginCaptures) + { + g_logger_error("Invalid backreference in regex '{}'. Cannot backreference '{}', only '{}' captures in the begin block.", endPattern, capture.captureIndex, numBeginCaptures); + onig_free(c.begin); + return res; + } + + capture.strReplaceStart = digitStart - 1; + capture.strReplaceEnd = i - 1; + + c.end.backrefs.push_back(capture); + parsingDigit = false; + } + else if (parsingDigit && i == (int)endPattern.size() - 1) + { + std::string substr = endPattern.substr(digitStart, i - digitStart + 1); + capture.captureIndex = std::atoi(substr.c_str()); + + if (capture.captureIndex < 0 || capture.captureIndex > numBeginCaptures) + { + g_logger_error("Invalid backreference in regex '{}'. Cannot backreference '{}', only '{}' captures in the begin block.", endPattern, capture.captureIndex, numBeginCaptures); + onig_free(c.begin); + return res; + } + + capture.strReplaceStart = digitStart - 1; + capture.strReplaceEnd = i; + + c.end.backrefs.push_back(capture); + parsingDigit = false; + } + } + } + + if (!c.end.isDynamic) + { + c.end.simpleRegex = onigFromString(json["end"], true); + + if (!c.end.simpleRegex) + { + onig_free(c.begin); + return res; + } + } + else + { + c.end.regexText = json["end"]; + } if (json.contains("name")) { diff --git a/assets/customGrammars/lua.grammar.json b/assets/customGrammars/lua.grammar.json index f4924271..b46723c6 100644 --- a/assets/customGrammars/lua.grammar.json +++ b/assets/customGrammars/lua.grammar.json @@ -318,6 +318,47 @@ } }, "name": "string.quoted.double.lua" + }, + { + "begin": "(?<=\\.cdef)\\s*(\\[(=*)\\[)", + "beginCaptures": { + "0": { + "name": "string.quoted.other.multiline.lua" + }, + "1": { + "name": "punctuation.definition.string.begin.lua" + } + }, + "contentName": "meta.embedded.lua", + "end": "(\\]\\2\\])[ \\t]*", + "endCaptures": { + "0": { + "name": "string.quoted.other.multiline.lua" + }, + "1": { + "name": "punctuation.definition.string.end.lua" + } + }, + "patterns": [ + { + "include": "source.c" + } + ] + }, + { + "begin": "(?