From 992c20ed4e0263fb10b7744baf4a6649d03bb4a8 Mon Sep 17 00:00:00 2001 From: Scott Anderson <662325+scottanderson@users.noreply.github.com> Date: Tue, 1 Sep 2026 01:27:20 -0400 Subject: [PATCH 1/2] feat: Add a pluggable text codec for string patterns Add StringEncodeDecode: an interface with decode(), encode(), and encodeLossy(). The host application sets one on the Evaluator. With none set, a string pattern keeps its raw-byte behavior. decode() and encode() return std::optional. A nullopt result marks a byte sequence, or a character, the named encoding cannot represent. encodeLossy() never fails. It substitutes a replacement character for anything the encoding cannot represent. PatternString routes reads and writes through the codec. getValue() and getBytesOf() throw core::err::E0004 on a nullopt result. A script can catch this error with try/catch. setValueLossy() writes through encodeLossy(), and clears the cached bytes and the cached display string; getBytesOf() otherwise reflects a write only after the next pattern run. setValue() clears the cached bytes the same way, on every pattern type, not just PatternString. formatDisplayValue() decodes through decode(). It returns the raw decoded text. The host application escapes the text for display. getBytesOf() caps the encoded result to the pattern's own size. getEncodingName() reads the string's own encoding attribute. PatternWideString keeps its fixed UTF-16 behavior. The codec does not apply to it. Add PatternLanguage::clearFormatCaches(). It clears every placed pattern's cached display value, across every section. --- lib/include/pl/core/evaluator.hpp | 11 +++ lib/include/pl/core/string_encode_decode.hpp | 31 +++++++ lib/include/pl/pattern_language.hpp | 8 ++ lib/include/pl/patterns/pattern.hpp | 1 + lib/include/pl/patterns/pattern_string.hpp | 96 +++++++++++++++++--- lib/source/pl/pattern_language.cpp | 7 ++ 6 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 lib/include/pl/core/string_encode_decode.hpp diff --git a/lib/include/pl/core/evaluator.hpp b/lib/include/pl/core/evaluator.hpp index 8d383757..4805b4b5 100644 --- a/lib/include/pl/core/evaluator.hpp +++ b/lib/include/pl/core/evaluator.hpp @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -207,6 +208,15 @@ namespace pl::core { void setDataSource(u64 baseAddress, size_t dataSize, std::function readerFunction, std::optional> writerFunction = std::nullopt); + // Sets the text codec for string patterns. Pass nullptr to go back to raw bytes. + void setStringEncodeDecode(std::shared_ptr codec) { + this->m_stringEncodeDecode = std::move(codec); + } + + [[nodiscard]] const std::shared_ptr& getStringEncodeDecode() const { + return this->m_stringEncodeDecode; + } + void setDataBaseAddress(u64 baseAddress) { this->m_dataBaseAddress = baseAddress; } @@ -556,6 +566,7 @@ namespace pl::core { std::vector> m_currentTemplateArguments; std::function m_dangerousFunctionCalledCallback = []{ return false; }; + std::shared_ptr m_stringEncodeDecode; std::function m_breakpointHitCallback = []{ }; std::atomic m_allowDangerousFunctions = DangerousFunctionPermission::Ask; ControlFlowStatement m_currControlFlowStatement = ControlFlowStatement::None; diff --git a/lib/include/pl/core/string_encode_decode.hpp b/lib/include/pl/core/string_encode_decode.hpp new file mode 100644 index 00000000..8c8e8fe5 --- /dev/null +++ b/lib/include/pl/core/string_encode_decode.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace pl::core { + + // A pluggable text codec for string patterns. The host application sets this on the + // Evaluator. With none set, string patterns keep their old raw-byte behavior. + // + // decode() and encode() are fallible: std::nullopt when `bytes`/`text` is not + // valid or representable under the named encoding. encodeLossy() never fails, + // substituting a replacement for anything it cannot represent. + class StringEncodeDecode { + public: + virtual ~StringEncodeDecode() = default; + + // `encoding` is empty when the pattern names no encoding; the codec picks + // its own default then. + [[nodiscard]] virtual std::optional decode(std::span bytes, std::string_view encoding) const = 0; + [[nodiscard]] virtual std::optional> encode(std::string_view text, std::string_view encoding) const = 0; + + [[nodiscard]] virtual std::vector encodeLossy(std::string_view text, std::string_view encoding) const = 0; + }; + +} diff --git a/lib/include/pl/pattern_language.hpp b/lib/include/pl/pattern_language.hpp index 2cffb219..ee36c4d4 100644 --- a/lib/include/pl/pattern_language.hpp +++ b/lib/include/pl/pattern_language.hpp @@ -294,6 +294,14 @@ namespace pl { */ void reset(); + /** + * @brief Clears every placed pattern's cached display value, across every section. + * Call this after changing something a pattern's formatted value depends on + * without re-running the pattern, such as the host application's declared + * string encoding. + */ + void clearFormatCaches(); + /** * @brief Checks whether the runtime is currently running * @return True if the runtime is running, false otherwise diff --git a/lib/include/pl/patterns/pattern.hpp b/lib/include/pl/patterns/pattern.hpp index 8fdeff5b..4710bbf7 100644 --- a/lib/include/pl/patterns/pattern.hpp +++ b/lib/include/pl/patterns/pattern.hpp @@ -484,6 +484,7 @@ namespace pl::ptrn { if (!result.empty()) { this->getEvaluator()->writeData(this->getOffset(), result.data(), result.size(), this->getSection()); this->clearFormatCache(); + this->clearByteCache(); } } diff --git a/lib/include/pl/patterns/pattern_string.hpp b/lib/include/pl/patterns/pattern_string.hpp index 1d42d350..f499deab 100644 --- a/lib/include/pl/patterns/pattern_string.hpp +++ b/lib/include/pl/patterns/pattern_string.hpp @@ -3,6 +3,9 @@ #include #include +#include + +#include namespace pl::ptrn { @@ -32,23 +35,76 @@ namespace pl::ptrn { } + // Empty when this pattern names no encoding. The codec then falls back to its own + // default. + [[nodiscard]] std::string getEncodingName() const { + if (const auto &arguments = this->getAttributeArguments("encoding"); !arguments.empty()) + return arguments[0].toString(true); + return ""; + } + std::string getValue(size_t size) const { if (size == 0) return ""; + auto *evaluator = this->getEvaluator(); + std::string buffer(size, '\x00'); - this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection()); + evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection()); + + if (const auto &codec = evaluator->getStringEncodeDecode(); codec != nullptr) { + const auto encoding = this->getEncodingName(); + auto decoded = codec->decode({ reinterpret_cast(buffer.data()), buffer.size() }, encoding); + if (!decoded.has_value()) + core::err::E0004.throwError(fmt::format("invalid byte sequence for encoding '{}'", encoding)); + + return *decoded; + } return buffer; } std::vector getBytesOf(const core::Token::Literal &value) const override { - if (auto stringValue = std::get_if(&value); stringValue != nullptr) - return { stringValue->begin(), stringValue->end() }; - else + if (auto stringValue = std::get_if(&value); stringValue != nullptr) { + std::vector bytes; + + if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr) { + const auto encoding = this->getEncodingName(); + auto encoded = codec->encode(*stringValue, encoding); + if (!encoded.has_value()) + core::err::E0004.throwError(fmt::format("text has no byte value in encoding '{}'", encoding)); + + bytes = *encoded; + } else { + bytes = { stringValue->begin(), stringValue->end() }; + } + + // This field owns a fixed number of bytes in the file. A longer write would + // overwrite whatever comes right after it. Truncate or pad with NUL to the + // field's own size, the same contract every other pattern type already has. + bytes.resize(this->getSize()); + return bytes; + } else return { }; } + // Force-writes `value`, substituting a replacement character for anything the + // pattern's encoding cannot represent. setValue() rejects such a value instead; + // an editor that offers an explicit lossy override calls this one directly. + void setValueLossy(const std::string &value) { + std::vector bytes; + + if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr) + bytes = codec->encodeLossy(value, this->getEncodingName()); + else + bytes = { value.begin(), value.end() }; + + bytes.resize(this->getSize()); + this->getEvaluator()->writeData(this->getOffset(), bytes.data(), bytes.size(), this->getSection()); + this->clearFormatCache(); + this->clearByteCache(); + } + [[nodiscard]] std::string getFormattedName() const override { return "String"; } @@ -67,23 +123,39 @@ namespace pl::ptrn { } std::string formatDisplayValue() override { - auto size = std::min(this->getSize(), 0x7F); + auto *evaluator = this->getEvaluator(); + const auto fullSize = this->getSize(); + auto size = std::min(fullSize, 0x7F); if (size == 0) return "\"\""; std::string buffer(size, 0x00); - this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection()); + evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection()); - const auto pos = buffer.find_last_not_of('\x00'); - if (pos == std::string::npos) - return "\"\""; + if (auto formatted = Pattern::callUserFormatFunc(buffer); formatted.has_value()) + return *formatted; + + const auto &codec = evaluator->getStringEncodeDecode(); + const auto truncatedSuffix = size > fullSize ? "(truncated)" : ""; + + // No codec configured keeps the old raw-byte display, unescaped by any + // encoding. A configured codec reports an undecodable buffer as invalid, + // rather than substituting a replacement character into the display. + if (codec == nullptr) { + const auto pos = buffer.find_last_not_of('\x00'); + if (pos == std::string::npos) + return "\"\""; + buffer.erase(pos + 1); - buffer.erase(pos + 1); + return fmt::format("\"{0}\" {1}", hlp::encodeByteString({ buffer.begin(), buffer.end() }), truncatedSuffix); + } - auto displayString = hlp::encodeByteString({ buffer.begin(), buffer.end() }); + auto decoded = codec->decode({ reinterpret_cast(buffer.data()), buffer.size() }, this->getEncodingName()); + if (!decoded.has_value()) + throw std::runtime_error("Invalid"); - return Pattern::callUserFormatFunc(buffer).value_or(fmt::format("\"{0}\" {1}", displayString, size > this->getSize() ? "(truncated)" : "")); + return fmt::format("\"{0}\" {1}", *decoded, truncatedSuffix); } std::shared_ptr getEntry(size_t index) const override { diff --git a/lib/source/pl/pattern_language.cpp b/lib/source/pl/pattern_language.cpp index f161e264..0b638f66 100644 --- a/lib/source/pl/pattern_language.cpp +++ b/lib/source/pl/pattern_language.cpp @@ -533,6 +533,13 @@ namespace pl { this->m_parserManager.setPatternLanguage(this); } + void PatternLanguage::clearFormatCaches() { + for (const auto &[section, patterns] : this->m_patterns) { + for (const auto &pattern : patterns) + pattern->clearFormatCache(); + } + } + void PatternLanguage::addFunction(const api::Namespace &ns, const std::string &name, api::FunctionParameterCount parameterCount, const api::FunctionCallback &func) { this->m_functions.emplace_back(ns, name, parameterCount, func, false); } From 9cf297031b40652080a1ed508f307f76864e57f1 Mon Sep 17 00:00:00 2001 From: Scott Anderson <662325+scottanderson@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:10:22 -0400 Subject: [PATCH 2/2] feat: Publish a per-run snapshot of string pattern encodings Add StringEncodingRegion: the address, size, and encoding of one string pattern. PatternLanguage builds a sorted snapshot of these after every successful run. It publishes the snapshot through an atomic shared_ptr. getStringEncodingRegions() returns the full snapshot. findStringEncoding() looks up one exact address and size. Both functions are safe to call from any thread. --- .../pl/core/string_encoding_region.hpp | 18 ++++++ lib/include/pl/pattern_language.hpp | 20 +++++++ lib/source/pl/pattern_language.cpp | 58 +++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 lib/include/pl/core/string_encoding_region.hpp diff --git a/lib/include/pl/core/string_encoding_region.hpp b/lib/include/pl/core/string_encoding_region.hpp new file mode 100644 index 00000000..d8dfc8df --- /dev/null +++ b/lib/include/pl/core/string_encoding_region.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include + +namespace pl::core { + + // The encoding a string pattern resolved to on the last successful run. Plain data, not a + // reference to the pattern. See PatternLanguage::getStringEncodingRegions(). + struct StringEncodingRegion { + u64 section; + u64 address; + u64 size; + std::string encoding; + }; + +} diff --git a/lib/include/pl/pattern_language.hpp b/lib/include/pl/pattern_language.hpp index ee36c4d4..14eaa4b7 100644 --- a/lib/include/pl/pattern_language.hpp +++ b/lib/include/pl/pattern_language.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -280,6 +281,22 @@ namespace pl { */ [[nodiscard]] std::vector getPatternsAtAddress(u64 address, u64 section = 0x00) const; + /** + * @brief Gets a snapshot of what encoding every string pattern resolved to on the last + * successful run. Unlike the pattern tree itself, this is safe to read from any thread. + * @return The snapshot. Never null; empty before the first successful run. + */ + [[nodiscard]] std::shared_ptr> getStringEncodingRegions() const; + + /** + * @brief Looks up the encoding a string pattern at an exact address and size resolved to + * @param address Address to check + * @param size Size to check + * @param section Section id + * @return The encoding name, or std::nullopt if no string pattern matches exactly + */ + [[nodiscard]] std::optional findStringEncoding(u64 address, u64 size, u64 section = 0x00) const; + /** * @brief Get the colors of all patterns that overlap with the given address @@ -421,6 +438,7 @@ namespace pl { private: void flattenPatterns(); + void buildStringEncodingRegions(); private: Internals m_internals; @@ -438,6 +456,8 @@ namespace pl { std::atomic m_flattenedPatternsValid = false; std::map> m_flattenedPatterns; std::thread m_flattenThread; + std::atomic>> m_stringEncodingRegions + = std::make_shared>(); std::vector> m_cleanupCallbacks; std::vector> m_currAST; diff --git a/lib/source/pl/pattern_language.cpp b/lib/source/pl/pattern_language.cpp index 0b638f66..ed5d68ef 100644 --- a/lib/source/pl/pattern_language.cpp +++ b/lib/source/pl/pattern_language.cpp @@ -11,6 +11,7 @@ #include #include +#include #include @@ -18,6 +19,9 @@ #include #include +#include +#include + namespace pl { static std::string getFunctionName(const api::Namespace &ns, const std::string &name) { @@ -81,6 +85,7 @@ namespace pl { this->m_patterns = std::move(other.m_patterns); this->m_flattenedPatterns = std::move(other.m_flattenedPatterns); + this->m_stringEncodingRegions.store(other.m_stringEncodingRegions.load()); this->m_cleanupCallbacks = std::move(other.m_cleanupCallbacks); this->m_currAST = std::move(other.m_currAST); @@ -314,6 +319,7 @@ namespace pl { this->reset(); } else { this->flattenPatterns(); + this->buildStringEncodingRegions(); this->m_patternsValid = true; } @@ -501,6 +507,7 @@ namespace pl { this->m_patterns.clear(); this->m_flattenedPatterns.clear(); this->m_flattenedPatternsValid = false; + this->m_stringEncodingRegions.store(std::make_shared>()); this->m_currError.reset(); this->m_compileErrors.clear(); @@ -594,6 +601,57 @@ namespace pl { } } + void PatternLanguage::buildStringEncodingRegions() { + auto regions = std::make_shared>(); + + std::function walk = [&](ptrn::Pattern *pattern) { + if (this->m_aborted) + return; + + if (auto string = dynamic_cast(pattern)) { + if (auto encoding = string->getEncodingName(); !encoding.empty()) + regions->push_back({ pattern->getSection(), pattern->getOffset(), pattern->getSize(), std::move(encoding) }); + } + + if (auto iterable = dynamic_cast(pattern)) { + for (const auto &child : iterable->getEntries()) + walk(child.get()); + } + }; + + for (const auto &[section, patterns] : this->m_patterns) { + for (const auto &pattern : patterns) { + if (this->m_aborted) + return; + + walk(pattern.get()); + } + } + + std::ranges::sort(*regions, {}, [](const core::StringEncodingRegion ®ion) { + return std::pair{ region.section, region.address }; + }); + + this->m_stringEncodingRegions.store(std::move(regions)); + } + + std::shared_ptr> PatternLanguage::getStringEncodingRegions() const { + return this->m_stringEncodingRegions.load(); + } + + std::optional PatternLanguage::findStringEncoding(u64 address, u64 size, u64 section) const { + auto regions = this->getStringEncodingRegions(); + + auto it = std::ranges::lower_bound(*regions, std::pair{ section, address }, {}, [](const core::StringEncodingRegion ®ion) { + return std::pair{ region.section, region.address }; + }); + + if (it != regions->end() && it->section == section && it->address == address && it->size == size) + return it->encoding; + + return std::nullopt; + } + std::vector PatternLanguage::getPatternsAtAddress(u64 address, u64 section) const { if (this->m_flattenedPatterns.empty() || !this->m_flattenedPatterns.contains(section)) return { };