feat: Add a pluggable text codec for string patterns

lib/include/pl/core/evaluator.hpp

@@ -13,6 +13,7 @@
 
 #include <pl/core/log_console.hpp>
 #include <pl/core/token.hpp>
+#include <pl/core/string_encode_decode.hpp>
 #include <pl/api.hpp>
 
 #include <pl/core/errors/runtime_errors.hpp>
@@ -207,6 +208,41 @@ namespace pl::core {
 
         void setDataSource(u64 baseAddress, size_t dataSize, std::function<void(u64, u8*, size_t)> readerFunction, std::optional<std::function<void(u64, const u8*, size_t)>> writerFunction = std::nullopt);
 
+        /**
+         * @brief Sets the text codec for string patterns
+         * @param codec Codec to set. Pass nullptr to go back to raw bytes.
+         */
+        void setStringEncodeDecode(std::shared_ptr<StringEncodeDecode> codec) {
+            this->m_stringEncodeDecode = std::move(codec);
+        }
+
+        /**
+         * @brief Gets the text codec set for string patterns
+         * @return The codec, or nullptr when none is set
+         */
+        [[nodiscard]] const std::shared_ptr<StringEncodeDecode>& getStringEncodeDecode() const {
+            return this->m_stringEncodeDecode;
+        }
+
+        /**
+         * @brief Sets the default encoding a string pattern with no [[encoding]]
+         * attribute of its own resolves to
+         * @param encoding Encoding name to set. Empty lets the codec pick its own
+         * default.
+         */
+        void setDefaultEncoding(std::string encoding) {
+            this->m_defaultEncoding = std::move(encoding);
+        }
+
+        /**
+         * @brief Gets the default encoding a string pattern with no [[encoding]]
+         * attribute of its own resolves to
+         * @return The encoding name, empty when none is set
+         */
+        [[nodiscard]] const std::string& getDefaultEncoding() const {
+            return this->m_defaultEncoding;
+        }
+
         void setDataBaseAddress(u64 baseAddress) {
             this->m_dataBaseAddress = baseAddress;
         }
@@ -556,6 +592,8 @@ namespace pl::core {
         std::vector<std::unique_ptr<ast::ASTNode>> m_currentTemplateArguments;
 
         std::function<bool()> m_dangerousFunctionCalledCallback = []{ return false; };
+        std::shared_ptr<StringEncodeDecode> m_stringEncodeDecode;
+        std::string m_defaultEncoding;
         std::function<void()> m_breakpointHitCallback = []{ };
         std::atomic<DangerousFunctionPermission> m_allowDangerousFunctions = DangerousFunctionPermission::Ask;
         ControlFlowStatement m_currControlFlowStatement = ControlFlowStatement::None;

lib/include/pl/core/string_encode_decode.hpp

@@ -0,0 +1,53 @@
+#pragma once
+
+#include <optional>
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <pl/helpers/types.hpp>
+
+namespace pl::core {
+
+    /**
+     * @brief A pluggable text codec for string patterns
+     * @note The host application sets this on the Evaluator. With none set, string
+     * patterns keep their old raw-byte behavior.
+     */
+    class StringEncodeDecode {
+    public:
+        virtual ~StringEncodeDecode() = default;
+
+        /**
+         * @brief Decodes `bytes` under `encoding`
+         * @param bytes Bytes to decode
+         * @param encoding Encoding to decode with; empty when the pattern names no
+         * encoding, in which case the codec picks its own default
+         * @return The decoded text, or std::nullopt when `bytes` is not valid under
+         * `encoding`
+         */
+        [[nodiscard]] virtual std::optional<std::string> decode(std::span<const u8> bytes, std::string_view encoding) const = 0;
+
+        /**
+         * @brief Encodes `text` under `encoding`
+         * @param text Text to encode
+         * @param encoding Encoding to encode with; empty when the pattern names no
+         * encoding, in which case the codec picks its own default
+         * @return The encoded bytes, or std::nullopt when `text` is not representable
+         * under `encoding`
+         */
+        [[nodiscard]] virtual std::optional<std::vector<u8>> encode(std::string_view text, std::string_view encoding) const = 0;
+
+        /**
+         * @brief Encodes `text` under `encoding`, never failing
+         * @param text Text to encode
+         * @param encoding Encoding to encode with; empty when the pattern names no
+         * encoding, in which case the codec picks its own default
+         * @return The encoded bytes, substituting a replacement for anything `encoding`
+         * cannot represent
+         */
+        [[nodiscard]] virtual std::vector<u8> encodeLossy(std::string_view text, std::string_view encoding) const = 0;
+    };
+
+}

lib/include/pl/pattern_language.hpp

@@ -294,6 +294,14 @@ namespace pl {
          */
         void reset();
 
+        /**
+         * @brief Clears every placed pattern's cached display value, across every section.
+         * Call this after changing something a pattern's formatted value depends on
+         * without re-running the pattern, such as the host application's declared
+         * string encoding.
+         */
+        void clearFormatCaches();
+
         /**
          * @brief Checks whether the runtime is currently running
          * @return True if the runtime is running, false otherwise

lib/include/pl/patterns/pattern.hpp

@@ -484,6 +484,7 @@ namespace pl::ptrn {
             if (!result.empty()) {
                 this->getEvaluator()->writeData(this->getOffset(), result.data(), result.size(), this->getSection());
                 this->clearFormatCache();
+                this->clearByteCache();
             }
         }
 

lib/include/pl/patterns/pattern_string.hpp

@@ -3,6 +3,7 @@
 #include <pl/patterns/pattern.hpp>
 
 #include <pl/patterns/pattern_character.hpp>
+#include <pl/core/errors/runtime_errors.hpp>
 
 namespace pl::ptrn {
 
@@ -32,23 +33,84 @@ namespace pl::ptrn {
 
         }
 
+        /**
+         * @brief Gets this string's own encoding
+         * @return This pattern's own [[encoding]] attribute, the evaluator's default
+         * encoding if none, or empty when neither is set. An empty result falls back
+         * to the codec's own default.
+         */
+        [[nodiscard]] std::string getEncodingName() const {
+            if (const auto &arguments = this->getAttributeArguments("encoding"); !arguments.empty())
+                return arguments[0].toString(true);
+            return this->getEvaluator()->getDefaultEncoding();
+        }
+
         std::string getValue(size_t size) const {
             if (size == 0)
                 return "";
 
+            auto *evaluator = this->getEvaluator();
+
             std::string buffer(size, '\x00');
-            this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection());
+            evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection());
+
+            if (const auto &codec = evaluator->getStringEncodeDecode(); codec != nullptr) {
+                const auto encoding = this->getEncodingName();
+                auto decoded = codec->decode({ reinterpret_cast<const u8*>(buffer.data()), buffer.size() }, encoding);
+                if (!decoded.has_value())
+                    core::err::E0004.throwError(fmt::format("invalid byte sequence for encoding '{}'", encoding));
+
+                return *decoded;
+            }
 
             return buffer;
         }
 
         std::vector<u8> getBytesOf(const core::Token::Literal &value) const override {
-            if (auto stringValue = std::get_if<std::string>(&value); stringValue != nullptr)
-                return { stringValue->begin(), stringValue->end() };
-            else
+            if (auto stringValue = std::get_if<std::string>(&value); stringValue != nullptr) {
+                std::vector<u8> bytes;
+
+                if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr) {
+                    const auto encoding = this->getEncodingName();
+                    auto encoded = codec->encode(*stringValue, encoding);
+                    if (!encoded.has_value())
+                        core::err::E0004.throwError(fmt::format("text has no byte value in encoding '{}'", encoding));
+
+                    bytes = *encoded;
+                } else {
+                    bytes = { stringValue->begin(), stringValue->end() };
+                }
+
+                // This field owns a fixed number of bytes in the file. A longer write would
+                // overwrite whatever comes right after it. Truncate or pad with NUL to the
+                // field's own size, the same contract every other pattern type already has.
+                bytes.resize(this->getSize());
+                return bytes;
+            } else
                 return { };
         }
 
+        /**
+         * @brief Force-writes `value`, substituting a replacement character for
+         * anything the pattern's encoding cannot represent
+         * @param value Value to write
+         * @note setValue() rejects such a value instead; an editor that offers an
+         * explicit lossy override calls this one directly.
+         */
+        void setValueLossy(const std::string &value) {
+            std::vector<u8> bytes;
+
+            if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr)
+                bytes = codec->encodeLossy(value, this->getEncodingName());
+            else
+                bytes = { value.begin(), value.end() };
+
+            bytes.resize(this->getSize());
+            this->getEvaluator()->writeData(this->getOffset(), bytes.data(), bytes.size(), this->getSection());
+            this->clearFormatCache();
+            this->clearByteCache();
+        }
+
         [[nodiscard]] std::string getFormattedName() const override {
             return "String";
         }
@@ -67,23 +129,72 @@ namespace pl::ptrn {
         }
 
         std::string formatDisplayValue() override {
-            auto size = std::min<size_t>(this->getSize(), 0x7F);
+            auto *evaluator = this->getEvaluator();
+            const auto fullSize = this->getSize();
+
+            // Read a little past DisplayBudget, so a multi-byte codepoint at the
+            // cutoff usually has the bytes to decode whole. The decode below still
+            // backs off further on failure; this only keeps that the common case.
+            constexpr size_t DisplayBudget = 0x7F;
+            auto size = std::min<size_t>(fullSize, DisplayBudget + 8);
 
             if (size == 0)
                 return "\"\"";
 
             std::string buffer(size, 0x00);
-            this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection());
-
-            const auto pos = buffer.find_last_not_of('\x00');
-            if (pos == std::string::npos)
-                return "\"\"";
-
-            buffer.erase(pos + 1);
-
-            auto displayString = hlp::encodeByteString({ buffer.begin(), buffer.end() });
-
-            return Pattern::callUserFormatFunc(buffer).value_or(fmt::format("\"{0}\" {1}", displayString, size > this->getSize() ? "(truncated)" : ""));
+            evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection());
+
+            if (auto formatted = Pattern::callUserFormatFunc(buffer); formatted.has_value())
+                return *formatted;
+
+            const auto &codec = evaluator->getStringEncodeDecode();
+            bool truncated = size < fullSize;
+
+            // No codec configured keeps the old raw-byte display, unescaped by any
+            // encoding. A configured codec reports an undecodable buffer as invalid,
+            // rather than substituting a replacement character into the display.
+            if (codec == nullptr) {
+                if (buffer.size() > DisplayBudget) {
+                    buffer.resize(DisplayBudget);
+                    truncated = true;
+                }
+
+                const auto pos = buffer.find_last_not_of('\x00');
+                if (pos == std::string::npos)
+                    return "\"\"";
+                buffer.erase(pos + 1);
+
+                return fmt::format("\"{0}\" {1}", hlp::encodeByteString({ buffer.begin(), buffer.end() }), truncated ? "(truncated)" : "");
+            }
+
+            const auto encoding = this->getEncodingName();
+
+            // The read above can itself end mid-codepoint. decode() fails on the whole
+            // buffer in that case, not just its incomplete tail, so back off a few
+            // bytes and retry rather than reporting a merely-truncated read as invalid.
+            std::optional<std::string> decoded;
+            for (size_t backoff = 0; backoff <= std::min<size_t>(4, buffer.size()); ++backoff) {
+                decoded = codec->decode({ reinterpret_cast<const u8*>(buffer.data()), buffer.size() - backoff }, encoding);
+                if (decoded.has_value()) {
+                    if (backoff > 0)
+                        truncated = true;
+                    break;
+                }
+            }
+            if (!decoded.has_value())
+                core::err::E0004.throwError(fmt::format("invalid byte sequence for encoding '{}'", encoding));
+
+            // Trim the decoded text itself, not the raw bytes, so a multi-byte
+            // codepoint at the cutoff stays whole instead of splitting mid-sequence.
+            if (decoded->size() > DisplayBudget) {
+                auto cut = DisplayBudget;
+                while (cut > 0 && (static_cast<u8>((*decoded)[cut]) & 0xC0) == 0x80)
+                    --cut;
+                decoded->resize(cut);
+                truncated = true;
+            }
+
+            return fmt::format("\"{0}\" {1}", *decoded, truncated ? "(truncated)" : "");
         }
 
         std::shared_ptr<Pattern> getEntry(size_t index) const override {

lib/source/pl/lib/std/pragmas.cpp

@@ -38,6 +38,11 @@ namespace pl::lib::libstd {
                 return false;
         });
 
+        runtime.addPragma("encoding", [](pl::PatternLanguage &runtime, const std::string &value) {
+            runtime.getInternals().evaluator->setDefaultEncoding(value);
+            return true;
+        });
+
         runtime.addPragma("eval_depth", [](pl::PatternLanguage &runtime, const std::string &value) {
             auto limit = parseLimit(value);
             if (!limit.has_value())

lib/source/pl/pattern_language.cpp

@@ -533,6 +533,13 @@ namespace pl {
         this->m_parserManager.setPatternLanguage(this);
     }
 
+    void PatternLanguage::clearFormatCaches() {
+        for (const auto &[section, patterns] : this->m_patterns) {
+            for (const auto &pattern : patterns)
+                pattern->clearFormatCache();
+        }
+    }
+
     void PatternLanguage::addFunction(const api::Namespace &ns, const std::string &name, api::FunctionParameterCount parameterCount, const api::FunctionCallback &func) {
         this->m_functions.emplace_back(ns, name, parameterCount, func, false);
     }