diff --git a/CMakeLists.txt b/CMakeLists.txt index 85efe3f4..21bb1001 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,10 +134,10 @@ CPMAddPackage( EXCLUDE_FROM_ALL FALSE ) CPMAddPackage( - NAME numeric + NAME ae-numeric GIT_REPOSITORY "https://github.com/aethernetio/aethernet-numeric.git" - GIT_TAG "main" - OPTIONS "AE_NUMERIC_INSTALL ${AE_INSTALL}" "AE_BUILD_TESTS ${AE_BUILD_TESTS}" + GIT_TAG "68fcbb3d7975adddf2be5fae4c8c22ef725da5e7" + OPTIONS "AE_NUMERIC_INSTALL ${AE_INSTALL}" "AE_BUILD_TESTS OFF" EXCLUDE_FROM_ALL FALSE ) CPMAddPackage( @@ -150,7 +150,7 @@ CPMAddPackage( CPMAddPackage( NAME aether-tele GIT_REPOSITORY "https://github.com/aethernetio/aether-tele.git" - GIT_TAG "main" + GIT_TAG "7424eb5bdd49e78bd73aa1e453594d3660c111c4" OPTIONS "AE_TELE_INSTALL ${AE_INSTALL}" "AE_TELE_BUILD_TESTS ${AE_BUILD_TESTS}" EXCLUDE_FROM_ALL FALSE @@ -188,7 +188,7 @@ target_link_libraries(${TARGET_NAME} PUBLIC gcem etl stdexec - numeric + ae-numeric aether::miscpp) message(STATUS "Aether build for CMAKE_SYSTEM_NAME: ${CMAKE_SYSTEM_NAME}") diff --git a/aether/ae_actions/telemetry.cpp b/aether/ae_actions/telemetry.cpp index c2e26cad..879c577a 100644 --- a/aether/ae_actions/telemetry.cpp +++ b/aether/ae_actions/telemetry.cpp @@ -24,7 +24,7 @@ # include "aether/aether.h" # include "aether/tele_statistics.h" -# include "aether/tiered_int_serializer.h" // IWYU pragma: keep +# include "aether/types/packed_size.h" // IWYU pragma: keep # include "aether/ae_actions/ae_actions_tele.h" diff --git a/aether/api_protocol/api_message.h b/aether/api_protocol/api_message.h index ab5dee9e..f778e08d 100644 --- a/aether/api_protocol/api_message.h +++ b/aether/api_protocol/api_message.h @@ -22,18 +22,15 @@ #include #include -#include - #include "aether-miscpp/serialization/serialization.h" +#include "aether/types/packed_size.h" #include "aether/vector_buffer.h" namespace ae { using MessageId = std::uint8_t; -using PackedSize = TieredInt; - class ApiParser; class ApiPacker; diff --git a/aether/obj/domain.h b/aether/obj/domain.h index dd989da2..e85e4255 100644 --- a/aether/obj/domain.h +++ b/aether/obj/domain.h @@ -31,7 +31,7 @@ #include "aether-miscpp/serialization/binary_archive.h" #include "aether-miscpp/serialization/serialization.h" #include "aether/ptr/ptr_view.h" -#include "aether/tiered_int_serializer.h" +#include "aether/types/packed_size.h" #include "aether/obj/idomain_storage.h" #include "aether/obj/obj_id.h" diff --git a/aether/server_keys.h b/aether/server_keys.h index 398c28b8..f98e78b6 100644 --- a/aether/server_keys.h +++ b/aether/server_keys.h @@ -20,8 +20,6 @@ #include #include -#include - #include "aether/crypto/crypto_nonce.h" #include "aether/crypto/key.h" #include "aether/types/server_id.h" diff --git a/aether/stream_api/sized_packet_gate.cpp b/aether/stream_api/sized_packet_gate.cpp index 117398da..132acb62 100644 --- a/aether/stream_api/sized_packet_gate.cpp +++ b/aether/stream_api/sized_packet_gate.cpp @@ -23,7 +23,7 @@ namespace ae { static constexpr std::size_t kSizedPacketOverhead = - sizeof(PacketSize::ValueType); // max for packet size + PacketSize::kMaxWireBytes; // max for packet size DataBuffer SizedPacketGate::WriteIn(DataBuffer&& buffer) { DataBuffer write_buffer; diff --git a/aether/stream_api/stream_api.cpp b/aether/stream_api/stream_api.cpp index 85aed297..5ff2a370 100644 --- a/aether/stream_api/stream_api.cpp +++ b/aether/stream_api/stream_api.cpp @@ -18,6 +18,8 @@ #include +#include "aether/types/packed_size.h" + namespace ae { StreamApiImpl::StreamApiImpl(ProtocolContext& protocol_context) : ApiClass{protocol_context}, stream{protocol_context} {} @@ -46,8 +48,7 @@ std::uint8_t StreamIdGenerator::GetNextServerStreamId() { static constexpr std::size_t kStreamMessageOverhead = 1 + 1 + - sizeof( - PackedSize::ValueType); // message code + stream id + child data size + PackedSize::kMaxWireBytes; // message code + stream id + child data size StreamApiGate::StreamApiGate(StreamApiImpl& stream_api, StreamId stream_id) : stream_id_{stream_id}, diff --git a/aether/tele_statistics.h b/aether/tele_statistics.h index 8b2e8d50..fe91b125 100644 --- a/aether/tele_statistics.h +++ b/aether/tele_statistics.h @@ -24,6 +24,7 @@ #include "aether/config.h" #include "aether/obj/obj.h" +#include "aether/types/packed_size.h" namespace ae { class TeleStatistics : public Obj { diff --git a/aether/tiered_int_serializer.h b/aether/tiered_int_serializer.h deleted file mode 100644 index eef3092d..00000000 --- a/aether/tiered_int_serializer.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2026 Aethernet Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef AETHER_TIERED_INT_SERIALIZER_H_ -#define AETHER_TIERED_INT_SERIALIZER_H_ - -#include "numeric/tiered_int.h" - -#include "aether-miscpp/serialization/binary_archive.h" - -namespace ae::seri { - -template -struct TIntWriter { - template - requires(std::is_integral_v) - TIntWriter& operator<<(T const& v) { - res = buffer.Write(seri::DataTag{v}); - return *this; - } - - B& buffer; - SeriResult res{Ok{seri::good}}; -}; - -template -struct TIntReader { - template - requires(std::is_integral_v) - TIntReader& operator>>(T& v) { - res = buffer.Read(seri::DataTag{v}); - return *this; - } - - B& buffer; - SeriResult res{Ok{seri::good}}; -}; - -template -struct Serializer, TieredInt> { - using Archive = BinaryArchive; - using TInt = TieredInt; - - SeriResult Seri(Archive& archive, Meta meta) const { - auto writer = TIntWriter{.buffer = archive.buffer()}; - meta.value.Serialize(writer); - return writer.res; - } - - SeriResult Deseri(Archive& archive, Meta meta) const { - auto reader = TIntReader{.buffer = archive.buffer()}; - auto r = meta.value.Deserialize(reader); - if (r != TierDeserializeRes::kFinished) { - return Error{read_eof}; - } - return reader.res; - } -}; -} // namespace ae::seri - -#endif // AETHER_TIERED_INT_SERIALIZER_H_ diff --git a/aether/transport/data_packet_collector.cpp b/aether/transport/data_packet_collector.cpp index 15c135ca..ec3f9a3c 100644 --- a/aether/transport/data_packet_collector.cpp +++ b/aether/transport/data_packet_collector.cpp @@ -74,10 +74,9 @@ std::pair StreamDataPacketCollector::GetPacketSize( std::uint8_t const* data, std::size_t size) { auto temp_buffer_size = temp_data_buffer_.size(); - // use no more than packet size may contain - auto use_max_size = sizeof(PacketSize::ValueType) < size - ? sizeof(PacketSize::ValueType) - : size; + // use no more than the max serialized packet-size header + auto use_max_size = + PacketSize::kMaxWireBytes < size ? PacketSize::kMaxWireBytes : size; temp_data_buffer_.insert(temp_data_buffer_.end(), data, data + use_max_size); diff --git a/aether/transport/data_packet_collector.h b/aether/transport/data_packet_collector.h index 17bdb41f..7d570021 100644 --- a/aether/transport/data_packet_collector.h +++ b/aether/transport/data_packet_collector.h @@ -24,14 +24,11 @@ #include #include -#include - +#include "aether/types/packed_size.h" #include "aether/types/data_buffer.h" namespace ae { -using PacketSize = TieredInt; - struct Packet { explicit Packet(std::size_t expected_size); Packet(Packet const&) = delete; diff --git a/aether/types/packed_size.h b/aether/types/packed_size.h new file mode 100644 index 00000000..90153c28 --- /dev/null +++ b/aether/types/packed_size.h @@ -0,0 +1,89 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_TYPES_PACKED_SIZE_H_ +#define AETHER_TYPES_PACKED_SIZE_H_ + +#include +#include + +#include "ae-numeric/tiered_int.h" +#include "ae-numeric/wire_io.h" + +#include "aether-miscpp/serialization/binary_archive.h" + +namespace ae { + +// Canonical legacy-compatible wire size encoding used by packet and message +// framing. Byte-for-byte equivalent of the old +// TieredInt. +using PackedSize = TieredInt; + +// Transport framing uses the same wire encoding; keep the historical name as +// an alias so packet code stays readable without duplicating the type. +using PacketSize = PackedSize; + +} // namespace ae + +namespace ae::seri { + +// BinaryArchive adapter for ae-numeric TieredInt via wire_traits / +// Serialize / Deserialize. Truncated input maps to read_eof (no exceptions, +// no heap allocations). The same wire_io entry points can later back +// FixedPoint / Exponential serializers without changing BinaryArchive. +template +struct Serializer, TieredInt> { + using Archive = BinaryArchive; + using TInt = TieredInt; + + SeriResult Seri(Archive& archive, Meta meta) const { + std::uint8_t buf[MaxWireBytes()]; + std::size_t const n = ae::Serialize(meta.value, buf); + return archive.buffer().Write(DataWriteTag{buf, n}); + } + + SeriResult Deseri(Archive& archive, Meta meta) const { + std::uint8_t buf[MaxWireBytes()]{}; + std::size_t len = 0; + + // Grow one byte at a time until WireBytesNeeded reports a complete value. + // Avoids reading past this field into the next wire value. + while (len < MaxWireBytes()) { + auto const read_res = + archive.buffer().Read(DataReadTag{buf + len, std::size_t{1}}); + if (!read_res) { + return read_res; + } + ++len; + + if (TInt::WireBytesNeeded(buf, len) == 0) { + continue; + } + + auto const decoded = ae::Deserialize(buf, len); + if (decoded.bytes_read == 0 || decoded.bytes_read != len) { + return Error{read_eof}; + } + meta.value = decoded.value; + return Ok{good}; + } + return Error{read_eof}; + } +}; + +} // namespace ae::seri + +#endif // AETHER_TYPES_PACKED_SIZE_H_ diff --git a/aether/vector_buffer.h b/aether/vector_buffer.h index 2550cc9f..13baa9b1 100644 --- a/aether/vector_buffer.h +++ b/aether/vector_buffer.h @@ -17,27 +17,25 @@ #ifndef AETHER_VECTOR_BUFFER_H_ #define AETHER_VECTOR_BUFFER_H_ -#include #include #include #include -#include // IWYU pragma: keep #include "aether-miscpp/serialization/binary_archive.h" -#include "aether/tiered_int_serializer.h" +#include "aether/types/packed_size.h" namespace ae { -template +template struct VectorBuffer { - using PackedSize = TieredInt; + using PackedSize = SizeT; seri::SeriResult Write(seri::SizeWriteTag tag) { - auto v = PackedSize{tag.size}; - auto writer = seri::TIntWriter{.buffer = *this}; - v.Serialize(writer); - return writer.res; + auto const v = PackedSize{tag.size}; + std::uint8_t buf[PackedSize::kMaxWireBytes]; + std::size_t const n = v.Serialize(buf); + return Write(seri::DataWriteTag{buf, n}); } seri::SeriResult Write(seri::DataWriteTag tag) { @@ -47,16 +45,21 @@ struct VectorBuffer { } seri::SeriResult Read(seri::SizeReadTag tag) { - auto v = PackedSize{}; - auto reader = seri::TIntReader{.buffer = *this}; - TierDeserializeRes r = v.Deserialize(reader); - if (r != TierDeserializeRes::kFinished) { + auto const available = buff.size() - read_offset; + std::size_t const n = + PackedSize::WireBytesNeeded(buff.data() + read_offset, available); + if (n == 0) { return Error{seri::read_eof}; } - if (!reader.res) { - return reader.res; + + PackedSize decoded{}; + std::size_t const bytes_read = + decoded.Deserialize(buff.data() + read_offset, n); + if (bytes_read == 0) { + return Error{seri::read_eof}; } - tag.size = static_cast(v); + read_offset += bytes_read; + tag.size = static_cast(decoded); return Ok{seri::good}; } diff --git a/tests/test-api-protocol/assert_packet.h b/tests/test-api-protocol/assert_packet.h index 54f1bd0c..3a68c47c 100644 --- a/tests/test-api-protocol/assert_packet.h +++ b/tests/test-api-protocol/assert_packet.h @@ -25,7 +25,7 @@ #include "aether-miscpp/serialization/binary_archive.h" #include "aether/api_protocol/api_message.h" -#include "aether/tiered_int_serializer.h" // IWYU pragma: export +#include "aether/types/packed_size.h" // IWYU pragma: export #if defined(__clang__) || defined(__GNUC__) # define FUNCTION_NAME __PRETTY_FUNCTION__ diff --git a/tests/test-object-system/CMakeLists.txt b/tests/test-object-system/CMakeLists.txt index 3ffffd85..0c315f42 100644 --- a/tests/test-object-system/CMakeLists.txt +++ b/tests/test-object-system/CMakeLists.txt @@ -48,7 +48,7 @@ if(NOT CM_PLATFORM) # for aether target_include_directories(${PROJECT_NAME} PRIVATE ${ROOT_DIR}) target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_LIST_DIR}) - target_link_libraries(${PROJECT_NAME} PRIVATE unity etl numeric aether::miscpp aether-tele) + target_link_libraries(${PROJECT_NAME} PRIVATE unity etl ae-numeric aether::miscpp aether-tele) target_compile_definitions(${PROJECT_NAME} PRIVATE "AE_DISTILLATION=1") target_compile_definitions(${PROJECT_NAME} PRIVATE "AE_PROJECT_VERSION=\"0.0.0\"") diff --git a/tests/test-transport/test-data-packet-collector.cpp b/tests/test-transport/test-data-packet-collector.cpp index 5b6519db..8b856433 100644 --- a/tests/test-transport/test-data-packet-collector.cpp +++ b/tests/test-transport/test-data-packet-collector.cpp @@ -17,11 +17,13 @@ #include #include +#include #include #include #include "aether-miscpp/serialization/binary_archive.h" +#include "aether/types/packed_size.h" #include "aether/transport/data_packet_collector.h" #include "aether/vector_buffer.h" @@ -152,6 +154,180 @@ void test_BigPacketPartially() { TEST_ASSERT(!p.empty()); } } + +struct GoldenSize { + std::uint64_t value; + std::uint8_t const* bytes; + std::size_t size; +}; + +// Legacy Aether TieredInt wire golden vectors (LE), independent of codec. +static constexpr std::uint8_t k250[] = {0xfa}; +static constexpr std::uint8_t k251[] = {0xfb, 0x00}; +static constexpr std::uint8_t k1514[] = {0xff, 0xef}; +static constexpr std::uint8_t k1515[] = {0xff, 0xf0, 0x00, 0x00}; +static constexpr std::uint8_t k1049834[] = {0xff, 0xff, 0xff, 0xfe}; +static constexpr std::uint8_t k1049835[] = {0xff, 0xff, 0x00, 0xff, + 0x00, 0x00, 0x00, 0x00}; + +static constexpr GoldenSize kGoldenSizes[] = { + {250, k250, sizeof(k250)}, + {251, k251, sizeof(k251)}, + {1514, k1514, sizeof(k1514)}, + {1515, k1515, sizeof(k1515)}, + {1049834, k1049834, sizeof(k1049834)}, + {1049835, k1049835, sizeof(k1049835)}, +}; + +std::vector PayloadForSize(std::uint64_t size) { + std::vector payload(static_cast(size)); + for (std::size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast(i & 0xffu); + } + return payload; +} + +void AssertPayload(std::vector const& got, + std::vector const& expected) { + TEST_ASSERT_EQUAL(expected.size(), got.size()); + if (!expected.empty()) { + TEST_ASSERT_EQUAL_UINT8_ARRAY(expected.data(), got.data(), expected.size()); + } +} + +void test_PackedSizeGoldenSerializeDeserialize() { + for (GoldenSize const& g : kGoldenSizes) { + std::vector encoded; + { + auto buffer = VectorBuffer{encoded}; + TEST_ASSERT(buffer.Write(seri::SizeWriteTag{static_cast(g.value)})); + } + TEST_ASSERT_EQUAL(g.size, encoded.size()); + TEST_ASSERT_EQUAL_UINT8_ARRAY(g.bytes, encoded.data(), g.size); + + { + auto buffer = VectorBuffer{encoded}; + std::size_t decoded{}; + TEST_ASSERT(buffer.Read(seri::SizeReadTag{decoded})); + TEST_ASSERT_EQUAL(static_cast(g.value), decoded); + } + + { + auto archive = + seri::BinaryArchive{seri::BinaryVectorBuffer<>{encoded}}; + PackedSize value{}; + TEST_ASSERT(archive.Load(value)); + TEST_ASSERT_EQUAL(static_cast(g.value), + static_cast(value)); + } + + { + std::vector roundtrip; + auto archive = + seri::BinaryArchive{seri::BinaryVectorBuffer<>{roundtrip}}; + TEST_ASSERT(archive.Save(PackedSize{g.value})); + TEST_ASSERT_EQUAL(g.size, roundtrip.size()); + TEST_ASSERT_EQUAL_UINT8_ARRAY(g.bytes, roundtrip.data(), g.size); + } + } +} + +void test_PacketFramingGoldenBoundaries() { + for (GoldenSize const& g : kGoldenSizes) { + auto payload = PayloadForSize(g.value); + auto frame = MakeStreamPacket(payload); + TEST_ASSERT_EQUAL(g.size + payload.size(), frame.size()); + TEST_ASSERT_EQUAL_UINT8_ARRAY(g.bytes, frame.data(), g.size); + + StreamDataPacketCollector collector; + collector.AddData(frame.data(), frame.size()); + auto got = collector.PopPacket(); + AssertPayload(got, payload); + TEST_ASSERT(collector.PopPacket().empty()); + } +} + +void FeedFragmented(StreamDataPacketCollector& collector, + std::uint8_t const* data, std::size_t size, + std::size_t chunk) { + std::size_t offset = 0; + while (offset < size) { + auto const n = (offset + chunk <= size) ? chunk : (size - offset); + collector.AddData(data + offset, n); + offset += n; + } +} + +void test_FragmentedHeaderAfterEachByte() { + for (GoldenSize const& g : kGoldenSizes) { + auto payload = PayloadForSize(g.value); + auto frame = MakeStreamPacket(payload); + TEST_ASSERT_EQUAL(g.size, frame.size() - payload.size()); + + StreamDataPacketCollector collector; + // Split the size header after every possible byte, then deliver payload. + for (std::size_t i = 0; i < g.size; ++i) { + collector.AddData(frame.data() + i, 1); + TEST_ASSERT(collector.PopPacket().empty()); + } + if (!payload.empty()) { + collector.AddData(frame.data() + g.size, payload.size()); + } + AssertPayload(collector.PopPacket(), payload); + TEST_ASSERT(collector.PopPacket().empty()); + } +} + +void test_WholeHeaderThenPayload() { + for (GoldenSize const& g : kGoldenSizes) { + auto payload = PayloadForSize(g.value); + auto frame = MakeStreamPacket(payload); + + StreamDataPacketCollector collector; + collector.AddData(frame.data(), g.size); + TEST_ASSERT(collector.PopPacket().empty()); + collector.AddData(frame.data() + g.size, payload.size()); + AssertPayload(collector.PopPacket(), payload); + } +} + +void test_MultipleFramesBackToBack() { + std::vector stream; + std::vector> payloads; + for (GoldenSize const& g : kGoldenSizes) { + auto payload = PayloadForSize(g.value % 64u); // keep multi-frame test light + // Re-encode with the actual small payload size (not g.value). + auto frame = MakeStreamPacket(payload); + stream.insert(stream.end(), frame.begin(), frame.end()); + payloads.push_back(std::move(payload)); + } + + StreamDataPacketCollector collector; + collector.AddData(stream.data(), stream.size()); + for (auto const& expected : payloads) { + AssertPayload(collector.PopPacket(), expected); + } + TEST_ASSERT(collector.PopPacket().empty()); +} + +void test_TierHeaderLengthTransitions() { + // Explicit 1 / 2 / 4 / 8-byte header transitions around tier boundaries. + for (GoldenSize const& g : kGoldenSizes) { + std::uint8_t buf[PackedSize::kMaxWireBytes]{}; + auto const n = PackedSize{g.value}.Serialize(buf); + TEST_ASSERT_EQUAL(g.size, n); + TEST_ASSERT_EQUAL_UINT8_ARRAY(g.bytes, buf, g.size); + + // Frame a small payload but feed the whole frame one byte at a time so + // collector state machines exercise header length transitions. + auto payload = PayloadForSize(static_cast(g.size)); + auto frame = MakeStreamPacket(payload); + StreamDataPacketCollector collector; + FeedFragmented(collector, frame.data(), frame.size(), 1); + AssertPayload(collector.PopPacket(), payload); + } +} + } // namespace ae::test_data_pc int test_data_packet_collector() { @@ -162,5 +338,11 @@ int test_data_packet_collector() { RUN_TEST(ae::test_data_pc::test_AddBigPacket); RUN_TEST(ae::test_data_pc::test_AddFewPacketInOne); RUN_TEST(ae::test_data_pc::test_BigPacketPartially); + RUN_TEST(ae::test_data_pc::test_PackedSizeGoldenSerializeDeserialize); + RUN_TEST(ae::test_data_pc::test_PacketFramingGoldenBoundaries); + RUN_TEST(ae::test_data_pc::test_FragmentedHeaderAfterEachByte); + RUN_TEST(ae::test_data_pc::test_WholeHeaderThenPayload); + RUN_TEST(ae::test_data_pc::test_MultipleFramesBackToBack); + RUN_TEST(ae::test_data_pc::test_TierHeaderLengthTransitions); return UNITY_END(); } diff --git a/tests/test-types/CMakeLists.txt b/tests/test-types/CMakeLists.txt index 84c30aae..9e14134e 100644 --- a/tests/test-types/CMakeLists.txt +++ b/tests/test-types/CMakeLists.txt @@ -36,7 +36,7 @@ if(NOT CM_PLATFORM) target_sources(${PROJECT_NAME} PRIVATE ${test_srcs}) # for aether target_include_directories(${PROJECT_NAME} PRIVATE ${ROOT_DIR}) - target_link_libraries(${PROJECT_NAME} PRIVATE unity gcem etl numeric aether::miscpp) + target_link_libraries(${PROJECT_NAME} PRIVATE unity gcem etl ae-numeric aether::miscpp) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") target_compile_options(${PROJECT_NAME} PUBLIC /Zc:preprocessor) endif()