From 28e4a971e27cc00b2af45546607f912fb96d8828 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:20:57 -0700 Subject: [PATCH 1/7] runtime: test container GC layouts --- testdata/gc.go | 151 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 1 deletion(-) diff --git a/testdata/gc.go b/testdata/gc.go index 456d763b4c..445290cbb4 100644 --- a/testdata/gc.go +++ b/testdata/gc.go @@ -1,6 +1,9 @@ package main -import "runtime" +import ( + "reflect" + "runtime" +) var xorshift32State uint32 = 1 @@ -19,6 +22,9 @@ func randuint32() uint32 { func main() { testNonPointerHeap() + testGlobalMapRoots() + testGlobalChannelRoots() + testReflectRoots() testKeepAlive() } @@ -74,3 +80,146 @@ func testKeepAlive() { var x int runtime.KeepAlive(&x) } + +type globalMapObject struct { + marker int + data [64]byte +} + +var globalMap = make(map[int]*globalMapObject) +var globalChannel chan *globalMapObject +var globalPointerSlice []*globalMapObject +var globalGCClobber any + +type globalMapLargeKey struct { + object *globalMapObject + data [129]byte +} + +type globalMapLargeValue struct { + object *globalMapObject + data [129]byte +} + +var globalLargeKeyMap = make(map[globalMapLargeKey]int) +var globalLargeValueMap = make(map[int]globalMapLargeValue) + +//go:noinline +func populateGlobalMaps() { + for i := 0; i < 32; i++ { + globalMap[i] = &globalMapObject{marker: 100 + i} + globalPointerSlice = append(globalPointerSlice, &globalMapObject{marker: 800 + i}) + } + globalLargeKeyMap[globalMapLargeKey{ + object: &globalMapObject{marker: 200}, + }] = 1 + globalLargeValueMap[0] = globalMapLargeValue{ + object: &globalMapObject{marker: 300}, + } +} + +func testGlobalMapRoots() { + populateGlobalMaps() + + runtime.GC() + for i := 0; i < 100; i++ { + globalGCClobber = new(globalMapObject) + } + runtime.GC() + + for i := 0; i < 32; i++ { + if globalMap[i].marker != 100+i { + panic("global map value was collected") + } + } + for key := range globalLargeKeyMap { + if key.object.marker != 200 { + panic("indirect global map key was collected") + } + } + if globalLargeValueMap[0].object.marker != 300 { + panic("indirect global map value was collected") + } + for i, object := range globalPointerSlice { + if object.marker != 800+i { + panic("global slice value was collected") + } + } +} + +//go:noinline +func populateGlobalChannel() { + globalChannel = make(chan *globalMapObject, 4) + globalChannel <- &globalMapObject{marker: 400} +} + +func testGlobalChannelRoots() { + populateGlobalChannel() + + runtime.GC() + for i := 0; i < 100; i++ { + globalGCClobber = new(globalMapObject) + } + runtime.GC() + + if (<-globalChannel).marker != 400 { + panic("global channel value was collected") + } +} + +type reflectRootObject struct { + marker int + child *reflectRootObject + data [128]byte +} + +type reflectMapKey struct { + object *reflectRootObject + data [129]byte +} + +type reflectMapValue struct { + object *reflectRootObject + data [129]byte +} + +type reflectRootMap map[reflectMapKey]reflectMapValue + +var globalReflectObject *reflectRootObject +var globalReflectMap reflectRootMap + +//go:noinline +func populateReflectRoots() { + value := reflect.New(reflect.TypeOf(reflectRootObject{})) + globalReflectObject = value.Interface().(*reflectRootObject) + globalReflectObject.child = &reflectRootObject{marker: 500} + + mapValue := reflect.MakeMapWithSize(reflect.TypeOf(globalReflectMap), 1) + mapValue.SetMapIndex( + reflect.ValueOf(reflectMapKey{object: &reflectRootObject{marker: 600}}), + reflect.ValueOf(reflectMapValue{object: &reflectRootObject{marker: 700}}), + ) + globalReflectMap = mapValue.Interface().(reflectRootMap) +} + +func testReflectRoots() { + populateReflectRoots() + + runtime.GC() + for i := 0; i < 100; i++ { + globalGCClobber = new(reflectRootObject) + } + runtime.GC() + + if globalReflectObject.child.marker != 500 { + panic("reflected object field was collected") + } + for key, value := range globalReflectMap { + if key.object.marker != 600 { + panic("reflected map key was collected") + } + if value.object.marker != 700 { + panic("reflected map value was collected") + } + } +} From 30615ea7fedcb9083a2ad89d61ca891b9fe5133d Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:21:19 -0700 Subject: [PATCH 2/7] runtime: require explicit GC layouts --- builder/testdata/binary-size.txt | 2 +- compiler/channel.go | 6 ++- compiler/interface.go | 8 ++- compiler/map.go | 69 +++++++++++++++++++++++++ compiler/testdata/go1.21.ll | 4 +- compiler/testdata/go1.27.ll | 2 +- compiler/testdata/large.ll | 9 ++-- compiler/testdata/zeromap.ll | 26 ++++++---- interp/memory.go | 12 ++--- interp/testdata/alloc.ll | 5 ++ interp/testdata/alloc.out.ll | 2 + src/internal/gclayout/gclayout.go | 13 +++-- src/internal/reflectlite/type.go | 31 +++++++++-- src/internal/reflectlite/value.go | 26 +++++----- src/internal/task/task_asyncify.go | 3 +- src/internal/task/task_stack.go | 3 +- src/reflect/value_test.go | 30 +++++++++++ src/runtime/arch_tinygowasm_malloc.go | 9 ++-- src/runtime/baremetal.go | 3 +- src/runtime/chan.go | 4 +- src/runtime/gc_blocks.go | 5 +- src/runtime/gc_leaking.go | 3 +- src/runtime/gc_precise.go | 9 ++-- src/runtime/hashmap.go | 39 +++++++++----- transform/testdata/allocs.ll | 26 +++++----- transform/testdata/allocs.out.ll | 6 +-- transform/testdata/gc-stackslots.ll | 16 +++--- transform/testdata/gc-stackslots.out.ll | 16 +++--- 28 files changed, 279 insertions(+), 108 deletions(-) diff --git a/builder/testdata/binary-size.txt b/builder/testdata/binary-size.txt index 363860cc93..ef8e6c1a42 100644 --- a/builder/testdata/binary-size.txt +++ b/builder/testdata/binary-size.txt @@ -1,4 +1,4 @@ target package code rodata data bss hifive1b examples/echo 4321 323 0 2268 microbit examples/serial 2842 382 8 2264 -wioterminal examples/pininterrupt 8039 1665 132 7496 +wioterminal examples/pininterrupt 8039 1669 132 7496 diff --git a/compiler/channel.go b/compiler/channel.go index a562e97e3a..e03995b52e 100644 --- a/compiler/channel.go +++ b/compiler/channel.go @@ -14,8 +14,10 @@ import ( ) func (b *builder) createMakeChan(expr *ssa.MakeChan) llvm.Value { - elementSize := b.targetData.TypeAllocSize(b.getLLVMType(expr.Type().Underlying().(*types.Chan).Elem())) + elementType := b.getLLVMType(expr.Type().Underlying().(*types.Chan).Elem()) + elementSize := b.targetData.TypeAllocSize(elementType) elementSizeValue := llvm.ConstInt(b.uintptrType, elementSize, false) + elementLayout := b.createObjectLayout(elementType, expr.Pos()) bufSize := b.getValue(expr.Size, getPos(expr)) b.createChanBoundsCheck(elementSize, bufSize, expr.Size.Type().Underlying().(*types.Basic), expr.Pos()) if bufSize.Type().IntTypeWidth() < b.uintptrType.IntTypeWidth() { @@ -23,7 +25,7 @@ func (b *builder) createMakeChan(expr *ssa.MakeChan) llvm.Value { } else if bufSize.Type().IntTypeWidth() > b.uintptrType.IntTypeWidth() { bufSize = b.CreateTrunc(bufSize, b.uintptrType, "") } - return b.createRuntimeCall("chanMake", []llvm.Value{elementSizeValue, bufSize}, "") + return b.createRuntimeCall("chanMake", []llvm.Value{elementSizeValue, bufSize, elementLayout}, "") } // createChanSend emits a pseudo chan send operation. It is lowered to the diff --git a/compiler/interface.go b/compiler/interface.go index 84f91cc448..10bffd7993 100644 --- a/compiler/interface.go +++ b/compiler/interface.go @@ -284,6 +284,7 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value { types.NewVar(token.NoPos, nil, "elementType", types.Typ[types.UnsafePointer]), types.NewVar(token.NoPos, nil, "length", types.Typ[types.Uintptr]), types.NewVar(token.NoPos, nil, "sliceOf", types.Typ[types.UnsafePointer]), + types.NewVar(token.NoPos, nil, "layout", types.Typ[types.UnsafePointer]), ) case *types.Map: typeFieldTypes = append(typeFieldTypes, @@ -291,6 +292,7 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value { types.NewVar(token.NoPos, nil, "ptrTo", types.Typ[types.UnsafePointer]), types.NewVar(token.NoPos, nil, "elementType", types.Typ[types.UnsafePointer]), types.NewVar(token.NoPos, nil, "keyType", types.Typ[types.UnsafePointer]), + types.NewVar(token.NoPos, nil, "hashmapTypeInfo", types.Typ[types.UnsafePointer]), ) case *types.Struct: typeFieldTypes = append(typeFieldTypes, @@ -299,6 +301,7 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value { types.NewVar(token.NoPos, nil, "pkgpath", types.Typ[types.UnsafePointer]), types.NewVar(token.NoPos, nil, "size", types.Typ[types.Uint32]), types.NewVar(token.NoPos, nil, "numFields", types.Typ[types.Uint16]), + types.NewVar(token.NoPos, nil, "layout", types.Typ[types.UnsafePointer]), types.NewVar(token.NoPos, nil, "fields", types.NewArray(c.getRuntimeType("structField"), int64(typ.NumFields()))), ) if len(methods) > 0 { @@ -418,6 +421,7 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value { c.getTypeCode(typ.Elem()), // elementType llvm.ConstInt(c.uintptrType, uint64(typ.Len()), false), // length c.getTypeCode(types.NewSlice(typ.Elem())), // slicePtr + c.createObjectLayout(c.getLLVMType(typ), token.NoPos), // layout } case *types.Map: typeFields = []llvm.Value{ @@ -425,6 +429,7 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value { c.getTypeCode(types.NewPointer(typ)), // ptrTo c.getTypeCode(typ.Elem()), // elem c.getTypeCode(typ.Key()), // key + c.getHashmapTypeInfo(typ, token.NoPos), // hashmapTypeInfo } case *types.Struct: var pkgpath string @@ -450,6 +455,7 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value { pkgPathPtr, llvm.ConstInt(c.ctx.Int32Type(), uint64(size), false), // size llvm.ConstInt(c.ctx.Int16Type(), uint64(typ.NumFields()), false), // numFields + c.createObjectLayout(llvmStructType, token.NoPos), // layout } structFieldType := c.getLLVMRuntimeType("structField") @@ -510,7 +516,7 @@ func (c *compilerContext) getTypeCode(typ types.Type) llvm.Value { typeFields = []llvm.Value{c.getTypeCode(types.NewPointer(typ))} // TODO: params, return values, etc } - // Prepend metadata byte. + // Prepend the common RawType field. typeFields = append([]llvm.Value{ llvm.ConstInt(c.ctx.Int8Type(), uint64(metabyte), false), }, typeFields...) diff --git a/compiler/map.go b/compiler/map.go index ea8a49f431..58dd476200 100644 --- a/compiler/map.go +++ b/compiler/map.go @@ -13,6 +13,12 @@ import ( const hashArrayUnrollLimit = 4 +const ( + hashmapBucketSlots = 8 + hashmapMaxKeySize = 128 + hashmapMaxValueSize = 128 +) + // createMakeMap creates a new map object (runtime.hashmap) by allocating and // initializing an appropriately sized object. func (b *builder) createMakeMap(expr *ssa.MakeMap) (llvm.Value, error) { @@ -25,6 +31,8 @@ func (b *builder) createMakeMap(expr *ssa.MakeMap) (llvm.Value, error) { valueSize := b.targetData.TypeAllocSize(llvmValueType) llvmKeySize := llvm.ConstInt(b.uintptrType, keySize, false) llvmValueSize := llvm.ConstInt(b.uintptrType, valueSize, false) + mapLayout := b.getHashmapTypeInfo(mapType, expr.Pos()) + sizeHint := llvm.ConstInt(b.uintptrType, 8, false) if expr.Reserve != nil { sizeHint = b.getValue(expr.Reserve, getPos(expr)) @@ -54,11 +62,72 @@ func (b *builder) createMakeMap(expr *ssa.MakeMap) (llvm.Value, error) { hashmap := b.createRuntimeCall("hashmapMakeGeneric", []llvm.Value{ llvmKeySize, llvmValueSize, sizeHint, + mapLayout, hashFn, equalFn, }, "") return hashmap, nil } +func (c *compilerContext) getHashmapTypeInfo(mapType *types.Map, pos token.Pos) llvm.Value { + llvmKeyType := c.getLLVMType(mapType.Key().Underlying()) + llvmValueType := c.getLLVMType(mapType.Elem().Underlying()) + keySize := c.targetData.TypeAllocSize(llvmKeyType) + valueSize := c.targetData.TypeAllocSize(llvmValueType) + keyLayout := c.createObjectLayout(llvmKeyType, pos) + valueLayout := c.createObjectLayout(llvmValueType, pos) + + llvmKeySlotType := llvmKeyType + if keySize > hashmapMaxKeySize { + llvmKeySlotType = c.dataPtrType + } + llvmValueSlotType := llvmValueType + if valueSize > hashmapMaxValueSize { + llvmValueSlotType = c.dataPtrType + } + + // Keep this in sync with runtime.hashmapBucket and + // runtime.hashmapBucketHeaderSize. + pointerSize := c.targetData.TypeAllocSize(c.dataPtrType) + headerSize := (uint64(8) + pointerSize + 7) &^ 7 + headerPadding := headerSize - uint64(8) - pointerSize + bucketFields := []llvm.Type{ + llvm.ArrayType(c.ctx.Int8Type(), hashmapBucketSlots), + c.dataPtrType, + } + if headerPadding != 0 { + bucketFields = append(bucketFields, llvm.ArrayType(c.ctx.Int8Type(), int(headerPadding))) + } + bucketFields = append(bucketFields, + llvm.ArrayType(llvmKeySlotType, hashmapBucketSlots), + llvm.ArrayType(llvmValueSlotType, hashmapBucketSlots), + ) + bucketType := c.ctx.StructType(bucketFields, true) + bucketSize := headerSize + + c.targetData.TypeAllocSize(llvmKeySlotType)*hashmapBucketSlots + + c.targetData.TypeAllocSize(llvmValueSlotType)*hashmapBucketSlots + if c.targetData.TypeAllocSize(bucketType) != bucketSize { + panic("compiler hashmap bucket layout does not match runtime") + } + bucketLayout := c.createObjectLayout(bucketType, pos) + mapLayoutName := "runtime.hashmapType:" + + hashmapCanonicalTypeName(mapType.Key()) + ":" + + hashmapCanonicalTypeName(mapType.Elem()) + mapLayout := c.mod.NamedGlobal(mapLayoutName) + if mapLayout.IsNil() { + initializer := c.ctx.ConstStruct([]llvm.Value{ + keyLayout, + valueLayout, + bucketLayout, + }, false) + mapLayout = llvm.AddGlobal(c.mod, initializer.Type(), mapLayoutName) + mapLayout.SetInitializer(initializer) + mapLayout.SetGlobalConstant(true) + mapLayout.SetUnnamedAddr(true) + mapLayout.SetLinkage(llvm.LinkOnceODRLinkage) + } + return mapLayout +} + // getRuntimeFunctionValue returns a TinyGo function value (with nil context) // for the named runtime function. func (b *builder) getRuntimeFunctionValue(name string, sig *types.Signature) llvm.Value { diff --git a/compiler/testdata/go1.21.ll b/compiler/testdata/go1.21.ll index 664309518e..00d7146dab 100644 --- a/compiler/testdata/go1.21.ll +++ b/compiler/testdata/go1.21.ll @@ -166,13 +166,13 @@ entry: } ; Function Attrs: nounwind -define hidden void @main.clearMap(ptr dereferenceable_or_null(48) %m, ptr %context) unnamed_addr #1 { +define hidden void @main.clearMap(ptr dereferenceable_or_null(52) %m, ptr %context) unnamed_addr #1 { entry: call void @runtime.hashmapClear(ptr %m, ptr undef) #4 ret void } -declare void @runtime.hashmapClear(ptr dereferenceable_or_null(48), ptr) #0 +declare void @runtime.hashmapClear(ptr dereferenceable_or_null(52), ptr) #0 attributes #0 = { "target-features"="+bulk-memory,+bulk-memory-opt,+call-indirect-overlong,+mutable-globals,+nontrapping-fptoint,+sign-ext,-multivalue,-reference-types" } attributes #1 = { nounwind "target-features"="+bulk-memory,+bulk-memory-opt,+call-indirect-overlong,+mutable-globals,+nontrapping-fptoint,+sign-ext,-multivalue,-reference-types" } diff --git a/compiler/testdata/go1.27.ll b/compiler/testdata/go1.27.ll index d52ba19a38..522cb54a93 100644 --- a/compiler/testdata/go1.27.ll +++ b/compiler/testdata/go1.27.ll @@ -14,7 +14,7 @@ target triple = "wasm32-unknown-wasi" @"main$string" = internal unnamed_addr constant [18 x i8] c"main.genericMethod", align 1 @"main$string.1" = internal unnamed_addr constant [7 x i8] c"Regular", align 1 @"pointer:named:main.genericMethod$methodset" = linkonce_odr unnamed_addr constant { i32, [1 x ptr], { ptr } } { i32 1, [1 x ptr] [ptr @"reflect/methods.Regular:func:{basic:int}{basic:int}"], { ptr } { ptr @"(*main.genericMethod).Regular" } } -@"reflect/types.type:struct:{}" = linkonce_odr constant { i8, i16, ptr, ptr, i32, i16, [0 x %runtime.structField] } { i8 90, i16 0, ptr @"reflect/types.type:pointer:struct:{}", ptr @"reflect/types.type.pkgpath.empty", i32 0, i16 0, [0 x %runtime.structField] zeroinitializer }, align 4 +@"reflect/types.type:struct:{}" = linkonce_odr constant { i8, i16, ptr, ptr, i32, i16, ptr, [0 x %runtime.structField] } { i8 90, i16 0, ptr @"reflect/types.type:pointer:struct:{}", ptr @"reflect/types.type.pkgpath.empty", i32 0, i16 0, ptr inttoptr (i32 3 to ptr), [0 x %runtime.structField] zeroinitializer }, align 4 @"reflect/types.type.pkgpath.empty" = linkonce_odr unnamed_addr constant [1 x i8] zeroinitializer, align 1 @"reflect/types.type:pointer:struct:{}" = linkonce_odr constant { i8, i16, ptr } { i8 -43, i16 0, ptr @"reflect/types.type:struct:{}" }, align 4 @"named:main.genericMethod$methodset" = linkonce_odr unnamed_addr constant { i32, [1 x ptr], { ptr } } { i32 1, [1 x ptr] [ptr @"reflect/methods.Regular:func:{basic:int}{basic:int}"], { ptr } { ptr @"(main.genericMethod).Regular$invoke" } } diff --git a/compiler/testdata/large.ll b/compiler/testdata/large.ll index cc1f8556b4..0026b7ab53 100644 --- a/compiler/testdata/large.ll +++ b/compiler/testdata/large.ll @@ -9,6 +9,7 @@ target triple = "wasm32-unknown-wasi" @"runtime/gc.layout:258-000000000000000000000000000000000000000000000000000000000000000002" = linkonce_odr unnamed_addr constant { i32, [33 x i8] } { i32 258, [33 x i8] c"\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\02" } @"reflect/types.typeid:named:main.largeValue" = external constant i8 +@"runtime.hashmapType:[1025]byte:[1025]byte" = linkonce_odr unnamed_addr constant { ptr, ptr, ptr } { ptr inttoptr (i32 3 to ptr), ptr inttoptr (i32 3 to ptr), ptr inttoptr (i32 67108137 to ptr) } @llvm.used = appending global [15 x ptr] [ptr @"(main.largeReceiver).makeLargeValue", ptr @"(main.largeReceiver).readLargeValue", ptr @main.makeLargeValue, ptr @main.makeZeroLargeValue, ptr @main.readLargeValue, ptr @main.deferLargeValue, ptr @main.goLargeValue, ptr @main.makeLargeResults, ptr @main.makeTwoLargeResults, ptr @main.makeMixedLargeResults, ptr @main.chooseLargeValue, ptr @main.makePointerLargeValue, ptr @main.useLargeMap, ptr @main.useLargeChannel, ptr @main.selectLargeChannel] @"main$string" = internal unnamed_addr constant [31 x i8] c"blocking select matched no case", align 1 @"main$pack" = internal unnamed_addr constant { %runtime._string } { %runtime._string { ptr @"main$string", i32 31 } } @@ -350,7 +351,7 @@ declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg) #7 define hidden i8 @main.useLargeMap(ptr readonly dereferenceable_or_null(1025) %key, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 { entry: %stackalloc = alloca i8, align 1 - %0 = call ptr @runtime.hashmapMakeGeneric(i32 1025, i32 1025, i32 1, ptr null, ptr nonnull @runtime.hash32, ptr null, ptr nonnull @runtime.memequal, ptr undef) #9 + %0 = call ptr @runtime.hashmapMakeGeneric(i32 1025, i32 1025, i32 1, ptr nonnull @"runtime.hashmapType:[1025]byte:[1025]byte", ptr null, ptr nonnull @runtime.hash32, ptr null, ptr nonnull @runtime.memequal, ptr undef) #9 call void @runtime.trackPointer(ptr %0, ptr nonnull %stackalloc, ptr undef) #9 call void @runtime.hashmapBinarySet(ptr %0, ptr %key, ptr %value, ptr undef) #9 %result = call align 1 dereferenceable(1025) ptr @runtime.alloc(i32 1025, ptr nonnull inttoptr (i32 3 to ptr), ptr undef) #9 @@ -381,11 +382,11 @@ declare i32 @runtime.hash32(ptr, i32, i32, ptr) #0 declare i1 @runtime.memequal(ptr, ptr, i32, ptr) #0 -declare ptr @runtime.hashmapMakeGeneric(i32, i32, i32, ptr, ptr, ptr, ptr, ptr) #0 +declare ptr @runtime.hashmapMakeGeneric(i32, i32, i32, ptr, ptr, ptr, ptr, ptr, ptr) #0 -declare void @runtime.hashmapBinarySet(ptr dereferenceable_or_null(48), ptr, ptr, ptr) #0 +declare void @runtime.hashmapBinarySet(ptr dereferenceable_or_null(52), ptr, ptr, ptr) #0 -declare i1 @runtime.hashmapBinaryGet(ptr dereferenceable_or_null(48), ptr, ptr, i32, ptr) #0 +declare i1 @runtime.hashmapBinaryGet(ptr dereferenceable_or_null(52), ptr, ptr, i32, ptr) #0 ; Function Attrs: nounwind define hidden i8 @main.useLargeChannel(ptr dereferenceable_or_null(36) %ch, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 { diff --git a/compiler/testdata/zeromap.ll b/compiler/testdata/zeromap.ll index 3becf83aad..2140a5031e 100644 --- a/compiler/testdata/zeromap.ll +++ b/compiler/testdata/zeromap.ll @@ -6,6 +6,12 @@ target triple = "wasm32-unknown-wasi" %main.hasPadding = type { i1, i32, i1 } %runtime._string = type { ptr, i32 } +@"runtime/gc.layout:44-545555550500" = linkonce_odr unnamed_addr constant { i32, [6 x i8] } { i32 44, [6 x i8] c"TUUU\05\00" } +@"runtime.hashmapType:struct{string; string}:int" = linkonce_odr unnamed_addr constant { ptr, ptr, ptr } { ptr inttoptr (i32 329 to ptr), ptr inttoptr (i32 3 to ptr), ptr @"runtime/gc.layout:44-545555550500" } +@"runtime.hashmapType:[2]string:int" = linkonce_odr unnamed_addr constant { ptr, ptr, ptr } { ptr inttoptr (i32 69 to ptr), ptr inttoptr (i32 3 to ptr), ptr @"runtime/gc.layout:44-545555550500" } +@"runtime/gc.layout:92-545555555555555555550500" = linkonce_odr unnamed_addr constant { i32, [12 x i8] } { i32 92, [12 x i8] c"TUUUUUUUUU\05\00" } +@"runtime.hashmapType:[5]string:int" = linkonce_odr unnamed_addr constant { ptr, ptr, ptr } { ptr inttoptr (i32 69 to ptr), ptr inttoptr (i32 3 to ptr), ptr @"runtime/gc.layout:92-545555555555555555550500" } + declare void @runtime.trackPointer(ptr nocapture readonly, ptr, ptr) #0 ; Function Attrs: nounwind @@ -15,7 +21,7 @@ entry: } ; Function Attrs: noinline nounwind -define hidden i32 @main.testZeroGet(ptr dereferenceable_or_null(48) %m, i1 %s.b1, i32 %s.i, i1 %s.b2, ptr %context) unnamed_addr #2 { +define hidden i32 @main.testZeroGet(ptr dereferenceable_or_null(52) %m, i1 %s.b1, i32 %s.i, i1 %s.b2, ptr %context) unnamed_addr #2 { entry: %hashmap.key = alloca %main.hasPadding, align 8 %hashmap.value = alloca i32, align 4 @@ -35,13 +41,13 @@ entry: ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.start.p0(ptr nocapture) #3 -declare i1 @runtime.hashmapGenericGet(ptr dereferenceable_or_null(48), ptr nocapture, ptr nocapture, i32, ptr) #0 +declare i1 @runtime.hashmapGenericGet(ptr dereferenceable_or_null(52), ptr nocapture, ptr nocapture, i32, ptr) #0 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.end.p0(ptr nocapture) #3 ; Function Attrs: noinline nounwind -define hidden void @main.testZeroSet(ptr dereferenceable_or_null(48) %m, i1 %s.b1, i32 %s.i, i1 %s.b2, ptr %context) unnamed_addr #2 { +define hidden void @main.testZeroSet(ptr dereferenceable_or_null(52) %m, i1 %s.b1, i32 %s.i, i1 %s.b2, ptr %context) unnamed_addr #2 { entry: %hashmap.key = alloca %main.hasPadding, align 8 %hashmap.value = alloca i32, align 4 @@ -58,10 +64,10 @@ entry: ret void } -declare void @runtime.hashmapGenericSet(ptr dereferenceable_or_null(48), ptr nocapture, ptr nocapture, ptr) #0 +declare void @runtime.hashmapGenericSet(ptr dereferenceable_or_null(52), ptr nocapture, ptr nocapture, ptr) #0 ; Function Attrs: noinline nounwind -define hidden i32 @main.testZeroArrayGet(ptr dereferenceable_or_null(48) %m, [2 x %main.hasPadding] %s, ptr %context) unnamed_addr #2 { +define hidden i32 @main.testZeroArrayGet(ptr dereferenceable_or_null(52) %m, [2 x %main.hasPadding] %s, ptr %context) unnamed_addr #2 { entry: %hashmap.key = alloca [2 x %main.hasPadding], align 8 %hashmap.value = alloca i32, align 4 @@ -80,7 +86,7 @@ entry: } ; Function Attrs: noinline nounwind -define hidden void @main.testZeroArraySet(ptr dereferenceable_or_null(48) %m, [2 x %main.hasPadding] %s, ptr %context) unnamed_addr #2 { +define hidden void @main.testZeroArraySet(ptr dereferenceable_or_null(52) %m, [2 x %main.hasPadding] %s, ptr %context) unnamed_addr #2 { entry: %hashmap.key = alloca [2 x %main.hasPadding], align 8 %hashmap.value = alloca i32, align 4 @@ -102,7 +108,7 @@ entry: define hidden ptr @main.makeStringStructMap(ptr %context) unnamed_addr #2 { entry: %stackalloc = alloca i8, align 1 - %0 = call ptr @runtime.hashmapMakeGeneric(i32 16, i32 4, i32 8, ptr null, ptr nonnull @"hashmapKeyHash.struct{string; string}", ptr null, ptr nonnull @"hashmapKeyEqual.struct{string; string}", ptr undef) #4 + %0 = call ptr @runtime.hashmapMakeGeneric(i32 16, i32 4, i32 8, ptr nonnull @"runtime.hashmapType:struct{string; string}:int", ptr null, ptr nonnull @"hashmapKeyHash.struct{string; string}", ptr null, ptr nonnull @"hashmapKeyEqual.struct{string; string}", ptr undef) #4 call void @runtime.trackPointer(ptr %0, ptr nonnull %stackalloc, ptr undef) #4 ret ptr %0 } @@ -145,13 +151,13 @@ entry: declare i1 @runtime.stringEqual(ptr readonly, i32, ptr readonly, i32, ptr) #0 -declare ptr @runtime.hashmapMakeGeneric(i32, i32, i32, ptr, ptr, ptr, ptr, ptr) #0 +declare ptr @runtime.hashmapMakeGeneric(i32, i32, i32, ptr, ptr, ptr, ptr, ptr, ptr) #0 ; Function Attrs: noinline nounwind define hidden ptr @main.makeShortStringArrayMap(ptr %context) unnamed_addr #2 { entry: %stackalloc = alloca i8, align 1 - %0 = call ptr @runtime.hashmapMakeGeneric(i32 16, i32 4, i32 8, ptr null, ptr nonnull @"hashmapKeyHash.[2]string", ptr null, ptr nonnull @"hashmapKeyEqual.[2]string", ptr undef) #4 + %0 = call ptr @runtime.hashmapMakeGeneric(i32 16, i32 4, i32 8, ptr nonnull @"runtime.hashmapType:[2]string:int", ptr null, ptr nonnull @"hashmapKeyHash.[2]string", ptr null, ptr nonnull @"hashmapKeyEqual.[2]string", ptr undef) #4 call void @runtime.trackPointer(ptr %0, ptr nonnull %stackalloc, ptr undef) #4 ret ptr %0 } @@ -194,7 +200,7 @@ entry: define hidden ptr @main.makeLongStringArrayMap(ptr %context) unnamed_addr #2 { entry: %stackalloc = alloca i8, align 1 - %0 = call ptr @runtime.hashmapMakeGeneric(i32 40, i32 4, i32 8, ptr null, ptr nonnull @"hashmapKeyHash.[5]string", ptr null, ptr nonnull @"hashmapKeyEqual.[5]string", ptr undef) #4 + %0 = call ptr @runtime.hashmapMakeGeneric(i32 40, i32 4, i32 8, ptr nonnull @"runtime.hashmapType:[5]string:int", ptr null, ptr nonnull @"hashmapKeyHash.[5]string", ptr null, ptr nonnull @"hashmapKeyEqual.[5]string", ptr undef) #4 call void @runtime.trackPointer(ptr %0, ptr nonnull %stackalloc, ptr undef) #4 ret ptr %0 } diff --git a/interp/memory.go b/interp/memory.go index 7c1eb2d335..c9c09898a1 100644 --- a/interp/memory.go +++ b/interp/memory.go @@ -1278,13 +1278,14 @@ func (r *runner) readObjectLayout(layoutValue value) (uint64, *big.Int) { // integer value, or can be nil. ptr, err := layoutValue.asPointer(r) if err == errIntegerAsPointer { - // It's an integer, which means it's a small object or unknown. + // It's an integer, which means it's a small object. layout := layoutValue.Uint(r) if layout == 0 { - // Nil pointer, which means the layout is unknown. - return 0, nil + panic("runtime.alloc called without a GC layout") } if layout%2 != 1 { + // Conservative layouts are reserved for stack storage and cannot + // reach interpreted heap allocations. // Sanity check: the least significant bit must be set. This is how // the runtime can separate pointers from integers. panic("unexpected layout") @@ -1331,11 +1332,6 @@ func (r *runner) readObjectLayout(layoutValue value) (uint64, *big.Int) { // have some additional repetition, for example in the buffer of a slice. func (r *runner) getLLVMTypeFromLayout(layoutValue value) llvm.Type { objectSizeWords, bitmap := r.readObjectLayout(layoutValue) - if bitmap == nil { - // No information available. - return llvm.Type{} - } - if bitmap.BitLen() == 0 { // There are no pointers in this object, so treat this as a raw byte // buffer. This is important because objects without pointers may have diff --git a/interp/testdata/alloc.ll b/interp/testdata/alloc.ll index 82fbb5b276..3e3c1ca228 100644 --- a/interp/testdata/alloc.ll +++ b/interp/testdata/alloc.ll @@ -11,6 +11,7 @@ target triple = "wasm32--wasi" @layout3 = global ptr null @layout4 = global ptr null @bigobj1 = global ptr null +@pointerFree10 = global ptr null declare ptr @runtime.alloc(i32, ptr) unnamed_addr @@ -49,5 +50,9 @@ define internal void @main.init() unnamed_addr { ; Large object that needs to be stored in a separate global. %bigobj1 = call ptr @runtime.alloc(i32 248, ptr @"runtime/gc.layout:62-2000000000000001") store ptr %bigobj1, ptr @bigobj1 + + ; Another pointer-free object. + %pointerFree10 = call ptr @runtime.alloc(i32 10, ptr inttoptr (i32 3 to ptr)) + store ptr %pointerFree10, ptr @pointerFree10 ret void } diff --git a/interp/testdata/alloc.out.ll b/interp/testdata/alloc.out.ll index b9da6291f6..641bad4ddc 100644 --- a/interp/testdata/alloc.out.ll +++ b/interp/testdata/alloc.out.ll @@ -10,6 +10,7 @@ target triple = "wasm32--wasi" @layout3 = local_unnamed_addr global ptr @"main$alloc.6" @layout4 = local_unnamed_addr global ptr @"main$alloc.7" @bigobj1 = local_unnamed_addr global ptr @"main$alloc.8" +@pointerFree10 = local_unnamed_addr global ptr @"main$alloc.9" @"main$alloc" = internal global [12 x i8] zeroinitializer, align 4 @"main$alloc.1" = internal global [7 x i8] zeroinitializer, align 4 @"main$alloc.2" = internal global [3 x i8] zeroinitializer, align 4 @@ -19,6 +20,7 @@ target triple = "wasm32--wasi" @"main$alloc.6" = internal global { ptr, ptr, ptr, i32, i32, ptr, ptr, i32, i32, i32, i32, i32, i32, ptr, ptr, i32, i32, i32, ptr, ptr, i32, i32, ptr, i32, i32, ptr } zeroinitializer, align 4 @"main$alloc.7" = internal global [3 x { ptr, ptr, ptr, i32, i32, ptr, ptr, i32, i32, i32, i32, i32, i32, ptr, ptr, i32, i32, i32, ptr, ptr, i32, i32, ptr, i32, i32, ptr }] zeroinitializer, align 4 @"main$alloc.8" = internal global { ptr, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, ptr } zeroinitializer, align 4 +@"main$alloc.9" = internal global [10 x i8] zeroinitializer, align 4 define void @runtime.initAll() unnamed_addr { ret void diff --git a/src/internal/gclayout/gclayout.go b/src/internal/gclayout/gclayout.go index d6235889ff..3ed750d138 100644 --- a/src/internal/gclayout/gclayout.go +++ b/src/internal/gclayout/gclayout.go @@ -17,10 +17,15 @@ const ( sizeShift = sizeBits + 1 - NoPtrs = Layout((0 << sizeShift) | (1 << 1) | 1) - Pointer = Layout((1 << sizeShift) | ((unsafe.Sizeof(unsafe.Pointer(nil)) / ptrAlign) << 1) | 1) - String = Layout((1 << sizeShift) | ((unsafe.Sizeof("") / ptrAlign) << 1) | 1) - Slice = Layout((1 << sizeShift) | ((unsafe.Sizeof([]byte{}) / ptrAlign) << 1) | 1) + NoPtrs = Layout((0 << sizeShift) | (1 << 1) | 1) + Pointer = Layout((1 << sizeShift) | ((unsafe.Sizeof(unsafe.Pointer(nil)) / ptrAlign) << 1) | 1) + PointerPair = Layout((3 << sizeShift) | ((2 * unsafe.Sizeof(unsafe.Pointer(nil)) / ptrAlign) << 1) | 1) + String = Layout((1 << sizeShift) | ((unsafe.Sizeof("") / ptrAlign) << 1) | 1) + Slice = Layout((1 << sizeShift) | ((unsafe.Sizeof([]byte{}) / ptrAlign) << 1) | 1) + + // Conservative is reserved for stack storage, which does not have an + // ordinary Go object layout. + Conservative = Layout(2) ) func (l Layout) AsPtr() unsafe.Pointer { return unsafe.Pointer(l) } diff --git a/src/internal/reflectlite/type.go b/src/internal/reflectlite/type.go index 5ced5d3573..0189530a70 100644 --- a/src/internal/reflectlite/type.go +++ b/src/internal/reflectlite/type.go @@ -166,6 +166,11 @@ type RawType struct { meta uint8 // metadata byte, contains kind and flags (see constants above) } +type basicType struct { + RawType + ptrTo *RawType +} + // All types that have an element type: named, chan, slice, array, map (but not // pointer because it doesn't have ptrTo). type elemType struct { @@ -200,6 +205,7 @@ type arrayType struct { elem *RawType arrayLen uintptr slicePtr *RawType + layout unsafe.Pointer } type mapType struct { @@ -208,6 +214,7 @@ type mapType struct { ptrTo *RawType elem *RawType key *RawType + typeInfo unsafe.Pointer } // namedType is the type descriptor for named types. The numMethod field uses @@ -243,6 +250,7 @@ type structType struct { pkgpath *byte size uint32 numField uint16 + layout unsafe.Pointer fields [1]structField // the remaining fields are all of type structField // methods methodSet follows after fields, only when numMethod & numMethodHasMethodSet != 0 } @@ -298,6 +306,8 @@ func pointerTo(t *RawType) *RawType { } switch t.Kind() { + case Bool, Int, Int8, Int16, Int32, Int64, Uint, Uint8, Uint16, Uint32, Uint64, Uintptr, Complex64, Complex128, Float32, Float64, String, UnsafePointer: + return (*basicType)(unsafe.Pointer(t)).ptrTo case Pointer: if tag := t.ptrtag(); tag < 3 { return (*RawType)(unsafe.Add(unsafe.Pointer(t), 1)) @@ -306,6 +316,8 @@ func pointerTo(t *RawType) *RawType { // TODO(dgryski): This is blocking https://github.com/tinygo-org/tinygo/issues/3131 // We need to be able to create types that match existing types to prevent typecode equality. panic("reflect: cannot make *****T type") + case Interface, Func: + return (*interfaceType)(unsafe.Pointer(t)).ptrTo case Struct: return (*structType)(unsafe.Pointer(t)).ptrTo default: @@ -729,6 +741,7 @@ func (t *RawType) Align() int { } func (r *RawType) gcLayout() unsafe.Pointer { + r = r.underlying() kind := r.Kind() if kind < String { @@ -736,16 +749,26 @@ func (r *RawType) gcLayout() unsafe.Pointer { } switch kind { - case Pointer, UnsafePointer, Chan, Map: - return gclayout.Pointer.AsPtr() case String: return gclayout.String.AsPtr() + case UnsafePointer, Chan, Pointer, Map: + return gclayout.Pointer.AsPtr() + case Interface, Func: + return gclayout.PointerPair.AsPtr() case Slice: return gclayout.Slice.AsPtr() + case Array: + return (*arrayType)(unsafe.Pointer(r)).layout + case Struct: + return (*structType)(unsafe.Pointer(r)).layout + default: + panic("reflect: invalid GC layout kind") } +} - // Unknown (for now); let the conservative pointer scanning handle it - return nil +func (r *RawType) hashmapTypeInfo() unsafe.Pointer { + r = r.underlying() + return (*mapType)(unsafe.Pointer(r)).typeInfo } // FieldAlign returns the alignment if this type is used in a struct field. It diff --git a/src/internal/reflectlite/value.go b/src/internal/reflectlite/value.go index 18ebe3df71..e0591ec86e 100644 --- a/src/internal/reflectlite/value.go +++ b/src/internal/reflectlite/value.go @@ -1,6 +1,7 @@ package reflectlite import ( + "internal/gclayout" "math" "unsafe" ) @@ -1644,7 +1645,7 @@ func makeInt(flags valueFlags, bits uint64, t *RawType) Value { ptr := unsafe.Pointer(&v.value) if size > unsafe.Sizeof(uintptr(0)) { - ptr = alloc(size, nil) + ptr = alloc(size, gclayout.NoPtrs.AsPtr()) v.value = ptr } @@ -1671,7 +1672,7 @@ func makeFloat(flags valueFlags, f float64, t *RawType) Value { ptr := unsafe.Pointer(&v.value) if size > unsafe.Sizeof(uintptr(0)) { - ptr = alloc(size, nil) + ptr = alloc(size, gclayout.NoPtrs.AsPtr()) v.value = ptr } @@ -1703,7 +1704,7 @@ func makeComplex(flags valueFlags, f complex128, t *RawType) Value { ptr := unsafe.Pointer(&v.value) if size > unsafe.Sizeof(uintptr(0)) { - ptr = alloc(size, nil) + ptr = alloc(size, gclayout.NoPtrs.AsPtr()) v.value = ptr } @@ -1834,7 +1835,7 @@ func Zero(typ Type) Value { return Value{ typecode: typ.(*RawType), - value: alloc(size, nil), + value: alloc(size, typ.(*RawType).gcLayout()), flags: valueFlagExported | valueFlagRO, } } @@ -1844,7 +1845,7 @@ func Zero(typ Type) Value { func New(typ Type) Value { return Value{ typecode: pointerTo(typ.(*RawType)), - value: alloc(typ.Size(), nil), + value: alloc(typ.Size(), typ.(*RawType).gcLayout()), flags: valueFlagExported, } } @@ -2203,13 +2204,13 @@ func (v Value) FieldByNameFunc(match func(string) bool) Value { } //go:linkname hashmapMake runtime.hashmapMake -func hashmapMake(keySize, valueSize uintptr, sizeHint uintptr, alg uint8) unsafe.Pointer +func hashmapMake(keySize, valueSize uintptr, sizeHint uintptr, typeInfo unsafe.Pointer, alg uint8) unsafe.Pointer //go:linkname hashmapMakeReflect runtime.hashmapMakeReflect -func hashmapMakeReflect(keySize, valueSize, sizeHint uintptr, keyType unsafe.Pointer) unsafe.Pointer +func hashmapMakeReflect(keySize, valueSize, sizeHint uintptr, typeInfo, keyType unsafe.Pointer) unsafe.Pointer //go:linkname chanMake runtime.chanMake -func chanMake(elementSize uintptr, bufSize uintptr) unsafe.Pointer +func chanMake(elementSize uintptr, bufSize uintptr, elementLayout unsafe.Pointer) unsafe.Pointer // MakeMapWithSize creates a new map with the specified type and initial space // for approximately n elements. @@ -2231,18 +2232,19 @@ func MakeMapWithSize(typ Type, n int) Value { key := typ.Key().(*RawType) val := typ.Elem().(*RawType) + typeInfo := typ.(*RawType).hashmapTypeInfo() var m unsafe.Pointer if key.Kind() == String { - m = hashmapMake(key.Size(), val.Size(), uintptr(n), hashmapAlgorithmString) + m = hashmapMake(key.Size(), val.Size(), uintptr(n), typeInfo, hashmapAlgorithmString) } else if key.isBinary() { - m = hashmapMake(key.Size(), val.Size(), uintptr(n), hashmapAlgorithmBinary) + m = hashmapMake(key.Size(), val.Size(), uintptr(n), typeInfo, hashmapAlgorithmBinary) } else { // Composite key type (struct with strings, floats, etc.). // Use runtime-generated hash/equal closures that walk the // type structure, matching the compiler-generated functions. - m = hashmapMakeReflect(key.Size(), val.Size(), uintptr(n), unsafe.Pointer(key)) + m = hashmapMakeReflect(key.Size(), val.Size(), uintptr(n), typeInfo, unsafe.Pointer(key)) } return Value{ @@ -2269,7 +2271,7 @@ func MakeChan(typ Type, size int) Value { panic("reflect.MakeChan: unidirectional channel type") } elem := typ.Elem().(*RawType) - ch := chanMake(elem.Size(), uintptr(size)) + ch := chanMake(elem.Size(), uintptr(size), elem.gcLayout()) return Value{ typecode: typ.(*RawType), value: ch, diff --git a/src/internal/task/task_asyncify.go b/src/internal/task/task_asyncify.go index 4d78e19373..3bc41b3a29 100644 --- a/src/internal/task/task_asyncify.go +++ b/src/internal/task/task_asyncify.go @@ -3,6 +3,7 @@ package task import ( + "internal/gclayout" "unsafe" ) @@ -73,7 +74,7 @@ func (s *state) initialize(fn uintptr, args unsafe.Pointer, stackSize uintptr) { s.args = args // Create a stack. - stack := runtime_alloc(stackSize, nil) + stack := runtime_alloc(stackSize, gclayout.Conservative.AsPtr()) // Set up the stack canary, a random number that should be checked when // switching from the task back to the scheduler. The stack canary pointer diff --git a/src/internal/task/task_stack.go b/src/internal/task/task_stack.go index 23f3b9097f..eaab211bbd 100644 --- a/src/internal/task/task_stack.go +++ b/src/internal/task/task_stack.go @@ -3,6 +3,7 @@ package task import ( + "internal/gclayout" "unsafe" ) @@ -36,7 +37,7 @@ func taskExit() { // initialize the state and prepare to call the specified function with the specified argument bundle. func (s *state) initialize(fn uintptr, args unsafe.Pointer, stackSize uintptr) { // Create a stack. - stack := runtime_alloc(stackSize, nil) + stack := runtime_alloc(stackSize, gclayout.Conservative.AsPtr()) // Set up the stack canary, a random number that should be checked when // switching from the task back to the scheduler. The stack canary pointer diff --git a/src/reflect/value_test.go b/src/reflect/value_test.go index b31f1e48e5..3456b29410 100644 --- a/src/reflect/value_test.go +++ b/src/reflect/value_test.go @@ -994,6 +994,20 @@ func TestTypeAssertPanic(t *testing.T) { }) } +type tinyMakeChanElement struct { + ptr *int + text string +} + +var tinyMakeChanChurn []*int + +//go:noinline +func fillTinyMakeChan(ch chan tinyMakeChanElement) { + value := new(int) + *value = 42 + ch <- tinyMakeChanElement{ptr: value, text: "hello"} +} + func TestTinyMakeChan(t *testing.T) { // Value.Send and Value.Recv are not implemented yet, so the channel is // exercised through Interface(): that proves MakeChan returns a working @@ -1027,6 +1041,22 @@ func TestTinyMakeChan(t *testing.T) { } }) + t.Run("buffered pointers survive GC", func(t *testing.T) { + v := MakeChan(TypeOf(make(chan tinyMakeChanElement)), 1) + ch := v.Interface().(chan tinyMakeChanElement) + fillTinyMakeChan(ch) + runtime.GC() + tinyMakeChanChurn = make([]*int, 128) + for i := range tinyMakeChanChurn { + tinyMakeChanChurn[i] = new(int) + } + + got := <-ch + if *got.ptr != 42 || got.text != "hello" { + t.Errorf("<-ch=%v, want {42 hello}", got) + } + }) + t.Run("unbuffered", func(t *testing.T) { v := MakeChan(TypeOf(make(chan string)), 0) if got, want := v.Cap(), 0; got != want { diff --git a/src/runtime/arch_tinygowasm_malloc.go b/src/runtime/arch_tinygowasm_malloc.go index df824881e2..694840af90 100644 --- a/src/runtime/arch_tinygowasm_malloc.go +++ b/src/runtime/arch_tinygowasm_malloc.go @@ -2,7 +2,10 @@ package runtime -import "unsafe" +import ( + "internal/gclayout" + "unsafe" +) // The below functions override the default allocator of wasi-libc. This ensures // code linked from other languages can allocate memory without colliding with @@ -21,7 +24,7 @@ func libc_malloc(size uintptr) unsafe.Pointer { if size == 0 { return nil } - ptr := alloc(size, nil) + ptr := alloc(size, gclayout.NoPtrs.AsPtr()) allocs[(*byte)(ptr)] = size return ptr } @@ -54,7 +57,7 @@ func libc_realloc(oldPtr unsafe.Pointer, size uintptr) unsafe.Pointer { // It's hard to optimize this to expand the current buffer with our GC, but // it is theoretically possible. For now, just always allocate fresh. // TODO: we could skip this if the new allocation is smaller than the old. - ptr := alloc(size, nil) + ptr := alloc(size, gclayout.NoPtrs.AsPtr()) if oldPtr != nil { if oldSize, ok := allocs[(*byte)(oldPtr)]; ok { diff --git a/src/runtime/baremetal.go b/src/runtime/baremetal.go index 6dd29e490d..4893297fe1 100644 --- a/src/runtime/baremetal.go +++ b/src/runtime/baremetal.go @@ -3,6 +3,7 @@ package runtime import ( + "internal/gclayout" "sync/atomic" "unsafe" ) @@ -11,7 +12,7 @@ import ( func libc_malloc(size uintptr) unsafe.Pointer { // Note: this zeroes the returned buffer which is not necessary. // The same goes for bytealg.MakeNoZero. - return alloc(size, nil) + return alloc(size, gclayout.NoPtrs.AsPtr()) } //export calloc diff --git a/src/runtime/chan.go b/src/runtime/chan.go index a85e9b6617..f425daf5d5 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -137,11 +137,11 @@ type chanSelectState struct { value unsafe.Pointer } -func chanMake(elementSize uintptr, bufSize uintptr) *channel { +func chanMake(elementSize uintptr, bufSize uintptr, elementLayout unsafe.Pointer) *channel { return &channel{ elementSize: elementSize, bufCap: bufSize, - buf: alloc(elementSize*bufSize, nil), + buf: alloc(elementSize*bufSize, elementLayout), } } diff --git a/src/runtime/gc_blocks.go b/src/runtime/gc_blocks.go index 3afed0a3eb..c27401d62d 100644 --- a/src/runtime/gc_blocks.go +++ b/src/runtime/gc_blocks.go @@ -31,6 +31,7 @@ package runtime // Moss. import ( + "internal/gclayout" "internal/reflectlite" "internal/task" "runtime/interrupt" @@ -501,7 +502,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer { if ptr == nil { - return alloc(size, nil) + return alloc(size, gclayout.NoPtrs.AsPtr()) } // Find the first block of the original allocation. @@ -526,7 +527,7 @@ func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer { } // Create a new allocation and copy the old data. - newAlloc := alloc(size, nil) + newAlloc := alloc(size, gclayout.NoPtrs.AsPtr()) memcpy(newAlloc, ptr, oldSize) free(ptr) diff --git a/src/runtime/gc_leaking.go b/src/runtime/gc_leaking.go index 839acd8d9c..3ebee0989b 100644 --- a/src/runtime/gc_leaking.go +++ b/src/runtime/gc_leaking.go @@ -7,6 +7,7 @@ package runtime // may be the only memory allocator possible. import ( + "internal/gclayout" "internal/task" "sync/atomic" "unsafe" @@ -69,7 +70,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { } func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer { - newAlloc := alloc(size, nil) + newAlloc := alloc(size, gclayout.NoPtrs.AsPtr()) if ptr == nil { return newAlloc } diff --git a/src/runtime/gc_precise.go b/src/runtime/gc_precise.go index 062cc46afa..7f05c4f094 100644 --- a/src/runtime/gc_precise.go +++ b/src/runtime/gc_precise.go @@ -55,7 +55,10 @@ package runtime -import "unsafe" +import ( + "internal/gclayout" + "unsafe" +) const sizeFieldBits = 4 + (unsafe.Sizeof(uintptr(0)) / 4) @@ -76,9 +79,7 @@ func (layout gcLayout) pointerFree() bool { // The length is rounded down to a multiple of the element size. func (layout gcLayout) scan(start, len uintptr) { switch { - case layout == 0: - // This is an unknown layout. - // Scan conservatively. + case layout == gcLayout(gclayout.Conservative): // NOTE: This is *NOT* equivalent to a slice of pointers on AVR. scanConservative(start, len) diff --git a/src/runtime/hashmap.go b/src/runtime/hashmap.go index 56405dd10e..5b43edd2d5 100644 --- a/src/runtime/hashmap.go +++ b/src/runtime/hashmap.go @@ -14,6 +14,7 @@ import ( // The underlying hashmap structure for Go. type hashmap struct { buckets unsafe.Pointer // pointer to array of buckets + typeInfo *hashmapTypeInfo seed uintptr count uintptr keySize uintptr @@ -26,6 +27,17 @@ type hashmap struct { keyHash func(key unsafe.Pointer, size, seed uintptr) uint32 } +type hashmapTypeInfo struct { + keyLayout unsafe.Pointer + valueLayout unsafe.Pointer + bucketLayout unsafe.Pointer +} + +//go:inline +func hashmapType(m *hashmap) *hashmapTypeInfo { + return m.typeInfo +} + const ( hashmapMaxKeySize = 128 hashmapMaxValueSize = 128 @@ -113,7 +125,7 @@ func hashmapTopHash(hash uint32) uint8 { } // Create a new hashmap with the given keySize and valueSize. -func hashmapMake(keySize, valueSize uintptr, sizeHint uintptr, alg uint8) *hashmap { +func hashmapMake(keySize, valueSize uintptr, sizeHint uintptr, typeInfo unsafe.Pointer, alg uint8) *hashmap { bucketBits := uint8(0) for hashmapHasSpaceToGrow(bucketBits) && hashmapOverLoadFactor(sizeHint, bucketBits) { bucketBits++ @@ -132,13 +144,14 @@ func hashmapMake(keySize, valueSize uintptr, sizeHint uintptr, alg uint8) *hashm } bucketBufSize := hashmapBucketHeaderSize + keySlotSize*8 + valueSlotSize*8 - buckets := alloc(bucketBufSize*(1< Date: Fri, 7 Aug 2026 10:32:00 -0700 Subject: [PATCH 3/7] runtime: precisely scan Wasm globals --- src/runtime/gc_custom.go | 1 - src/runtime/gc_globals_blocks.go | 22 +++ src/runtime/gc_globals_custom.go | 18 +++ src/runtime/gc_globals_range.go | 40 ++++++ src/runtime/gc_stack_portable.go | 2 +- transform/gc.go | 181 +++++++++++++++++++++++- transform/testdata/gc-stackslots.ll | 11 ++ transform/testdata/gc-stackslots.out.ll | 31 ++++ 8 files changed, 301 insertions(+), 5 deletions(-) create mode 100644 src/runtime/gc_globals_blocks.go create mode 100644 src/runtime/gc_globals_custom.go create mode 100644 src/runtime/gc_globals_range.go diff --git a/src/runtime/gc_custom.go b/src/runtime/gc_custom.go index 0125f1688b..12f1b2e12d 100644 --- a/src/runtime/gc_custom.go +++ b/src/runtime/gc_custom.go @@ -48,7 +48,6 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer func free(ptr unsafe.Pointer) // markRoots is called with the start and end addresses to scan for references. -// It is currently only called with the top and bottom of the stack. func markRoots(start, end uintptr) // GC is called to explicitly run garbage collection. diff --git a/src/runtime/gc_globals_blocks.go b/src/runtime/gc_globals_blocks.go new file mode 100644 index 0000000000..b9cb94cb8f --- /dev/null +++ b/src/runtime/gc_globals_blocks.go @@ -0,0 +1,22 @@ +//go:build (gc.conservative || gc.precise) && tinygo.wasm + +package runtime + +import "unsafe" + +func markGlobals() { + for i := uintptr(0); i < gcGlobalRootCount(); i++ { + addr := gcGlobalRoot(i) + size := gcGlobalRootSize(i) + for offset := uintptr(0); offset < size; offset += unsafe.Sizeof(uintptr(0)) { + slot := unsafe.Add(addr, offset) + markRoot(uintptr(slot), *(*uintptr)(slot)) + } + } +} + +// These functions are generated by the compiler from the pointer layouts of +// mutable globals. Each range contains only pointer slots. +func gcGlobalRootCount() uintptr +func gcGlobalRoot(index uintptr) unsafe.Pointer +func gcGlobalRootSize(index uintptr) uintptr diff --git a/src/runtime/gc_globals_custom.go b/src/runtime/gc_globals_custom.go new file mode 100644 index 0000000000..bf612e6ac1 --- /dev/null +++ b/src/runtime/gc_globals_custom.go @@ -0,0 +1,18 @@ +//go:build gc.custom && tinygo.wasm + +package runtime + +import "unsafe" + +func markGlobals() { + for i := uintptr(0); i < gcGlobalRootCount(); i++ { + start := uintptr(gcGlobalRoot(i)) + markRoots(start, start+gcGlobalRootSize(i)) + } +} + +// These functions are generated by the compiler from the pointer layouts of +// mutable globals. Each range contains only pointer slots. +func gcGlobalRootCount() uintptr +func gcGlobalRoot(index uintptr) unsafe.Pointer +func gcGlobalRootSize(index uintptr) uintptr diff --git a/src/runtime/gc_globals_range.go b/src/runtime/gc_globals_range.go new file mode 100644 index 0000000000..5702e2b1ca --- /dev/null +++ b/src/runtime/gc_globals_range.go @@ -0,0 +1,40 @@ +//go:build gc.boehm && tinygo.wasm + +package runtime + +import "unsafe" + +func markGlobals() { + rangeCount := gcGlobalRootCount() + if rangeCount == 0 { + return + } + + var rootCount uintptr + for i := uintptr(0); i < rangeCount; i++ { + rootCount += gcGlobalRootSize(i) / unsafe.Sizeof(uintptr(0)) + } + + // markRoots only accepts a range, so copy all global pointers into + // contiguous scratch space for marking. + roots := unsafe.Slice((*uintptr)(gcGlobalRootValues()), rootCount) + var rootIndex uintptr + for i := uintptr(0); i < rangeCount; i++ { + addr := gcGlobalRoot(i) + size := gcGlobalRootSize(i) + for offset := uintptr(0); offset < size; offset += unsafe.Sizeof(uintptr(0)) { + roots[rootIndex] = *(*uintptr)(unsafe.Add(addr, offset)) + rootIndex++ + } + } + + start := uintptr(unsafe.Pointer(&roots[0])) + markRoots(start, start+rootCount*unsafe.Sizeof(roots[0])) +} + +// These functions are generated by the compiler from the pointer layouts of +// mutable globals. Each range contains only pointer slots. +func gcGlobalRootCount() uintptr +func gcGlobalRoot(index uintptr) unsafe.Pointer +func gcGlobalRootSize(index uintptr) uintptr +func gcGlobalRootValues() unsafe.Pointer diff --git a/src/runtime/gc_stack_portable.go b/src/runtime/gc_stack_portable.go index 04162bb07a..fdf0a7cae9 100644 --- a/src/runtime/gc_stack_portable.go +++ b/src/runtime/gc_stack_portable.go @@ -10,7 +10,7 @@ import ( func gcMarkReachable() { markStack() - findGlobals(markRoots) + markGlobals() } //go:extern runtime.stackChainStart diff --git a/transform/gc.go b/transform/gc.go index abbe3cb7bb..5a36d73e7e 100644 --- a/transform/gc.go +++ b/transform/gc.go @@ -1,6 +1,8 @@ package transform import ( + "strings" + "tinygo.org/x/go-llvm" ) @@ -12,6 +14,8 @@ const shiftExcludeArgMem = 2 // MakeGCStackSlots converts all calls to runtime.trackPointer to explicit // stores to stack slots that are scannable by the GC. func MakeGCStackSlots(mod llvm.Module) bool { + hasGlobalRoots := makeGCGlobalRoots(mod) + // Check whether there are allocations at all. alloc := mod.NamedFunction("runtime.alloc") if alloc.IsNil() { @@ -26,12 +30,12 @@ func MakeGCStackSlots(mod llvm.Module) bool { stackChainStart.SetInitializer(llvm.ConstNull(stackChainStart.GlobalValueType())) stackChainStart.SetGlobalConstant(true) } - return false + return hasGlobalRoots } trackPointer := mod.NamedFunction("runtime.trackPointer") if trackPointer.IsNil() || trackPointer.FirstUse().IsNil() { - return false // nothing to do + return hasGlobalRoots } ctx := mod.Context() @@ -107,7 +111,7 @@ func MakeGCStackSlots(mod llvm.Module) bool { for _, use := range getUses(trackPointer) { use.EraseFromParentAsInstruction() } - return false + return hasGlobalRoots } stackChainStart.SetLinkage(llvm.InternalLinkage) stackChainStartType := stackChainStart.GlobalValueType() @@ -285,6 +289,177 @@ func MakeGCStackSlots(mod llvm.Module) bool { return true } +func makeGCGlobalRoots(mod llvm.Module) bool { + rootCount := mod.NamedFunction("runtime.gcGlobalRootCount") + rootAt := mod.NamedFunction("runtime.gcGlobalRoot") + rootSize := mod.NamedFunction("runtime.gcGlobalRootSize") + rootValues := mod.NamedFunction("runtime.gcGlobalRootValues") + if rootCount.IsNil() || rootAt.IsNil() || rootSize.IsNil() || + !rootCount.FirstBasicBlock().IsNil() || + !rootAt.FirstBasicBlock().IsNil() || + !rootSize.FirstBasicBlock().IsNil() { + return false + } + if !rootValues.IsNil() && !rootValues.FirstBasicBlock().IsNil() { + return false + } + + ctx := mod.Context() + uintptrType := rootCount.GlobalValueType().ReturnType() + targetData := llvm.NewTargetData(mod.DataLayout()) + defer targetData.Dispose() + var roots []gcGlobalRootRange + for global := mod.FirstGlobal(); !global.IsNil(); global = llvm.NextGlobal(global) { + if strings.HasPrefix(global.Name(), "llvm.") || + global.IsGlobalConstant() || + global.Initializer().IsNil() || + !gcTypeHasPointers(global.GlobalValueType()) { + continue + } + roots = appendGCGlobalRootRanges(roots, global, global.GlobalValueType(), targetData, ctx.Int8Type(), uintptrType) + } + + ptrType := rootAt.GlobalValueType().ReturnType() + rootType := ctx.StructType([]llvm.Type{ptrType, uintptrType}, false) + rootInitializers := make([]llvm.Value, len(roots)) + for i, root := range roots { + rootInitializers[i] = llvm.ConstNamedStruct(rootType, []llvm.Value{ + root.address, + llvm.ConstInt(uintptrType, root.size, false), + }) + } + rootArrayType := llvm.ArrayType(rootType, len(roots)) + rootArray := llvm.AddGlobal(mod, rootArrayType, "runtime.gcGlobalRoots") + rootArray.SetInitializer(llvm.ConstArray(rootType, rootInitializers)) + rootArray.SetGlobalConstant(true) + rootArray.SetLinkage(llvm.InternalLinkage) + + builder := ctx.NewBuilder() + defer builder.Dispose() + + entry := ctx.AddBasicBlock(rootCount, "entry") + builder.SetInsertPointAtEnd(entry) + builder.CreateRet(llvm.ConstInt(rootCount.GlobalValueType().ReturnType(), uint64(len(roots)), false)) + + entry = ctx.AddBasicBlock(rootAt, "entry") + builder.SetInsertPointAtEnd(entry) + index := rootAt.FirstParam() + root := builder.CreateInBoundsGEP(rootArrayType, rootArray, []llvm.Value{ + llvm.ConstInt(ctx.Int32Type(), 0, false), + index, + }, "") + addr := builder.CreateStructGEP(rootType, root, 0, "") + builder.CreateRet(builder.CreateLoad(ptrType, addr, "")) + + entry = ctx.AddBasicBlock(rootSize, "entry") + builder.SetInsertPointAtEnd(entry) + index = rootSize.FirstParam() + root = builder.CreateInBoundsGEP(rootArrayType, rootArray, []llvm.Value{ + llvm.ConstInt(ctx.Int32Type(), 0, false), + index, + }, "") + size := builder.CreateStructGEP(rootType, root, 1, "") + builder.CreateRet(builder.CreateLoad(uintptrType, size, "")) + + if !rootValues.IsNil() { + pointerSize := uint64(targetData.PointerSize()) + var rootValueCount uint64 + for _, root := range roots { + rootValueCount += root.size / pointerSize + } + rootValueArray := llvm.AddGlobal(mod, llvm.ArrayType(uintptrType, int(rootValueCount)), "runtime.gcGlobalRootValueArray") + rootValueArray.SetInitializer(llvm.ConstNull(rootValueArray.GlobalValueType())) + rootValueArray.SetLinkage(llvm.InternalLinkage) + + entry = ctx.AddBasicBlock(rootValues, "entry") + builder.SetInsertPointAtEnd(entry) + builder.CreateRet(rootValueArray) + } + + return true +} + +// gcGlobalRootRange is a contiguous range of pointer slots. +// It never includes padding or non-pointer fields. +type gcGlobalRootRange struct { + address llvm.Value + size uint64 +} + +func appendGCGlobalRootRanges(roots []gcGlobalRootRange, global llvm.Value, typ llvm.Type, targetData llvm.TargetData, i8Type, uintptrType llvm.Type) []gcGlobalRootRange { + var offsets []uint64 + offsets = appendGCGlobalRootOffsets(offsets, typ, targetData, 0) + if len(offsets) == 0 { + return roots + } + + pointerSize := uint64(targetData.PointerSize()) + rangeStart := offsets[0] + rangeEnd := rangeStart + pointerSize + for _, offset := range offsets[1:] { + if offset == rangeEnd { + rangeEnd += pointerSize + continue + } + roots = appendGCGlobalRootRange(roots, global, rangeStart, rangeEnd-rangeStart, pointerSize, i8Type, uintptrType) + rangeStart = offset + rangeEnd = offset + pointerSize + } + return appendGCGlobalRootRange(roots, global, rangeStart, rangeEnd-rangeStart, pointerSize, i8Type, uintptrType) +} + +func appendGCGlobalRootRange(roots []gcGlobalRootRange, global llvm.Value, offset, size, pointerSize uint64, i8Type, uintptrType llvm.Type) []gcGlobalRootRange { + if offset%pointerSize != 0 || size%pointerSize != 0 { + panic("global root range is not pointer aligned") + } + address := global + if offset != 0 { + address = llvm.ConstGEP(i8Type, global, []llvm.Value{ + llvm.ConstInt(uintptrType, offset, false), + }) + } + return append(roots, gcGlobalRootRange{address: address, size: size}) +} + +func appendGCGlobalRootOffsets(offsets []uint64, typ llvm.Type, targetData llvm.TargetData, baseOffset uint64) []uint64 { + switch typ.TypeKind() { + case llvm.PointerTypeKind: + return append(offsets, baseOffset) + case llvm.StructTypeKind: + for i, fieldType := range typ.StructElementTypes() { + if gcTypeHasPointers(fieldType) { + fieldOffset := targetData.ElementOffset(typ, i) + offsets = appendGCGlobalRootOffsets(offsets, fieldType, targetData, baseOffset+fieldOffset) + } + } + case llvm.ArrayTypeKind: + elemType := typ.ElementType() + if gcTypeHasPointers(elemType) { + elemSize := targetData.TypeAllocSize(elemType) + for i := 0; i < typ.ArrayLength(); i++ { + offsets = appendGCGlobalRootOffsets(offsets, elemType, targetData, baseOffset+uint64(i)*elemSize) + } + } + } + return offsets +} + +func gcTypeHasPointers(typ llvm.Type) bool { + switch typ.TypeKind() { + case llvm.PointerTypeKind: + return true + case llvm.StructTypeKind: + for _, field := range typ.StructElementTypes() { + if gcTypeHasPointers(field) { + return true + } + } + case llvm.ArrayTypeKind: + return typ.ArrayLength() != 0 && gcTypeHasPointers(typ.ElementType()) + } + return false +} + // markParentFunctions traverses all parent function calls (recursively) and // adds them to the set of marked functions. It only considers function calls: // any other uses of such a function is ignored. diff --git a/transform/testdata/gc-stackslots.ll b/transform/testdata/gc-stackslots.ll index 8da5110cbd..912051901b 100644 --- a/transform/testdata/gc-stackslots.ll +++ b/transform/testdata/gc-stackslots.ll @@ -5,11 +5,22 @@ target triple = "wasm32-unknown-unknown-wasm" @someGlobal = global i8 3 @ptrGlobal = global ptr null @arrGlobal = global [8 x i8] zeroinitializer +@structGlobal = global {ptr, i32, [2 x ptr]} zeroinitializer +@ptrArrayGlobal = global [8 x ptr] zeroinitializer +@constantPtrGlobal = constant ptr @someGlobal declare void @runtime.trackPointer(ptr nocapture readonly) declare noalias nonnull ptr @runtime.alloc(i32, ptr) +declare i32 @runtime.gcGlobalRootCount() + +declare ptr @runtime.gcGlobalRoot(i32) + +declare i32 @runtime.gcGlobalRootSize(i32) + +declare ptr @runtime.gcGlobalRootValues() + ; Generic function that returns a pointer (that must be tracked). define ptr @getPointer() { ret ptr @someGlobal diff --git a/transform/testdata/gc-stackslots.out.ll b/transform/testdata/gc-stackslots.out.ll index 28094c3954..5fdad3f6a1 100644 --- a/transform/testdata/gc-stackslots.out.ll +++ b/transform/testdata/gc-stackslots.out.ll @@ -5,11 +5,42 @@ target triple = "wasm32-unknown-unknown-wasm" @someGlobal = global i8 3 @ptrGlobal = global ptr null @arrGlobal = global [8 x i8] zeroinitializer +@structGlobal = global { ptr, i32, [2 x ptr] } zeroinitializer +@ptrArrayGlobal = global [8 x ptr] zeroinitializer +@constantPtrGlobal = constant ptr @someGlobal +@runtime.gcGlobalRoots = internal constant [4 x { ptr, i32 }] [{ ptr, i32 } { ptr @ptrGlobal, i32 4 }, { ptr, i32 } { ptr @structGlobal, i32 4 }, { ptr, i32 } { ptr getelementptr (i8, ptr @structGlobal, i32 8), i32 8 }, { ptr, i32 } { ptr @ptrArrayGlobal, i32 32 }] +@runtime.gcGlobalRootValueArray = internal global [12 x i32] zeroinitializer declare void @runtime.trackPointer(ptr nocapture readonly) declare noalias nonnull ptr @runtime.alloc(i32, ptr) +define i32 @runtime.gcGlobalRootCount() { +entry: + ret i32 4 +} + +define ptr @runtime.gcGlobalRoot(i32 %0) { +entry: + %1 = getelementptr inbounds [4 x { ptr, i32 }], ptr @runtime.gcGlobalRoots, i32 0, i32 %0 + %2 = getelementptr inbounds nuw { ptr, i32 }, ptr %1, i32 0, i32 0 + %3 = load ptr, ptr %2, align 4 + ret ptr %3 +} + +define i32 @runtime.gcGlobalRootSize(i32 %0) { +entry: + %1 = getelementptr inbounds [4 x { ptr, i32 }], ptr @runtime.gcGlobalRoots, i32 0, i32 %0 + %2 = getelementptr inbounds nuw { ptr, i32 }, ptr %1, i32 0, i32 1 + %3 = load i32, ptr %2, align 4 + ret i32 %3 +} + +define ptr @runtime.gcGlobalRootValues() { +entry: + ret ptr @runtime.gcGlobalRootValueArray +} + define ptr @getPointer() { ret ptr @someGlobal } From 6bd0ff3b9b18fbceaf72034e0fa014bf272b0bdf Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:24:01 -0700 Subject: [PATCH 4/7] runtime: eagerly mark Boehm global roots --- src/runtime/gc_boehm.go | 3 +++ src/runtime/gc_globals_range.go | 30 +++++-------------------- transform/testdata/gc-stackslots.ll | 2 -- transform/testdata/gc-stackslots.out.ll | 6 ----- 4 files changed, 8 insertions(+), 33 deletions(-) diff --git a/src/runtime/gc_boehm.go b/src/runtime/gc_boehm.go index 66a78697fe..02ee0aaa93 100644 --- a/src/runtime/gc_boehm.go +++ b/src/runtime/gc_boehm.go @@ -165,6 +165,9 @@ func libgc_size(ptr uintptr) uintptr //export GC_push_all func libgc_push_all(bottom, top uintptr) +//export GC_push_all_eager +func libgc_push_all_eager(bottom, top uintptr) + //export GC_push_all_stack func libgc_push_all_stack(bottom, top uintptr) diff --git a/src/runtime/gc_globals_range.go b/src/runtime/gc_globals_range.go index 5702e2b1ca..2f7d0ac738 100644 --- a/src/runtime/gc_globals_range.go +++ b/src/runtime/gc_globals_range.go @@ -5,31 +5,12 @@ package runtime import "unsafe" func markGlobals() { - rangeCount := gcGlobalRootCount() - if rangeCount == 0 { - return + for i := uintptr(0); i < gcGlobalRootCount(); i++ { + addr := gcGlobalRoot(uintptr(i)) + // GC_push_all queues one range per call and overflows Boehm's mark + // stack for programs with thousands of roots. Scan each range now. + libgc_push_all_eager(uintptr(addr), uintptr(addr)+gcGlobalRootSize(i)) } - - var rootCount uintptr - for i := uintptr(0); i < rangeCount; i++ { - rootCount += gcGlobalRootSize(i) / unsafe.Sizeof(uintptr(0)) - } - - // markRoots only accepts a range, so copy all global pointers into - // contiguous scratch space for marking. - roots := unsafe.Slice((*uintptr)(gcGlobalRootValues()), rootCount) - var rootIndex uintptr - for i := uintptr(0); i < rangeCount; i++ { - addr := gcGlobalRoot(i) - size := gcGlobalRootSize(i) - for offset := uintptr(0); offset < size; offset += unsafe.Sizeof(uintptr(0)) { - roots[rootIndex] = *(*uintptr)(unsafe.Add(addr, offset)) - rootIndex++ - } - } - - start := uintptr(unsafe.Pointer(&roots[0])) - markRoots(start, start+rootCount*unsafe.Sizeof(roots[0])) } // These functions are generated by the compiler from the pointer layouts of @@ -37,4 +18,3 @@ func markGlobals() { func gcGlobalRootCount() uintptr func gcGlobalRoot(index uintptr) unsafe.Pointer func gcGlobalRootSize(index uintptr) uintptr -func gcGlobalRootValues() unsafe.Pointer diff --git a/transform/testdata/gc-stackslots.ll b/transform/testdata/gc-stackslots.ll index 912051901b..2ec36367f5 100644 --- a/transform/testdata/gc-stackslots.ll +++ b/transform/testdata/gc-stackslots.ll @@ -19,8 +19,6 @@ declare ptr @runtime.gcGlobalRoot(i32) declare i32 @runtime.gcGlobalRootSize(i32) -declare ptr @runtime.gcGlobalRootValues() - ; Generic function that returns a pointer (that must be tracked). define ptr @getPointer() { ret ptr @someGlobal diff --git a/transform/testdata/gc-stackslots.out.ll b/transform/testdata/gc-stackslots.out.ll index 5fdad3f6a1..b3fc586e5a 100644 --- a/transform/testdata/gc-stackslots.out.ll +++ b/transform/testdata/gc-stackslots.out.ll @@ -9,7 +9,6 @@ target triple = "wasm32-unknown-unknown-wasm" @ptrArrayGlobal = global [8 x ptr] zeroinitializer @constantPtrGlobal = constant ptr @someGlobal @runtime.gcGlobalRoots = internal constant [4 x { ptr, i32 }] [{ ptr, i32 } { ptr @ptrGlobal, i32 4 }, { ptr, i32 } { ptr @structGlobal, i32 4 }, { ptr, i32 } { ptr getelementptr (i8, ptr @structGlobal, i32 8), i32 8 }, { ptr, i32 } { ptr @ptrArrayGlobal, i32 32 }] -@runtime.gcGlobalRootValueArray = internal global [12 x i32] zeroinitializer declare void @runtime.trackPointer(ptr nocapture readonly) @@ -36,11 +35,6 @@ entry: ret i32 %3 } -define ptr @runtime.gcGlobalRootValues() { -entry: - ret ptr @runtime.gcGlobalRootValueArray -} - define ptr @getPointer() { ret ptr @someGlobal } From 9c8c7946e9f5dd99b3f5eabb6886000d775c81ba Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:35:11 -0700 Subject: [PATCH 5/7] test: reduce AVR map memory pressure --- testdata/map.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/testdata/map.go b/testdata/map.go index f5be02ab06..0f19ed0976 100644 --- a/testdata/map.go +++ b/testdata/map.go @@ -123,8 +123,15 @@ func main() { println(`structMap[{"tau", 6.28}]:`, structMap[namedFloat{"tau", 6.28}]) // test preallocated map - squares := make(map[int]int, 200) - testBigMap(squares, 100) + mapSize := 200 + mapEntries := 100 + if unsafe.Sizeof(uintptr(0)) < 4 { + // Leave enough heap for the rest of this test on low-memory devices. + mapSize = 100 + mapEntries = 50 + } + squares := make(map[int]int, mapSize) + testBigMap(squares, mapEntries) println("tested preallocated map") // test growing maps From 920380d0591b7e569350cd2ea4a8f32aada89bf0 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:08:32 -0700 Subject: [PATCH 6/7] runtime: precisely scan globals on all platforms --- builder/build.go | 3 + builder/testdata/binary-size.txt | 6 +- src/runtime/gc_globals.go | 15 --- src/runtime/gc_globals_blocks.go | 2 +- ...c_globals_range.go => gc_globals_boehm.go} | 2 +- src/runtime/gc_globals_custom.go | 2 +- src/runtime/gc_globals_none.go | 6 ++ src/runtime/gc_stack_cores.go | 4 +- src/runtime/gc_stack_raw.go | 2 +- src/runtime/gc_stack_threads.go | 2 +- src/runtime/os_darwin.go | 91 ------------------- src/runtime/os_linux.go | 90 ------------------ src/runtime/os_windows.go | 30 ------ src/runtime/os_windows_pe.go | 63 ------------- src/runtime/runtime_nintendoswitch.go | 25 ----- transform/optimizer.go | 64 +++++++++++++ transform/optimizer_internal_test.go | 33 +++++++ transform/testdata/optimizer-alloc-uses.ll | 20 ++++ 18 files changed, 136 insertions(+), 324 deletions(-) delete mode 100644 src/runtime/gc_globals.go rename src/runtime/{gc_globals_range.go => gc_globals_boehm.go} (94%) create mode 100644 src/runtime/gc_globals_none.go delete mode 100644 src/runtime/os_windows_pe.go create mode 100644 transform/optimizer_internal_test.go create mode 100644 transform/testdata/optimizer-alloc-uses.ll diff --git a/builder/build.go b/builder/build.go index 974ddc2a37..155a9e1db1 100644 --- a/builder/build.go +++ b/builder/build.go @@ -1293,6 +1293,9 @@ func makeGlobalsModule(ctx llvm.Context, globals map[string]map[string]string, m global := llvm.AddGlobal(mod, stringType, globalName) global.SetInitializer(initializer) global.SetAlignment(targetData.PrefTypeAlignment(stringType)) + // Keep external linkage for module resolution. Hidden visibility permits internalization. + // See https://llvm.org/docs/LangRef.html#visibility-styles. + global.SetVisibility(llvm.HiddenVisibility) } } diff --git a/builder/testdata/binary-size.txt b/builder/testdata/binary-size.txt index ef8e6c1a42..3a3cd1cc06 100644 --- a/builder/testdata/binary-size.txt +++ b/builder/testdata/binary-size.txt @@ -1,4 +1,4 @@ target package code rodata data bss -hifive1b examples/echo 4321 323 0 2268 -microbit examples/serial 2842 382 8 2264 -wioterminal examples/pininterrupt 8039 1669 132 7496 +hifive1b examples/echo 4405 323 0 2268 +microbit examples/serial 2922 382 8 2264 +wioterminal examples/pininterrupt 8251 1717 148 7496 diff --git a/src/runtime/gc_globals.go b/src/runtime/gc_globals.go deleted file mode 100644 index 58e70ca3e1..0000000000 --- a/src/runtime/gc_globals.go +++ /dev/null @@ -1,15 +0,0 @@ -//go:build (baremetal || tinygo.wasm) && !uefi - -package runtime - -// This file implements findGlobals for all systems where the start and end of -// the globals section can be found through linker-defined symbols. - -// findGlobals finds all globals (which are reachable by definition) and calls -// the callback for them. -// -// This implementation marks all globals conservatively and assumes it can use -// linker-defined symbols for the start and end of the .data section. -func findGlobals(found func(start, end uintptr)) { - found(globalsStart, globalsEnd) -} diff --git a/src/runtime/gc_globals_blocks.go b/src/runtime/gc_globals_blocks.go index b9cb94cb8f..9aacf0b27b 100644 --- a/src/runtime/gc_globals_blocks.go +++ b/src/runtime/gc_globals_blocks.go @@ -1,4 +1,4 @@ -//go:build (gc.conservative || gc.precise) && tinygo.wasm +//go:build gc.conservative || gc.precise package runtime diff --git a/src/runtime/gc_globals_range.go b/src/runtime/gc_globals_boehm.go similarity index 94% rename from src/runtime/gc_globals_range.go rename to src/runtime/gc_globals_boehm.go index 2f7d0ac738..92057f50ea 100644 --- a/src/runtime/gc_globals_range.go +++ b/src/runtime/gc_globals_boehm.go @@ -1,4 +1,4 @@ -//go:build gc.boehm && tinygo.wasm +//go:build gc.boehm package runtime diff --git a/src/runtime/gc_globals_custom.go b/src/runtime/gc_globals_custom.go index bf612e6ac1..6b3a720607 100644 --- a/src/runtime/gc_globals_custom.go +++ b/src/runtime/gc_globals_custom.go @@ -1,4 +1,4 @@ -//go:build gc.custom && tinygo.wasm +//go:build gc.custom package runtime diff --git a/src/runtime/gc_globals_none.go b/src/runtime/gc_globals_none.go new file mode 100644 index 0000000000..264406ae08 --- /dev/null +++ b/src/runtime/gc_globals_none.go @@ -0,0 +1,6 @@ +//go:build gc.leaking || gc.none + +package runtime + +func markGlobals() { +} diff --git a/src/runtime/gc_stack_cores.go b/src/runtime/gc_stack_cores.go index 9100109a2c..66aff00879 100644 --- a/src/runtime/gc_stack_cores.go +++ b/src/runtime/gc_stack_cores.go @@ -26,7 +26,7 @@ func gcMarkReachable() { } // Scan globals. - findGlobals(markRoots) + markGlobals() // Nothing more to do: the other cores haven't started yet. return @@ -57,7 +57,7 @@ func gcMarkReachable() { } // Scan globals. - findGlobals(markRoots) + markGlobals() // Signal each core in turn that they can scan the stack. for i := uint32(0); i < numCPU; i++ { diff --git a/src/runtime/gc_stack_raw.go b/src/runtime/gc_stack_raw.go index 03c37696a9..95d4b0a59f 100644 --- a/src/runtime/gc_stack_raw.go +++ b/src/runtime/gc_stack_raw.go @@ -12,7 +12,7 @@ var gcScanState atomic.Uint32 func gcMarkReachable() { markStack() - findGlobals(markRoots) + markGlobals() } // markStack marks all root pointers found on the stack. diff --git a/src/runtime/gc_stack_threads.go b/src/runtime/gc_stack_threads.go index a2b06486f5..0e58644a84 100644 --- a/src/runtime/gc_stack_threads.go +++ b/src/runtime/gc_stack_threads.go @@ -13,7 +13,7 @@ func gcMarkReachable() { // //go:linkname gcScanGlobals internal/task.gcScanGlobals func gcScanGlobals() { - findGlobals(markRoots) + markGlobals() } // Function called from assembly with all registers pushed, to actually scan the diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go index 6a151af807..c807cdeb2a 100644 --- a/src/runtime/os_darwin.go +++ b/src/runtime/os_darwin.go @@ -31,97 +31,6 @@ const ( sig_SIGSEGV = 11 ) -// https://opensource.apple.com/source/xnu/xnu-7195.141.2/EXTERNAL_HEADERS/mach-o/loader.h.auto.html -type machHeader struct { - magic uint32 - cputype uint32 - cpusubtype uint32 - filetype uint32 - ncmds uint32 - sizeofcmds uint32 - flags uint32 - reserved uint32 -} - -// Struct for the LC_SEGMENT_64 load command. -type segmentLoadCommand struct { - cmd uint32 // LC_SEGMENT_64 - cmdsize uint32 - segname [16]byte - vmaddr uintptr - vmsize uintptr - fileoff uintptr - filesize uintptr - maxprot uint32 - initprot uint32 - nsects uint32 - flags uint32 -} - -// MachO header of the currently running process. -// -//go:extern _mh_execute_header -var libc_mh_execute_header machHeader - -// Find global variables in .data/.bss sections. -// The MachO linker doesn't seem to provide symbols for the start and end of the -// data section. There is get_etext, get_edata, and get_end, but these are -// undocumented and don't work with ASLR (which is enabled by default). -// Therefore, read the MachO header directly. -func findGlobals(found func(start, end uintptr)) { - // Here is a useful blog post to understand the MachO file format: - // https://h3adsh0tzz.com/2020/01/macho-file-format/ - - const ( - MH_MAGIC_64 = 0xfeedfacf - LC_SEGMENT_64 = 0x19 - VM_PROT_WRITE = 0x02 - ) - - // Sanity check that we're actually looking at a MachO header. - if gcAsserts && libc_mh_execute_header.magic != MH_MAGIC_64 { - runtimeFatal("gc: unexpected MachO header") - } - - // Iterate through the load commands. - // Because we're only interested in LC_SEGMENT_64 load commands, cast the - // pointer to that struct in advance. - var offset uintptr - var hasOffset bool - cmd := (*segmentLoadCommand)(unsafe.Pointer(uintptr(unsafe.Pointer(&libc_mh_execute_header)) + unsafe.Sizeof(machHeader{}))) - for i := libc_mh_execute_header.ncmds; i != 0; i-- { - if cmd.cmd == LC_SEGMENT_64 { - if cmd.fileoff == 0 && cmd.nsects != 0 { - // Detect ASLR offset by checking fileoff and nsects. This - // locates the __TEXT segment. This matches getsectiondata: - // https://opensource.apple.com/source/cctools/cctools-973.0.1/libmacho/getsecbyname.c.auto.html - offset = uintptr(unsafe.Pointer(&libc_mh_execute_header)) - cmd.vmaddr - hasOffset = true - } - if cmd.maxprot&VM_PROT_WRITE != 0 { - // Found a writable segment, which may contain Go globals. - if gcAsserts && !hasOffset { - // No ASLR offset detected. Did the __TEXT segment come - // after the __DATA segment? - // Note that when ASLR is disabled (for example, when - // running inside lldb), the offset is zero. That's why we - // need a separate hasOffset for this assert. - runtimeFatal("gc: did not detect ASLR offset") - } - // Scan this segment for GC roots. - // This could be improved by only reading the memory areas - // covered by sections. That would reduce the amount of memory - // scanned a little bit (up to a single VM page). - found(offset+cmd.vmaddr, offset+cmd.vmaddr+cmd.vmsize) - } - } - - // Move on to the next load command (which may or may not be a - // LC_SEGMENT_64). - cmd = (*segmentLoadCommand)(unsafe.Add(unsafe.Pointer(cmd), cmd.cmdsize)) - } -} - func hardwareRand() (n uint64, ok bool) { n |= uint64(libc_arc4random()) n |= uint64(libc_arc4random()) << 32 diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index a99a2ad290..9a8f6b203f 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -34,101 +34,11 @@ const ( sig_SIGSEGV = linux_SIGSEGV ) -// For the definition of the various header structs, see: -// https://refspecs.linuxfoundation.org/elf/elf.pdf -// Also useful: -// https://en.wikipedia.org/wiki/Executable_and_Linkable_Format -type elfHeader struct { - ident_magic uint32 - ident_class uint8 - ident_data uint8 - ident_version uint8 - ident_osabi uint8 - ident_abiversion uint8 - _ [7]byte // reserved - filetype uint16 - machine uint16 - version uint32 - entry uintptr - phoff uintptr - shoff uintptr - flags uint32 - ehsize uint16 - phentsize uint16 - phnum uint16 - shentsize uint16 - shnum uint16 - shstrndx uint16 -} - -type elfProgramHeader64 struct { - _type uint32 - flags uint32 - offset uintptr - vaddr uintptr - paddr uintptr - filesz uintptr - memsz uintptr - align uintptr -} - -type elfProgramHeader32 struct { - _type uint32 - offset uintptr - vaddr uintptr - paddr uintptr - filesz uintptr - memsz uintptr - flags uint32 - align uintptr -} - -// ELF header of the currently running process. -// -//go:extern __ehdr_start -var ehdr_start elfHeader - // int *__errno_location(void); // //export __errno_location func libc_errno_location() *int32 -// findGlobals finds globals in the .data/.bss sections. -// It parses the ELF program header to find writable segments. -func findGlobals(found func(start, end uintptr)) { - // Relevant constants from the ELF specification. - // See: https://refspecs.linuxfoundation.org/elf/elf.pdf - const ( - PT_LOAD = 1 - PF_W = 0x2 // program flag: write access - ) - - headerPtr := unsafe.Pointer(uintptr(unsafe.Pointer(&ehdr_start)) + ehdr_start.phoff) - for i := 0; i < int(ehdr_start.phnum); i++ { - // Look for a writable segment and scan its contents. - // There is a little bit of duplication here, which is unfortunate. But - // the alternative would be to put elfProgramHeader in separate files - // which is IMHO a lot uglier. If only the ELF spec was consistent - // between 32-bit and 64-bit... - if TargetBits == 64 { - header := (*elfProgramHeader64)(headerPtr) - if header._type == PT_LOAD && header.flags&PF_W != 0 { - start := header.vaddr - end := start + header.memsz - found(start, end) - } - } else { - header := (*elfProgramHeader32)(headerPtr) - if header._type == PT_LOAD && header.flags&PF_W != 0 { - start := header.vaddr - end := start + header.memsz - found(start, end) - } - } - headerPtr = unsafe.Add(headerPtr, ehdr_start.phentsize) - } -} - //export getpagesize func libc_getpagesize() int diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index c6d2218509..d6af0a2868 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -6,36 +6,6 @@ const GOOS = "windows" const zeroSizeAllocPtr uintptr = 16 // part of the first protected page -//export GetModuleHandleExA -func _GetModuleHandleExA(dwFlags uint32, lpModuleName unsafe.Pointer, phModule **exeHeader) bool - -// Mark global variables. -// Unfortunately, the linker doesn't provide symbols for the start and end of -// the data/bss sections. Therefore these addresses need to be determined at -// runtime. This might seem complex and it kind of is, but it only compiles to -// around 160 bytes of amd64 instructions. -// Most of this function is based on the documentation in -// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format. -func findGlobals(found func(start, end uintptr)) { - // Constants used in this function. - const ( - // https://docs.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-getmodulehandleexa - GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT = 0x00000002 - ) - - if module == nil { - // Obtain a handle to the currently executing image. What we're getting - // here is really just __ImageBase, but it's probably better to obtain - // it using GetModuleHandle to account for ASLR etc. - result := _GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, nil, &module) - if gcAsserts && (!result || module.signature != 0x5A4D) { // 0x4D5A is "MZ" - runtimeFatal("cannot get module handle") - } - } - - findGlobalsForPE(found) -} - type systeminfo struct { anon0 [4]byte dwpagesize uint32 diff --git a/src/runtime/os_windows_pe.go b/src/runtime/os_windows_pe.go deleted file mode 100644 index 61f2f7fa0b..0000000000 --- a/src/runtime/os_windows_pe.go +++ /dev/null @@ -1,63 +0,0 @@ -//go:build windows || uefi - -package runtime - -import "unsafe" - -// MS-DOS stub with PE header offset: -// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#ms-dos-stub-image-only -type exeHeader struct { - signature uint16 - _ [58]byte // skip DOS header - peHeader uint32 // at offset 0x3C -} - -// COFF file header: -// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#file-headers -type peHeader struct { - magic uint32 - machine uint16 - numberOfSections uint16 - timeDateStamp uint32 - pointerToSymbolTable uint32 - numberOfSymbols uint32 - sizeOfOptionalHeader uint16 - characteristics uint16 -} - -// COFF section header: -// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#section-table-section-headers -type peSection struct { - name [8]byte - virtualSize uint32 - virtualAddress uint32 - sizeOfRawData uint32 - pointerToRawData uint32 - pointerToRelocations uint32 - pointerToLinenumbers uint32 - numberOfRelocations uint16 - numberOfLinenumbers uint16 - characteristics uint32 -} - -var module *exeHeader - -func findGlobalsForPE(found func(start, end uintptr)) { - // https://docs.microsoft.com/en-us/windows/win32/debug/pe-format - const imageSCNMemWrite = 0x80000000 - - pe := (*peHeader)(unsafe.Add(unsafe.Pointer(module), module.peHeader)) - if gcAsserts && pe.magic != 0x00004550 { // 0x4550 is "PE" - runtimeFatal("cannot find PE header") - } - - section := (*peSection)(unsafe.Pointer(uintptr(unsafe.Pointer(pe)) + uintptr(pe.sizeOfOptionalHeader) + unsafe.Sizeof(peHeader{}))) - for i := 0; i < int(pe.numberOfSections); i++ { - if section.characteristics&imageSCNMemWrite != 0 { - start := uintptr(unsafe.Pointer(module)) + uintptr(section.virtualAddress) - end := uintptr(unsafe.Pointer(module)) + uintptr(section.virtualAddress) + uintptr(section.virtualSize) - found(start, end) - } - section = (*peSection)(unsafe.Add(unsafe.Pointer(section), unsafe.Sizeof(peSection{}))) - } -} diff --git a/src/runtime/runtime_nintendoswitch.go b/src/runtime/runtime_nintendoswitch.go index 02b7e8c1ce..031e4bd2d0 100644 --- a/src/runtime/runtime_nintendoswitch.go +++ b/src/runtime/runtime_nintendoswitch.go @@ -257,31 +257,6 @@ func getHeapEnd() uintptr { return heapEnd } -//go:extern __data_start -var dataStartSymbol [0]byte - -//go:extern __data_end -var dataEndSymbol [0]byte - -//go:extern __bss_start -var bssStartSymbol [0]byte - -//go:extern __bss_end -var bssEndSymbol [0]byte - -// Find global variables. -// The linker script provides __*_start and __*_end symbols that can be used to -// scan the given sections. They are already aligned so don't need to be -// manually aligned here. -func findGlobals(found func(start, end uintptr)) { - dataStart := uintptr(unsafe.Pointer(&dataStartSymbol)) - dataEnd := uintptr(unsafe.Pointer(&dataEndSymbol)) - found(dataStart, dataEnd) - bssStart := uintptr(unsafe.Pointer(&bssStartSymbol)) - bssEnd := uintptr(unsafe.Pointer(&bssEndSymbol)) - found(bssStart, bssEnd) -} - // getContextPtr returns the hblauncher context // this is externally linked by gonx func getContextPtr() uintptr { diff --git a/transform/optimizer.go b/transform/optimizer.go index 150a9a77cb..c860a65621 100644 --- a/transform/optimizer.go +++ b/transform/optimizer.go @@ -58,7 +58,9 @@ func Optimize(mod llvm.Module, config *compileopts.Config) []error { // LLVM 17 doesn't have the no-verify-fixpoint flag. optPasses = "globaldce,globalopt,ipsccp,instcombine,adce,function-attrs" } + blockGlobalAllocPromotion(mod) err := mod.RunPasses(optPasses, llvm.TargetMachine{}, po) + removeGlobalAllocPromotionMarker(mod) if err != nil { return []error{fmt.Errorf("could not build pass pipeline: %w", err)} } @@ -80,7 +82,9 @@ func Optimize(mod llvm.Module, config *compileopts.Config) []error { // After interfaces are lowered, there are many more opportunities for // interprocedural optimizations. To get them to work, function // attributes have to be updated first. + blockGlobalAllocPromotion(mod) err = mod.RunPasses(optPasses, llvm.TargetMachine{}, po) + removeGlobalAllocPromotionMarker(mod) if err != nil { return []error{fmt.Errorf("could not build pass pipeline: %w", err)} } @@ -162,7 +166,9 @@ func Optimize(mod llvm.Module, config *compileopts.Config) []error { po := llvm.NewPassBuilderOptions() defer po.Dispose() passes := fmt.Sprintf("thinlto-pre-link<%s>", optLevel) + blockGlobalAllocPromotion(mod) err := mod.RunPasses(passes, llvm.TargetMachine{}, po) + removeGlobalAllocPromotionMarker(mod) if err != nil { return []error{fmt.Errorf("could not build pass pipeline: %w", err)} } @@ -177,6 +183,64 @@ func Optimize(mod llvm.Module, config *compileopts.Config) []error { return nil } +func blockGlobalAllocPromotion(mod llvm.Module) { + ctx := mod.Context() + ptrType := llvm.PointerType(ctx.Int8Type(), 0) + marker := llvm.AddFunction(mod, "tinygo.gc.alloc.marker", llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptrType}, false)) + + builder := ctx.NewBuilder() + defer builder.Dispose() + var marked bool + for _, name := range []string{"runtime.alloc", "runtime.alloc_noheap"} { + alloc := mod.NamedFunction(name) + if alloc.IsNil() { + continue + } + for _, call := range getUses(alloc) { + if call.IsACallInst().IsNil() || call.CalledValue() != alloc { + continue + } + if isPointerFreeAllocation(call) { + continue + } + + // GlobalOpt may otherwise turn this allocation into an untyped + // global, hiding its pointer fields from makeGCGlobalRoots. + next := llvm.NextInstruction(call) + if next.IsNil() { + continue + } + builder.SetInsertPointBefore(next) + builder.CreateCall(marker.GlobalValueType(), marker, []llvm.Value{call}, "") + marked = true + } + } + if !marked { + marker.EraseFromParentAsFunction() + } +} + +func isPointerFreeAllocation(call llvm.Value) bool { + const noPointerLayout = 3 + + layout := call.Operand(1) + return !layout.IsAConstantExpr().IsNil() && + layout.Opcode() == llvm.IntToPtr && + !layout.Operand(0).IsAConstantInt().IsNil() && + layout.Operand(0).ZExtValue() == noPointerLayout +} + +func removeGlobalAllocPromotionMarker(mod llvm.Module) { + marker := mod.NamedFunction("tinygo.gc.alloc.marker") + if marker.IsNil() { + return + } + for _, call := range getUses(marker) { + call.EraseFromParentAsInstruction() + } + marker.EraseFromParentAsFunction() +} + // functionsUsedInTransform is a list of function symbols that may be used // during TinyGo optimization passes so they have to be marked as external // linkage until all TinyGo passes have finished. diff --git a/transform/optimizer_internal_test.go b/transform/optimizer_internal_test.go new file mode 100644 index 0000000000..3358706dd0 --- /dev/null +++ b/transform/optimizer_internal_test.go @@ -0,0 +1,33 @@ +package transform + +import ( + "testing" + + "tinygo.org/x/go-llvm" +) + +func TestBlockGlobalAllocPromotionUses(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + buf, err := llvm.NewMemoryBufferFromFile("testdata/optimizer-alloc-uses.ll") + if err != nil { + t.Fatal(err) + } + mod, err := ctx.ParseIR(buf) + if err != nil { + t.Fatal(err) + } + defer mod.Dispose() + + blockGlobalAllocPromotion(mod) + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + marker := mod.NamedFunction("tinygo.gc.alloc.marker") + if marker.IsNil() { + t.Fatal("allocation marker was not created") + } + if uses := getUses(marker); len(uses) != 1 { + t.Fatalf("got %d marker uses, want 1", len(uses)) + } +} diff --git a/transform/testdata/optimizer-alloc-uses.ll b/transform/testdata/optimizer-alloc-uses.ll new file mode 100644 index 0000000000..d2bed01c30 --- /dev/null +++ b/transform/testdata/optimizer-alloc-uses.ll @@ -0,0 +1,20 @@ +target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128" +target triple = "wasm32-unknown-unknown-wasm" + +@allocFunction = constant ptr @runtime.alloc + +declare ptr @runtime.alloc(i32, ptr) + +declare void @use(ptr) + +define void @passAllocator() { +entry: + call void @use(ptr @runtime.alloc) + ret void +} + +define ptr @allocate() { +entry: + %allocation = call ptr @runtime.alloc(i32 4, ptr inttoptr (i32 5 to ptr)) + ret ptr %allocation +} From 0f5793117336cc66a3de498b78c118b0008a2b12 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:32:45 -0700 Subject: [PATCH 7/7] runtime: preserve malloc allocations until free C malloc storage has explicit lifetime: it must remain allocated until free even when no GC-visible pointer references it. Treating it as an ordinary NoPtrs allocation breaks bare-metal C object graphs, while conservatively scanning arbitrary C bytes creates false Go roots. Add allocManual/freeManual so collectors can represent pointer-free, explicitly managed storage. Block GC keeps these objects permanently marked and releases their blocks on free; Boehm uses atomic uncollectable allocations; leaking and custom collectors provide equivalent behavior. Wasm keeps its allocation map only for validation and sizes, and WASIp2 realloc now copies min(oldSize, newSize). Bump the Boehm library cache version because enabling atomic uncollectable allocations changes its compiled flags and exported API. Also handle zero-size and overflowing allocations, serialize allocation registries, reject Go finalizers on manual storage, and add CGo regressions for C pointer graphs, hidden until-free allocations, repeated free/reuse, and allocation edge cases. --- builder/bdwgc.go | 9 ++- builder/testdata/binary-size.txt | 6 +- compileopts/config.go | 2 +- src/runtime/arch_tinygowasm_malloc.go | 65 ++++++++++------ src/runtime/baremetal.go | 9 ++- src/runtime/gc_blocks.go | 104 +++++++++++++++++-------- src/runtime/gc_boehm.go | 23 ++++++ src/runtime/gc_custom.go | 9 +-- src/runtime/gc_leaking.go | 13 ---- src/runtime/gc_manual.go | 11 +++ src/runtime/gc_manual_custom.go | 35 +++++++++ src/runtime/gc_manual_leaking.go | 15 ++++ src/runtime/gc_none.go | 2 - src/runtime/runtime_wasip2.go | 17 ++++- testdata/cgo/main.c | 105 ++++++++++++++++++++++++++ testdata/cgo/main.go | 31 ++++++++ testdata/cgo/main.h | 14 ++++ testdata/cgo/out.txt | 6 ++ 18 files changed, 385 insertions(+), 91 deletions(-) create mode 100644 src/runtime/gc_manual.go create mode 100644 src/runtime/gc_manual_custom.go create mode 100644 src/runtime/gc_manual_leaking.go diff --git a/builder/bdwgc.go b/builder/bdwgc.go index b03b154203..508e9470b8 100644 --- a/builder/bdwgc.go +++ b/builder/bdwgc.go @@ -30,10 +30,11 @@ var BoehmGC = Library{ // Use a minimal environment. "-DNO_MSGBOX_ON_ERROR", // don't call MessageBoxA on Windows "-DDONT_USE_ATEXIT", - "-DNO_GETENV", // smaller binary, more predictable configuration - "-DNO_CLOCK", // don't use system clock - "-DNO_DEBUGGING", // reduce code size - "-DGC_NO_FINALIZATION", // finalization is not used at the moment + "-DNO_GETENV", // smaller binary, more predictable configuration + "-DNO_CLOCK", // don't use system clock + "-DNO_DEBUGGING", // reduce code size + "-DGC_NO_FINALIZATION", // finalization is not used at the moment + "-DGC_ATOMIC_UNCOLLECTABLE", // pointer-free storage retained until GC_free // Special flag to work around the lack of __data_start in ld.lld. // TODO: try to fix this in LLVM/lld directly so we don't have to diff --git a/builder/testdata/binary-size.txt b/builder/testdata/binary-size.txt index 3a3cd1cc06..8dada1b980 100644 --- a/builder/testdata/binary-size.txt +++ b/builder/testdata/binary-size.txt @@ -1,4 +1,4 @@ target package code rodata data bss -hifive1b examples/echo 4405 323 0 2268 -microbit examples/serial 2922 382 8 2264 -wioterminal examples/pininterrupt 8251 1717 148 7496 +hifive1b examples/echo 4533 323 0 2268 +microbit examples/serial 3002 382 8 2264 +wioterminal examples/pininterrupt 8331 1717 148 7496 diff --git a/compileopts/config.go b/compileopts/config.go index 777cb782ea..dc59f9686e 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -24,7 +24,7 @@ import ( // library path in advance in several places). var libVersions = map[string]int{ "musl": 3, - "bdwgc": 2, + "bdwgc": 3, "picolibc": 2, "wasmbuiltins": 1, } diff --git a/src/runtime/arch_tinygowasm_malloc.go b/src/runtime/arch_tinygowasm_malloc.go index 694840af90..1877aa575e 100644 --- a/src/runtime/arch_tinygowasm_malloc.go +++ b/src/runtime/arch_tinygowasm_malloc.go @@ -3,7 +3,7 @@ package runtime import ( - "internal/gclayout" + "internal/task" "unsafe" ) @@ -11,21 +11,21 @@ import ( // code linked from other languages can allocate memory without colliding with // our GC allocations. -// Map of allocations, where the key is the allocated pointer and the value is -// the size of the allocation. -// TODO: make this a map[unsafe.Pointer]uintptr, since that results in slightly -// smaller binaries. But for that to work, unsafe.Pointer needs to be seen as a -// binary key (which it is not at the moment). -// See https://github.com/tinygo-org/tinygo/pull/4898 for details. -var allocs = make(map[*byte]uintptr) +// Map of allocations, where the key is the allocation address and the value is +// its size. Integer keys intentionally do not act as GC roots: manual +// allocations are retained by the allocator until free. +var allocs = make(map[uintptr]uintptr) +var allocsLock task.PMutex //export malloc func libc_malloc(size uintptr) unsafe.Pointer { if size == 0 { return nil } - ptr := alloc(size, gclayout.NoPtrs.AsPtr()) - allocs[(*byte)(ptr)] = size + ptr := allocManual(size) + allocsLock.Lock() + allocs[uintptr(ptr)] = size + allocsLock.Unlock() return ptr } @@ -34,16 +34,22 @@ func libc_free(ptr unsafe.Pointer) { if ptr == nil { return } - if _, ok := allocs[(*byte)(ptr)]; ok { - delete(allocs, (*byte)(ptr)) + allocsLock.Lock() + if _, ok := allocs[uintptr(ptr)]; ok { + delete(allocs, uintptr(ptr)) + allocsLock.Unlock() + freeManual(ptr) } else { + allocsLock.Unlock() runtimeFatal("free: invalid pointer") } } //export calloc func libc_calloc(nmemb, size uintptr) unsafe.Pointer { - // No difference between calloc and malloc. + if size != 0 && nmemb > ^uintptr(0)/size { + return nil + } return libc_malloc(nmemb * size) } @@ -54,22 +60,37 @@ func libc_realloc(oldPtr unsafe.Pointer, size uintptr) unsafe.Pointer { return nil } + var oldSize uintptr + if oldPtr != nil { + allocsLock.Lock() + var ok bool + oldSize, ok = allocs[uintptr(oldPtr)] + allocsLock.Unlock() + if !ok { + runtimeFatal("realloc: invalid pointer") + } + } + // It's hard to optimize this to expand the current buffer with our GC, but // it is theoretically possible. For now, just always allocate fresh. // TODO: we could skip this if the new allocation is smaller than the old. - ptr := alloc(size, gclayout.NoPtrs.AsPtr()) + ptr := allocManual(size) + allocsLock.Lock() if oldPtr != nil { - if oldSize, ok := allocs[(*byte)(oldPtr)]; ok { - oldBuf := unsafe.Slice((*byte)(oldPtr), oldSize) - newBuf := unsafe.Slice((*byte)(ptr), size) - copy(newBuf, oldBuf) - delete(allocs, (*byte)(oldPtr)) - } else { + if currentSize, ok := allocs[uintptr(oldPtr)]; !ok || currentSize != oldSize { + allocsLock.Unlock() runtimeFatal("realloc: invalid pointer") } + oldBuf := unsafe.Slice((*byte)(oldPtr), oldSize) + newBuf := unsafe.Slice((*byte)(ptr), size) + copy(newBuf, oldBuf) + delete(allocs, uintptr(oldPtr)) + } + allocs[uintptr(ptr)] = size + allocsLock.Unlock() + if oldPtr != nil { + freeManual(oldPtr) } - - allocs[(*byte)(ptr)] = size return ptr } diff --git a/src/runtime/baremetal.go b/src/runtime/baremetal.go index 4893297fe1..d2963a4964 100644 --- a/src/runtime/baremetal.go +++ b/src/runtime/baremetal.go @@ -3,7 +3,6 @@ package runtime import ( - "internal/gclayout" "sync/atomic" "unsafe" ) @@ -12,18 +11,20 @@ import ( func libc_malloc(size uintptr) unsafe.Pointer { // Note: this zeroes the returned buffer which is not necessary. // The same goes for bytealg.MakeNoZero. - return alloc(size, gclayout.NoPtrs.AsPtr()) + return allocManual(size) } //export calloc func libc_calloc(nmemb, size uintptr) unsafe.Pointer { - // No difference between calloc and malloc. + if size != 0 && nmemb > ^uintptr(0)/size { + return nil + } return libc_malloc(nmemb * size) } //export free func libc_free(ptr unsafe.Pointer) { - free(ptr) + freeManual(ptr) } //export runtime_putchar diff --git a/src/runtime/gc_blocks.go b/src/runtime/gc_blocks.go index c27401d62d..72f1dd041f 100644 --- a/src/runtime/gc_blocks.go +++ b/src/runtime/gc_blocks.go @@ -205,8 +205,9 @@ func (b gcBlock) free() { // objHeader is a structure appended to every heap object to hold metadata. type objHeader struct { - // next is the next object to scan after this. - next *objHeader + // next links the GC scan list. Manual allocations remain permanently marked + // and use the otherwise invalid value 1 as an until-free marker. + next uintptr // layout holds the layout bitmap used to find pointers in the object. layout gcLayout @@ -482,6 +483,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { // Create the object header. size -= unsafe.Sizeof(objHeader{}) header := (*objHeader)(unsafe.Add(pointer, size)) + header.next = 0 header.layout = parseGCLayout(layout) // We've claimed this allocation, now we can unlock the heap. @@ -500,42 +502,61 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { return pointer } -func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer { - if ptr == nil { - return alloc(size, gclayout.NoPtrs.AsPtr()) +// allocManual allocates pointer-free memory that remains live until freeManual. +func allocManual(size uintptr) unsafe.Pointer { + if size == 0 { + return alloc_zero(size, gclayout.NoPtrs.AsPtr()) } + ptr := alloc(size, gclayout.NoPtrs.AsPtr()) - // Find the first block of the original allocation. - firstBlock := blockFromAddr(uintptr(ptr)) - - // Find the last block of the original allocation. - lastBlock := firstBlock.findHead() + gcLock.Lock() + head := blockFromAddr(uintptr(ptr)).findHead() + head.setState(blockStateMark) + header := (*objHeader)(unsafe.Add(head.pointer(), bytesPerBlock-unsafe.Sizeof(objHeader{}))) + header.next = 1 + gcLock.Unlock() + return ptr +} - // Calculate the size of the original allocation body. - oldSize := uintptr(lastBlock-firstBlock)*bytesPerBlock + (bytesPerBlock - unsafe.Sizeof(objHeader{})) +func free(ptr unsafe.Pointer) { + if ptr == nil { + return + } - if size <= oldSize { - // The requested size is less than the old size. - // There are likely scenarios for this: - // - The caller intended to grow the allocation, but the original size - // was rounded up by alloc to a multiple of the block size. - // The rounded size is already sufficient. - // - The caller intended to shrink the allocation. - // We currently ignore this case. - // Either way, the current allocation can be left alone. - return ptr + gcLock.Lock() + addr := uintptr(ptr) + if !isOnHeap(addr) || (addr-heapStart)%bytesPerBlock != 0 { + gcLock.Unlock() + runtimeFatal("free: invalid pointer") } - // Create a new allocation and copy the old data. - newAlloc := alloc(size, gclayout.NoPtrs.AsPtr()) - memcpy(newAlloc, ptr, oldSize) - free(ptr) + firstBlock := blockFromAddr(addr) + state := firstBlock.state() + if state != blockStateTail && state != blockStateHead && state != blockStateMark { + gcLock.Unlock() + runtimeFatal("free: invalid pointer") + } - return newAlloc -} + allocationStart := firstBlock + for allocationStart != 0 && (allocationStart-1).state() == blockStateTail { + allocationStart-- + } + if allocationStart != firstBlock { + gcLock.Unlock() + runtimeFatal("free: invalid pointer") + } -func free(ptr unsafe.Pointer) { - // TODO: free blocks on request, when the compiler knows they're unused. + lastBlock := firstBlock.findHead() + header := (*objHeader)(unsafe.Add(lastBlock.pointer(), bytesPerBlock-unsafe.Sizeof(objHeader{}))) + if header.next != 1 { + gcLock.Unlock() + runtimeFatal("free: invalid pointer") + } + for block := firstBlock; block <= lastBlock; block++ { + block.free() + } + insertFreeRange(firstBlock.pointer(), uintptr(lastBlock-firstBlock+1)) + gcLock.Unlock() } // GC performs a garbage collection cycle. @@ -666,7 +687,7 @@ func finishMark() { if obj == nil { return } - scanList = obj.next + scanList = (*objHeader)(unsafe.Pointer(obj.next)) // Check if the object may contain pointers. if obj.layout.pointerFree() { @@ -724,7 +745,7 @@ func markRoot(addr, root uintptr) { // Add the object to the scan list. header := (*objHeader)(unsafe.Add(head.pointer(), bytesPerBlock-unsafe.Sizeof(objHeader{}))) - header.next = scanList + header.next = uintptr(unsafe.Pointer(scanList)) scanList = header } @@ -758,7 +779,10 @@ func sweep() uintptr { // Unmark the next head. block-- - block.unmark() + header := (*objHeader)(unsafe.Add(block.pointer(), bytesPerBlock-unsafe.Sizeof(objHeader{}))) + if header.next != 1 { + block.unmark() + } // Skip the tail. for block > 0 && (block-1).state() == blockStateTail { @@ -903,5 +927,19 @@ func SetFinalizer(obj interface{}, finalizer interface{}) { // A nil pointer has nothing to finalize. return } + + gcLock.Lock() + addr := uintptr(objPtr) + manual := false + if isOnHeap(addr) { + head := blockFromAddr(addr).findHead() + header := (*objHeader)(unsafe.Add(head.pointer(), bytesPerBlock-unsafe.Sizeof(objHeader{}))) + manual = header.next == 1 + } + gcLock.Unlock() + if manual && finalizer != nil { + runtimeFatal("runtime.SetFinalizer: manual allocation") + } + registerFinalizer(uintptr(objPtr), finalizer) } diff --git a/src/runtime/gc_boehm.go b/src/runtime/gc_boehm.go index 02ee0aaa93..c4f4fc7283 100644 --- a/src/runtime/gc_boehm.go +++ b/src/runtime/gc_boehm.go @@ -98,8 +98,28 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { return ptr } +func allocManual(size uintptr) unsafe.Pointer { + if size == 0 { + return alloc_zero(size, gclayout.NoPtrs.AsPtr()) + } + + gcLock.Lock() + ptr := libgc_malloc_atomic_uncollectable(size) + gcResumeWorld() + gcLock.Unlock() + if ptr == nil { + runtimeFatal("gc: out of memory") + return nil + } + memzero(ptr, size) + return ptr +} + func free(ptr unsafe.Pointer) { + gcLock.Lock() libgc_free(ptr) + gcResumeWorld() + gcLock.Unlock() } func GC() { @@ -153,6 +173,9 @@ func libgc_malloc(uintptr) unsafe.Pointer //export GC_malloc_atomic func libgc_malloc_atomic(uintptr) unsafe.Pointer +//export GC_malloc_atomic_uncollectable +func libgc_malloc_atomic_uncollectable(uintptr) unsafe.Pointer + //export GC_free func libgc_free(unsafe.Pointer) diff --git a/src/runtime/gc_custom.go b/src/runtime/gc_custom.go index 12f1b2e12d..86224bbbd3 100644 --- a/src/runtime/gc_custom.go +++ b/src/runtime/gc_custom.go @@ -23,14 +23,7 @@ package runtime // - func SetFinalizer(obj interface{}, finalizer interface{}) // - func ReadMemStats(ms *runtime.MemStats) // -// -// In addition, if targeting wasi, the following functions should be exported for interoperability -// with wasi libraries that use them. Note, this requires the export directive, not go:linkname. -// -// - func malloc(size uintptr) unsafe.Pointer -// - func free(ptr unsafe.Pointer) -// - func calloc(nmemb, size uintptr) unsafe.Pointer -// - func realloc(oldPtr unsafe.Pointer, size uintptr) unsafe.Pointer +// The compiler provides the global root ranges used by markRoots. import ( "unsafe" diff --git a/src/runtime/gc_leaking.go b/src/runtime/gc_leaking.go index 3ebee0989b..ce19a54b5f 100644 --- a/src/runtime/gc_leaking.go +++ b/src/runtime/gc_leaking.go @@ -7,7 +7,6 @@ package runtime // may be the only memory allocator possible. import ( - "internal/gclayout" "internal/task" "sync/atomic" "unsafe" @@ -69,18 +68,6 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { return pointer } -func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer { - newAlloc := alloc(size, gclayout.NoPtrs.AsPtr()) - if ptr == nil { - return newAlloc - } - // according to POSIX everything beyond the previous pointer's - // size will have indeterminate values so we can just copy garbage - memcpy(newAlloc, ptr, size) - - return newAlloc -} - func free(ptr unsafe.Pointer) { // Memory is never freed. } diff --git a/src/runtime/gc_manual.go b/src/runtime/gc_manual.go new file mode 100644 index 0000000000..25f7adc54c --- /dev/null +++ b/src/runtime/gc_manual.go @@ -0,0 +1,11 @@ +//go:build !gc.custom + +package runtime + +import "unsafe" + +func freeManual(ptr unsafe.Pointer) { + if ptr != nil && ptr != unsafe.Pointer(zeroSizeAllocPtr) { + free(ptr) + } +} diff --git a/src/runtime/gc_manual_custom.go b/src/runtime/gc_manual_custom.go new file mode 100644 index 0000000000..4f6adc6488 --- /dev/null +++ b/src/runtime/gc_manual_custom.go @@ -0,0 +1,35 @@ +//go:build gc.custom + +package runtime + +import ( + "internal/gclayout" + "internal/task" + "unsafe" +) + +// Custom collectors retain manual allocations through ordinary typed roots so +// the custom GC interface does not need an additional allocation primitive. +var manualAllocs = make(map[*byte]struct{}) +var manualAllocsLock task.PMutex + +func allocManual(size uintptr) unsafe.Pointer { + if size == 0 { + return alloc_zero(size, gclayout.NoPtrs.AsPtr()) + } + ptr := alloc(size, gclayout.NoPtrs.AsPtr()) + manualAllocsLock.Lock() + manualAllocs[(*byte)(ptr)] = struct{}{} + manualAllocsLock.Unlock() + return ptr +} + +func freeManual(ptr unsafe.Pointer) { + if ptr == nil || ptr == unsafe.Pointer(zeroSizeAllocPtr) { + return + } + manualAllocsLock.Lock() + delete(manualAllocs, (*byte)(ptr)) + manualAllocsLock.Unlock() + free(ptr) +} diff --git a/src/runtime/gc_manual_leaking.go b/src/runtime/gc_manual_leaking.go new file mode 100644 index 0000000000..5813587383 --- /dev/null +++ b/src/runtime/gc_manual_leaking.go @@ -0,0 +1,15 @@ +//go:build gc.leaking || gc.none + +package runtime + +import ( + "internal/gclayout" + "unsafe" +) + +func allocManual(size uintptr) unsafe.Pointer { + if size == 0 { + return alloc_zero(size, gclayout.NoPtrs.AsPtr()) + } + return alloc(size, gclayout.NoPtrs.AsPtr()) +} diff --git a/src/runtime/gc_none.go b/src/runtime/gc_none.go index ce9649c719..8634308d9d 100644 --- a/src/runtime/gc_none.go +++ b/src/runtime/gc_none.go @@ -22,8 +22,6 @@ func scanCurrentStack() {} func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer -func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer - func free(ptr unsafe.Pointer) { // Nothing to free when nothing gets allocated. } diff --git a/src/runtime/runtime_wasip2.go b/src/runtime/runtime_wasip2.go index 46ce3d853b..5dfe4de62d 100644 --- a/src/runtime/runtime_wasip2.go +++ b/src/runtime/runtime_wasip2.go @@ -31,7 +31,22 @@ func os_runtime_args() []string { //export cabi_realloc func cabi_realloc(ptr, oldsize, align, newsize unsafe.Pointer) unsafe.Pointer { - return realloc(ptr, uintptr(newsize)) + size := uintptr(newsize) + if size == 0 { + freeManual(ptr) + return nil + } + + newPtr := allocManual(size) + if ptr != nil { + copySize := uintptr(oldsize) + if copySize > size { + copySize = size + } + memcpy(newPtr, ptr, copySize) + freeManual(ptr) + } + return newPtr } func ticksToNanoseconds(ticks timeUnit) int64 { diff --git a/testdata/cgo/main.c b/testdata/cgo/main.c index 94b338dda2..1de4b32506 100644 --- a/testdata/cgo/main.c +++ b/testdata/cgo/main.c @@ -1,6 +1,7 @@ #include #include "main.h" #include +#include int global = 3; bool globalBool = 1; @@ -82,3 +83,107 @@ int set_errno(int err) { errno = err; return -1; } + +typedef struct malloc_node { + struct malloc_node *next; + int value; +} malloc_node; + +void *makeMallocChain(void) { + malloc_node *tail = malloc(sizeof(malloc_node)); + tail->next = NULL; + tail->value = 42; + + malloc_node *head = malloc(sizeof(malloc_node)); + head->next = tail; + head->value = 1; + return head; +} + +void clobberMalloc(void) { +#if defined(__AVR__) + return; +#else + malloc_node *nodes[64]; + for (int i = 0; i < 64; i++) { + nodes[i] = malloc(sizeof(malloc_node)); + nodes[i]->next = NULL; + nodes[i]->value = 0; + } + for (int i = 0; i < 64; i++) { + free(nodes[i]); + } +#endif +} + +int mallocChainValue(void *ptr) { + return ((malloc_node *)ptr)->next->value; +} + +#define MALLOC_HIDE_MASK ((uintptr_t)0x5a5a5a5a) + +__attribute__((noinline)) uintptr_t makeHiddenMalloc(void) { + malloc_node *node = malloc(sizeof(malloc_node)); + node->next = NULL; + node->value = 84; + return (uintptr_t)node ^ MALLOC_HIDE_MASK; +} + +int hiddenMallocValue(uintptr_t hidden) { + malloc_node *node = (malloc_node *)(hidden ^ MALLOC_HIDE_MASK); + return node->value; +} + +void freeHiddenMalloc(uintptr_t hidden) { + free((void *)(hidden ^ MALLOC_HIDE_MASK)); +} + +void mallocFreeStress(void) { +#if defined(__AVR__) + const int count = 32; +#else + const int count = 1024; +#endif + for (int i = 0; i < count; i++) { + char *ptr = malloc(1024); + ptr[0] = (char)i; + free(ptr); + } +} + +void mallocZero(void) { + free(malloc(0)); +} + +int reallocPreservesContents(void) { + int *ptr = malloc(sizeof(int)); + *ptr = 42; + ptr = realloc(ptr, 2 * sizeof(int)); + int value = ptr[0]; + free(ptr); + return value; +} + +__attribute__((noinline)) void *callCalloc(size_t nmemb, size_t size) { + return calloc(nmemb, size); +} + +int callocOverflowReturnsNull(void) { +#if defined(__linux__) || defined(_WIN32) || defined(__APPLE__) + return 1; +#else + volatile size_t nmemb = (size_t)-1; + return callCalloc(nmemb, 2) == NULL; +#endif +} + +__attribute__((noinline)) void clobberStack(void) { +#if defined(__AVR__) + return; +#else + volatile uintptr_t values[128]; + for (int i = 0; i < 128; i++) { + values[i] = 0; + } +#endif +} diff --git a/testdata/cgo/main.go b/testdata/cgo/main.go index 38d11386a9..bf75337b91 100644 --- a/testdata/cgo/main.go +++ b/testdata/cgo/main.go @@ -19,6 +19,7 @@ import "C" import "C" import ( + "runtime" "syscall" "unsafe" ) @@ -171,6 +172,36 @@ func main() { println("len(C.GoBytes(C.CBytes(nil),0)):", len(C.GoBytes(C.CBytes(nil), 0))) println(`rountrip CBytes:`, C.GoString((*C.char)(C.CBytes([]byte("hello\000"))))) + // malloc allocations remain live until free, even when C pointers are the + // only links between them. + mallocChain := C.makeMallocChain() + runtime.GC() + C.clobberMalloc() + println("malloc chain:", C.mallocChainValue(mallocChain)) + + // malloc lifetime ends at free, not when the allocation becomes invisible + // to the GC. Encode the address so neither Go nor C exposes a pointer root. + hiddenMallocChan := make(chan C.uintptr_t, 1) + hiddenMallocDone := make(chan struct{}) + go func() { + hiddenMallocChan <- C.makeHiddenMalloc() + close(hiddenMallocDone) + }() + hiddenMalloc := <-hiddenMallocChan + <-hiddenMallocDone + C.clobberStack() + runtime.GC() + C.clobberMalloc() + println("hidden malloc:", C.hiddenMallocValue(hiddenMalloc)) + C.freeHiddenMalloc(hiddenMalloc) + + C.mallocFreeStress() + println("malloc/free stress: ok") + C.mallocZero() + println("malloc zero: ok") + println("realloc preserves contents:", C.reallocPreservesContents()) + println("calloc overflow:", C.callocOverflowReturnsNull() != 0) + // Check that errno is returned from the second return value, and that it // matches the errno value that was just set. _, errno := C.set_errno(C.EINVAL) diff --git a/testdata/cgo/main.h b/testdata/cgo/main.h index 3942497f23..7e0b432aa4 100644 --- a/testdata/cgo/main.h +++ b/testdata/cgo/main.h @@ -1,4 +1,5 @@ #include +#include #include #include @@ -157,3 +158,16 @@ double doSqrt(double); void printf_single_int(char *format, int arg); int set_errno(int err); + +void *makeMallocChain(void); +void clobberMalloc(void); +int mallocChainValue(void *ptr); +uintptr_t makeHiddenMalloc(void); +int hiddenMallocValue(uintptr_t hidden); +void freeHiddenMalloc(uintptr_t hidden); +void mallocFreeStress(void); +void mallocZero(void); +int reallocPreservesContents(void); +void *callCalloc(size_t nmemb, size_t size); +int callocOverflowReturnsNull(void); +void clobberStack(void); diff --git a/testdata/cgo/out.txt b/testdata/cgo/out.txt index 1d63f5e82f..1fe617a579 100644 --- a/testdata/cgo/out.txt +++ b/testdata/cgo/out.txt @@ -75,6 +75,12 @@ len(C.GoStringN(nil, 0)): 0 len(C.GoBytes(nil, 0)): 0 len(C.GoBytes(C.CBytes(nil),0)): 0 rountrip CBytes: hello +malloc chain: 42 +hidden malloc: 84 +malloc/free stress: ok +malloc zero: ok +realloc preserves contents: 42 +calloc overflow: true EINVAL: true EAGAIN: true copied string: foobar