diff --git a/builder/testdata/binary-size.txt b/builder/testdata/binary-size.txt index 363860cc93..1372a6204c 100644 --- a/builder/testdata/binary-size.txt +++ b/builder/testdata/binary-size.txt @@ -1,4 +1,4 @@ target package code rodata data bss hifive1b examples/echo 4321 323 0 2268 -microbit examples/serial 2842 382 8 2264 -wioterminal examples/pininterrupt 8039 1665 132 7496 +microbit examples/serial 2846 382 8 2264 +wioterminal examples/pininterrupt 8075 1665 132 7504 diff --git a/compiler/testdata/channel.ll b/compiler/testdata/channel.ll index 31466ead0f..fb3dd5c349 100644 --- a/compiler/testdata/channel.ll +++ b/compiler/testdata/channel.ll @@ -15,7 +15,7 @@ entry: } ; Function Attrs: nounwind -define hidden void @main.chanIntSend(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #1 { +define hidden void @main.chanIntSend(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #1 { entry: %chan.op = alloca %runtime.channelOp, align 8 %chan.value = alloca i32, align 4 @@ -31,13 +31,13 @@ entry: ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.start.p0(ptr nocapture) #2 -declare void @runtime.chanSend(ptr dereferenceable_or_null(36), ptr, ptr dereferenceable_or_null(16), ptr) #0 +declare void @runtime.chanSend(ptr dereferenceable_or_null(40), ptr, ptr dereferenceable_or_null(16), ptr) #0 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.end.p0(ptr nocapture) #2 ; Function Attrs: nounwind -define hidden void @main.chanIntRecv(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #1 { +define hidden void @main.chanIntRecv(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #1 { entry: %chan.op = alloca %runtime.channelOp, align 8 %chan.value = alloca i32, align 4 @@ -49,10 +49,10 @@ entry: ret void } -declare i1 @runtime.chanRecv(ptr dereferenceable_or_null(36), ptr, ptr dereferenceable_or_null(16), ptr) #0 +declare i1 @runtime.chanRecv(ptr dereferenceable_or_null(40), ptr, ptr dereferenceable_or_null(16), ptr) #0 ; Function Attrs: nounwind -define hidden void @main.chanZeroSend(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #1 { +define hidden void @main.chanZeroSend(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #1 { entry: %chan.op = alloca %runtime.channelOp, align 8 call void @llvm.lifetime.start.p0(ptr nonnull %chan.op) @@ -62,7 +62,7 @@ entry: } ; Function Attrs: nounwind -define hidden void @main.chanZeroRecv(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #1 { +define hidden void @main.chanZeroRecv(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #1 { entry: %chan.op = alloca %runtime.channelOp, align 8 call void @llvm.lifetime.start.p0(ptr nonnull %chan.op) @@ -72,7 +72,7 @@ entry: } ; Function Attrs: nounwind -define hidden void @main.selectZeroRecv(ptr dereferenceable_or_null(36) %ch1, ptr dereferenceable_or_null(36) %ch2, ptr %context) unnamed_addr #1 { +define hidden void @main.selectZeroRecv(ptr dereferenceable_or_null(40) %ch1, ptr dereferenceable_or_null(40) %ch2, ptr %context) unnamed_addr #1 { entry: %select.states.alloca = alloca [2 x %runtime.chanSelectState], align 8 %select.send.value = alloca i32, align 4 diff --git a/compiler/testdata/goroutine-cortex-m-qemu-tasks.ll b/compiler/testdata/goroutine-cortex-m-qemu-tasks.ll index bf117fa975..2479d5e34b 100644 --- a/compiler/testdata/goroutine-cortex-m-qemu-tasks.ll +++ b/compiler/testdata/goroutine-cortex-m-qemu-tasks.ll @@ -146,13 +146,13 @@ declare i32 @llvm.umin.i32(i32, i32) #7 declare void @llvm.memmove.p0.p0.i32(ptr nocapture writeonly, ptr nocapture readonly, i32, i1 immarg) #8 ; Function Attrs: nounwind -define hidden void @main.closeBuiltinGoroutine(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #0 { +define hidden void @main.closeBuiltinGoroutine(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #0 { entry: call void @runtime.chanClose(ptr %ch, ptr undef) #11 ret void } -declare void @runtime.chanClose(ptr dereferenceable_or_null(36), ptr) #1 +declare void @runtime.chanClose(ptr dereferenceable_or_null(40), ptr) #1 ; Function Attrs: nounwind define hidden void @main.startInterfaceMethod(ptr %itf.typecode, ptr %itf.value, ptr %context) unnamed_addr #0 { diff --git a/compiler/testdata/goroutine-wasm-asyncify.ll b/compiler/testdata/goroutine-wasm-asyncify.ll index 062ec1a423..f39fbed1aa 100644 --- a/compiler/testdata/goroutine-wasm-asyncify.ll +++ b/compiler/testdata/goroutine-wasm-asyncify.ll @@ -155,13 +155,13 @@ declare i32 @llvm.umin.i32(i32, i32) #7 declare void @llvm.memmove.p0.p0.i32(ptr nocapture writeonly, ptr nocapture readonly, i32, i1 immarg) #8 ; Function Attrs: nounwind -define hidden void @main.closeBuiltinGoroutine(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #1 { +define hidden void @main.closeBuiltinGoroutine(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #1 { entry: call void @runtime.chanClose(ptr %ch, ptr undef) #11 ret void } -declare void @runtime.chanClose(ptr dereferenceable_or_null(36), ptr) #0 +declare void @runtime.chanClose(ptr dereferenceable_or_null(40), ptr) #0 ; Function Attrs: nounwind define hidden void @main.startInterfaceMethod(ptr %itf.typecode, ptr %itf.value, ptr %context) unnamed_addr #1 { diff --git a/compiler/testdata/large.ll b/compiler/testdata/large.ll index cc1f8556b4..937978b1cc 100644 --- a/compiler/testdata/large.ll +++ b/compiler/testdata/large.ll @@ -388,7 +388,7 @@ declare void @runtime.hashmapBinarySet(ptr dereferenceable_or_null(48), ptr, ptr declare i1 @runtime.hashmapBinaryGet(ptr dereferenceable_or_null(48), ptr, ptr, i32, ptr) #0 ; Function Attrs: nounwind -define hidden i8 @main.useLargeChannel(ptr dereferenceable_or_null(36) %ch, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 { +define hidden i8 @main.useLargeChannel(ptr dereferenceable_or_null(40) %ch, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 { entry: %chan.op1 = alloca %runtime.channelOp, align 8 %chan.op = alloca %runtime.channelOp, align 8 @@ -425,15 +425,15 @@ if.then: ; preds = %entry ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.start.p0(ptr nocapture) #8 -declare void @runtime.chanSend(ptr dereferenceable_or_null(36), ptr, ptr dereferenceable_or_null(16), ptr) #0 +declare void @runtime.chanSend(ptr dereferenceable_or_null(40), ptr, ptr dereferenceable_or_null(16), ptr) #0 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.end.p0(ptr nocapture) #8 -declare i1 @runtime.chanRecv(ptr dereferenceable_or_null(36), ptr, ptr dereferenceable_or_null(16), ptr) #0 +declare i1 @runtime.chanRecv(ptr dereferenceable_or_null(40), ptr, ptr dereferenceable_or_null(16), ptr) #0 ; Function Attrs: nounwind -define hidden i8 @main.selectLargeChannel(ptr dereferenceable_or_null(36) %ch, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 { +define hidden i8 @main.selectLargeChannel(ptr dereferenceable_or_null(40) %ch, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 { entry: %select.block.alloca = alloca [2 x %runtime.channelOp], align 8 %select.states.alloca = alloca [2 x %runtime.chanSelectState], align 8 diff --git a/compiler/testdata/pragma.ll b/compiler/testdata/pragma.ll index d8b446b89d..339dfc9a13 100644 --- a/compiler/testdata/pragma.ll +++ b/compiler/testdata/pragma.ll @@ -84,10 +84,10 @@ entry: declare void @main.undefinedFunctionNotInSection(ptr) #0 -declare void @main.doesNotEscapeParam(ptr nocapture dereferenceable_or_null(4), ptr nocapture, i32, i32, ptr nocapture dereferenceable_or_null(36), ptr nocapture, ptr) #0 +declare void @main.doesNotEscapeParam(ptr nocapture dereferenceable_or_null(4), ptr nocapture, i32, i32, ptr nocapture dereferenceable_or_null(40), ptr nocapture, ptr) #0 ; Function Attrs: nounwind -define hidden void @main.stillEscapes(ptr dereferenceable_or_null(4) %a, ptr %b.data, i32 %b.len, i32 %b.cap, ptr dereferenceable_or_null(36) %c, ptr %d, ptr %context) unnamed_addr #1 { +define hidden void @main.stillEscapes(ptr dereferenceable_or_null(4) %a, ptr %b.data, i32 %b.len, i32 %b.cap, ptr dereferenceable_or_null(40) %c, ptr %d, ptr %context) unnamed_addr #1 { entry: ret void } diff --git a/make/test.mk b/make/test.mk index d1fc4f2638..0fa5bf8f76 100644 --- a/make/test.mk +++ b/make/test.mk @@ -223,6 +223,9 @@ tinygo-test: @# TestUnmarshalNestingLimit{Slice,Struct}: encoding/asn1 nesting limit added in @# https://github.com/golang/go/commit/6a6d115f9a7422b2fa081ba6f567eefb4a099462 $(TINYGO) test $(TEST_ADDITIONAL_FLAGS) $(TEST_SKIP_FLAG) $(filter-out encoding/xml,$(TEST_PACKAGES_HOST)) $(TEST_PACKAGES_SLOW) + $(TINYGO) test $(TEST_ADDITIONAL_FLAGS) -skip='^(TestReflectFuncOf|TestChannelMovedOutOfBubble|TestTimerFromInsideBubble|TestWaitGroupMovedIntoBubble|TestWaitGroupMovedOutOfBubble|TestWaitGroupMovedBetweenBubblesWithNonZeroCount)$$' internal/synctest + $(TINYGO) test $(TEST_ADDITIONAL_FLAGS) -skip='^(TestFatal|TestError|TestVerboseError|TestSkip|TestVerboseSkip|TestHelper|TestHTTPTransport100Continue)$$' testing/synctest + $(TINYGO) test $(TEST_ADDITIONAL_FLAGS) -run='^TestSynctestMarshal$$' encoding/json ifeq ($(TEST_ENCODING_XML),true) $(TINYGO) test $(TEST_ADDITIONAL_FLAGS) $(TEST_SKIP_FLAG) -stack-size=16MB encoding/xml endif diff --git a/src/internal/task/task.go b/src/internal/task/task.go index 55f89fa9fd..04538cf323 100644 --- a/src/internal/task/task.go +++ b/src/internal/task/task.go @@ -30,9 +30,15 @@ type Task struct { // since it falls into the padding of the FipsIndicator bit above. RunState uint8 + // SynctestBlocked is set while this task is durably blocked. + SynctestBlocked bool + // DeferFrame stores a pointer to the (stack allocated) defer frame of the // goroutine that is used for the recover builtin. DeferFrame unsafe.Pointer + + // SynctestBubble identifies the synctest bubble this task belongs to. + SynctestBubble unsafe.Pointer } const ( @@ -74,3 +80,34 @@ func runtime_alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer //go:linkname scheduleTask runtime.scheduleTask func scheduleTask(*Task) + +func inheritSynctest(t *Task) { + if !synctestIsEnabled() { + return + } + parent := Current() + if parent == nil || parent.SynctestBubble == nil { + return + } + t.SynctestBubble = parent.SynctestBubble + synctestTaskCreated(t) +} + +func exitSynctest(t *Task) { + if !synctestIsEnabled() { + return + } + if t.SynctestBubble != nil { + synctestTaskExited(t) + t.SynctestBubble = nil + } +} + +//go:linkname synctestTaskCreated runtime.synctestTaskCreated +func synctestTaskCreated(*Task) + +//go:linkname synctestTaskExited runtime.synctestTaskExited +func synctestTaskExited(*Task) + +//go:linkname synctestIsEnabled runtime.synctestIsEnabled +func synctestIsEnabled() bool diff --git a/src/internal/task/task_asyncify.go b/src/internal/task/task_asyncify.go index 4d78e19373..8aa4446d39 100644 --- a/src/internal/task/task_asyncify.go +++ b/src/internal/task/task_asyncify.go @@ -55,6 +55,7 @@ type stackState struct { // The new goroutine is immediately started. func start(fn uintptr, args unsafe.Pointer, stackSize uintptr) { t := &Task{} + inheritSynctest(t) addLiveTask(t) t.state.initialize(fn, args, stackSize) scheduleTask(t) diff --git a/src/internal/task/task_exit.go b/src/internal/task/task_exit.go index 40f988058c..0e4cd6026d 100644 --- a/src/internal/task/task_exit.go +++ b/src/internal/task/task_exit.go @@ -24,6 +24,7 @@ func Exit() { func exit(goexit bool) { t := Current() + exitSynctest(t) remaining := atomic.AddUint32(&liveTasks, ^uint32(0)) if t == mainTask { if goexit { diff --git a/src/internal/task/task_stack.go b/src/internal/task/task_stack.go index 23f3b9097f..5a3c10d3ad 100644 --- a/src/internal/task/task_stack.go +++ b/src/internal/task/task_stack.go @@ -69,6 +69,7 @@ var startTask [0]uint8 // The new goroutine is scheduled to run later. func start(fn uintptr, args unsafe.Pointer, stackSize uintptr) { t := &Task{} + inheritSynctest(t) addLiveTask(t) t.state.initialize(fn, args, stackSize) scheduleTask(t) diff --git a/src/internal/task/task_threads.go b/src/internal/task/task_threads.go index 52374dec87..7960106d49 100644 --- a/src/internal/task/task_threads.go +++ b/src/internal/task/task_threads.go @@ -100,6 +100,7 @@ func (t *Task) Resume() { // Start a new OS thread. func start(fn uintptr, args unsafe.Pointer, stackSize uintptr) { t := &Task{} + inheritSynctest(t) t.state.id = atomic.AddUintptr(&goroutineID, 1) if verbose { println("*** start: ", t.state.id, "from", Current().state.id) @@ -132,6 +133,8 @@ func taskExited(t *Task) { } func exit(t *Task) bool { + exitSynctest(t) + // Remove from the queue. // TODO: this can be made more efficient by using a doubly linked list. activeTaskLock.Lock() diff --git a/src/runtime/chan.go b/src/runtime/chan.go index a85e9b6617..6af0fc3ccb 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -61,6 +61,7 @@ type channel struct { receivers chanQueue lock task.PMutex buf unsafe.Pointer + synctest unsafe.Pointer } const ( @@ -142,6 +143,23 @@ func chanMake(elementSize uintptr, bufSize uintptr) *channel { elementSize: elementSize, bufCap: bufSize, buf: alloc(elementSize*bufSize, nil), + synctest: currentTaskSynctestBubble(), + } +} + +func currentTaskSynctestBubble() unsafe.Pointer { + if !synctestEnabled { + return nil + } + if current := task.Current(); current != nil { + return current.SynctestBubble + } + return nil +} + +func (ch *channel) checkSynctest(op string) { + if synctestEnabled && ch.synctest != nil && currentTaskSynctestBubble() != ch.synctest { + runtimeFatal(op + " synctest channel from outside bubble") } } @@ -231,8 +249,10 @@ func (ch *channel) trySend(value unsafe.Pointer) (sent bool, wake *task.Task) { func chanSend(ch *channel, value unsafe.Pointer, op *channelOp) { if ch == nil { // A nil channel blocks forever. Do not schedule this goroutine again. + synctestTaskBlock(task.Current()) deadlock() } + ch.checkSynctest("send on") mask := interrupt.Disable() ch.lock.Lock() @@ -254,6 +274,9 @@ func chanSend(ch *channel, value unsafe.Pointer, op *channelOp) { op.index = 0 op.value = value ch.senders.push(op) + if synctestEnabled && ch.synctest != nil { + synctestTaskBlock(t) + } ch.lock.Unlock() interrupt.Restore(mask) @@ -311,8 +334,10 @@ func (ch *channel) tryRecv(value unsafe.Pointer) (received, ok bool, wake *task. func chanRecv(ch *channel, value unsafe.Pointer, op *channelOp) bool { if ch == nil { // A nil channel blocks forever. Do not schedule this goroutine again. + synctestTaskBlock(task.Current()) deadlock() } + ch.checkSynctest("receive on") mask := interrupt.Disable() ch.lock.Lock() @@ -334,6 +359,9 @@ func chanRecv(ch *channel, value unsafe.Pointer, op *channelOp) bool { op.task = t op.index = 0 ch.receivers.push(op) + if synctestEnabled && ch.synctest != nil { + synctestTaskBlock(t) + } ch.lock.Unlock() interrupt.Restore(mask) @@ -351,6 +379,7 @@ func chanClose(ch *channel) { // Not allowed by the language spec. runtimePanic("close of nil channel") } + ch.checkSynctest("close of") mask := interrupt.Disable() ch.lock.Lock() @@ -457,6 +486,12 @@ func unlockAllStates(states []chanSelectState) { // The 'ops' slice must be set if (and only if) this is a blocking select. func chanSelect(recvbuf unsafe.Pointer, states []chanSelectState, ops []channelOp) (uint32, bool) { mask := interrupt.Disable() + var currentBubble unsafe.Pointer + var synctestDurable bool + if synctestEnabled { + currentBubble = currentTaskSynctestBubble() + synctestDurable = currentBubble != nil + } // Lock everything. chanSelectLock.Lock() @@ -475,6 +510,17 @@ func chanSelect(recvbuf unsafe.Pointer, states []chanSelectState, ops []channelO // operation. continue } + if synctestEnabled { + if state.ch.synctest != nil && state.ch.synctest != currentBubble { + unlockAllStates(states) + chanSelectLock.Unlock() + interrupt.Restore(mask) + runtimeFatal("select on synctest channel from outside bubble") + } + if state.ch.synctest == nil { + synctestDurable = false + } + } if state.value == nil { // chan receive if received, ok, sender := state.ch.tryRecv(recvbuf); received { @@ -528,6 +574,9 @@ func chanSelect(recvbuf unsafe.Pointer, states []chanSelectState, ops []channelO state.ch.senders.push(op) } } + if synctestDurable { + synctestTaskBlock(t) + } // Now we wait until one of the send/receive operations can proceed. unlockAllStates(states) diff --git a/src/runtime/scheduler_cooperative.go b/src/runtime/scheduler_cooperative.go index a69247c84e..93f69b4ef5 100644 --- a/src/runtime/scheduler_cooperative.go +++ b/src/runtime/scheduler_cooperative.go @@ -56,7 +56,9 @@ var finalizerIdleGC func() bool func deadlock() { // Keep permanently blocked tasks reachable so their suspended stacks remain // GC roots, but never put them back on the runnable queue. - deadlockedTasks.Push(task.Current()) + current := task.Current() + synctestTaskBlock(current) + deadlockedTasks.Push(current) task.Pause() runtimeFatal("unreachable") } @@ -94,6 +96,7 @@ func goexit() { // Add this task to the end of the run queue. func scheduleTask(t *task.Task) { + synctestTaskWake(t) runqueue.Push(t) } @@ -297,6 +300,9 @@ func sleep(duration int64) { if duration <= 0 { return } + if synctestSleep(duration) { + return + } addSleepTask(task.Current(), nanosecondsToTicks(duration)) task.Pause() } diff --git a/src/runtime/scheduler_cores.go b/src/runtime/scheduler_cores.go index 853812bd23..0eeb0cc5d4 100644 --- a/src/runtime/scheduler_cores.go +++ b/src/runtime/scheduler_cores.go @@ -28,6 +28,7 @@ var ( func deadlock() { // Call yield without requesting a wakeup. + synctestTaskBlock(task.Current()) task.Pause() trap() } @@ -39,6 +40,7 @@ func goexit() { // Mark the given task as ready to resume. // This is allowed even if the task isn't paused yet, but will pause soon. func scheduleTask(t *task.Task) { + synctestTaskWake(t) schedulerLock.Lock() switch t.RunState { case task.RunStatePaused: @@ -153,6 +155,9 @@ func sleep(duration int64) { if duration <= 0 { return } + if synctestSleep(duration) { + return + } wakeup := ticks() + nanosecondsToTicks(duration) diff --git a/src/runtime/scheduler_none.go b/src/runtime/scheduler_none.go index 7f75ef8e98..a7a6baffe1 100644 --- a/src/runtime/scheduler_none.go +++ b/src/runtime/scheduler_none.go @@ -47,6 +47,7 @@ func goexit() { } func scheduleTask(t *task.Task) { + synctestTaskWake(t) // Pause() will panic, so this should not be reachable. } diff --git a/src/runtime/scheduler_threads.go b/src/runtime/scheduler_threads.go index 5b6638b98f..d7b191ef60 100644 --- a/src/runtime/scheduler_threads.go +++ b/src/runtime/scheduler_threads.go @@ -35,11 +35,15 @@ func sleep(duration int64) { if duration <= 0 { return } + if synctestSleep(duration) { + return + } sleepTicks(nanosecondsToTicks(duration)) } func deadlock() { + synctestTaskBlock(task.Current()) task.Pause() } @@ -48,6 +52,7 @@ func goexit() { } func scheduleTask(t *task.Task) { + synctestTaskWake(t) t.Resume() } diff --git a/src/runtime/synctest.go b/src/runtime/synctest.go index fa11c991fc..889dad57a0 100644 --- a/src/runtime/synctest.go +++ b/src/runtime/synctest.go @@ -1,15 +1,491 @@ package runtime -// Dummy implementation of synctest functions (we don't support synctest at the -// moment). +import ( + "internal/task" + "unsafe" +) + +const synctestBaseTime = 946684800000000000 + +var synctestEnabled bool + +func synctestIsEnabled() bool { + return synctestEnabled +} + +type synctestBubble struct { + lock task.PMutex + + timers *timerNode + + root *task.Task + main *task.Task + waiter *task.Task + + total int + running int + active int + + rootSleeping bool + waiting bool + done bool + now int64 + timerSeq uint32 +} + +type synctestAssociation struct { + next *synctestAssociation + ptr unsafe.Pointer + bubble *synctestBubble +} + +var ( + synctestAssociationsLock task.PMutex + synctestAssociations *synctestAssociation +) + +func taskSynctestBubble(t *task.Task) *synctestBubble { + if t == nil || t.SynctestBubble == nil { + return nil + } + return (*synctestBubble)(t.SynctestBubble) +} + +func currentSynctestBubble() *synctestBubble { + if !synctestEnabled { + return nil + } + return taskSynctestBubble(task.Current()) +} + +func (bubble *synctestBubble) wakeLocked() *task.Task { + if bubble.running != 0 || bubble.active != 0 { + return nil + } + if bubble.timers != nil && bubble.timers.timer.when <= bubble.now { + if bubble.rootSleeping { + bubble.rootSleeping = false + return bubble.root + } + return nil + } + if bubble.waiter != nil { + waiter := bubble.waiter + bubble.waiter = nil + return waiter + } + if bubble.rootSleeping { + bubble.rootSleeping = false + return bubble.root + } + return nil +} + +func (bubble *synctestBubble) time() int64 { + bubble.lock.Lock() + now := bubble.now + bubble.lock.Unlock() + return now +} + +func (bubble *synctestBubble) addTimer(tn *timerNode) { + bubble.lock.Lock() + if tn.timer.when <= bubble.now { + bubble.lock.Unlock() + tn.callback(tn, 0) + return + } + bubble.timerSeq++ + insertBeforeEqual := (bubble.timerSeq/2)&1 != 0 + queue := &bubble.timers + for *queue != nil { + if (*queue).timer.when > tn.timer.when { + break + } + if insertBeforeEqual && (*queue).timer.when == tn.timer.when { + break + } + queue = &(*queue).next + } + tn.next = *queue + *queue = tn + bubble.lock.Unlock() +} + +func (bubble *synctestBubble) removeTimer(tim *timer) *timerNode { + bubble.lock.Lock() + defer bubble.lock.Unlock() + for queue := &bubble.timers; *queue != nil; queue = &(*queue).next { + if (*queue).timer == tim { + node := *queue + *queue = node.next + node.next = nil + return node + } + } + return nil +} + +func (bubble *synctestBubble) checkTimerAccess(op string) { + if currentSynctestBubble() != bubble { + runtimeFatal(op + " of synctest timer from outside bubble") + } +} + +func synctestWakeTaskTimer(tn *timerNode, delta int64) { + scheduleTask(tn.timer.arg.(*task.Task)) +} + +func synctestSleep(duration int64) bool { + if !synctestEnabled { + return false + } + current := task.Current() + bubble := taskSynctestBubble(current) + if bubble == nil { + return false + } + + bubble.lock.Lock() + when := bubble.now + duration + bubble.lock.Unlock() + tim := &timer{ + when: when, + arg: current, + synctest: bubble, + } + bubble.addTimer(&timerNode{ + timer: tim, + callback: synctestWakeTaskTimer, + }) + synctestTaskBlock(current) + task.Pause() + return true +} + +func synctestTaskCreated(t *task.Task) { + if !synctestEnabled { + return + } + bubble := taskSynctestBubble(t) + bubble.lock.Lock() + bubble.total++ + bubble.running++ + bubble.lock.Unlock() +} + +func synctestTaskExited(t *task.Task) { + if !synctestEnabled { + return + } + bubble := taskSynctestBubble(t) + bubble.lock.Lock() + if t.SynctestBlocked { + t.SynctestBlocked = false + } else { + bubble.running-- + } + bubble.total-- + if t == bubble.main { + bubble.done = true + } + if bubble.running < 0 || bubble.total < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid task count") + } + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } +} + +func synctestTaskWake(t *task.Task) { + if !synctestEnabled { + return + } + bubble := taskSynctestBubble(t) + if bubble == nil { + return + } + bubble.lock.Lock() + if t.SynctestBlocked { + t.SynctestBlocked = false + bubble.running++ + } + bubble.lock.Unlock() +} + +func synctestTaskBlock(t *task.Task) { + if !synctestEnabled { + return + } + bubble := taskSynctestBubble(t) + if bubble == nil { + return + } + bubble.lock.Lock() + if !t.SynctestBlocked { + t.SynctestBlocked = true + bubble.running-- + } + if bubble.running < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid running task count") + } + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } +} + +func synctestTaskBlockBegin(t *task.Task) bool { + if !synctestEnabled { + return false + } + bubble := taskSynctestBubble(t) + if bubble == nil { + return false + } + bubble.lock.Lock() + bubble.active++ + bubble.lock.Unlock() + return true +} + +func synctestTaskBlockEnd(t *task.Task, blocked bool) { + bubble := taskSynctestBubble(t) + if bubble == nil { + return + } + bubble.lock.Lock() + if blocked && !t.SynctestBlocked { + t.SynctestBlocked = true + bubble.running-- + } + bubble.active-- + if bubble.running < 0 || bubble.active < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid block transition") + } + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } +} + +func synctestBlockBegin(t *task.Task) bool { + return synctestTaskBlockBegin(t) +} + +func synctestBlockEnd(t *task.Task, blocked bool) { + synctestTaskBlockEnd(t, blocked) +} + +func synctestBlock(t *task.Task) { + synctestTaskBlock(t) +} + +//go:linkname synctest_run internal/synctest.Run +func synctest_run(f func()) { + synctestEnabled = true + root := task.Current() + if root.SynctestBubble != nil { + panic("synctest.Run called from within a synctest bubble") + } + + bubble := &synctestBubble{ + root: root, + now: synctestBaseTime, + } + + root.SynctestBubble = unsafe.Pointer(bubble) + go func() { + bubble.lock.Lock() + bubble.main = task.Current() + bubble.lock.Unlock() + f() + }() + root.SynctestBubble = nil + + for { + bubble.lock.Lock() + if bubble.total == 0 { + bubble.lock.Unlock() + return + } + if bubble.running == 0 && bubble.active == 0 { + if bubble.timers != nil && !bubble.done { + timer := bubble.timers + bubble.timers = timer.next + timer.next = nil + if timer.timer.when > bubble.now { + bubble.now = timer.timer.when + } + bubble.lock.Unlock() + + root.SynctestBubble = unsafe.Pointer(bubble) + timer.callback(timer, 0) + root.SynctestBubble = nil + continue + } + if bubble.waiter != nil { + waiter := bubble.waiter + bubble.waiter = nil + bubble.lock.Unlock() + scheduleTask(waiter) + continue + } + done := bubble.done + bubble.lock.Unlock() + if done { + panic("deadlock: main bubble goroutine has exited but blocked goroutines remain") + } + panic("deadlock: all goroutines in bubble are blocked") + } + bubble.rootSleeping = true + bubble.lock.Unlock() + task.Pause() + } +} + +//go:linkname synctest_wait internal/synctest.Wait +func synctest_wait() { + current := task.Current() + bubble := taskSynctestBubble(current) + if bubble == nil { + panic("goroutine is not in a bubble") + } + + bubble.lock.Lock() + if bubble.waiting { + bubble.lock.Unlock() + panic("wait already in progress") + } + bubble.waiting = true + current.SynctestBlocked = true + bubble.running-- + dueTimer := bubble.timers != nil && bubble.timers.timer.when <= bubble.now + if bubble.running == 0 && bubble.active == 0 && !dueTimer { + current.SynctestBlocked = false + bubble.running++ + bubble.waiting = false + bubble.lock.Unlock() + return + } + bubble.waiter = current + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } + + task.Pause() + + bubble.lock.Lock() + bubble.waiting = false + bubble.lock.Unlock() +} + +//go:linkname synctest_isInBubble internal/synctest.IsInBubble +func synctest_isInBubble() bool { + return currentSynctestBubble() != nil +} + +//go:linkname synctest_associate internal/synctest.associate +func synctest_associate(p unsafe.Pointer) int { + bubble := currentSynctestBubble() + if bubble == nil { + panic("goroutine is not in a bubble") + } + + synctestAssociationsLock.Lock() + for assoc := synctestAssociations; assoc != nil; assoc = assoc.next { + if assoc.ptr == p { + synctestAssociationsLock.Unlock() + if assoc.bubble == bubble { + return 1 + } + return 2 + } + } + synctestAssociations = &synctestAssociation{ + next: synctestAssociations, + ptr: p, + bubble: bubble, + } + synctestAssociationsLock.Unlock() + return 1 +} + +//go:linkname synctest_disassociate internal/synctest.disassociate +func synctest_disassociate(p unsafe.Pointer) { + synctestAssociationsLock.Lock() + for assoc := &synctestAssociations; *assoc != nil; assoc = &(*assoc).next { + if (*assoc).ptr == p { + *assoc = (*assoc).next + break + } + } + synctestAssociationsLock.Unlock() +} + +//go:linkname synctest_isAssociated internal/synctest.isAssociated +func synctest_isAssociated(p unsafe.Pointer) bool { + bubble := currentSynctestBubble() + if bubble == nil { + return false + } + + synctestAssociationsLock.Lock() + defer synctestAssociationsLock.Unlock() + for assoc := synctestAssociations; assoc != nil; assoc = assoc.next { + if assoc.ptr == p { + return assoc.bubble == bubble + } + } + return false +} //go:linkname synctest_acquire internal/synctest.acquire func synctest_acquire() any { - // Dummy: we don't support synctest. - return nil + bubble := currentSynctestBubble() + if bubble == nil { + return nil + } + bubble.lock.Lock() + bubble.active++ + bubble.lock.Unlock() + return bubble } //go:linkname synctest_release internal/synctest.release -func synctest_release(sg any) { - // Dummy: we don't support synctest. +func synctest_release(value any) { + bubble := value.(*synctestBubble) + bubble.lock.Lock() + bubble.active-- + if bubble.active < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid active count") + } + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } +} + +//go:linkname synctest_inBubble internal/synctest.inBubble +func synctest_inBubble(value any, f func()) { + current := task.Current() + if current.SynctestBubble != nil { + panic("goroutine is already bubbled") + } + current.SynctestBubble = unsafe.Pointer(value.(*synctestBubble)) + defer func() { + current.SynctestBubble = nil + }() + f() } diff --git a/src/runtime/time.go b/src/runtime/time.go index e0a0346a50..3c41d38b12 100644 --- a/src/runtime/time.go +++ b/src/runtime/time.go @@ -11,6 +11,8 @@ type timer struct { // Callback from the time package. f func(arg any, seq uintptr, delta int64) arg any + + synctest *synctestBubble } func (tim *timer) callCallback(delta int64) { @@ -21,39 +23,56 @@ func (tim *timer) callCallback(delta int64) { // the same as time.Timer and time.Ticker so it can be used as-is in the time // package. type timeTimer struct { - c unsafe.Pointer // <-chan time.Time - init bool + c unsafe.Pointer // <-chan time.Time + initTimer bool timer } //go:linkname newTimer time.newTimer func newTimer(when, period int64, f func(arg any, seq uintptr, delta int64), arg any, c unsafe.Pointer) *timeTimer { + bubble := currentSynctestBubble() tim := &timeTimer{ - c: c, - init: true, + c: c, + initTimer: true, timer: timer{ - when: when, - period: period, - f: f, - arg: arg, + when: when, + period: period, + f: f, + arg: arg, + synctest: bubble, }, } scheduleLog("new timer") - addTimer(&timerNode{ + node := &timerNode{ timer: &tim.timer, callback: timerCallback, - }) + } + if bubble != nil { + bubble.addTimer(node) + } else { + addTimer(node) + } return tim } //go:linkname stopTimer time.stopTimer func stopTimer(tim *timeTimer) bool { + if tim.timer.synctest != nil { + tim.timer.synctest.checkTimerAccess("stop") + return tim.timer.synctest.removeTimer(&tim.timer) != nil + } return removeTimer(&tim.timer) != nil } //go:linkname resetTimer time.resetTimer func resetTimer(t *timeTimer, when, period int64) bool { - n := removeTimer(&t.timer) + var n *timerNode + if t.timer.synctest != nil { + t.timer.synctest.checkTimerAccess("reset") + n = t.timer.synctest.removeTimer(&t.timer) + } else { + n = removeTimer(&t.timer) + } removed := n != nil if n == nil { n = new(timerNode) @@ -62,20 +81,28 @@ func resetTimer(t *timeTimer, when, period int64) bool { t.timer.period = period n.timer = &t.timer n.callback = timerCallback - addTimer(n) + if t.timer.synctest != nil { + t.timer.synctest.addTimer(n) + } else { + addTimer(n) + } return removed } //go:linkname time_runtimeNano time.runtimeNano func time_runtimeNano() int64 { - // Note: we're ignoring sync groups here (package testing/synctest). - // See: https://github.com/golang/go/issues/67434 + if bubble := currentSynctestBubble(); bubble != nil { + return bubble.time() + } return nanotime() } //go:linkname time_runtimeNow time.runtimeNow func time_runtimeNow() (sec int64, nsec int32, mono int64) { - // Also ignoring the sync group here, like time_runtimeNano above. + if bubble := currentSynctestBubble(); bubble != nil { + now := bubble.time() + return now / 1e9, int32(now % 1e9), 0 + } return now() } @@ -121,12 +148,16 @@ func timerCallback(tn *timerNode, delta int64) { // If this is a periodic timer (a ticker), re-add it to the queue. if tn.timer.period != 0 { - reAddTimer(tn) + if tn.timer.synctest != nil { + tn.timer.when += tn.timer.period + tn.timer.synctest.addTimer(tn) + } else { + reAddTimer(tn) + } } } //go:linkname time_runtimeIsBubbled time.runtimeIsBubbled func time_runtimeIsBubbled() bool { - // We don't currently support bubbles. - return false + return currentSynctestBubble() != nil } diff --git a/src/sync/cond.go b/src/sync/cond.go index 139d8e0229..23c2624ccc 100644 --- a/src/sync/cond.go +++ b/src/sync/cond.go @@ -31,6 +31,9 @@ func (c *Cond) trySignal() bool { // Pop a blocked task off of the stack, and schedule it if applicable. t := c.blocked.Pop() if t != nil { + if t.SynctestBubble != nil && task.Current().SynctestBubble != t.SynctestBubble { + runtimeFatal("semaphore wake of synctest goroutine from outside bubble") + } dataPtr := (*task.Uint32)(unsafe.Pointer(&t.Data)) // The data value is 0 when the task is not yet sleeping, and 1 when it is. @@ -71,6 +74,8 @@ func (c *Cond) Wait() { c.blocked.Push(t) c.lock.Unlock() + transition := synctestBlockBegin(t) + // Temporarily unlock L. c.L.Unlock() @@ -79,6 +84,9 @@ func (c *Cond) Wait() { // If we were signaled while unlocking, immediately complete. if dataPtr.Swap(1) != 0 { + if transition { + synctestBlockEnd(t, false) + } // The data value was already 1, so we got a signal already (and weren't // scheduled because trySignal was the first to change the value). return @@ -87,8 +95,17 @@ func (c *Cond) Wait() { // We were the first to change the value from 0 to 1, meaning we did not get // a signal during the call to Unlock(). So we wait until we do get a // signal. + if transition { + synctestBlockEnd(t, true) + } task.Pause() } //go:linkname scheduleTask runtime.scheduleTask func scheduleTask(*task.Task) + +//go:linkname synctestBlockBegin runtime.synctestBlockBegin +func synctestBlockBegin(*task.Task) bool + +//go:linkname synctestBlockEnd runtime.synctestBlockEnd +func synctestBlockEnd(*task.Task, bool) diff --git a/src/sync/waitgroup.go b/src/sync/waitgroup.go index 9a564c5f7b..d7190ad66e 100644 --- a/src/sync/waitgroup.go +++ b/src/sync/waitgroup.go @@ -1,68 +1,102 @@ package sync -import "internal/task" +import ( + "internal/task" + "unsafe" +) type WaitGroup struct { - futex task.Futex + futex task.Futex + lock task.PMutex + waiters task.Stack + counter int + waiting int + synctest unsafe.Pointer } func (wg *WaitGroup) Add(delta int) { - switch { - case delta > 0: - // Delta is positive. + if !synctestIsEnabled() { + wg.addPlain(delta) + return + } + + currentBubble := task.Current().SynctestBubble + wg.lock.Lock() + if currentBubble == nil && wg.synctest == nil { + wg.lock.Unlock() + wg.addPlain(delta) + return + } + if wg.synctest == nil && wg.futex.Load() != 0 { + wg.lock.Unlock() + runtimeFatal("sync: WaitGroup.Add called from inside and outside synctest bubble") + } + if currentBubble != nil { + if wg.synctest == nil { + wg.synctest = currentBubble + } else if wg.synctest != currentBubble { + wg.lock.Unlock() + runtimeFatal("sync: WaitGroup.Add called from multiple synctest bubbles") + } + } else if wg.synctest != nil { + wg.lock.Unlock() + runtimeFatal("sync: WaitGroup.Add called from inside and outside synctest bubble") + } + + if delta > 0 && wg.counter == 0 && wg.waiting != 0 { + wg.lock.Unlock() + panic("sync: WaitGroup misuse: Add called concurrently with Wait") + } + if delta > 0 && wg.counter > int(^uint32(0)>>1)-delta { + wg.lock.Unlock() + panic("sync: WaitGroup counter overflowed") + } + wg.counter += delta + if wg.counter < 0 { + wg.lock.Unlock() + panic("sync: negative WaitGroup counter") + } + if wg.counter != 0 { + wg.lock.Unlock() + return + } + + waiters := wg.waiters.Queue() + if wg.waiting == 0 { + wg.synctest = nil + } + wg.lock.Unlock() + + for waiter := waiters.Pop(); waiter != nil; waiter = waiters.Pop() { + scheduleTask(waiter) + } +} + +func (wg *WaitGroup) addPlain(delta int) { + if delta > 0 { for { - // Check for overflow. counter := wg.futex.Load() - if uint32(delta) > (^uint32(0))-counter { + if uint32(delta) > ^uint32(0)-counter { panic("sync: WaitGroup counter overflowed") } - - // Add to the counter. if wg.futex.CompareAndSwap(counter, counter+uint32(delta)) { - // Successfully added. return } } - default: - // Delta is negative (or zero). - for { - counter := wg.futex.Load() - - // Check for underflow. - if uint32(-delta) > counter { - panic("sync: negative WaitGroup counter") - } - - // Subtract from the counter. - if !wg.futex.CompareAndSwap(counter, counter-uint32(-delta)) { - // Could not swap, trying again. - continue - } - - // If the counter is zero, everything is done and the waiters should - // be resumed. - // When there are multiple thread, there is a chance for the counter - // to go to zero, WakeAll to be called, and then the counter to be - // incremented again before a waiting goroutine has a chance to - // check the new (zero) value. However the last increment is - // explicitly given in the docs as something that should not be - // done: - // - // > Note that calls with a positive delta that occur when the - // > counter is zero must happen before a Wait. - // - // So we're fine here. - if counter-uint32(-delta) == 0 { - // TODO: this is not the most efficient implementation possible - // because we wake up all waiters unconditionally, even if there - // might be none. Though since the common usage is for this to - // be called with at least one waiter, it's probably fine. - wg.futex.WakeAll() - } + } - // Successfully swapped (and woken all waiting tasks if needed). - return + for { + counter := wg.futex.Load() + if uint32(-delta) > counter { + panic("sync: negative WaitGroup counter") + } + if !wg.futex.CompareAndSwap(counter, counter-uint32(-delta)) { + continue } + if counter-uint32(-delta) == 0 { + wg.futex.WakeAll() + } + return } } @@ -71,15 +105,54 @@ func (wg *WaitGroup) Done() { } func (wg *WaitGroup) Wait() { + if !synctestIsEnabled() { + wg.waitPlain() + return + } + + wg.lock.Lock() + if wg.synctest == nil { + wg.lock.Unlock() + wg.waitPlain() + return + } + current := task.Current() + if wg.counter == 0 { + if wg.waiting == 0 { + wg.synctest = nil + } + wg.lock.Unlock() + return + } + wg.waiting++ + wg.waiters.Push(current) + if wg.synctest != nil && wg.synctest == current.SynctestBubble { + synctestBlock(current) + } + wg.lock.Unlock() + + task.Pause() + + wg.lock.Lock() + wg.waiting-- + if wg.counter != 0 { + wg.lock.Unlock() + panic("sync: WaitGroup is reused before previous Wait has returned") + } + if wg.waiting == 0 { + wg.synctest = nil + } + wg.lock.Unlock() +} + +func (wg *WaitGroup) waitPlain() { for { counter := wg.futex.Load() if counter == 0 { - return // everything already finished + return } - if wg.futex.Wait(counter) { - // Successfully woken by WakeAll (in wg.Add). - break + return } } } @@ -91,3 +164,9 @@ func (wg *WaitGroup) Go(f func()) { f() }() } + +//go:linkname synctestBlock runtime.synctestBlock +func synctestBlock(*task.Task) + +//go:linkname synctestIsEnabled runtime.synctestIsEnabled +func synctestIsEnabled() bool diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index 7ac1ea47d4..66003f84f8 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -8,6 +8,18 @@ import ( "reflect" ) +func TestSynctestDuringCleanup(t *T) { + parent := &T{} + parent.cleanupStarted = true + defer func() { + const want = "testing: synctest.Run called during t.Cleanup" + if got := recover(); got != want { + t.Errorf("panic = %v, want %q", got, want) + } + }() + testingSynctestTest(parent, func(*T) {}) +} + func TestCleanup(t *T) { var cleanups []int t.Run("test", func(t *T) { diff --git a/src/testing/testing.go b/src/testing/testing.go index 60d8fab645..874a0ccd09 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -77,6 +77,8 @@ type common struct { start time.Time // Time test or benchmark started duration time.Duration + cleanupStarted bool + tempDir string tempDirErr error tempDirSeq int32 @@ -184,7 +186,8 @@ var _ TB = (*B)(nil) // Logs are accumulated during execution and dumped to standard output when done. type T struct { common - context *testContext // For running tests and subtests. + context *testContext // For running tests and subtests. + isSynctest bool } // Name returns the name of the running test or benchmark. @@ -460,6 +463,11 @@ func (c *common) Chdir(dir string) { // runCleanup is called at the end of the test. func (c *common) runCleanup() { + c.cleanupStarted = true + if c.cancelCtx != nil { + c.cancelCtx() + c.cancelCtx = nil + } for { var cleanup func() if len(c.cleanups) > 0 { @@ -470,15 +478,15 @@ func (c *common) runCleanup() { if cleanup == nil { return } - if c.cancelCtx != nil { - c.cancelCtx() - } cleanup() } } // Parallel is not implemented, it is only provided for compatibility. func (t *T) Parallel() { + if t.isSynctest { + panic("testing: t.Parallel called inside synctest bubble") + } // Unimplemented. } @@ -504,9 +512,41 @@ func tRunner(t *T, fn func(t *T)) { t.finished = true } +//go:linkname testingSynctestTest testing/synctest.testingSynctestTest +func testingSynctestTest(t *T, f func(*T)) bool { + if t.cleanupStarted { + panic("testing: synctest.Run called during t.Cleanup") + } + + ctx, cancelCtx := context.WithCancel(context.Background()) + synctestT := T{ + common: common{ + output: &logger{logToStdout: flagVerbose}, + name: t.name, + parent: &t.common, + level: t.level + 1, + ctx: ctx, + cancelCtx: cancelCtx, + }, + context: t.context, + isSynctest: true, + } + + done := make(chan struct{}) + go func() { + defer close(done) + tRunner(&synctestT, f) + }() + <-done + return !synctestT.failed +} + // Run runs f as a subtest of t called name. It waits until the subtest is finished // and returns whether the subtest succeeded. func (t *T) Run(name string, f func(t *T)) bool { + if t.isSynctest { + panic("testing: t.Run called inside synctest bubble") + } t.hasSub = true testName, ok, _ := t.context.match.fullName(&t.common, name) if !ok { @@ -550,6 +590,9 @@ func (t *T) Run(name string, f func(t *T)) bool { // // Not Implemented. func (t *T) Deadline() (deadline time.Time, ok bool) { + if t.isSynctest { + panic("testing: t.Deadline called inside synctest bubble") + } deadline = t.context.deadline return deadline, !deadline.IsZero() }