We're using com.google.cloud:google-cloud-pubsub:1.152.0 in production (with Temurin JRE version 25).
Our Kotlin wrapper for pub/sub is defined like this:
class PubSubPublisher(private val config: PubSubConfig) : AutoCloseable {
private val publisher: Publisher
private var channel: ManagedChannel? = null
init {
// [...]
val publisherBuilder = Publisher.newBuilder(topicName).setCredentialsProvider(credentialsProvider)
if (config.endpoint != null) {
channel = ManagedChannelBuilder.forTarget(config.endpoint).usePlaintext().build()
val channelProvider: TransportChannelProvider =
FixedTransportChannelProvider.create(
GrpcTransportChannel.create(channel),
)
publisherBuilder.setChannelProvider(channelProvider)
}
publisher = publisherBuilder.build()
}
override fun close() {
channel?.shutdown()
publisher.shutdown()
publisher.awaitTermination(config.timeout.inWholeMilliseconds, TimeUnit.MILLISECONDS)
}
}
The following thread dump, captured hours after the job was done, shows that the main thread is blocked because publisher.shutdown() never completes:
Full thread dump OpenJDK 64-Bit Server VM (25.0.3+9-LTS mixed mode, sharing):
Threads class SMR info:
_java_thread_list=0x000078f944002f80, length=27, elements={
0x000078f9b802aba0, 0x000078f9b8073fd0, 0x000078f9b80757d0, 0x000078f9b80771f0,
0x000078f9b8078ae0, 0x000078f9b807a330, 0x000078f9b807c100, 0x000078f9b807dac0,
0x000078f9b80a6f10, 0x000078f9b80a9550, 0x000078f9b8b2f7b0, 0x000078f9b8c924c0,
0x000078f9b8f52220, 0x000078f958041430, 0x000078f9b90721f0, 0x000078f93c0012b0,
0x000078f9b9135610, 0x000078f9b9185c60, 0x000078f968001290, 0x000078f96c000f40,
0x000078f968002790, 0x000078f96c002700, 0x000078f918000f40, 0x000078f91c000f40,
0x000078f968003630, 0x000078f96c003470, 0x000078f968013b10
}
"main" #3 [12] prio=5 os_prio=0 cpu=8723.33ms elapsed=711365.58s tid=0x000078f9b802aba0 nid=12 in Object.wait() [0x000078f9bce2d000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait0(java.base@25.0.3/Native Method)
- waiting on <0x00000000ca65d670> (a com.google.cloud.pubsub.v1.Waiter)
at java.lang.Object.wait(java.base@25.0.3/Unknown Source)
at java.lang.Object.wait(java.base@25.0.3/Unknown Source)
at com.google.cloud.pubsub.v1.Waiter.waitComplete(Waiter.java:45)
- locked <0x00000000ca65d670> (a com.google.cloud.pubsub.v1.Waiter)
at com.google.cloud.pubsub.v1.Publisher.shutdown(Publisher.java:681)
at xxx.reporting.pubsub.PubSubPublisher.close(PubSubPublisher.kt:62)
at xxx.reporting.result.ResultReportPublisher.close(ResultReportPublisher.kt:42)
at xxx.schema.Main$run$1.invokeSuspend(Main.kt:120)
at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:34)
at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:100)
at kotlinx.coroutines.EventLoopImplBase.processNextEvent(EventLoop.common.kt:256)
at kotlinx.coroutines.BlockingCoroutine.joinBlocking(Builders.kt:54)
at kotlinx.coroutines.BuildersKt__BuildersKt.runBlockingImpl(Builders.kt:30)
at kotlinx.coroutines.BuildersKt.runBlockingImpl(Unknown Source)
at kotlinx.coroutines.BuildersKt__Builders_concurrentKt.runBlockingK(Builders.concurrent.kt:172)
at kotlinx.coroutines.BuildersKt.runBlockingK(Unknown Source)
at kotlinx.coroutines.BuildersKt__Builders_concurrentKt.runBlockingK$default(Builders.concurrent.kt:157)
at kotlinx.coroutines.BuildersKt.runBlockingK$default(Unknown Source)
at xxx.schema.Main.run(Main.kt:45)
at xxx.schema.MainKt.main(Main.kt:209)
at xxx.schema.MainKt.main(Main.kt)
"Reference Handler" #6 [15] daemon prio=10 os_prio=0 cpu=8.20ms elapsed=711365.56s tid=0x000078f9b8073fd0 nid=15 waiting on condition [0x000078f9b2131000]
java.lang.Thread.State: RUNNABLE
at java.lang.ref.Reference.waitForReferencePendingList(java.base@25.0.3/Native Method)
at java.lang.ref.Reference.processPendingReferences(java.base@25.0.3/Unknown Source)
at java.lang.ref.Reference$ReferenceHandler.run(java.base@25.0.3/Unknown Source)
"Finalizer" #7 [16] daemon prio=8 os_prio=0 cpu=0.19ms elapsed=711365.56s tid=0x000078f9b80757d0 nid=16 in Object.wait() [0x000078f9a2ffe000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait0(java.base@25.0.3/Native Method)
- waiting on <0x00000000c9f8de98> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.Object.wait(java.base@25.0.3/Unknown Source)
at java.lang.Object.wait(java.base@25.0.3/Unknown Source)
at java.lang.ref.ReferenceQueue.remove0(java.base@25.0.3/Unknown Source)
at java.lang.ref.ReferenceQueue.remove(java.base@25.0.3/Unknown Source)
- locked <0x00000000c9f8de98> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.Finalizer$FinalizerThread.run(java.base@25.0.3/Unknown Source)
"Signal Dispatcher" #8 [17] daemon prio=9 os_prio=0 cpu=0.22ms elapsed=711365.56s tid=0x000078f9b80771f0 nid=17 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Service Thread" #9 [18] daemon prio=9 os_prio=0 cpu=23816.66ms elapsed=711365.56s tid=0x000078f9b8078ae0 nid=18 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Monitor Deflation Thread" #10 [19] daemon prio=9 os_prio=0 cpu=86439.05ms elapsed=711365.56s tid=0x000078f9b807a330 nid=19 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" #11 [20] daemon prio=9 os_prio=0 cpu=43115.44ms elapsed=711365.56s tid=0x000078f9b807c100 nid=20 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
No compile task
"C1 CompilerThread0" #12 [21] daemon prio=9 os_prio=0 cpu=9421.35ms elapsed=711365.56s tid=0x000078f9b807dac0 nid=21 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
No compile task
"Notification Thread" #13 [22] daemon prio=9 os_prio=0 cpu=132.89ms elapsed=711365.55s tid=0x000078f9b80a6f10 nid=22 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Common-Cleaner" #14 [23] daemon prio=8 os_prio=0 cpu=655.92ms elapsed=711365.55s tid=0x000078f9b80a9550 nid=23 in Object.wait() [0x000078f9a28fe000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait0(java.base@25.0.3/Native Method)
- waiting on <no object reference available>
at java.lang.Object.wait(java.base@25.0.3/Unknown Source)
at java.lang.ref.ReferenceQueue.remove0(java.base@25.0.3/Unknown Source)
at java.lang.ref.ReferenceQueue.remove(java.base@25.0.3/Unknown Source)
- locked <0x00000000c9f8df50> (a java.lang.ref.ReferenceQueue$Lock)
at jdk.internal.ref.CleanerImpl.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
at jdk.internal.misc.InnocuousThread.run(java.base@25.0.3/Unknown Source)
"Gax-1" #16 [24] daemon prio=5 os_prio=0 cpu=1444.81ms elapsed=711361.94s tid=0x000078f9b8b2f7b0 nid=24 waiting on condition [0x000078f9a27fe000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000ca3f10a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Gax-2" #17 [25] daemon prio=5 os_prio=0 cpu=1466.29ms elapsed=711361.75s tid=0x000078f9b8c924c0 nid=25 waiting on condition [0x000078f9a217c000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000ca65af80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"mysql-cj-abandoned-connection-cleanup" #20 [27] daemon prio=5 os_prio=0 cpu=7939.51ms elapsed=711359.04s tid=0x000078f9b8f52220 nid=27 in Object.wait() [0x000078f9a1f7c000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait0(java.base@25.0.3/Native Method)
- waiting on <no object reference available>
at java.lang.Object.wait(java.base@25.0.3/Unknown Source)
at java.lang.ref.ReferenceQueue.remove0(java.base@25.0.3/Unknown Source)
at java.lang.ref.ReferenceQueue.remove(java.base@25.0.3/Unknown Source)
- locked <0x00000000cac33688> (a java.lang.ref.ReferenceQueue$Lock)
at com.mysql.cj.jdbc.AbandonedConnectionCleanupThread.run(AbandonedConnectionCleanupThread.java:84)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"JNA Cleaner" #23 [31] daemon prio=5 os_prio=0 cpu=1889.80ms elapsed=711358.69s tid=0x000078f958041430 nid=31 in Object.wait() [0x000078f9a1d7c000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait0(java.base@25.0.3/Native Method)
- waiting on <no object reference available>
at java.lang.Object.wait(java.base@25.0.3/Unknown Source)
at java.lang.ref.ReferenceQueue.remove0(java.base@25.0.3/Unknown Source)
at java.lang.ref.ReferenceQueue.remove(java.base@25.0.3/Unknown Source)
- locked <0x00000000caf429b0> (a java.lang.ref.ReferenceQueue$Lock)
at com.sun.jna.internal.Cleaner$CleanerThread.run(Cleaner.java:154)
"Connection evictor" #24 [32] daemon prio=5 os_prio=0 cpu=2831.57ms elapsed=711358.36s tid=0x000078f9b90721f0 nid=32 waiting on condition [0x000078f9a19fe000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleepNanos0(java.base@25.0.3/Native Method)
at java.lang.Thread.sleepNanos(java.base@25.0.3/Unknown Source)
at java.lang.Thread.sleep(java.base@25.0.3/Unknown Source)
at xxx.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Connection evictor" #33 [41] daemon prio=5 os_prio=0 cpu=1352.65ms elapsed=711357.94s tid=0x000078f93c0012b0 nid=41 waiting on condition [0x000078f9a0dfe000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleepNanos0(java.base@25.0.3/Native Method)
at java.lang.Thread.sleepNanos(java.base@25.0.3/Unknown Source)
at java.lang.Thread.sleep(java.base@25.0.3/Unknown Source)
at xxx.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Connection evictor" #34 [42] daemon prio=5 os_prio=0 cpu=1308.17ms elapsed=711356.98s tid=0x000078f9b9135610 nid=42 waiting on condition [0x000078f9a14fe000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleepNanos0(java.base@25.0.3/Native Method)
at java.lang.Thread.sleepNanos(java.base@25.0.3/Unknown Source)
at java.lang.Thread.sleep(java.base@25.0.3/Unknown Source)
at xxx.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"HikariPool-1:housekeeper" #38 [45] daemon prio=5 os_prio=0 cpu=5461.43ms elapsed=711356.03s tid=0x000078f9b9185c60 nid=45 waiting on condition [0x000078f9a0efe000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000cad953d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Gax-3" #45 [52] daemon prio=5 os_prio=0 cpu=1018.43ms elapsed=711351.94s tid=0x000078f968001290 nid=52 waiting on condition [0x000078f9a09fe000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000ca3f10a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Gax-4" #46 [53] daemon prio=5 os_prio=0 cpu=1366.39ms elapsed=711351.75s tid=0x000078f96c000f40 nid=53 waiting on condition [0x000078f9a08fe000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000ca65af80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Gax-5" #52 [59] daemon prio=5 os_prio=0 cpu=1423.40ms elapsed=711341.94s tid=0x000078f968002790 nid=59 waiting on condition [0x000078f9a07fe000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000ca3f10a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Gax-6" #54 [61] daemon prio=5 os_prio=0 cpu=1390.12ms elapsed=711341.75s tid=0x000078f96c002700 nid=61 waiting on condition [0x000078f9a05fe000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000ca65af80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Gax-7" #57 [64] daemon prio=5 os_prio=0 cpu=1160.21ms elapsed=711331.94s tid=0x000078f918000f40 nid=64 waiting on condition [0x000078f9a02fe000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000ca3f10a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Gax-8" #58 [65] daemon prio=5 os_prio=0 cpu=973.96ms elapsed=711331.75s tid=0x000078f91c000f40 nid=65 waiting on condition [0x000078f9a01fe000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000ca65af80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Gax-9" #61 [68] daemon prio=5 os_prio=0 cpu=1811.37ms elapsed=711321.94s tid=0x000078f968003630 nid=68 waiting on condition [0x000078f8f7ffe000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000ca3f10a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"Gax-10" #62 [69] daemon prio=5 os_prio=0 cpu=1539.54ms elapsed=711321.75s tid=0x000078f96c003470 nid=69 waiting on condition [0x000078f8f7efe000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@25.0.3/Native Method)
- parking to wait for <0x00000000ca65af80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@25.0.3/Unknown Source)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@25.0.3/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@25.0.3/Unknown Source)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
"grpc-default-worker-ELG-1-1" #82 [89] daemon prio=5 os_prio=0 cpu=260.56ms elapsed=711174.36s tid=0x000078f968013b10 nid=89 runnable [0x000078f9a0afe000]
java.lang.Thread.State: RUNNABLE
at io.grpc.netty.shaded.io.netty.channel.epoll.Native.epollWait(Native Method)
at io.grpc.netty.shaded.io.netty.channel.epoll.Native.epollWait(Native.java:220)
at io.grpc.netty.shaded.io.netty.channel.epoll.Native.epollWait(Native.java:213)
at io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
at io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
at io.grpc.netty.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.grpc.netty.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.grpc.netty.shaded.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.runWith(java.base@25.0.3/Unknown Source)
at java.lang.Thread.run(java.base@25.0.3/Unknown Source)
Are we misusing the Publisher here?
Is this a termination bug?
We're using
com.google.cloud:google-cloud-pubsub:1.152.0in production (with Temurin JRE version 25).Our Kotlin wrapper for pub/sub is defined like this:
The following thread dump, captured hours after the job was done, shows that the main thread is blocked because
publisher.shutdown()never completes:Are we misusing the
Publisherhere?Is this a termination bug?