I am trying to get a custom provider within Keycloak to work with a clustered VertX solution. And have been breaking my head on the problem for the past week. In older version of Keycloak this didn't use to be an issue, but since the introduction of Quarkus and with it, its bundled version of VertX I have been unable to get things working.
There are always two outcomes,
Arc.Container() and get the correct VertX which correctly joins the Hazelcast cluster but throws an Exception as soon as I try to send a message to the EventBus using for example publish():Caused by: java.lang.NoSuchMethodError: 'java.util.concurrent.ExecutorService io.vertx.core.impl.VertxInternal.getWorkerPool()'
last handler in the pipeline did not handle the exception.: io.netty.channel.ChannelPipelineException: io.vertx.core.net.impl.VertxHandler.handlerAdded() has thrown an exception; removed.
2023-07-20T19:54:04.958126142Z at io.netty.channel.DefaultChannelPipeline.callHandlerAdded0(DefaultChannelPipeline.java:624)
2023-07-20T19:54:04.958134097Z at io.netty.channel.DefaultChannelPipeline.addLast(DefaultChannelPipeline.java:223)
2023-07-20T19:54:04.958138785Z at io.netty.channel.DefaultChannelPipeline.addLast(DefaultChannelPipeline.java:195)
2023-07-20T19:54:04.958142132Z at io.vertx.core.net.impl.NetClientImpl.connected(NetClientImpl.java:313)
2023-07-20T19:54:04.958145217Z at io.vertx.core.net.impl.NetClientImpl.lambda$connectInternal2$2(NetClientImpl.java:271)
2023-07-20T19:54:04.958148243Z at io.vertx.core.impl.ContextInternal.dispatch(ContextInternal.java:264)
2023-07-20T19:54:04.958151119Z at io.vertx.core.net.impl.ChannelProvider.connected(ChannelProvider.java:175)
2023-07-20T19:54:04.958154044Z at io.vertx.core.net.impl.ChannelProvider.lambda$handleConnect$0(ChannelProvider.java:158)
2023-07-20T19:54:04.958156949Z at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
2023-07-20T19:54:04.958159885Z at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
2023-07-20T19:54:04.958162790Z at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
2023-07-20T19:54:04.958165836Z at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
2023-07-20T19:54:04.958168772Z at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
2023-07-20T19:54:04.958171657Z at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625)
2023-07-20T19:54:04.958174653Z at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105)
2023-07-20T19:54:04.958177588Z at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)
2023-07-20T19:54:04.958180604Z at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:300)
2023-07-20T19:54:04.958183619Z at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:335)
2023-07-20T19:54:04.958199038Z at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
2023-07-20T19:54:04.958204529Z at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
2023-07-20T19:54:04.958210690Z at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
2023-07-20T19:54:04.958216261Z at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
2023-07-20T19:54:04.958222412Z at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
2023-07-20T19:54:04.958228433Z at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
2023-07-20T19:54:04.958234575Z at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
2023-07-20T19:54:04.958240216Z at java.base/java.lang.Thread.run(Thread.java:833)
2023-07-20T19:54:04.958244904Z Caused by: java.lang.NoSuchMethodError: 'java.util.concurrent.ExecutorService io.vertx.core.impl.VertxInternal.getWorkerPool()'
2023-07-20T19:54:04.958250525Z at io.vertx.spi.cluster.hazelcast.impl.SubsOpSerializer.execute(SubsOpSerializer.java:60)
2023-07-20T19:54:04.958255665Z at io.vertx.spi.cluster.hazelcast.HazelcastClusterManager.addRegistration(HazelcastClusterManager.java:361)
2023-07-20T19:54:04.958261455Z at io.vertx.core.eventbus.impl.clustered.ClusteredEventBus.onLocalRegistration(ClusteredEventBus.java:143)
2023-07-20T19:54:04.958266745Z at io.vertx.core.eventbus.impl.EventBusImpl.addRegistration(EventBusImpl.java:262)
2023-07-20T19:54:04.958272015Z at io.vertx.core.eventbus.impl.HandlerRegistration.register(HandlerRegistration.java:63)
2023-07-20T19:54:04.958277906Z at io.vertx.core.eventbus.impl.MessageConsumerImpl.handler(MessageConsumerImpl.java:226)
2023-07-20T19:54:04.958284469Z at io.vertx.core.net.impl.NetSocketImpl.registerEventBusHandler(NetSocketImpl.java:108)
2023-07-20T19:54:04.958290039Z at io.vertx.core.net.impl.NetClientImpl.lambda$connected$8(NetClientImpl.java:309)
2023-07-20T19:54:04.958295529Z at io.vertx.core.net.impl.VertxHandler.setConnection(VertxHandler.java:82)
2023-07-20T19:54:04.958301080Z at io.vertx.core.net.impl.VertxHandler.handlerAdded(VertxHandler.java:88)
2023-07-20T19:54:04.958306209Z at io.netty.channel.AbstractChannelHandlerContext.callHandlerAdded(AbstractChannelHandlerContext.java:1114)
2023-07-20T19:54:04.958316639Z at io.netty.channel.DefaultChannelPipeline.callHandlerAdded0(DefaultChannelPipeline.java:609)
2023-07-20T19:54:04.958323031Z ... 25 more
2023-07-20T19:54:04.958329142Z
This is more or less my setup:
Dockerfile:
FROM quay.io/keycloak/keycloak:21.1.2 as builder
# Enable health and metrics support
ENV KC_HEALTH_ENABLED=false
# Configure a database vendor
ENV KC_DB=postgres
ENV JDBC_PARAMS="connectTimeout=30000"
WORKDIR /opt/keycloak
COPY --chown=keycloak:keycloak ./extensions/package.module-all.jar /opt/keycloak/providers/package.module-all.jar
RUN /opt/keycloak/bin/kc.sh build
FROM quay.io/keycloak/keycloak:21.1.2
COPY --from=builder /opt/keycloak/ /opt/keycloak/
ENTRYPOINT ["/opt/keycloak/bin/kc.sh"]
build.gradle
plugins {
id 'java'
id 'org.jetbrains.kotlin.jvm' version '1.9.0'
id 'com.github.johnrengelman.shadow' version '8.1.1'
id 'io.quarkus'
}
version ''
repositories {
mavenCentral()
jcenter()
}
test {
useJUnitPlatform()
}
shadowJar {
zip64 = true
destinationDirectory.set(file("$rootDir/../keycloak/extensions"))
}
compileKotlin {
kotlinOptions {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
}
}
compileTestKotlin {
kotlinOptions {
jvmTarget = "11"
}
}
dependencies {
compileOnly group: 'com.google.auto.service', name: 'auto-service', version: '1.0-rc7'
compileOnly group: 'org.keycloak', name: 'keycloak-server-spi', version: keycloakVersion
compileOnly group: 'org.keycloak', name: 'keycloak-core', version: keycloakVersion
compileOnly group: 'org.keycloak', name: 'keycloak-services', version: keycloakVersion
compileOnly group: 'org.keycloak', name: 'keycloak-server-spi-private', version: keycloakVersion
compileOnly group: 'org.keycloak', name: 'keycloak-model-jpa', version: keycloakVersion
compileOnly group: 'io.vertx', name: 'vertx-core', version: vertxVersion
compileOnly group: 'io.vertx', name: 'vertx-auth-common', version: vertxVersion
compileOnly group: 'io.vertx', name: 'vertx-bridge-common', version: vertxVersion
compileOnly group: 'io.vertx', name: 'vertx-codegen', version: vertxVersion
compileOnly group: 'io.vertx', name: 'vertx-web', version: vertxVersion
compileOnly group: 'io.vertx', name: 'vertx-web-common', version: vertxVersion
compileOnly group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jdk8', version: '1.7.10'
compileOnly group: 'org.jetbrains.kotlinx', name: 'kotlinx-coroutines-core', version: '1.6.4'
compileOnly enforcedPlatform("${quarkusPlatformGroupId}:${quarkusPlatformArtifactId}:${quarkusPlatformVersion}")
compileOnly group: 'io.quarkus', name: 'quarkus-vertx'
compileOnly group: 'io.quarkus', name: 'quarkus-arc'
implementation group: 'io.vertx', name: 'vertx-hazelcast', version: vertxVersion, {
exclude(group: 'io.vertx')
}
implementation group: 'io.vertx', name: 'vertx-lang-kotlin', version: vertxVersion, {
exclude(group: 'io.vertx')
}
implementation group: 'io.vertx', name: 'vertx-lang-kotlin-coroutines', version: vertxVersion, {
exclude(group: 'io.vertx')
}
implementation group: 'io.vertx', name: 'vertx-config', version: vertxVersion, {
exclude(group: 'io.vertx')
}
}
EventBusProviderImpl
@AutoService(EventBusProvider::class)
class EventBusProviderImpl : EventBusProvider {
override fun get(): EventBus {
if (instance != null) {
return instance as EventBus
}
setupEventBus()
return instance!!
}
companion object {
private var instance: EventBus? = null
private fun setupEventBus() {
runBlocking {
val container = Arc.container()
val vertxInjectable = container.select(Vertx::class.java)
val vertx = vertxInjectable.get()
vertx.eventBus().publish("test", "{\"test\":\"boo\"}")
instance = vertx.eventBus()
}
}
}
}
Any idea's or pointers would be more than welcome as the whole Quarkus thing is a tad alien to me compared to the previous JBoss based way of working.
After a lot more fighting with it today, I finally got it to work. The trick is to ensure that you rename just enough packages through the shadowJar relocate method to fool Quarkus in accepting the bundled VertX while not breaking Hazelcast who wants to find the exact same named class when sending events back and forth across the cluster.
As such this is the setup I settled on in the end:
build.gradle
plugins {
id 'java'
id 'org.jetbrains.kotlin.jvm' version '1.9.0'
id 'com.github.johnrengelman.shadow' version '8.1.1'
}
version ''
repositories {
mavenCentral()
jcenter()
}
test {
useJUnitPlatform()
}
shadowJar {
zip64 = true
destinationDirectory.set(file("$rootDir/../keycloak/extensions"))
mergeServiceFiles()
relocate 'io.vertx.core', 'nl.custom.module.bundled.io.vertx.core'
relocate 'io.vertx.web-client', 'nl.custom.module.bundled.io.vertx.web-client'
relocate 'io.vertx.lang-kotlin', 'nl.custom.module.bundled.io.vertx.lang-kotlin'
relocate 'io.vertx.lang-kotlin-coroutines', 'nl.custom.module.bundled.io.vertx.lang-kotlin-coroutines'
relocate 'io.vertx.config', 'nl.custom.module.bundled.io.vertx.config'
relocate 'io.vertx.kotlin', 'nl.custom.module.bundled.io.vertx.kotlin'
relocate 'io.vertx.ext', 'nl.custom.module.bundled.io.vertx.ext'
}
compileKotlin {
kotlinOptions {
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
}
}
compileTestKotlin {
kotlinOptions {
jvmTarget = "11"
}
}
assemble.dependsOn shadowJar
dependencies {
compileOnly group: 'com.google.auto.service', name: 'auto-service', version: '1.0-rc7'
compileOnly group: 'org.keycloak', name: 'keycloak-server-spi', version: keycloakVersion
compileOnly group: 'org.keycloak', name: 'keycloak-core', version: keycloakVersion
compileOnly group: 'org.keycloak', name: 'keycloak-services', version: keycloakVersion
compileOnly group: 'org.keycloak', name: 'keycloak-server-spi-private', version: keycloakVersion
compileOnly group: 'org.keycloak', name: 'keycloak-model-jpa', version: keycloakVersion
compileOnly group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jdk8', version: '1.7.10'
compileOnly group: 'org.jetbrains.kotlinx', name: 'kotlinx-coroutines-core', version: '1.6.4'
implementation group: 'io.vertx', name: 'vertx-core', version: vertxVersion
implementation group: 'io.vertx', name: 'vertx-web-client', version: vertxVersion
implementation group: 'io.vertx', name: 'vertx-hazelcast', version: vertxVersion
implementation group: 'io.vertx', name: 'vertx-lang-kotlin', version: vertxVersion
implementation group: 'io.vertx', name: 'vertx-lang-kotlin-coroutines', version: vertxVersion
implementation group: 'io.vertx', name: 'vertx-config', version: vertxVersion
testRuntimeOnly group:'org.junit.jupiter', name:'junit-jupiter-engine', version:'5.7.0'
testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.7.0'
testImplementation group: 'io.vertx', name: 'vertx-junit5', version: vertxVersion
}
After discovering the relocation feature of shadowJar it fell into place quite quickly. Even though the errors about the mismatching classes where very tedious to debug.