Found one Java-level deadlock: ============================= "main": waiting for ownable synchronizer 0x00000000f0dec8b8, (a java.util.concurrent.locks.ReentrantLock$NonfairSync), which is held by "ecos-zookeeper-events" "ecos-zookeeper-events": waiting to lock monitor 0x00007f977c045e00 (object 0x00000000f38d8a40, a java.util.concurrent.ConcurrentHashMap$ReservationNode), which is held by "main" zk-event -> setValue -> lock -> fireEvent -> createRecordsDao -> getWorkspaceSystemId -> CaffeineCache -> BLOCKED syncAllTypes -> getWorkspaceSystemId -> CaffeineCache -> getSystemId в бд -> getTypeInfoData -> EcosRegistryImpl.getValue -> BLOCKED "ecos-zookeeper-events" #33 [43] prio=5 os_prio=0 cpu=3560.86ms elapsed=22484.07s tid=0x00007f982e44a1a0 nid=43 waiting for monitor entry [0x00007f97d5422000] java.lang.Thread.State: BLOCKED (on object monitor) at java.util.concurrent.ConcurrentHashMap.compute(java.base@24.0.2/Unknown Source) - waiting to lock <0x00000000f38d8a40> (a java.util.concurrent.ConcurrentHashMap$ReservationNode) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2688) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2670) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:112) at com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:60) at ru.citeck.ecos.model.lib.workspace.WorkspaceServiceImpl.getWorkspaceSystemId(WorkspaceServiceImpl.kt:146) at ru.citeck.ecos.model.lib.workspace.WorkspaceServiceImpl.getPrefixForIdInWorkspace(WorkspaceServiceImpl.kt:346) at ru.citeck.ecos.model.lib.workspace.WorkspaceServiceImpl.addWsPrefixToId(WorkspaceServiceImpl.kt:170) at ru.citeck.ecos.model.type.service.utils.EModelTypeUtils.createId(EModelTypeUtils.kt:101) at ru.citeck.ecos.model.type.service.utils.EModelTypeUtils.getEmodelSourceTableId(EModelTypeUtils.kt:80) at ru.citeck.ecos.model.type.config.TypesConfig.createRecordsDao(TypesConfig.kt:90) at ru.citeck.ecos.model.type.config.TypesConfig.init$lambda$3(TypesConfig.kt:78) at ru.citeck.ecos.model.type.config.TypesConfig$$Lambda/0x00007f97d82763c0.invoke(Unknown Source) at ru.citeck.ecos.webapp.lib.registry.EcosRegistryImpl.listenEvents$lambda$20(EcosRegistryImpl.kt:179) at ru.citeck.ecos.webapp.lib.registry.EcosRegistryImpl$$Lambda/0x00007f97d81c9e60.invoke(Unknown Source) at ru.citeck.ecos.webapp.lib.registry.EcosRegistryImpl.fireEvent(EcosRegistryImpl.kt:193) at ru.citeck.ecos.webapp.lib.registry.EcosRegistryImpl.setValue(EcosRegistryImpl.kt:144) at ru.citeck.ecos.webapp.lib.registry.MutableEcosRegistryDelegate.setValue(MutableEcosRegistryDelegate.kt:21) at ru.citeck.ecos.webapp.lib.registry.EcosRegistryImpl$checkInitializationInSyncBlock$initializersRegistry$1.setValue(EcosRegistryImpl.kt:73) at ru.citeck.ecos.webapp.lib.registry.init.ZkRegistryInitializer.processZkEvent(ZkRegistryInitializer.kt:228) at ru.citeck.ecos.webapp.lib.registry.init.ZkRegistryInitializer.init$lambda$13(ZkRegistryInitializer.kt:178) at ru.citeck.ecos.webapp.lib.registry.init.ZkRegistryInitializer$$Lambda/0x00007f97d820b8f8.invoke(Unknown Source) at ru.citeck.ecos.zookeeper.client.EcosZooKeeperClient._init_$lambda$7(EcosZooKeeperClient.kt:94) at ru.citeck.ecos.zookeeper.client.EcosZooKeeperClient$$Lambda/0x00007f97d774ec10.invoke(Unknown Source) at kotlin.concurrent.ThreadsKt$thread$thread$1.run(Thread.kt:30) "main" #3 [12] prio=5 os_prio=0 cpu=43811.88ms elapsed=22496.81s tid=0x00007f982c02d7b0 nid=12 waiting on condition [0x00007f9831102000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@24.0.2/Native Method) - parking to wait for <0x00000000f0dec8b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@24.0.2/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@24.0.2/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@24.0.2/Unknown Source) at java.util.concurrent.locks.ReentrantLock$Sync.lock(java.base@24.0.2/Unknown Source) at java.util.concurrent.locks.ReentrantLock.lock(java.base@24.0.2/Unknown Source) at ru.citeck.ecos.webapp.lib.registry.EcosRegistryImpl.checkInitialization(EcosRegistryImpl.kt:42) at ru.citeck.ecos.webapp.lib.registry.EcosRegistryImpl.getValueWithMeta(EcosRegistryImpl.kt:163) at ru.citeck.ecos.webapp.lib.registry.EcosRegistryImpl.getValue(EcosRegistryImpl.kt:159) at ru.citeck.ecos.webapp.lib.model.type.registry.EcosTypesRegistry.getTypeInfo(EcosTypesRegistry.kt:74) at ru.citeck.ecos.webapp.lib.spring.context.model.ModelServiceFactoryConfig$createTypesRepo$1.getTypeInfo(ModelServiceFactoryConfig.kt:115) at ru.citeck.ecos.data.sql.ecostype.DbEcosModelService.getTypeInfo(DbEcosModelService.kt:94) at ru.citeck.ecos.data.sql.records.dao.query.DbFindQueryContext$DbQueryTypeInfoDataImpl.(DbFindQueryContext.kt:318) at ru.citeck.ecos.data.sql.records.dao.query.DbFindQueryContext.getTypeInfoData$lambda$12(DbFindQueryContext.kt:275) at ru.citeck.ecos.data.sql.records.dao.query.DbFindQueryContext$$Lambda/0x00007f97d81d7850.invoke(Unknown Source) at ru.citeck.ecos.data.sql.records.dao.query.DbFindQueryContext.getTypeInfoData$lambda$13(DbFindQueryContext.kt:274) at ru.citeck.ecos.data.sql.records.dao.query.DbFindQueryContext$$Lambda/0x00007f97d81d7a88.apply(Unknown Source) at java.util.HashMap.computeIfAbsent(java.base@24.0.2/Unknown Source) at ru.citeck.ecos.data.sql.records.dao.query.DbFindQueryContext.getTypeInfoData(DbFindQueryContext.kt:274) at ru.citeck.ecos.data.sql.records.dao.query.DbFindQueryContext.(DbFindQueryContext.kt:46) at ru.citeck.ecos.data.sql.records.dao.atts.DbRecordsAttsDao.getRecordsAtts(DbRecordsAttsDao.kt:28) at ru.citeck.ecos.data.sql.records.DbRecordsDao.getRecordsAtts(DbRecordsDao.kt:143) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getAttsFromRecordsDao(LocalRecordsResolverImpl.kt:676) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getAttsFromSource(LocalRecordsResolverImpl.kt:652) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getRefsAttsImpl(LocalRecordsResolverImpl.kt:639) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getRecordsAttsImpl$ecos_records(LocalRecordsResolverImpl.kt:605) at ru.citeck.ecos.records3.record.resolver.interceptor.GetRecordsAttsInterceptorsChain.invoke(LocalRecordsInterceptor.kt:100) at ru.citeck.ecos.records3.record.resolver.interceptor.obs.ObservableRecordsInterceptor.getRecordsAtts(ObservableRecordsInterceptor.kt:51) at ru.citeck.ecos.records3.record.resolver.interceptor.GetRecordsAttsInterceptorsChain.invoke(LocalRecordsInterceptor.kt:98) at ru.citeck.ecos.records3.record.resolver.interceptor.AuditRecordsInterceptor.getRecordsAtts(AuditRecordsInterceptor.kt:108) at ru.citeck.ecos.records3.record.resolver.interceptor.GetRecordsAttsInterceptorsChain.invoke(LocalRecordsInterceptor.kt:98) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getRecordsAtts$lambda$24(LocalRecordsResolverImpl.kt:588) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl$$Lambda/0x00007f97d843b328.invoke(Unknown Source) at ru.citeck.ecos.records3.record.atts.RecordAttsServiceImpl.doWithSchema$lambda$0(RecordAttsServiceImpl.kt:65) at ru.citeck.ecos.records3.record.atts.RecordAttsServiceImpl$$Lambda/0x00007f97d7ee2ab0.invoke(Unknown Source) at ru.citeck.ecos.records3.record.atts.schema.resolver.AttContext$Companion.doWithCtx(AttContext.kt:82) at ru.citeck.ecos.records3.record.atts.RecordAttsServiceImpl.doWithSchema(RecordAttsServiceImpl.kt:44) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getRecordsAtts(LocalRecordsResolverImpl.kt:583) at ru.citeck.ecos.records3.record.resolver.LocalRemoteResolver.loadAttsForRefs(LocalRemoteResolver.kt:303) at ru.citeck.ecos.records3.record.resolver.LocalRemoteResolver.loadAttsForRefsWithCache(LocalRemoteResolver.kt:260) at ru.citeck.ecos.records3.record.resolver.LocalRemoteResolver.getAtts(LocalRemoteResolver.kt:150) at ru.citeck.ecos.records3.RecordsServiceImpl.getAtts$lambda$8(RecordsServiceImpl.kt:66) at ru.citeck.ecos.records3.RecordsServiceImpl$$Lambda/0x00007f97d7ee6b28.invoke(Unknown Source) at ru.citeck.ecos.records3.record.request.RequestContext$Companion.doWithCtx(RequestContext.kt:196) at ru.citeck.ecos.records3.RecordsServiceImpl.getAtts(RecordsServiceImpl.kt:64) at ru.citeck.ecos.records3.record.dao.impl.proxy.RecordsDaoProxy.getRecordsAtts$lambda$0(RecordsDaoProxy.kt:60) at ru.citeck.ecos.records3.record.dao.impl.proxy.RecordsDaoProxy$$Lambda/0x00007f97d8441e18.invoke(Unknown Source) at ru.citeck.ecos.records3.record.dao.impl.proxy.RecordsDaoProxy.doWithSourceIdMapping$lambda$8(RecordsDaoProxy.kt:118) at ru.citeck.ecos.records3.record.dao.impl.proxy.RecordsDaoProxy$$Lambda/0x00007f97d81d1ee0.invoke(Unknown Source) at ru.citeck.ecos.records3.record.request.RequestContext$Companion.doWithCtx(RequestContext.kt:196) at ru.citeck.ecos.records3.record.request.RequestContext$Companion.doWithCtx(RequestContext.kt:102) at ru.citeck.ecos.records3.record.dao.impl.proxy.RecordsDaoProxy.doWithSourceIdMapping(RecordsDaoProxy.kt:115) at ru.citeck.ecos.records3.record.dao.impl.proxy.RecordsDaoProxy.getRecordsAtts(RecordsDaoProxy.kt:59) at ru.citeck.ecos.model.domain.workspace.api.records.WorkspaceProxyDao.getRecordsAtts(WorkspaceProxyDao.kt:308) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getAttsFromRecordsDao(LocalRecordsResolverImpl.kt:676) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getAttsFromSource(LocalRecordsResolverImpl.kt:652) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getRefsAttsImpl(LocalRecordsResolverImpl.kt:639) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getRecordsAttsImpl$ecos_records(LocalRecordsResolverImpl.kt:605) at ru.citeck.ecos.records3.record.resolver.interceptor.GetRecordsAttsInterceptorsChain.invoke(LocalRecordsInterceptor.kt:100) at ru.citeck.ecos.records3.record.resolver.interceptor.obs.ObservableRecordsInterceptor.getRecordsAtts(ObservableRecordsInterceptor.kt:51) at ru.citeck.ecos.records3.record.resolver.interceptor.GetRecordsAttsInterceptorsChain.invoke(LocalRecordsInterceptor.kt:98) at ru.citeck.ecos.records3.record.resolver.interceptor.AuditRecordsInterceptor.getRecordsAtts(AuditRecordsInterceptor.kt:108) at ru.citeck.ecos.records3.record.resolver.interceptor.GetRecordsAttsInterceptorsChain.invoke(LocalRecordsInterceptor.kt:98) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getRecordsAtts$lambda$24(LocalRecordsResolverImpl.kt:588) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl$$Lambda/0x00007f97d843b328.invoke(Unknown Source) at ru.citeck.ecos.records3.record.atts.RecordAttsServiceImpl.doWithSchema$lambda$0(RecordAttsServiceImpl.kt:65) at ru.citeck.ecos.records3.record.atts.RecordAttsServiceImpl$$Lambda/0x00007f97d7ee2ab0.invoke(Unknown Source) at ru.citeck.ecos.records3.record.atts.schema.resolver.AttContext$Companion.doWithCtx(AttContext.kt:82) at ru.citeck.ecos.records3.record.atts.RecordAttsServiceImpl.doWithSchema(RecordAttsServiceImpl.kt:44) at ru.citeck.ecos.records3.record.resolver.LocalRecordsResolverImpl.getRecordsAtts(LocalRecordsResolverImpl.kt:583) at ru.citeck.ecos.records3.record.resolver.LocalRemoteResolver.loadAttsForRefs(LocalRemoteResolver.kt:303) at ru.citeck.ecos.records3.record.resolver.LocalRemoteResolver.loadAttsForRefsWithCache(LocalRemoteResolver.kt:260) at ru.citeck.ecos.records3.record.resolver.LocalRemoteResolver.getAtts(LocalRemoteResolver.kt:150) at ru.citeck.ecos.records3.RecordsServiceImpl.getAtts$lambda$8(RecordsServiceImpl.kt:66) at ru.citeck.ecos.records3.RecordsServiceImpl$$Lambda/0x00007f97d7ee6b28.invoke(Unknown Source) at ru.citeck.ecos.records3.record.request.RequestContext$Companion.doWithCtx(RequestContext.kt:196) at ru.citeck.ecos.records3.RecordsServiceImpl.getAtts(RecordsServiceImpl.kt:64) at ru.citeck.ecos.records3.AbstractRecordsService.getAtts(AbstractRecordsService.kt:73) at ru.citeck.ecos.records3.AbstractRecordsService.getAtt(AbstractRecordsService.kt:64) at ru.citeck.ecos.model.domain.workspace.service.EmodelWorkspaceService.wsSystemIdCache$lambda$2$lambda$1(EmodelWorkspaceService.kt:58) at ru.citeck.ecos.model.domain.workspace.service.EmodelWorkspaceService$$Lambda/0x00007f97d843a120.invoke(Unknown Source) at ru.citeck.ecos.context.lib.auth.component.DefaultAuthComponent.runAs(DefaultAuthComponent.kt:60) at ru.citeck.ecos.context.lib.auth.AuthContext.runAs(AuthContext.kt:187) at ru.citeck.ecos.context.lib.auth.AuthContext.runAsSystem(AuthContext.kt:152) at ru.citeck.ecos.model.domain.workspace.service.EmodelWorkspaceService.wsSystemIdCache$lambda$2(EmodelWorkspaceService.kt:57) at ru.citeck.ecos.model.domain.workspace.service.EmodelWorkspaceService$$Lambda/0x00007f97d7f4cfe0.invoke(Unknown Source) at ru.citeck.ecos.model.domain.workspace.service.EmodelWorkspaceService.wsSystemIdCache$lambda$3(EmodelWorkspaceService.kt:56) at ru.citeck.ecos.model.domain.workspace.service.EmodelWorkspaceService$$Lambda/0x00007f97d7f4d218.load(Unknown Source) at com.hazelcast.hibernate.shaded.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$2(LocalLoadingCache.java:145) at com.hazelcast.hibernate.shaded.caffeine.cache.LocalLoadingCache$$Lambda/0x00007f97d7f52f50.apply(Unknown Source) at com.hazelcast.hibernate.shaded.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2406) at com.hazelcast.hibernate.shaded.caffeine.cache.BoundedLocalCache$$Lambda/0x00007f97d8439ed8.apply(Unknown Source) at java.util.concurrent.ConcurrentHashMap.compute(java.base@24.0.2/Unknown Source) - locked <0x00000000f38d8a60> (a java.util.concurrent.ConcurrentHashMap$ReservationNode) at com.hazelcast.hibernate.shaded.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2404) at com.hazelcast.hibernate.shaded.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2387) at com.hazelcast.hibernate.shaded.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) at com.hazelcast.hibernate.shaded.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:56) at ru.citeck.ecos.model.domain.workspace.service.EmodelWorkspaceService.getSystemId(EmodelWorkspaceService.kt:108) at ru.citeck.ecos.model.domain.workspace.service.CustomWorkspaceApi.mapIdentifiers(CustomWorkspaceApi.kt:38) at ru.citeck.ecos.model.lib.ModelServiceFactory$createWorkspaceApi$1.mapIdentifiers(ModelServiceFactory.kt:130) at ru.citeck.ecos.model.lib.workspace.WorkspaceServiceImpl.workspaceSysIdCache$lambda$2(WorkspaceServiceImpl.kt:41) at ru.citeck.ecos.model.lib.workspace.WorkspaceServiceImpl$$Lambda/0x00007f97d7f2a7c8.invoke(Unknown Source) at ru.citeck.ecos.model.lib.workspace.WorkspaceServiceImpl.workspaceSysIdCache$lambda$3(WorkspaceServiceImpl.kt:40) at ru.citeck.ecos.model.lib.workspace.WorkspaceServiceImpl$$Lambda/0x00007f97d7f2aa00.load(Unknown Source) at com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$0(LocalLoadingCache.java:185) at com.github.benmanes.caffeine.cache.LocalLoadingCache$$Lambda/0x00007f97d7f2a570.apply(Unknown Source) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$0(BoundedLocalCache.java:2690) at com.github.benmanes.caffeine.cache.BoundedLocalCache$$Lambda/0x00007f97d8263480.apply(Unknown Source) at java.util.concurrent.ConcurrentHashMap.compute(java.base@24.0.2/Unknown Source) - locked <0x00000000f38d8a40> (a java.util.concurrent.ConcurrentHashMap$ReservationNode) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2688) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2670) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:112) at com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:60) at ru.citeck.ecos.model.lib.workspace.WorkspaceServiceImpl.getWorkspaceSystemId(WorkspaceServiceImpl.kt:146) at ru.citeck.ecos.model.lib.workspace.WorkspaceServiceImpl.getPrefixForIdInWorkspace(WorkspaceServiceImpl.kt:346) at ru.citeck.ecos.model.lib.workspace.WorkspaceServiceImpl.addWsPrefixToId(WorkspaceServiceImpl.kt:170) at ru.citeck.ecos.model.type.service.resolver.TypeDefResolver.getResolvedByParentType(TypeDefResolver.kt:88) at ru.citeck.ecos.model.type.service.resolver.TypeDefResolver.getResolvedTypes(TypeDefResolver.kt:70) at ru.citeck.ecos.model.type.service.resolver.TypeDefResolver.getResolvedTypesWithMeta$lambda$0(TypeDefResolver.kt:56) at ru.citeck.ecos.model.type.service.resolver.TypeDefResolver$$Lambda/0x00007f97d8257238.invoke(Unknown Source) at ru.citeck.ecos.model.type.service.resolver.TypeDefResolver.doWithMeta(TypeDefResolver.kt:574) at ru.citeck.ecos.model.type.service.resolver.TypeDefResolver.getResolvedTypesWithMeta(TypeDefResolver.kt:56) at ru.citeck.ecos.model.type.service.TypesRegistryInitializer.syncAllTypes(TypesRegistryInitializer.kt:155) at ru.citeck.ecos.model.type.service.TypesRegistryInitializer.init$lambda$1(TypesRegistryInitializer.kt:80) at ru.citeck.ecos.model.type.service.TypesRegistryInitializer$$Lambda/0x00007f97d826f850.invoke(Unknown Source) at ru.citeck.ecos.webapp.lib.spring.context.context.EcosWebAppContextImpl.invokeActions(EcosWebAppContextImpl.kt:158) at ru.citeck.ecos.webapp.lib.spring.context.context.EcosWebAppContextImpl.applicationContextWasLoaded(EcosWebAppContextImpl.kt:148) at ru.citeck.ecos.webapp.lib.spring.EcosSpringApplication.run$lambda$2(EcosSpringApplication.kt:39) at ru.citeck.ecos.webapp.lib.spring.EcosSpringApplication$$Lambda/0x00007f97d712c6d0.invoke(Unknown Source) at ru.citeck.ecos.context.lib.auth.component.DefaultAuthComponent.runAs(DefaultAuthComponent.kt:60) at ru.citeck.ecos.context.lib.auth.AuthContext.runAs(AuthContext.kt:187) at ru.citeck.ecos.context.lib.auth.AuthContext.runAsSystem(AuthContext.kt:152) at ru.citeck.ecos.webapp.lib.spring.EcosSpringApplication.run(EcosSpringApplication.kt:36) at ru.citeck.ecos.model.EcosModelApp.main(EcosModelApp.java:24)