JDK1.8下多线程使用JDBC加载ClickHouse和hive驱动问题

JDK1.8下多线程使用JDBC加载CH和hive驱动问题

在线程池里并行加载ClickHouse和Hive驱动时，发现程序无反应。通过日志发现均卡在驱动加载部分。猜测加载驱动时发生问题。随即通过下方程序进行验证调试。

现象重现

public class DriverLock {public static void main(String[] args) {InnerCHThread chThread = new InnerCHThread();chThread.setName("chThread");InnerHiveThread hiveThread = new InnerHiveThread();hiveThread.setName("hiveThread");chThread.start();hiveThread.start();}public static class InnerCHThread extends Thread {@Overridepublic void run() {try {Class.forName("com.clickhouse.jdbc.ClickHouseDriver", true, this.getClass().getClassLoader());System.out.println("com.clickhouse.jdbc.ClickHouseDriver");} catch (ClassNotFoundException e) {e.printStackTrace();}}}public static class InnerHiveThread extends Thread {@Overridepublic void run() {try {Class.forName("org.apache.hive.jdbc.HiveDriver", true, this.getClass().getClassLoader());System.out.println("org.apache.hive.jdbc.HiveDriver");} catch (ClassNotFoundException e) {e.printStackTrace();}}}
}

线程dump

jstack命令

PS E:\\code\\myself\\china-unicorn> jstack 13100
2023-03-29 14:40:17
Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.333-b02 mixed mode):"DestroyJavaVM" #13 prio=5 os_prio=0 tid=0x000001c2329d9000 nid=0x3eb0 waiting on condition [0x0000000000000000]java.lang.Thread.State: RUNNABLE"hiveThread" #12 prio=5 os_prio=0 tid=0x000001c24f7b2000 nid=0x23a4 in Object.wait() [0x00000001016fd000]java.lang.Thread.State: RUNNABLEat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)at java.lang.reflect.Constructor.newInstance(Constructor.java:423)at java.lang.Class.newInstance(Class.java:442)at java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380)at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404)at java.util.ServiceLoader$1.next(ServiceLoader.java:480)at java.sql.DriverManager$2.run(DriverManager.java:603)at java.sql.DriverManager$2.run(DriverManager.java:583)at java.security.AccessController.doPrivileged(Native Method)at java.sql.DriverManager.loadInitialDrivers(DriverManager.java:583)at java.sql.DriverManager.<clinit>(DriverManager.java:101)at org.apache.hive.jdbc.HiveDriver.<clinit>(HiveDriver.java:44)at java.lang.Class.forName0(Native Method)at java.lang.Class.forName(Class.java:348)at com.donny.bigdata.surveillance.common.DriverLock$InnerHiveThread.run(DriverLock.java:35)"chThread" #11 prio=5 os_prio=0 tid=0x000001c24f7b1800 nid=0x39e8 in Object.wait() [0x00000001015fe000]java.lang.Thread.State: RUNNABLEat com.clickhouse.jdbc.ClickHouseDriver.<clinit>(ClickHouseDriver.java:70)at java.lang.Class.forName0(Native Method)at java.lang.Class.forName(Class.java:348)at com.donny.bigdata.surveillance.common.DriverLock$InnerCHThread.run(DriverLock.java:23)"Service Thread" #10 daemon prio=9 os_prio=0 tid=0x000001c24f64e000 nid=0x2e68 runnable [0x0000000000000000]java.lang.Thread.State: RUNNABLE"C1 CompilerThread2" #9 daemon prio=9 os_prio=2 tid=0x000001c24f637000 nid=0x316c waiting on condition [0x0000000000000000]java.lang.Thread.State: RUNNABLE"C2 CompilerThread1" #8 daemon prio=9 os_prio=2 tid=0x000001c24f5de000 nid=0x5b08 waiting on condition [0x0000000000000000]java.lang.Thread.State: RUNNABLE"C2 CompilerThread0" #7 daemon prio=9 os_prio=2 tid=0x000001c24f5db000 nid=0x4c60 waiting on condition [0x0000000000000000]java.lang.Thread.State: RUNNABLE"Monitor Ctrl-Break" #6 daemon prio=5 os_prio=0 tid=0x000001c24f5a6000 nid=0x4474 runnable [0x0000000100ffe000]java.lang.Thread.State: RUNNABLEat java.net.SocketInputStream.socketRead0(Native Method)at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)at java.net.SocketInputStream.read(SocketInputStream.java:171)at java.net.SocketInputStream.read(SocketInputStream.java:141)at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284)at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326)at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)- locked <0x000000076bc2fa10> (a java.io.InputStreamReader)at java.io.InputStreamReader.read(InputStreamReader.java:184)at java.io.BufferedReader.fill(BufferedReader.java:161)at java.io.BufferedReader.readLine(BufferedReader.java:324)- locked <0x000000076bc2fa10> (a java.io.InputStreamReader)at java.io.BufferedReader.readLine(BufferedReader.java:389)at com.intellij.rt.execution.application.AppMainV2$1.run(AppMainV2.java:49)"Attach Listener" #5 daemon prio=5 os_prio=2 tid=0x000001c24ddb5000 nid=0x3774 waiting on condition [0x0000000000000000]java.lang.Thread.State: RUNNABLE"Signal Dispatcher" #4 daemon prio=9 os_prio=2 tid=0x000001c24dd5f000 nid=0x41d4 runnable [0x0000000000000000]java.lang.Thread.State: RUNNABLE"Finalizer" #3 daemon prio=8 os_prio=1 tid=0x000001c24d61c800 nid=0x3430 in Object.wait() [0x0000000100cff000]java.lang.Thread.State: WAITING (on object monitor)at java.lang.Object.wait(Native Method)- waiting on <0x000000076b388ee8> (a java.lang.ref.ReferenceQueue$Lock)at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:150)- locked <0x000000076b388ee8> (a java.lang.ref.ReferenceQueue$Lock)at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:171)at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216)"Reference Handler" #2 daemon prio=10 os_prio=2 tid=0x000001c24dd46800 nid=0x3f8c in Object.wait() [0x0000000100bff000]java.lang.Thread.State: WAITING (on object monitor)at java.lang.Object.wait(Native Method)- waiting on <0x000000076b386c00> (a java.lang.ref.Reference$Lock)at java.lang.Object.wait(Object.java:502)at java.lang.ref.Reference.tryHandlePending(Reference.java:191)- locked <0x000000076b386c00> (a java.lang.ref.Reference$Lock)at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)"VM Thread" os_prio=2 tid=0x000001c24dd20800 nid=0x3b50 runnable"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x000001c2329ee000 nid=0x414 runnable"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x000001c2329ef800 nid=0x33d0 runnable"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x000001c2329f1000 nid=0x54a4 runnable"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x000001c2329f3000 nid=0x4ea8 runnable"VM Periodic Task Thread" os_prio=2 tid=0x000001c24f7af000 nid=0x5720 waiting on conditionJNI global references: 315

jstack -m 命令

PS E:\\code\\myself\\china-unicorn> jstack -m 13100
Attaching to process ID 13100, please wait...
Debugger attached successfully.
Server compiler detected.
JVM version is 25.333-b02
Deadlock Detection:No deadlocks found.----------------- 0 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 1 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 2 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 3 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 4 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 5 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 6 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 7 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 8 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 9 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 10 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
0x000001c24dd60080              ????????
----------------- 11 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 12 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
0x3659be07190c00c6              ????????
----------------- 13 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
0x0065006800630061              ????????
----------------- 14 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 15 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 16 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 17 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14
----------------- 18 -----------------
0x00007ffafbdaaa24      ntdll!NtWaitForSingleObject + 0x14

通过dump文件没有发下明显的死锁。

转而查询运行中的线程，驱动类HiveDriver和ClickHouseDriver都通过java.sql.DriverManager.registerDriver方法进行注册的，都借助了类DriverManager。

DriverManager加载驱动过程

DriverManager是JDK提供的一个驱动管理类。对其加载驱动时的过程分析。DriverManager初始化的时候会先加载loadInitialDrivers方法。

static {loadInitialDrivers();println("JDBC DriverManager initialized");
}

在loadInitialDrivers方法中会通过类ServiceLoader来获取实现了Driver接口的驱动类(会扫描classpath下的满足要求的驱动类)

    private static void loadInitialDrivers() {String drivers;try {drivers = AccessController.doPrivileged(new PrivilegedAction<String>() {public String run() {return System.getProperty("jdbc.drivers");}});} catch (Exception ex) {drivers = null;}AccessController.doPrivileged(new PrivilegedAction<Void>() {public Void run() {ServiceLoader<Driver> loadedDrivers = ServiceLoader.load(Driver.class);Iterator<Driver> driversIterator = loadedDrivers.iterator();try{// 当线程hiveThread执行完对hive驱动类的加载之后，迭代器中也加入了ch的驱动类while(driversIterator.hasNext()) {driversIterator.next();}} catch(Throwable t) {// Do nothing}return null;}});println("DriverManager.initialize: jdbc.drivers = " + drivers);// 一般逻辑（系统属性jdbc.drivers为空），直接跳出了。if (drivers == null || drivers.equals("")) {return;}// 有jdbc.drivers环境配置的时候，会一次尝试对jdbc.drivers中的服务类进行类加载String[] driversList = drivers.split(":");println("number of Drivers:" + driversList.length);for (String aDriver : driversList) {try {println("DriverManager.Initialize: loading " + aDriver);Class.forName(aDriver, true,ClassLoader.getSystemClassLoader());} catch (Exception ex) {println("DriverManager.Initialize: load failed: " + ex);}}}

在ServiceLoader中两个对象providers，lookupIterator和方法iterator()，providers是已经完成加载的驱动服务类集合对象，lookupIterator等待加载的驱动服务类集合的迭代器对象。

// 已经完成加载的驱动服务类集合对象
private LinkedHashMap<String,S> providers = new LinkedHashMap<>();
// 等待加载的驱动服务类集合的迭代器对象
private LazyIterator lookupIterator;public Iterator<S> iterator() {return new Iterator<S>() {Iterator<Map.Entry<String,S>> knownProviders= providers.entrySet().iterator();public boolean hasNext() {// 直接跳过已加载的驱动服务类if (knownProviders.hasNext())return true;return lookupIterator.hasNext();}public S next() {// 优先遍历已加载的驱动服务类if (knownProviders.hasNext())return knownProviders.next().getValue();return lookupIterator.next();}public void remove() {throw new UnsupportedOperationException();}};}

LazyIterator中有重要方法 hasNextService被driversIterator.hasNext()调用，nextService被 driversIterator.next()调用

        private boolean hasNextService() {// 下一个服务的名不为空，说明还有服务if (nextName != null) {return true;}if (configs == null) {try {String fullName = PREFIX + service.getName();if (loader == null)configs = ClassLoader.getSystemResources(fullName);elseconfigs = loader.getResources(fullName);} catch (IOException x) {fail(service, "Error locating configuration files", x);}}while ((pending == null) || !pending.hasNext()) {if (!configs.hasMoreElements()) {return false;}pending = parse(service, configs.nextElement());}nextName = pending.next();return true;}private S nextService() {if (!hasNextService())throw new NoSuchElementException();String cn = nextName;nextName = null;Class<?> c = null;try {c = Class.forName(cn, false, loader);} catch (ClassNotFoundException x) {fail(service,"Provider " + cn + " not found");}if (!service.isAssignableFrom(c)) {fail(service,"Provider " + cn  + " not a subtype");}try {// 卡住的方法// 对驱动类的实例创建（驱动类的加载）S p = service.cast(c.newInstance());providers.put(cn, p);return p;} catch (Throwable x) {fail(service,"Provider " + cn + " could not be instantiated",x);}throw new Error();          // This cannot happen}

分析猜想

当chThread和hiveThread分别去加载驱动服务的时候，当hiveThread线程执行到实例化CH驱动服务类的时候正好与chThread线程实例化CH驱动服务类，形成了竞争关系，造成了死锁。

实验1

引入多个jar，只指明一个驱动服务类进行主动加载

import java.sql.Driver;
import java.sql.DriverManager;/* 多线程的场景下DriverManager注册驱动死锁 @author 1792998761@qq.com* @date 2023/3/29 10:55*/
public class DriverLock {public static void main(String[] args) {try {Class.forName("org.apache.hive.jdbc.HiveDriver");} catch (ClassNotFoundException e) {e.printStackTrace();}java.util.Enumeration<Driver> drivers = DriverManager.getDrivers();while (drivers.hasMoreElements()) {System.out.println(drivers.nextElement().toString());}}}

结果

com.clickhouse.jdbc.ClickHouseDriver@15975490
org.apache.derby.jdbc.AutoloadedDriver40@7c16905e
org.apache.hive.jdbc.HiveDriver@3e6fa38a

实验2

引入多个jar，不指明驱动服务类进行主动加载，只主动调用DriverManager.getDrivers()

import java.sql.Driver;
import java.sql.DriverManager;/* 多线程的场景下DriverManager注册驱动死锁 @author 1792998761@qq.com* @date 2023/3/29 10:55*/
public class DriverLock {public static void main(String[] args) {java.util.Enumeration<Driver> drivers = DriverManager.getDrivers();while (drivers.hasMoreElements()) {System.out.println(drivers.nextElement().toString());}}
}

结果

org.apache.hive.jdbc.HiveDriver@763d9750
com.clickhouse.jdbc.ClickHouseDriver@6b143ee9
org.apache.derby.jdbc.AutoloadedDriver40@2a2d45ba

实验3

多线程下，调用DriverManager.getDrivers()

package com.donny.bigdata.surveillance.common;import java.sql.Driver;
import java.sql.DriverManager;/* 多线程的场景下DriverManager注册驱动死锁 @author 1792998761@qq.com* @date 2023/3/29 10:55*/
public class DriverLock {public static void main(String[] args) {InnerCHThread chThread = new InnerCHThread();chThread.setName("chThread");InnerHiveThread hiveThread = new InnerHiveThread();hiveThread.setName("hiveThread");chThread.start();hiveThread.start();}public static class InnerCHThread extends Thread {@Overridepublic void run() {try {java.util.Enumeration<Driver> drivers = DriverManager.getDrivers();while (drivers.hasMoreElements()) {System.out.println(drivers.nextElement().toString());}} catch (Exception e) {e.printStackTrace();}}}public static class InnerHiveThread extends Thread {@Overridepublic void run() {try {java.util.Enumeration<Driver> drivers = DriverManager.getDrivers();while (drivers.hasMoreElements()) {System.out.println(drivers.nextElement().toString());}} catch (Exception e) {e.printStackTrace();}}}
}

结果

org.apache.hive.jdbc.HiveDriver@489c2bd0
com.clickhouse.jdbc.ClickHouseDriver@15fddb00
org.apache.derby.jdbc.AutoloadedDriver40@7c57eb6f
org.apache.hive.jdbc.HiveDriver@489c2bd0
com.clickhouse.jdbc.ClickHouseDriver@15fddb00
org.apache.derby.jdbc.AutoloadedDriver40@7c57eb6f

小结

• DriverManager类在加载驱动服务类的时候，会扫描classpath下所有合适的类进行类加载。

• 对于多线程中通过DriverManager类指明加载类很容易就造成死锁。

解决方案

1. 在非多线程场景下使用DriverManager，提前进行服务类加载。

JVM深度分析

JDK的sql设计不合理导致的驱动类初始化死锁问题

加载多个JdbcDriver造成死锁