This is on behalf of tobytbzhang@tencent.com
After JDK-8290025, some tests using ShenandoahGC failed on the lock rank check between AdapterHandlerLibrary_lock and ShenandoahRequestedGC_lock
Reproduce: test the following tests with -XX:+UseShenandoahGC
```
compiler/codecache/jmx/InitialAndMaxUsageTest.java
compiler/codecache/OverflowCodeCacheTest.java
compiler/profiling/spectrapredefineclass/Launcher.java
compiler/profiling/spectrapredefineclass_classloaders/Launcher.java
```
Symptom
```
#
# A fatal error has been detected by the Java Runtime Environment:
#
# Internal Error (/data1/ws/jdk/src/hotspot/share/runtime/mutex.cpp:454), pid=2018566, tid=2022220
# assert(false) failed: Attempting to acquire lock ShenandoahRequestedGC_lock/safepoint-1 out of order with lock AdapterHandlerLibrary_lock/safepoint-1 -- possible deadlock
#
# JRE version: OpenJDK Runtime Environment (20.0) (slowdebug build 20-internal-adhoc.root.jdk)
# Java VM: OpenJDK 64-Bit Server VM (slowdebug 20-internal-adhoc.root.jdk, mixed mode, sharing, tiered, compressed oops, compressed class ptrs, shenandoah gc, linux-amd64)
# Problematic frame:
# V [libjvm.so+0x106fd6a] Mutex::check_rank(Thread*)+0x426
#
Current thread (0x00007faf18318c90): JavaThread "MainThread" [_thread_in_vm, id=2022220, stack(0x00007faf00113000,0x00007faf00214000)]
Stack: [0x00007faf00113000,0x00007faf00214000], sp=0x00007faf002117c0, free space=1017k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
V [libjvm.so+0x106fd6a] Mutex::check_rank(Thread*)+0x426 (mutex.cpp:454)
V [libjvm.so+0x106ea2b] Mutex::lock(Thread*)+0x83 (mutex.cpp:120)
V [libjvm.so+0x106ea9f] Mutex::lock()+0x23 (mutex.cpp:132)
V [libjvm.so+0x351434] MutexLocker::MutexLocker(Mutex*, Mutex::SafepointCheckFlag)+0x5a (mutexLocker.hpp:205)
V [libjvm.so+0x3514ad] MonitorLocker::MonitorLocker(Monitor*, Mutex::SafepointCheckFlag)+0x29 (mutexLocker.hpp:244)
V [libjvm.so+0x120dd66] ShenandoahControlThread::handle_requested_gc(GCCause::Cause)+0x2e (shenandoahControlThread.cpp:509)
V [libjvm.so+0x120dd35] ShenandoahControlThread::request_gc(GCCause::Cause)+0xeb (shenandoahControlThread.cpp:495)
V [libjvm.so+0x1221d31] ShenandoahHeap::collect(GCCause::Cause)+0x2b (shenandoahHeap.cpp:1152)
V [libjvm.so+0x875f2c] CodeCache::gc_on_allocation()+0x13c (codeCache.cpp:796)
V [libjvm.so+0x86a63c] AdapterBlob::create(CodeBuffer*)+0x1e (codeBlob.cpp:310)
V [libjvm.so+0x11cb95b] AdapterHandlerLibrary::create_adapter(AdapterBlob*&, int, BasicType*, bool)+0x217 (sharedRuntime.cpp:2904)
V [libjvm.so+0x11cb6c7] AdapterHandlerLibrary::get_adapter(methodHandle const&)+0x2cd (sharedRuntime.cpp:2853)
V [libjvm.so+0x10333e1] Method::make_adapters(methodHandle const&, JavaThread*)+0x1d (method.cpp:1252)
V [libjvm.so+0x103336f] Method::link_method(methodHandle const&, JavaThread*)+0x1d1 (method.cpp:1236)
V [libjvm.so+0xc0d82f] InstanceKlass::link_methods(JavaThread*)+0x8f (instanceKlass.cpp:943)
V [libjvm.so+0xc0d4c7] InstanceKlass::link_class_impl(JavaThread*)+0x4e5 (instanceKlass.cpp:886)
V [libjvm.so+0xc0ce1d] InstanceKlass::link_class(JavaThread*)+0x7f (instanceKlass.cpp:751)
V [libjvm.so+0xc0dcc1] InstanceKlass::initialize_impl(JavaThread*)+0x61 (instanceKlass.cpp:1020)
V [libjvm.so+0xc0cd0e] InstanceKlass::initialize(JavaThread*)+0x40 (instanceKlass.cpp:733)
V [libjvm.so+0xc365fa] InterpreterRuntime::_new(JavaThread*, ConstantPool*, int)+0x13a (interpreterRuntime.cpp:226)
j compiler.codecache.OverflowCodeCacheTest.test()V+166
j compiler.codecache.OverflowCodeCacheTest.main([Ljava/lang/String;)V+49
j java.lang.invoke.LambdaForm$DMH+0x0000000801002800.invokeStatic(Ljava/lang/Object;Ljava/lang/Object;)V+10 java.base@20-internal
j java.lang.invoke.LambdaForm$MH+0x0000000801003c00.invoke(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;+33 java.base@20-internal
j java.lang.invoke.Invokers$Holder.invokeExact_MT(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;+20 java.base@20-internal
j jdk.internal.reflect.DirectMethodHandleAccessor.invokeImpl(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;+55 java.base@20-internal
j jdk.internal.reflect.DirectMethodHandleAccessor.invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;+23 java.base@20-internal
j java.lang.reflect.Method.invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;+102 java.base@20-internal
j com.sun.javatest.regtest.agent.MainWrapper$MainThread.run()V+134
j java.lang.Thread.run()V+13 java.base@20-internal
v ~StubRoutines::call_stub 0x00007faf00e97d27
V [libjvm.so+0xc46895] JavaCalls::call_helper(JavaValue*, methodHandle const&, JavaCallArguments*, JavaThread*)+0x59d (javaCalls.cpp:417)
V [libjvm.so+0x10de464] os::os_exception_wrapper(void (*)(JavaValue*, methodHandle const&, JavaCallArguments*, JavaThread*), JavaValue*, methodHandle const&, JavaCallArguments*, JavaThread*)+0x32 (os_linux.cpp:4880)
V [libjvm.so+0xc462f4] JavaCalls::call(JavaValue*, methodHandle const&, JavaCallArguments*, JavaThread*)+0x3a (javaCalls.cpp:331)
V [libjvm.so+0xc45370] JavaCalls::call_virtual(JavaValue*, Klass*, Symbol*, Symbol*, JavaCallArguments*, JavaThread*)+0x19a (javaCalls.cpp:187)
V [libjvm.so+0xc4547d] JavaCalls::call_virtual(JavaValue*, Handle, Klass*, Symbol*, Symbol*, JavaThread*)+0x95 (javaCalls.cpp:193)
V [libjvm.so+0xd53f23] thread_entry(JavaThread*, JavaThread*)+0x91 (jvm.cpp:2876)
V [libjvm.so+0xc5fe60] JavaThread::thread_main_inner()+0x164 (javaThread.cpp:700)
V [libjvm.so+0xc5fcf0] JavaThread::run()+0x204 (javaThread.cpp:683)
V [libjvm.so+0x13bb874] Thread::call_run()+0x180 (thread.cpp:224)
V [libjvm.so+0x10d4c07] thread_native_entry(Thread*)+0x18f (os_linux.cpp:710)
Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)
j compiler.codecache.OverflowCodeCacheTest.test()V+166
j compiler.codecache.OverflowCodeCacheTest.main([Ljava/lang/String;)V+49
j java.lang.invoke.LambdaForm$DMH+0x0000000801002800.invokeStatic(Ljava/lang/Object;Ljava/lang/Object;)V+10 java.base@20-internal
j java.lang.invoke.LambdaForm$MH+0x0000000801003c00.invoke(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;+33 java.base@20-internal
j java.lang.invoke.Invokers$Holder.invokeExact_MT(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;+20 java.base@20-internal
j jdk.internal.reflect.DirectMethodHandleAccessor.invokeImpl(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;+55 java.base@20-internal
j jdk.internal.reflect.DirectMethodHandleAccessor.invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;+23 java.base@20-internal
j java.lang.reflect.Method.invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;+102 java.base@20-internal
j com.sun.javatest.regtest.agent.MainWrapper$MainThread.run()V+134
j java.lang.Thread.run()V+13 java.base@20-internal
v ~StubRoutines::call_stub 0x00007faf00e97d27
```