United StatesChange Country, Oracle Worldwide Web Sites Communities I am a... I want to...
Bug ID: JDK-7037756 Deadlock in compiler thread similiar to 6789220
JDK-7037756 : Deadlock in compiler thread similiar to 6789220

Details
Type:
Bug
Submit Date:
2011-04-19
Status:
Closed
Updated Date:
2011-07-29
Project Name:
JDK
Resolved Date:
2011-05-10
Component:
hotspot
OS:
generic
Sub-Component:
gc
CPU:
generic
Priority:
P2
Resolution:
Fixed
Affected Versions:
hs21
Fixed Versions:
hs21 (b11)

Related Reports
Backport:
Relates:

Sub Tasks

Description
C2 intermittenly hangs when compiling method. The symptoms are similiar to 6789220. Stack trace:

-----------------  lwp# 6 / thread# 6  --------------------
 feeca187 lwp_cond_wait (816e048, 816e030, 0, 0)
 fe9e398d void os::PlatformEvent::park() (816e000, 8065fc0, 8b1b001, 1) + bd
 fe9acc64 void Monitor::ILock(Thread*) (8065fc0, 816d000, b697d598, fe9adfa0) + 554
 fe9ae26f void Monitor::lock(Thread*) (8065fc0, 816d000, b697d5c8, fe5bdb5d) + 4c7
 fe5bdc92 void CompileBroker::compile_method_base(methodHandle,int,int,methodHandle,int,const char*,Thread*) (816d7b4, ffffffff, 4, 0, 0, fec0f6e4) + 146
 fe5be34b nmethod*CompileBroker::compile_method(methodHandle,int,int,methodHandle,int,const char*,Thread*) (816d7b4, ffffffff, 3, 0, 0, fec0f6e4) + 597
 fe135e6e void LinkResolver::resolve_invoke(CallInfo&,Handle,constantPoolHandle,int,Bytecodes::Code,Thread*) (b697d870, 0, 816d7a4, 7, b8, 816d000) + 576
 fe13953b void InterpreterRuntime::resolve_invoke(JavaThread*,Bytecodes::Code) (816d000, b8) + 8d3
 fac1215b * java/lang/ref/Reference$ReferenceHandler.run()V+31 (line 254)
 fac003d1 * StubRoutines (1)
 fe15d9c3 void JavaCalls::call_helper(JavaValue*,methodHandle*,JavaCallArguments*,Thread*) (b697dd30, b697dc44, b697dc50, 816d000, b697dca0, 816d790) + 52f
 fe15dd8c void os::os_exception_wrapper(void(*)(JavaValue*,methodHandle*,JavaCallArguments*,Thread*),JavaValue*,methodHandle*,JavaCallArguments*,Thread*) (fe15d494, b697dd30, b69
7dc44, b697dc50, 816d000, 816d000) + 18
 fe726e29 void JavaCalls::call_virtual(JavaValue*,Handle,KlassHandle,Symbol*,Symbol*,Thread*) (b697dd30, 816d790, 816d794, 806b8e0, 8090c18) + 171
 fe2326ea void thread_entry(JavaThread*,Thread*) (816d000) + 116
 fe22d0b7 void JavaThread::run() (816d000, fed05038, 0, fe9d93ea) + 2ff
 fe9d985c java_start (816d000) + 4c8
 feec7045 _thr_setup (fddd2200) + 4e
 feec7330 _lwp_start (fddd2200, 0, 0, b697dff8, feec7330, fddd2200)
...
-----------------  lwp# 12 / thread# 12  --------------------
 feeca187 lwp_cond_wait (81c3048, 81c3030, 0, 0)
 fe9e398d void os::PlatformEvent::park() (81c3000, 81c2400, 816d000, fe149404) + bd
 fe1492c2 void ObjectMonitor::enter(Thread*) (816f444, 81c2400, 81c2400, feabbf35) + 59a
 feabc02f void ObjectSynchronizer::fast_enter(Handle,BasicLock*,bool,Thread*) (81c2c20, b623dc18, 0, 81c2400) + 10b
 fe1bdb21 void instanceRefKlass::acquire_pending_list_lock(BasicLock*) (b623dc18) + 89
 feb3c41d bool VM_GC_Operation::doit_prologue() (b623dc00, 0, 0, fe1bd098) + 19
 fe1bd21d void VMThread::execute(VM_Operation*) (b623dc00, 1a, 0, 1) + 199
 fe9f9d9d HeapWord*ParallelScavengeHeap::mem_allocate(unsigned,bool,bool,bool*) (8069c08, 1a, 0, 0, b623dc90, 1a) + 2a5
 fe70a95a instanceOopDesc*instanceMirrorKlass::allocate_instance(KlassHandle,Thread*) (b6a0ffb0, 81c2c00, 81c2400, fe17cd61) + 106
 fe17cdc8 oopDesc*java_lang_Class::create_mirror(KlassHandle,Thread*) (81c2c00, 81c2400, 81c2400, 81c2bfc) + 78
 fe1f8da0 void arrayKlass::complete_create_array_klass(arrayKlassHandle,KlassHandle,Thread*) (81c2c00, 81c2bf4, 81c2400, 3f1) + dc
 fe1f22be klassOopDesc*objArrayKlassKlass::allocate_objArray_klass(int,KlassHandle,Thread*) (b6a000e8, 1, 81c2be0, 81c2400) + 8ea
 fe6f51bb klassOopDesc*instanceKlass::array_klass_impl(bool,int,Thread*) (b6cb8178, 0, 1, 81c2400, b623ec20, 81c2be0) + 15f
 fe6f5544 klassOopDesc*instanceKlass::array_klass_impl(bool,Thread*) (b6cb8178, 0, 81c2400, fe2157e9) + 1c
 fe215cac ciObjArrayKlass*ciObjArrayKlass::make(ciKlass*) (919b268, b623df4c, 1, fe573739) + 4e0
 fe574e1b bool ciTypeFlow::StateVector::apply_one_bytecode(ciBytecodeStream*) (81d04c8, b623dfc0, 0, fe13ab75) + 16f3
 fe13ae95 void ciTypeFlow::flow_block(ciTypeFlow::Block*,ciTypeFlow::StateVector*,ciTypeFlow::JsrSet*) (919be90, 919c050, 81d04c8, 81d04e8) + 331
 fe579e61 void ciTypeFlow::df_flow_types(ciTypeFlow::Block*,bool,ciTypeFlow::StateVector*,ciTypeFlow::JsrSet*) (919be90, 919c050, 1, 81d04c8, 81d04e8, 81d0510) + f0d
 fe1753cf void ciTypeFlow::do_flow() (919be90, b623ec20, 919b208, ffffffff) + 75f
 fe141144 ciTypeFlow*ciMethod::get_flow_analysis() (919b208, feca6000, b623e658, fe15cb43) + d4
 fe15cb79 CallGenerator*CallGenerator::for_inline(ciMethod*,float) (919b208, 3f800000, 3ff, fe5b5eae) + 45
 fe5b6444 Compile::Compile #Nvariant 1(ciEnv*,C2Compiler*,ciMethod*,int,bool,bool) (b623e6a0, b623ec20, 8063bc0, 919b208, ffffffff, 1) + a20
 fe1e802a void C2Compiler::compile_method(ciEnv*,ciMethod*,int) (8063bc0, b623ec20, 919b208, ffffffff) + ca
 fe1e8e0f void CompileBroker::invoke_compiler_on_method(CompileTask*) (87485f0, feca6000, b623eec8, fe280d70) + b1b
 fe281237 void CompileBroker::compiler_thread_loop() (7, feca6000, b623ef48, fe22d0b7, 81c2400, 81c2400) + a23
 fe284657 void compiler_thread_entry(JavaThread*,Thread*) (81c2400) + 1b
 fe22d0b7 void JavaThread::run() (81c2400, fed05038, 0, fe9d93ea) + 2ff
 fe9d985c java_start (81c2400) + 4c8
 feec7045 _thr_setup (fddd5200) + 4e
 feec7330 _lwp_start (fddd5200, 0, 0, b623eff8, feec7330, fddd5200)

                                    

Comments
SUGGESTED FIX

I think we need something similar to the following logic in compile_method_base

bool got_lock = false;
if (THREAD->is_reference_handler()) {
  got_lock = queue->lock()->try_lock();
} else {
  queue->lock()->lock();
  got_lock = true;
}

if (got_lock) {
  ...

   queue->lock()->unlock();
}

There are some early exits out of the scope containing the origial MutexLocker and we need to insert unlock calls there.
                                     
2011-04-21
EVALUATION

Another deadlock between the reference handler thread and both compiler threads.
                                     
2011-04-21
EVALUATION

http://hg.openjdk.java.net/jdk7/hotspot-gc/hotspot/rev/86ebb26bcdeb
                                     
2011-04-28
SUGGESTED FIX

An alternative fix was suggested by Tom Rodriguez:

Do not block in CompileBroker::compile_method_base() if the current/requesting thread owns the pending lits lock i.e. just return. There is a query routine in instanceRefKlass.{ch}pp that does precisely this.

The postive aspect of this alternative fix is that the fix for 6789220 would no longer be required - if the current thread owns the pending list lock and we just return then the current thread would no longer be waiting on a compilation.

The slight down side is that methods where we request a compilation while holding the lock would no longer be compiled. An analysis of the ReferenceHandler::run() method shows this number of methods to be very few (approx 2).

Performance measurements of throughput and pause times with some dacapo and SPECjvm2008 benchmarks that create a lot (> 50K) of references don't show any impact from the alternate fix.
                                     
2011-04-28
EVALUATION

http://hg.openjdk.java.net/jdk7/hotspot-rt/hotspot/rev/86ebb26bcdeb
                                     
2011-05-05



Hardware and Software, Engineered to Work Together