Stress test cause crashes with
#
# A fatal error has been detected by the Java Runtime Environment:
#
# Internal Error (/scratch/mesos/slaves/00f4d7f9-7805-4b6a-aef8-9bb130db2435-S438/frameworks/1735e8a2-a1db-478c-8104-60c8b0af87dd-0196/executors/0308a46f-9717-4049-a800-d91b0568f68b/runs/8b166c2e-b35b-4aa5-a171-bc93872413d3/workspace/open/src/hotspot/share/runtime/handles.cpp:35), pid=36670, tid=39683
# assert(_no_handle_mark_nesting == 0) failed: allocating handle inside NoHandleMark
#
# JRE version: Java(TM) SE Runtime Environment (14.0+7) (fastdebug build 14-ea+7-204)
# Java VM: Java HotSpot(TM) 64-Bit Server VM (fastdebug 14-ea+7-204, mixed mode, sharing, tiered, compressed oops, g1 gc, bsd-amd64)
# Core dump will be written. Default location: /cores/core.36670
#
# If you would like to submit a bug report, please visit:
# http://bugreport.java.com/bugreport/crash.jsp
#
--------------- S U M M A R Y ------------
Command Line: -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:MaxRAMPercentage=6 -XX:MaxRAMPercentage=50 -XX:+HeapDumpOnOutOfMemoryError -XX:+CrashOnOutOfMemoryError -Djava.net.preferIPv6Addresses=false -XX:+DisplayVMOutputToStderr -XX:+UsePerfData -Xlog:gc*,gc+heap=debug:gc.log:uptime,timemillis,level,tags -XX:+DisableExplicitGC -XX:+StartAttachListener -Djava.io.tmpdir=/Volumes/Mesos/mesos/work_dir/slaves/00f4d7f9-7805-4b6a-aef8-9bb130db2435-S30/frameworks/1735e8a2-a1db-478c-8104-60c8b0af87dd-0196/executors/44b90806-9674-4f0e-be9b-9281d4a6f059/runs/19a22e3d-464f-422c-86d3-058d17d50012/testoutput/test-support/jtreg_closed_test_hotspot_jtreg_applications_sparkexamples_SparkExamples24H_java/scratch/0/java.io.tmpdir -Duser.home=/Volumes/Mesos/mesos/work_dir/slaves/00f4d7f9-7805-4b6a-aef8-9bb130db2435-S30/frameworks/1735e8a2-a1db-478c-8104-60c8b0af87dd-0196/executors/44b90806-9674-4f0e-be9b-9281d4a6f059/runs/19a22e3d-464f-422c-86d3-058d17d50012/testoutput/test-support/jtreg_closed_test_hotspot_jtreg_applications_sparkexamples_SparkExamples24H_java/scratch/0/user.home -Dhadoop.root.logger=WARN,console -DSEED=1000 --add-exports=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/java.util=ALL-UNNAMED --add-exports=java.base/java.util.concurrent=ALL-UNNAMED --add-exports=java.base/java.nio=ALL-UNNAMED -Dhadoop.home.dir=/Volumes/Mesos/mesos/work_dir/jib-master/install/org/apache/hadoop/common/hadoop/3.1.1/hadoop-3.1.1.tar.gz/hadoop-3.1.1 -Dspark.master=local[9] -Dspark.sql.warehouse.dir=spark_tmp -Duser.country=US -Duser.language=en applications.kitchensink.process.stress.Main /Volumes/Mesos/mesos/work_dir/slaves/00f4d7f9-7805-4b6a-aef8-9bb130db2435-S30/frameworks/1735e8a2-a1db-478c-8104-60c8b0af87dd-0196/executors/44b90806-9674-4f0e-be9b-9281d4a6f059/runs/19a22e3d-464f-422c-86d3-058d17d50012/testoutput/test-support/jtreg_closed_test_hotspot_jtreg_applications_sparkexamples_SparkExamples24H_java/scratch/0/kitchensink.final.properties
Host: MacPro6,1 x86_64 3700 MHz, 8 cores, 16G, Darwin 18.2.0
Time: Sun Jul 28 02:18:37 2019 GMT elapsed time: 71002 seconds (0d 19h 43m 22s)
--------------- T H R E A D ---------------
Current thread (0x00007fbb050d4800): JavaThread "SparkStressModule" [_thread_in_vm, id=39683, stack(0x000070000721a000,0x000070000731a000)] _threads_hazard_ptr=0x00007fbad40e0ae0, _nested_threads_hazard_ptr_cnt=0
Stack: [0x000070000721a000,0x000070000731a000], sp=0x0000700007314aa0, free space=1002k
Native frames: (J=compiled Java code, A=aot compiled Java code, j=interpreted, Vv=VM code, C=native code)
V [libjvm.dylib+0xc11f81] VMError::report_and_die(int, char const*, char const*, __va_list_tag*, Thread*, unsigned char*, void*, void*, char const*, int, unsigned long)+0x6e5
V [libjvm.dylib+0xc1269d] VMError::report_and_die(Thread*, void*, char const*, int, char const*, char const*, __va_list_tag*)+0x47
V [libjvm.dylib+0x41ab68] report_vm_error(char const*, int, char const*, char const*, ...)+0x145
V [libjvm.dylib+0x59f816] HandleArea::allocate_handle(oop)+0x6a
V [libjvm.dylib+0x41d041] ConstantOopReadValue::ConstantOopReadValue(DebugInfoReadStream*)+0x10d
V [libjvm.dylib+0x41c4b7] ScopeValue::read_from(DebugInfoReadStream*)+0x139
V [libjvm.dylib+0x41c221] ObjectValue::read_object(DebugInfoReadStream*)+0x1d
V [libjvm.dylib+0x41c1ae] DebugInfoReadStream::read_object_value(bool)+0x196
V [libjvm.dylib+0xabc1ae] ScopeDesc::decode_object_values(int)+0xca
V [libjvm.dylib+0xabc0b8] ScopeDesc::ScopeDesc(CompiledMethod const*, int, int, bool, bool, bool)+0x48
V [libjvm.dylib+0x3ad9d1] CompiledMethod::scope_desc_at(unsigned char*)+0x9d
V [libjvm.dylib+0xc0476a] compiledVFrame::compiledVFrame(frame const*, RegisterMap const*, JavaThread*, CompiledMethod*)+0x64
V [libjvm.dylib+0xbfe67f] vframe::new_vframe(frame const*, RegisterMap const*, JavaThread*)+0xc7
V [libjvm.dylib+0xbfe815] vframe::sender() const+0xd5
V [libjvm.dylib+0xc04b80] compiledVFrame::sender() const+0x10a
V [libjvm.dylib+0xbfe875] vframe::java_sender() const+0x21
V [libjvm.dylib+0x2080dd] get_or_compute_monitor_info(JavaThread*)+0x110
V [libjvm.dylib+0x209967] BiasedLocking::walk_stack_and_revoke(oop, JavaThread*)+0x44f
V [libjvm.dylib+0x20b6d6] RevokeOneBias::do_thread(Thread*)+0x212
V [libjvm.dylib+0x5a0754] HandshakeThreadsOperation::do_handshake(JavaThread*)+0x112
V [libjvm.dylib+0x5a0b87] HandshakeState::process_self_inner(JavaThread*)+0x12f
V [libjvm.dylib+0xab9075] SafepointMechanism::block_or_handshake(JavaThread*)+0x4f
V [libjvm.dylib+0xab9092] SafepointMechanism::block_if_requested_slow(JavaThread*)+0xe
V [libjvm.dylib+0x9b58a3] ThreadBlockInVMWithDeadlockCheck::~ThreadBlockInVMWithDeadlockCheck()+0x83
V [libjvm.dylib+0x9b4468] Monitor::lock(Thread*)+0x1e8
V [libjvm.dylib+0xb8d837] Thread::check_for_dangling_thread_pointer(Thread*)+0x9f
V [libjvm.dylib+0xb8e03a] Thread::is_interrupted(Thread*, bool)+0x12
V [libjvm.dylib+0x716fd9] JVM_IsInterrupted+0x115
J 1039 java.lang.Thread.isInterrupted(Z)Z java.base@14-ea (0 bytes) @ 0x000000011e83dc02 [0x000000011e83dac0+0x0000000000000142]
J 51640 c2 java.util.concurrent.ThreadPoolExecutor.interruptIdleWorkers(Z)V java.base@14-ea (120 bytes) @ 0x00000001223ef228 [0x00000001223ee220+0x0000000000001008]
J 54599 c2 java.util.concurrent.ThreadPoolExecutor.shutdown()V java.base@14-ea (45 bytes) @ 0x00000001229d6614 [0x00000001229d5ec0+0x0000000000000754]
J 93267 c2 org.apache.spark.SparkContext.stop()V (321 bytes) @ 0x00000001225fa4d8 [0x00000001225f81c0+0x0000000000002318]
j org.apache.spark.examples.graphx.SynthBenchmark$.main([Ljava/lang/String;)V+949
j org.apache.spark.examples.graphx.SynthBenchmark.main([Ljava/lang/String;)V+4
v ~StubRoutines::call_stub
V [libjvm.dylib+0x60f819] JavaCalls::call_helper(JavaValue*, methodHandle const&, JavaCallArguments*, Thread*)+0x3c5
V [libjvm.dylib+0xa9861b] invoke(InstanceKlass*, methodHandle const&, Handle, bool, objArrayHandle, BasicType, objArrayHandle, bool, Thread*)+0xc9b
V [libjvm.dylib+0xa9772c] Reflection::invoke_method(oop, Handle, objArrayHandle, Thread*)+0x288
V [libjvm.dylib+0x71c6d4] JVM_InvokeMethod+0x422
J 5889 jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Ljava/lang/reflect/Method;Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object; java.base@14-ea (0 bytes) @ 0x000000011ec0dbe7 [0x000000011ec0da60+0x0000000000000187]
J 19940 c2 jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object; java.base@14-ea (104 bytes) @ 0x000000011f76bca0 [0x000000011f76b440+0x0000000000000860]
J 96989 c2 applications.kitchensink.process.stress.modules.SparkStressModule.runExample(Ljava/lang/String;)V (181 bytes) @ 0x0000000121453b8c [0x0000000121450da0+0x0000000000002dec]
j applications.kitchensink.process.stress.modules.SparkStressModule.execute()V+58
j applications.kitchensink.process.stress.modules.StressModule.run()V+109
v ~StubRoutines::call_stub
V [libjvm.dylib+0x60f819] JavaCalls::call_helper(JavaValue*, methodHandle const&, JavaCallArguments*, Thread*)+0x3c5
V [libjvm.dylib+0x60e309] JavaCalls::call_virtual(JavaValue*, Klass*, Symbol*, Symbol*, JavaCallArguments*, Thread*)+0x1f1
V [libjvm.dylib+0x60e424] JavaCalls::call_virtual(JavaValue*, Handle, Klass*, Symbol*, Symbol*, Thread*)+0x58
V [libjvm.dylib+0x714e61] thread_entry(JavaThread*, Thread*)+0x13f
V [libjvm.dylib+0xb90cf5] JavaThread::thread_main_inner()+0x1a1
V [libjvm.dylib+0xb908a8] JavaThread::run()+0x2aa
V [libjvm.dylib+0xb8d35f] Thread::call_run()+0x11b
V [libjvm.dylib+0x9f80dc] thread_native_entry(Thread*)+0x13a
C [libsystem_pthread.dylib+0x3305] _pthread_body+0x7e
C [libsystem_pthread.dylib+0x626f] _pthread_start+0x46
C [libsystem_pthread.dylib+0x2415] thread_start+0xd
Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)
J 1039 java.lang.Thread.isInterrupted(Z)Z java.base@14-ea (0 bytes) @ 0x000000011e83db8f [0x000000011e83dac0+0x00000000000000cf]
J 51640 c2 java.util.concurrent.ThreadPoolExecutor.interruptIdleWorkers(Z)V java.base@14-ea (120 bytes) @ 0x00000001223ef228 [0x00000001223ee220+0x0000000000001008]
J 54599 c2 java.util.concurrent.ThreadPoolExecutor.shutdown()V java.base@14-ea (45 bytes) @ 0x00000001229d6614 [0x00000001229d5ec0+0x0000000000000754]
J 93267 c2 org.apache.spark.SparkContext.stop()V (321 bytes) @ 0x00000001225fa4d8 [0x00000001225f81c0+0x0000000000002318]
j org.apache.spark.examples.graphx.SynthBenchmark$.main([Ljava/lang/String;)V+949
j org.apache.spark.examples.graphx.SynthBenchmark.main([Ljava/lang/String;)V+4
v ~StubRoutines::call_stub
J 5889 jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Ljava/lang/reflect/Method;Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object; java.base@14-ea (0 bytes) @ 0x000000011ec0db6e [0x000000011ec0da60+0x000000000000010e]
J 19940 c2 jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object; java.base@14-ea (104 bytes) @ 0x000000011f76bca0 [0x000000011f76b440+0x0000000000000860]
J 96989 c2 applications.kitchensink.process.stress.modules.SparkStressModule.runExample(Ljava/lang/String;)V (181 bytes) @ 0x0000000121453b8c [0x0000000121450da0+0x0000000000002dec]
j applications.kitchensink.process.stress.modules.SparkStressModule.execute()V+58
j applications.kitchensink.process.stress.modules.StressModule.run()V+109
v ~StubRoutines::call_stub