Customer is experiencing a core dump in the compiler thread:
And this is issue is easily reproduceable.
#
# An unexpected error has been detected by Java Runtime Environment:
#
# SIGSEGV (0xb) at pc=0xfe6c9d45, pid=14086, tid=20
#
# Java VM: Java HotSpot(TM) Server VM (10.0-b19 mixed mode solaris-x86)
# Problematic frame:
# V [libjvm.so+0xc9d45]
#
# If you would like to submit a bug report, please visit:
# http://java.sun.com/webapps/bugreport/crash.jsp
# The crash happened outside the Java Virtual Machine in native code.
# See problematic frame for where to report the bug.
#
--------------- T H R E A D ---------------
Current thread (0x082d5000): JavaThread "CompilerThread1" daemon [_thread_in_native, id=20, stack(0xf239d000,0xf23dd000)]
siginfo:si_signo=SIGSEGV: si_errno=0, si_code=1 (SEGV_MAPERR), si_addr=0x00000004
Found capture file: /xtpenv2/test/xtp/run_dir/log/replayData/Uqdf1_replayCapture
Using Group/IfAddr/Port: 224.2.17.36/10.10.10.63/61214
Data Feed: Uqdf1
Args: -showCmd -sendRate 1000 -XX:+PrintCompilation
------------------------------------------------------------------------------
Command Line: java -DXtpReplayUqdf1 -Djava.net.preferIPv4Stack=true -XX:+PrintCompilation -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -classpath /xtpenv2/test/xtp/run_dir/classes/XTP.jar:/xtpenv2/test/xtp/run_dir/classes/commons-net-1.4.0.jar:/xtpenv2/test/xtp/run_dir/classes/mdx_idl.jar:/xtpenv2/test/infra/run_dir/classes/ffimpl.jar:/xtpenv2/test/infra/run_dir/classes/ffpersist.jar:/xtpenv2/test/infra/run_dir/classes/FoundationFramework.jar:/xtpenv2/test/infra/run_dir/classes/infrastructure.jar:/xtpenv2/test/infra/run_dir/classes/testclasses12.jar:/usr/local/jdk1.6.0_04/lib/tools.jar:/xtpenv2/test/infra/run_dir/classes/common.jar:/xtpenv2/test/infra/run_dir/classes/InfraVerity.jar:/xtpenv2/test/infra/run_dir/classes/InfraVerityIDLClasses.jar:/xtpenv2/test/infra/run_dir/classes/InfraVerityIDL.jar:/xtpenv2/test/infra/run_dir/classes/InfraUtility.jar:/xtpenv2/test/infra/run_dir/classes/jakarta-regexp-1.2.jar:/xtpenv2/test/infra/run_dir/classes/OMGBaseClasses.jar:/xtpenv2/test/infra/run_dir/classes/OMGServiceClasses.jar:/xtpenv2/test/infra/run_dir/classes/SystemManagement.jar:/xtpenv2/test/infra/run_dir/classes/SystemManagementIDLClasses.jar:/xtpenv2/test/infra/run_dir/classes/xml4j.jar:/xtpenv2/test/infra/run_dir/classes/DirectoryService.jar:/xtpenv2/test/infra/run_dir/classes/IDLCompiler.jar:/xtpenv2/test/infra/run_dir/classes/MessagingSystem.jar:/xtpenv2/test/infra/run_dir/classes/MessagingSystemIDL.jar:/xtpenv2/test/infra/run_dir/classes/LoggingService.jar:/xtpenv2/test/infra/run_dir/classes/LoggingServiceIDLClasses.jar:/xtpenv2/test/infra/run_dir/classes/SecurityService.jar:/xtpenv2/test/infra/run_dir/classes/SecurityServiceIDLClasses.jar:/xtpenv2/test/infra/run_dir/classes/SessionManagementService.jar:/xtpenv2/test/infra/run_dir/classes/Rollout.jar:/xtpenv2/test/infra/run_dir/classes/jgl.jar:/xtpenv2/test/infra/run_dir/classes/jms.jar:/xtpenv2/test/infra/run_dir/classes/activemq-4.2.jar:/xtpenv2/test/infra/run_dir/classes/concurrency.jar:/xtpenv2/test/infra/run_dir/classes/ldapjdk.jar:/xtpenv2/test/infra/run_dir/classes/jndi.jar:/xtpenv2/test/infra/run_dir/classes/ldap.jar:/xtpenv2/test/infra/run_dir/classes/providerutil.jar:/xtpenv2/test/infra/run_dir/classes/fscontext.jar:/xtpenv2/test/infra/run_dir/classes/js.jar:/xtpenv2/test/infra/run_dir/classes/jstools.jar:/xtpenv2/test/infra/run_dir/classes/objectwave.jar:/xtpenv2/test/tradeeng/run_dir/classes/message.jar:/xtpenv2/test/tradeeng/run_dir/classes/connectionServer.jar:/xtpenv2/test/tradeeng/run_dir/classes/CommonFacilities.jar:/xtpenv2/test/tradeeng/run_dir/classes/server_common.jar:/xtpenv2/test/tradeeng/run_dir/classes/server_impls.jar:/xtpenv2/test/tradeeng/run_dir/classes/server_proxies.jar:/xtpenv2/test/tradeeng/run_dir/classes/server_interfaces.jar:/xtpenv2/test/tradeeng/run_dir/classes/server_idl.jar:/xtpenv2/test/tradeeng/run_dir/classes/event_idl.jar:/xtpenv2/test/tradeeng/run_dir/classes/event_interfaces.jar:/xtpenv2/test/tradeeng/run_dir/classes/domain_idl.jar:/xtpenv2/test/tradeeng/run_dir/classes/domain_impls.jar:/xtpenv2/test/tradeeng/run_dir/classes/domain_interfaces.jar:/xtpenv2/test/tradeeng/run_dir/classes/client_idl.jar:/xtpenv2/test/tradeeng/run_dir/classes/cmi_idl.jar:/xtpenv2/test/tradeeng/run_dir/classes/eis_impls.jar:/xtpenv2/test/tradeeng/run_dir/classes/applappl.jar:/xtpenv2/test/infra/run_dir/classes/jhall.jar:/xtpenv2/test/infra/run_dir/classes/jawall.jar:/xtpenv2/test/infra/run_dir/classes/CBOEUtility.jar:/opt/Talarian/ss68/java/lib/ss.jar:/xtpenv2/test/xtp/run_dir/classes/XTPTest.jar:/xtpenv2/test/xtp/run_dir/classes/junit.jar com.cboe.externalIntegrationServices.xtp.replay.perf.DataFeedPerfReplayTool /xtpenv2/test/xtp/run_dir/log/replayData/Uqdf1_replayCapture 224.2.17.36/10.10.10.63/61214 -dataFeed Uqdf1 -sendBuff 1000000 -sendRate 1000
------------------------------------------------------------------------------
time(seconds) unlimited
file(blocks) unlimited
data(kbytes) unlimited
stack(kbytes) 8192
coredump(blocks) unlimited
nofiles(descriptors) 512
vmemory(kbytes) unlimited
1 java.lang.String::hashCode (60 bytes)
2 java.lang.String::charAt (33 bytes)
3 java.lang.String::indexOf (151 bytes)
4 java.lang.String::lastIndexOf (156 bytes)
5 java.lang.String::indexOf (166 bytes)
6 java.lang.String::replace (142 bytes)
7 sun.net.www.ParseUtil::encodePath (336 bytes)
8 java.lang.String::equals (88 bytes)
9 com.cboe.externalIntegrationServices.xtp.translate.utils.MessageSwitch::getShort (26 bytes)
1% com.cboe.externalIntegrationServices.xtp.translate.utils.MessageSwitch::<init> @ 85 (171 bytes)
2% com.cboe.externalIntegrationServices.xtp.translate.utils.ExchangeMapper::<init> @ 71 (135 bytes)
10 com.cboe.externalIntegrationServices.xtp.translate.utils.MessageSwitch::<init> (171 bytes)
11 com.cboe.externalIntegrationServices.xtp.translate.utils.ExchangeMapper::<init> (135 bytes)
12 java.lang.Object::<init> (1 bytes)
--- n java.lang.System::arraycopy (static)
13 java.util.ArrayList::ensureCapacity (58 bytes)
14 java.util.ArrayList::add (100 bytes)
15 com.cboe.externalIntegrationServices.xtp.translate.utils.PriceParserUtil::<init> (91 bytes)
16 com.cboe.externalIntegrationServices.xtp.translate.utils.PriceDenomCodeParser::map (10 bytes)
16 made not entrant (2) com.cboe.externalIntegrationServices.xtp.translate.utils.PriceDenomCodeParser::map (10 bytes)
3% com.cboe.externalIntegrationServices.xtp.translate.utils.PriceDenomCodeParser::<clinit> @ 25 (305 bytes)
3% made not entrant (2) com.cboe.externalIntegrationServices.xtp.translate.utils.PriceDenomCodeParser::<clinit> @ 25 (305 bytes)
17 com.cboe.externalIntegrationServices.xtp.translate.fast.field.Fast_Primitive_Field::setArray (21 bytes)
18 java.util.ArrayList::add (29 bytes)
GroupAddr: 224.2.17.36
AdapterAddr: 10.10.10.63
Interface: <bge1> bge1
Using multicast.
Created DatagramPacket (send): len(1000) SockAddr(/224.2.17.36:61214)
Created UNBOUND socket for port 61214
Connect bypassed.
Set SendBuffSize to 1000000
Actual Socket Buff Sizes: Send(1000000)
Set send interface to /10.10.10.63:61214
Started Message Rate Throttling...
19 java.nio.Buffer::nextGetIndex (31 bytes)
20 java.nio.Bits::swap (15 bytes)
21 java.nio.DirectByteBuffer::ix (10 bytes)
22 ! com.cboe.externalIntegrationServices.xtp.utils.ByteBufferDataReader::readByte (26 bytes)
23 com.cboe.lwt.byteUtil.ByteIterator::set (11 bytes)
24 com.cboe.lwt.byteUtil.ByteIterator::fill (93 bytes)
25 com.cboe.lwt.byteUtil.ByteIterator::prev (12 bytes)
26 com.cboe.externalIntegrationServices.xtp.utils.ByteIteratorUtils::intToAscii (195 bytes)
27 java.nio.Bits::swap (21 bytes)
28 com.cboe.externalIntegrationServices.xtp.translate.framework.parser.MessageConverter::formatMessage (36 bytes)
29 java.nio.Buffer::nextGetIndex (38 bytes)
30 com.cboe.externalIntegrationServices.xtp.utils.ByteUtils::intToLeftPaddedAsciiBytes (32 bytes)
Environment Info :
Java Version : Java HotSpot(TM) Server VM (10.0-b19) for solaris-x86 JRE (1.6.0_04-b12).
Operating System Info : Solaris 10.
The dumps are at : /net/cores.central/cores/dir33/831682
Initial dumps given : pkg_java_core.tar.gz and hs_err_pid14086.log
Reproduceable Test Case : /net/cores.central/cores/dir33/831682/TestCase/ExperimentsStandalone.tar.Z
As the temporary workaround, we suggested to exclude this method from compiling:
com/cboe/externalIntegrationServices/xtp/utils/ByteUtils.intToLeftPaddedAsciiBytes
The above exclude compilation works well, does not crash. But its not an acceptable solution for Customer.
Customer has reproduced this in a small, low-dependency test program.
Run loop for a few hundred thousand iterations calling the method to coax the JIT compiler into compiling and boom!
It core-dumps every time.
One thing found interesting from customer side : Copied the exact same code that's blowing up into the test program
as "local" methods versus calling them in another class and those versions of the methods do not cause a crash.
And its pretty sure the "local" methods were compiled.
A test program is shipped too. This crashes every time on customer machines.
Test Case Given : /net/cores.central/cores/dir33/831682/TestCase/ExperimentsStandalone.tar.Z
The above given test case file contains a directory tree whose top-level directory is "ExperimentsStandalone".
This has everything needed to reproduce this problem. There are 4 classes, but 2 of the classes are
just dependency-related stuff which customer didn't want to touch.
There is also an ANT build script and a shell-script to run the Class to demonstrate the problem.
Here is what's in the above package under the top-level directory:
1) The code is already compiled and jarred in release/classes/ExperimentsStandalone.jar.
It used ANT to compile and create the .jar file. If want to re-build, customer has included the build.xml
ANT script to do so. They normally do not use javac at the command line, so they have not provided
scripts to build that way.
2) The source code is in the java directory. Under that is com/cboe/... etc.
The main program to invoke the test to demonstrate the crash is com.cboe.experiments.hotSpotCrash.HotSpotCompilerCrash
3) Customer has also supplied a script in the top level directory: runCrashExperiment.
This script runs HotSpotCompilerCrash, setting the necessary classpath to point to the release/classes/ExperimentsStandalone.jar file.
Run this script from the top-level directory; it has relative paths for the classpath.
If no arguments are given to the script, it deletes the .hotspot_compiler file if it exists and
allows the Hotspot compiler to run. Use this mode to cause the HotSpot compiler to crash.
If -excludeCompile is given as the first argument, it will write a .hotspot_compiler file in the current-dir
with the exclude statement you suggested.
Customer ran with the .hotspot_compiler file to exclude intToLeftPaddedAsciiBytes as suggested.
The program survives this (does not crash).
On a hunch, customer temporarily modified the script to write a .hotspot_compiler file to exclude a different method: intToAsciiBytes, which is called by intToLeftPaddedAsciiBytes.
When they exclude intToAsciiBytes but do NOT exclude intToLeftPaddedAsciiBytes, the program survives this as well when calling intToLeftPaddedAsciiBytes in a hard loop.
Unfortunately, they do not think it would be an acceptable work-around to exclude compilation of these methods in production because they are called quite a lot.
This would hurt the performance.
Customer had previously tested a few other scenarios that might be of interest, and the code still exists to run these scenarios, but the
calls to the code are currently commented out in the HotSpotCompilerCrash.runAllTest() method which is the method that runs the different scenarios.
One test that is commented out, for example, is where when copied ByteUtils.intToAsciiBytes and ByteUtils.intToLeftPaddedAsciiBytes directly into HotSpotCompilerCrash.
The code is still there but the calls to test it are commented out. This also crashes on intToLeftPaddedAsciiBytes, incidentally.
For what it's worth, the method really causing the problem is intToAsciiBytes because customer suspect this method is relatively complex to optimize.
However, it appears there may be some strange interaction with optimizing it and intToLeftPaddedAsciiBytes.
The tests with the intToAsciiBytes method should help explain why customer has another test commented-out that
tests Integer.toString. It is because intToAsciiBytes is a modified copy of Integer.getChars() which Integer.toString() calls.
Finally, customer did some more testing (which can be replicated if needed) where they commented out various bits of code in intToLeftPaddedAsciiBytes.
Customer has isolated this down to the for loop. They commented-out the for-loop (but not the if) in intToAsciiBytes, and it survives a run.
They even tried a refactoring of the for loop into a while loop, and it still crashes !! Of course this negates the functionality, but at least
they have isolated the code that HotSpot doesn't like.
Here is some more information regarding this that may be useful or at least interesting if not a bit puzzling.
Customer has tried a few refactorings to try to work around this problem and found the following:
Here is the original code for method that produces the compiler crash:
public static int intToLeftPaddedAsciiBytes(int p_value,byte[] p_bytes,byte p_pad) {
int startOffset = intToAsciiBytes(p_value,p_bytes);
if (startOffset > 0) {
for(int i = 0; i < startOffset; i++) {
p_bytes[i] = p_pad;
}
}
return startOffset;
}
-------------------------------------------------------------------------------------------------------------------------------------
NONE of the refactorings below produce the crash, and they are all functionally the similar.
-------------------------------------------------------------------------------------------------------------------------------------
public static int intToLeftPaddedAsciiBytes(int p_value,byte[] p_bytes,byte p_pad) {
int startOffset = intToAsciiBytes(p_value,p_bytes);
if (startOffset <= 0)
return startOffset;
else
for(int i = 0; i < startOffset; i++)
p_bytes[i] = p_pad;
return startOffset;
}
public static int intToLeftPaddedAsciiBytes(int p_value,byte[] p_bytes,byte p_pad) {
int startOffset = intToAsciiBytes(p_value,p_bytes);
for(int i = startOffset-1; i >= 0; i--)
p_bytes[i] = p_pad;
return startOffset;
}
public static int intToLeftPaddedAsciiBytes(int p_value,byte[] p_bytes,byte p_pad) {
int startOffset = intToAsciiBytes(p_value,p_bytes);
Arrays.fill(p_bytes, 0, startOffset, p_pad);
return startOffset;
}
Here's a minimal program which reproduces the crash. Moving the declaration of q or r into the loop causes the crash to disappear.
public class crash {
public static void main(String[] args) {
for (int i = 0; i < 100000; i++) {
intToLeftPaddedAsciiBytes();
}
}
public static int intToLeftPaddedAsciiBytes() {
int offset = 40;
int q;
int r;
int i = 100;
int result = 1;
while (offset > 0) {
q = (i * 52429) >>> (16+3);
r = i - (q * 10);
offset--;
i = q;
if (i == 0) {
break;
}
}
if (offset > 0) {
for(int j = 0; j < offset; j++) {
result++;
}
}
return result;
}
}