The SystemDictionary maps name/loader/protectiondomain triples to Klasses.  It never holds reference array klasses (those are linked from the base klass) nor adorned names such as "LString;".  I added an assert that nobody looks for strings with leading '[' or names like 'Lblah;'.  Code looking for such names is probably broken.  I found & fixed the few places which did such lookups. 

constantPoolOop::klass_at_if_loaded was one such place; this code did such lookups exclusively, hence always returned a NULL in the past. Hence callers of it (compiler threads and a few JVM calls, e.g. JVM_ConstantPoolGetKlassAtIfLoaded) always returned a NULL.  For the compiler threads, the CI goes ahead and retries the lookup with the adornment stripped off (and recursively on array bases).

The other major searcher-of-bogus-names was the add_loader_constraint code.  This code used to incorrectly install broken loader constraints (the klass field is NULL) if the lookup first happened on an array-of-FOO instead of the base FOO class. This in turns implies you can craft a program with incompatible FOO's being confused (make an array-of-FOO-via-Loader1, make an array-of-FOO-via-Loader2, pass 'em together somewhere and the lack of constraint means the two FOO bases are not forced to be identical), which in turn is a short step away from breaking type-safety and 'rooting' the JVM.

The included microbenchmark creates such broken loader constraints, and manifests itself as an endless deopt problem for C2.
Unpack the tar file, compile ALL java files, including K.java; copy K.class to "store\K.impl" - note the backslash.
Run with "-server -XX:+TraceDeoptimization".

Cliff


==== //java/main-dev/java/hotspot5/src/share/vm/memory/systemDictionary.cpp#32 - /home/cliffc/HotSpot/cliffc-bugs/hotspot5/src/share/vm/memory/systemDictionary.cpp ====
@@ -946,6 +946,12 @@
                                 Handle protection_domain,
                                 TRAPS) { 
+ debug_only( int len = class_name->utf8_length() );
+ assert0( len < 2 || // no arrays of refs in system dictionary, so probably an error to ask
+ class_name->byte_at(0) != '[' ||
+ !(class_name->byte_at(1) == '[' || class_name->byte_at(1) == 'L'));
+ assert0( len < 2 || // already stripped class marker so probably an error to ask
+ !(class_name->byte_at(0) == 'L' &&
+ class_name->byte_at(len-1) == ';'));
+
   unsigned int d_hash = dictionary()->compute_hash(class_name, class_loader);
   int d_index = dictionary()->hash_to_index(d_hash);
 
@@ -1376,6 +1382,10 @@
                                       symbolHandle class_name,
                                       Handle class_loader) {
   assert_locked_or_safepoint(SystemDictionary_lock);
+  assert0( class_name->byte_at(0) != '[' || // no arrays of refs in system dictionary
+           !(class_name->byte_at(1) == '[' || class_name->byte_at(1) == 'L'));
+  assert0( !(class_name->byte_at(0) == 'L' && // Assert already stripped class marker
+             class_name->byte_at(class_name->utf8_length()-1) == ';'));
 
   klassOop k = dictionary()->find_class(index, hash, class_name, class_loader);
   return k;
@@ -2102,6 +2112,18 @@
 bool SystemDictionary::add_loader_constraint_impl(symbolHandle class_name,
                                                   Handle class_loader1,
                                                   Handle class_loader2, Thread* THREAD) {
+  if (class_name->byte_at(0) == '[' &&
+      (class_name->byte_at(1) == '[' || class_name->byte_at(1) == 'L')) {
+    ResourceMark rm;
+    const char *p = class_name->as_utf8()+1;
+    int len = class_name->utf8_length()-1;
+    if( p[0] == 'L' && p[len-1] == ';' ) {
+      p++; len-=2;
+    }
+    class_name = oopFactory::new_symbol(p, len, THREAD);
+    return add_loader_constraint_impl(class_name,class_loader1,class_loader2,THREAD);
+  }
+
   unsigned int d_hash1 = dictionary()->compute_hash(class_name, class_loader1);
   int d_index1 = dictionary()->hash_to_index(d_hash1);
 
==== //java/main-dev/java/hotspot5/src/share/vm/oops/constantPoolOop.cpp#3 - /home/cliffc/HotSpot/cliffc-bugs/hotspot5/src/share/vm/oops/constantPoolOop.cpp ====
@@ -74,6 +74,13 @@
     assert(entry->is_symbol(), "must be either symbol or klass");
     Thread *thread = Thread::current();
     symbolHandle name (thread, (symbolOop)entry);
+    if( name->byte_at(0) == '[' && // no arrays of refs in system dictionary
+        (name->byte_at(1) == '[' || name->byte_at(1) == 'L'))
+      return NULL;              // No ref-arrays in the system dictionary, so fail now
+    if( name->byte_at(0) == 'L' && // no arrays of refs in system dictionary
+        (name->byte_at(name->utf8_length()-1) == ';'))
+      return NULL;              // No adorned names in system dictionary, so fail now
+
     oop loader = instanceKlass::cast(this_oop->pool_holder())->class_loader();
     oop protection_domain = Klass::cast(this_oop->pool_holder())->protection_domain();
     Handle h_prot (thread, protection_domain);
==== //java/main-dev/java/hotspot5/src/share/vm/opto/parse3.cpp#9 - /home/cliffc/HotSpot/cliffc-bugs/hotspot5/src/share/vm/opto/