When implementing JDK-8358340, I stumbled upon a weird assert:

$ CONF=linux-x86_64-server-fastdebug make test TEST=gc/shenandoah/TestAllocIntArrays.java JTREG=REPEAT_COUNT=10

#  Internal Error (/home/shade/trunks/jdk/src/hotspot/share/memory/iterator.inline.hpp:59), pid=1222144, tid=1222159
#  assert(AOTLinkedClassBulkLoader::is_pending_aot_linked_class(k)) failed: sanity: java.lang.module.ModuleDescriptor$Modifier

Stack: [0x000071e2112d3000,0x000071e2113d3000],  sp=0x000071e2113d1930,  free space=1018k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
V  [libjvm.so+0x3bad37]  ClaimMetadataVisitingOopIterateClosure::do_klass(Klass*)+0x427  (iterator.inline.hpp:59)
V  [libjvm.so+0x1afeb11]  void InstanceMirrorKlass::oop_oop_iterate_bounded<narrowOop, ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >(oop, ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, MemRegion)+0x9d1  (devirtualizer.inline.hpp:126)
V  [libjvm.so+0x1afec75]  void OopOopIterateBoundedDispatch<ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >::Table::oop_oop_iterate_bounded<InstanceMirrorKlass, narrowOop>(ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, oop, Klass*, MemRegion)+0x65  (iterator.inline.hpp:182)
V  [libjvm.so+0x1af6285]  void ShenandoahScanRemembered::process_clusters<ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >(unsigned long, unsigned long, HeapWordImpl**, ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, bool, unsigned int)+0x8d5  (iterator.inline.hpp:306)
V  [libjvm.so+0x1af0c8d]  ShenandoahScanRememberedTask::do_work(unsigned int)+0xb1d  (shenandoahScanRemembered.inline.hpp:359)
V  [libjvm.so+0x1af0e4c]  ShenandoahScanRememberedTask::work(unsigned int)+0xcc  (shenandoahScanRemembered.cpp:664)
V  [libjvm.so+0x1e52348]  WorkerThread::run()+0x88  (workerThread.cpp:69)
V  [libjvm.so+0x1cf321a]  Thread::call_run()+0xba  (thread.cpp:243)
V  [libjvm.so+0x17a78c8]  thread_native_entry(Thread*)+0x128  (os_linux.cpp:868)
C  [libc.so.6+0x9caa4]

The assert effectively says that we are encountering the class with nullptr CLD, e.g. the class is not yet loaded. Printing out the class name shows class names that are likely not used in the test at all (something like CurrencyProvider, etc). So I believe this is a class that is loaded from CDS archived heap.

Notice that we are walking InstanceMirrorKlass. Fuller crash trace with slowdebug:

Stack: [0x00007394d70f1000,0x00007394d71f1000],  sp=0x00007394d71ef780,  free space=1017k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
V  [libjvm.so+0x486ac5]  ClaimMetadataVisitingOopIterateClosure::do_klass(Klass*)+0x87  (iterator.inline.hpp:58)
V  [libjvm.so+0x193cac9]  EnableIf<!std::is_same<ClaimMetadataVisitingOopIterateClosure, OopIterateClosure>::value, void>::type call_do_klass<ClaimMetadataVisitingOopIterateClosure, OopIterateClosure, ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >(void (ClaimMetadataVisitingOopIterateClosure::*)(Klass*), void (OopIterateClosure::*)(Klass*), ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, Klass*)+0x46  (devirtualizer.inline.hpp:126)
V  [libjvm.so+0x1959991]  void Devirtualizer::do_klass<ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >(ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, Klass*)+0x5d  (devirtualizer.inline.hpp:131)
V  [libjvm.so+0x19a166e]  void InstanceMirrorKlass::oop_oop_iterate_bounded<narrowOop, ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >(oopDesc*, ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, MemRegion)+0x376  (instanceMirrorKlass.inline.hpp:132)
V  [libjvm.so+0x19a0406]  void OopOopIterateBoundedDispatch<ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >::Table::oop_oop_iterate_bounded<InstanceMirrorKlass, narrowOop>(ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, oopDesc*, Klass*, MemRegion)+0x48  (iterator.inline.hpp:181)
V  [libjvm.so+0x199d8e5]  void OopIteratorClosureDispatch::oop_oop_iterate<ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >(ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, oopDesc*, Klass*, MemRegion)+0x4b  (iterator.inline.hpp:305)
V  [libjvm.so+0x199d80f]  unsigned long oopDesc::oop_iterate_size<ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >(ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, MemRegion)+0x75  (oop.inline.hpp:397)
V  [libjvm.so+0x199d1eb]  void ShenandoahScanRemembered::process_clusters<ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >(unsigned long, unsigned long, HeapWordImpl**, ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, bool, unsigned int)+0x112b  (shenandoahScanRemembered.inline.hpp:241)
V  [libjvm.so+0x199bd5a]  void ShenandoahScanRemembered::process_region_slice<ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2> >(ShenandoahHeapRegion*, unsigned long, unsigned long, HeapWordImpl**, ShenandoahMarkRefsClosure<(ShenandoahGenerationType)2>*, bool, unsigned int)+0x2a6  (shenandoahScanRemembered.inline.hpp:363)
V  [libjvm.so+0x1997b80]  ShenandoahScanRememberedTask::do_work(unsigned int)+0x3be  (shenandoahScanRemembered.cpp:700)
V  [libjvm.so+0x199773d]  ShenandoahScanRememberedTask::work(unsigned int)+0x9d  (shenandoahScanRemembered.cpp:664)
V  [libjvm.so+0x1cd861d]  WorkerTaskDispatcher::worker_run_task()+0x95  (workerThread.cpp:69)
V  [libjvm.so+0x1cd8d74]  WorkerThread::run()+0x34  (workerThread.cpp:200)
V  [libjvm.so+0x1b579b5]  Thread::call_run()+0x1bd  (thread.cpp:243)

I believe this whole thing is a variant of JDK-8253081, but for Generational Shenandoah. After initial CDS load, we proceed to run the application, and at _some point_ we scan a dirty card, and _by accident_ discover a Java mirror for a class that is not yet loaded. We then go into InstanceMirrorKlass::oop_oop_iterate_bounded, which resolves the Klass* from that Java mirror, which turns out to be not yet loaded. So its CLD is nullptr, and the related visitor asserts.