This seems to be a regression caused by JDK-8020801.
Here's the stack trace of WLS running on jdk9/dev with a custom system class loader:
Error occurred during initialization of VM
java.lang.Error: java.lang.reflect.InvocationTargetException
at java.lang.ClassLoader.initSystemClassLoader(java.base@9-internal/ClassLoader.java:1952)
at java.lang.System.initPhase3(java.base@9-internal/System.java:2048)
Caused by: java.lang.reflect.InvocationTargetException
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(java.base@9-internal/Native Method)
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(java.base@9-internal/NativeConstructorAccessorImpl.java:62)
at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(java.base@9-internal/DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(java.base@9-internal/Constructor.java:487)
at java.lang.ClassLoader.initSystemClassLoader(java.base@9-internal/ClassLoader.java:1950)
at java.lang.System.initPhase3(java.base@9-internal/System.java:2048)
Caused by: java.lang.InternalError: getSystemClassLoader should only be called after VM booted
at java.lang.ClassLoader.getSystemClassLoader(java.base@9-internal/ClassLoader.java:1902)
at jdk.internal.reflect.Reflection.isExtClassLoader(java.base@9-internal/Reflection.java:325)
at jdk.internal.reflect.Reflection.isCallerSensitive(java.base@9-internal/Reflection.java:318)
at jdk.internal.reflect.ReflectionFactory.newMethodAccessor(java.base@9-internal/ReflectionFactory.java:182)
at java.lang.reflect.Method.acquireMethodAccessor(java.base@9-internal/Method.java:629)
at java.lang.reflect.Method.invoke(java.base@9-internal/Method.java:561)
at java.lang.Class.getEnumConstantsShared(java.base@9-internal/Class.java:3516)
at java.lang.Class.enumConstantDirectory(java.base@9-internal/Class.java:3538)
at java.lang.Enum.valueOf(java.base@9-internal/Enum.java:234)
at com.oracle.classloader.weblogic.LaunchClassLoader$IndexType.valueOf(LaunchClassLoader.java:154)
at com.oracle.classloader.weblogic.LaunchClassLoader.cacheFile(LaunchClassLoader.java:126)
at com.oracle.classloader.weblogic.LaunchClassLoader.access$100(LaunchClassLoader.java:22)
at com.oracle.classloader.weblogic.LaunchClassLoader$SystemLaunchConfiguration.<init>(LaunchClassLoader.java:73)
at com.oracle.classloader.weblogic.LaunchClassLoader.<init>(LaunchClassLoader.java:37)
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(java.base@9-internal/Native Method)
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(java.base@9-internal/NativeConstructorAccessorImpl.java:62)
at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(java.base@9-internal/DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(java.base@9-internal/Constructor.java:487)
at java.lang.ClassLoader.initSystemClassLoader(java.base@9-internal/ClassLoader.java:1950)
at java.lang.System.initPhase3(java.base@9-internal/System.java:2048)