Latest patch for the lambda form caching feature in 8u40 (JDK-8046703) causes an OutOfMemoryError.
Patch link: http://cr.openjdk.java.net/~vlivanov/lfc/2014-07-16
JDK WS patched: http://hg.openjdk.java.net/jdk9/dev/jdk

Sequential creation of method handles garbage causes OOME in patched jdk. NO OOME happens in original JDK 9 b27.

Here is the code to check for OOME (run it with limited heap, e.g. -Xmx10m):

import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;

/**
  *
  * @author kshefov
  */
public class LFCOOMTest {

     static Random RNG = new Random();
     static final MethodHandles.Lookup lookup = MethodHandles.lookup();

     /**
      * @param args the command line arguments
      */
     public static void main(String[] args) throws
ClassNotFoundException, NoSuchMethodException, IllegalAccessException,
IllegalArgumentException, InvocationTargetException {
         int arity;
         MethodType mt;
         MethodHandle mh;
         while (true) {
             arity = RNG.nextInt(127);
             mt = randomMethodTypeGenerator(arity);
             mh = methodHandleGenerator(mt.returnType(),
mt.parameterList());
         }
     }

     private static MethodHandle methodHandleGenerator(Class<?>
returnType, List<Class<?>> argTypes) throws NoSuchMethodException,
IllegalAccessException {
         MethodHandle result;
         Class<?> clazz = Class1.class;
         String returnTypeStr = returnType.getSimpleName();
         result = lookup.findStatic(clazz, "return" + returnTypeStr,
MethodType.methodType(returnType));
         return MethodHandles.dropArguments(result, 0, argTypes);

     }

     private static MethodType randomMethodTypeGenerator(int arity) {
         List<Class<?>> list = new ArrayList<>(arity);
         Class<?> argType;
         for (int i = 0; i < arity; i++) {
             argType = classes[RNG.nextInt(classes.length - 1)];
             list.add(argType);
         }
         Class<?> rtype = classes[RNG.nextInt(classes.length)];
         return MethodType.methodType(rtype, list);
     }

     private static final Class<?> classes[] = {
         Object.class,
         int.class,
         boolean.class,
         byte.class,
         short.class,
         char.class,
         long.class,
         float.class,
         double.class,
         void.class
     };

     private static class Class1 {

         static Object returnObject() {
             return 1;
         }

         static int returnint() {
             return 1;
         }

         static char returnchar() {
             return 1;
         }

         static boolean returnboolean() {
             return true;
         }

         static byte returnbyte() {
             return 1;
         }

         static short returnshort() {
             return 1;
         }

         static long returnlong() {
             return 1;
         }

         static float returnfloat() {
             return 1;
         }

         static double returndouble() {
             return 1;
         }

         static void returnvoid() {

         }

     }

}