FULL PRODUCT VERSION :
$ java -version
java version "1.6.0_20"
Java(TM) SE Runtime Environment (build 1.6.0_20-b02)
Java HotSpot(TM) Server VM (build 16.3-b01, mixed mode)

ADDITIONAL OS VERSION INFORMATION :
$ uname -a
Linux gibson 2.6.33-ARCH #1 SMP PREEMPT Thu May 13 12:06:25 CEST 2010 i686 Intel(R) Core(TM)2 CPU 6420 @ 2.13GHz GenuineIntel GNU/Linux

A DESCRIPTION OF THE PROBLEM :
When storing directly allocated byte buffers as soft references, the JVM crashes with OutOfMemoryException instead of reclaiming the buffers, even when they are softly reachable. This does *not* happen however, when the DirectByteBuffer instances are stored as WeakReferences. Please see the attached sample code. This limitation however does not occur with HeapByteBuffer instances, which get properly GCed regardless of reference type (soft/weak).

STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
Run the test code provided. In the code provided, I run the exact same code, only varying the type of ByteBuffer allocated, and the type of Reference used to store the buffers. To verify that the crash is not an artifact of the order of the test run,  comment out all the doTest invocations except the last one (which invokes a direct byte buffer test with soft references). The OOM error still persists.


ERROR MESSAGES/STACK TRACES THAT OCCUR :
HeapByteBuffer test with weak references did 87 clearings
HeapByteBuffer test with soft references did 89 clearings
DirectByteBuffer test with weak references did 85 clearings
Exception in thread "main" java.lang.OutOfMemoryError: Direct buffer memory
	at java.nio.Bits.reserveMemory(Bits.java:633)
	at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:95)
	at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:288)
	at Test$BufferPool.take(Test.java:68)
	at Test.doTest(Test.java:95)
	at Test.main(Test.java:88)


REPRODUCIBILITY :
This bug can be reproduced always.

---------- BEGIN SOURCE ----------
import java.lang.ref.Reference;
import java.lang.ref.ReferenceQueue;
import java.lang.ref.SoftReference;
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
import java.util.Comparator;
import java.util.PriorityQueue;


public class Test {
    static interface ByteBufferRef {
        public int capacity();
    }
    static class ByteBufferWeakRef extends WeakReference<ByteBuffer> implements ByteBufferRef {
        final int capacity;
        public ByteBufferWeakRef(ByteBuffer buf, ReferenceQueue<ByteBuffer> queue) {
            super(buf, queue);
            this.capacity = buf.capacity();
        }
        public int capacity() { return capacity; }
    }
    static class ByteBufferSoftRef extends SoftReference<ByteBuffer> implements ByteBufferRef {
        final int capacity;
        public ByteBufferSoftRef(ByteBuffer buf, ReferenceQueue<ByteBuffer> queue) {
            super(buf, queue);
            this.capacity = buf.capacity();
        }
        public int capacity() { return capacity; }
    }
    static class BufferPool {
        static final int getCapacity(Reference<ByteBuffer> ref) {
            return ((ByteBufferRef)ref).capacity();
        }
        static final Comparator<Reference<ByteBuffer>> comparator = new Comparator<Reference<ByteBuffer>>() {
            public int compare(Reference<ByteBuffer> b1, Reference<ByteBuffer> b2) {
                return getCapacity(b2) - getCapacity(b1);
            }
        };
        
        private final boolean isDirect;
        private final boolean isSoft;
        public BufferPool(boolean isDirect, boolean isSoft) {
            this.isDirect = isDirect;
            this.isSoft = isSoft;
        }
        
        private int numClearings = 0;
        public int getNumClearings() {
            return numClearings;
        }
        
        private final PriorityQueue<Reference<ByteBuffer>> bufferQueue = new PriorityQueue<Reference<ByteBuffer>>(16, comparator);
        private final ReferenceQueue<ByteBuffer> refQueue = new ReferenceQueue<ByteBuffer>();
        
        public ByteBuffer take(int cap) {
            for (;;) {
                Reference<? extends ByteBuffer> buf = refQueue.poll();
                if (buf == null)
                    break;
                numClearings++;
            }
            Reference<ByteBuffer> candidate = bufferQueue.peek();
            if (candidate == null || getCapacity(candidate) < cap) {
                return isDirect ? ByteBuffer.allocateDirect(cap) : ByteBuffer.allocate(cap);
            } else {
                candidate = bufferQueue.poll();
                ByteBuffer buf = candidate.get();
                if (buf == null)
                    return take(cap);
                buf.clear();
                return buf;
            }
        }
        
        public void release(ByteBuffer buf) {
            bufferQueue.offer(isSoft ? new ByteBufferSoftRef(buf, refQueue) : new ByteBufferWeakRef(buf, refQueue));
        }
    }
    
    public static void main(String[] args) {
        doTest(false, false);
        doTest(false, true);
        doTest(true, false);
        doTest(true, true);
    }

    static void doTest(boolean isDirect, boolean isSoft) {
        BufferPool pool = new BufferPool(isDirect, isSoft);
        for (int i = 1; i <= 100; i++) {
            int size = i * 8 * 16000;
            ByteBuffer buf = pool.take(size);
            pool.release(buf);
        }
        String mode = isDirect ? "Direct" : "Heap";
        String refType = isSoft ? "soft" : "weak";
        System.out.println(mode + "ByteBuffer test with " + refType + " references did " + pool.getNumClearings() + " clearings");
    }
}

---------- END SOURCE ----------