FULL PRODUCT VERSION :
java version "1.6.0_20"
Java(TM) SE Runtime Environment (build 1.6.0_20-b02)
Java HotSpot(TM) Client VM (build 16.3-b01, mixed mode, sharing)

ADDITIONAL OS VERSION INFORMATION :
all (Java code bug)

EXTRA RELEVANT SYSTEM CONFIGURATION :
  Buggy java code is
com/sun/org/apache/xml/internal/security/utils/UnsyncByteArrayOutputStream
(rt.jar / src.zip)

A DESCRIPTION OF THE PROBLEM :
Java code

com/sun/org/apache/xml/internal/security/utils/UnsyncByteArrayOutputStream

expands the internal byte[] buf; with help of function void expandSize(); which does not know the real size to use; it enlarges the buffer size by factor 2. If on write access the actual size is too small but larger than the result of applying factor 2 the ArrayOutOfBoundsException raises on a call to

System.arraycopy()

with invalid bounds. Bug in functions public void write(byte[] arg0);
and public void write(byte[] arg0, int arg1, int arg2);



STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
Verifying an XML signature (reference.validate()) with data referenced larger than 16KB and cacheReference set to true raises an ArrayOutOfBoundsException.

EXPECTED VERSUS ACTUAL BEHAVIOR :
EXPECTED -
Retrieval of referenced data.
ACTUAL -
ArrayOutOfBoundsException

REPRODUCIBILITY :
This bug can be reproduced always.

---------- BEGIN SOURCE ----------
package xmldsigbug;

import java.io.File;
import java.security.Key;
import java.security.PublicKey;
import java.security.cert.X509Certificate;
import java.util.Iterator;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.crypto.AlgorithmMethod;
import javax.xml.crypto.KeySelector;
import javax.xml.crypto.KeySelectorException;
import javax.xml.crypto.KeySelectorResult;
import javax.xml.crypto.XMLCryptoContext;
import javax.xml.crypto.XMLStructure;
import javax.xml.crypto.dsig.Reference;
import javax.xml.crypto.dsig.SignatureMethod;
import javax.xml.crypto.dsig.XMLSignature;
import javax.xml.crypto.dsig.XMLSignatureFactory;
import javax.xml.crypto.dsig.dom.DOMValidateContext;
import javax.xml.crypto.dsig.keyinfo.KeyInfo;
import javax.xml.crypto.dsig.keyinfo.X509Data;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;

/** sample program to demonstrate bug in UnsyncByteArrayOutputStream.java
 * Java: 1.6.0_20 and earlier; xml-Security-1.4.3 and earlier
 *
 * UnsyncByteArrayOutputStream is triggered by a caching version of
 * DigesterOutputStream and used if javax.xml.crypto.dsig.cacheReference
 * is set to true. The bug 'ArrayIndexOutOfBoundException' happens
 * when reference.validate() is called.
 *
 * @author a.menke
 */
public class Main
{
  public static final String TESTFILE = "xmlSignatureWithReferenceDataLargerThan16kb.xml";
  /**
   * @param args the command line arguments
   */
  public static void main( String[] args )
  {
    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
    dbf.setNamespaceAware( true );

    try
    {
      File file = new File(TESTFILE);
      
      DocumentBuilder db = dbf.newDocumentBuilder();
      Document doc = db.parse( file );

      XMLSignatureFactory facXMLSignature = XMLSignatureFactory.getInstance( "DOM" );

//Code Sample 5
// Find Signature element.
      NodeList nlSignatures = doc.getElementsByTagNameNS( XMLSignature.XMLNS, "Signature" );
      if( nlSignatures.getLength() == 0 )
      {
        throw new Exception( "Cannot find Signature element" );
      }

      for( int ii = 0; ii < nlSignatures.getLength(); ++ii )
      {
// Create a DOMValidateContext and specify a KeySelector
// and document context.
        DOMValidateContext valContext = new DOMValidateContext( new X509KeySelector(), nlSignatures.item( ii ) );
        valContext.setBaseURI( "file:///" + file.getAbsolutePath().replace( '\\', '/' ).replace( ' ', '+' ) );

//Code Sample 7
// if you disable the following line all works fine: caching disabled
        valContext.setProperty( "javax.xml.crypto.dsig.cacheReference", Boolean.TRUE );
// Unmarshal the XMLSignature.
        XMLSignature signature = facXMLSignature.unmarshalXMLSignature( valContext );
// Validate the XMLSignature.
// not neccesary
//        boolean coreValidity = signature.validate( valContext );

        Iterator itReferences = signature.getSignedInfo().getReferences().iterator();
        for( int jj = 0; itReferences.hasNext(); ++jj )
        {
          Reference reference = (Reference)itReferences.next();
// here we get get the bug if cacheReference is enabled and the referenced data is larger than 2*8*1024byte
          boolean refValid = reference.validate( valContext );
        }
      }
    }
    catch( Exception ex )
    {
      Logger.getLogger( Main.class.getName() ).log( Level.SEVERE, null, ex );
    }
  }
    /* Code Samples from
   * http://java.sun.com/developer/technicalArticles/xml/dig_signature_api/
   */
  // Code Sample 8
  public static class X509KeySelector extends KeySelector
  {
    public KeySelectorResult select( KeyInfo keyInfo,
                                     KeySelector.Purpose purpose,
                                     AlgorithmMethod method,
                                     XMLCryptoContext context )
            throws KeySelectorException
    {
      if( null == keyInfo || null == purpose || null == method || null == context )
      {
        return null;
      }

      /** should iterate through keyInfo and
       * MUST preselsect the public key without
       * further knowledge of the signature
       */
      Iterator ki = keyInfo.getContent().iterator();
      while( ki.hasNext() )
      {
        XMLStructure info = (XMLStructure) ki.next();
        if( !(info instanceof X509Data) )
          continue;
        X509Data x509Data = (X509Data) info;
        Iterator xi = x509Data.getContent().iterator();
        while( xi.hasNext() )
        {
          Object o = xi.next();
          if( !(o instanceof X509Certificate) )
            continue;
          final PublicKey key = ((X509Certificate) o).getPublicKey();
          // Make sure the algorithm isReference compatible
          // with the method.
          if( algEquals( method.getAlgorithm(), key.getAlgorithm() ) )
          {
            return new KeySelectorResult()
            {
              @Override
              public Key getKey()
              {
                return key;
              }
            };
          }
        }
      }
      throw new KeySelectorException( "No key found!" );
    }

    static boolean algEquals( String algURIMethod, String algNameKey )
    {
      if(   (algNameKey.equalsIgnoreCase( "DSA" )
             && algURIMethod.equalsIgnoreCase( SignatureMethod.DSA_SHA1 ))
         || (algNameKey.equalsIgnoreCase( "RSA" )
             && (algURIMethod.equalsIgnoreCase( SignatureMethod.RSA_SHA1 )
                 || algURIMethod.equalsIgnoreCase( "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256" )
                )
            )
        )
      {
        return true;
      }
      else
      {
        return false;
      }
    }
  }
}

---------- END SOURCE ----------

CUSTOMER SUBMITTED WORKAROUND :
Workaround:
Using setURIDereferencer() with an own implementation of an URIDereferencer without that problem.


FIX:

A good fix is hard to decide: it might be that caching is a question of design.
A quick fix with a correct byte buffer might exhaust memory on large sizes.
A better approach might be the use of a user supplied stream object.
Another approach questionaire an other architecture where the 'hack' on
DigesterOutputStream to cache the data is superflous