JDK-8211426 : No available authentication scheme: TLS 1.3 with only a DSA certificate
  • Type: Bug
  • Component: security-libs
  • Sub-Component: javax.net.ssl
  • Affected Version: 11,12
  • Priority: P3
  • Status: Resolved
  • Resolution: Won't Fix
  • OS: generic
  • CPU: generic
  • Submitted: 2018-10-02
  • Updated: 2019-07-18
  • Resolved: 2019-01-16
Related Reports
Relates :  
Description
ADDITIONAL SYSTEM INFORMATION :
Mac OS 10.11.6

A DESCRIPTION OF THE PROBLEM :
I have a simple program which creates an SSL-protected connection between a client thread and a server thread. The program succeeds on Java 9 and earlier. However, the program fails with the following error when run on Java 11:

  javax.net.ssl.SSLHandshakeException: Received fatal alert: handshake_failure

REGRESSION : Last worked in version 10

STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
To reproduce the issue, download the sslHandshake.tar file attached to https://issues.apache.org/jira/browse/DERBY-6998, unpack the tarball, and follow the instructions in its README file.

EXPECTED VERSUS ACTUAL BEHAVIOR :
EXPECTED -
     [java] Startup args = [startServer, startClient]
     [java] ServerThread starting...
     [java] Host address = localhost/127.0.0.1
     [java] ClientThread starting...
     [java] Host address = localhost/127.0.0.1
     [java] ServerThread successfully accepted connection request 0
     [java] ClientThread initiating handshake...
     [java] Hooray! Successfully shook the server's hand!
     [java] ClientThread exiting...
     [java] ServerThread read 'Hello, world!' from the socket!
     [java] ServerThread exiting...

ACTUAL -
     [java] Startup args = [startServer, startClient]
     [java] ServerThread starting...
     [java] Host address = localhost/127.0.0.1
     [java] ClientThread starting...
     [java] Host address = localhost/127.0.0.1
     [java] ServerThread successfully accepted connection request 0
     [java] ClientThread initiating handshake...
     [java] Oops! Handshake failed!
     [java] Oops! Socket failed!javax.net.ssl.SSLHandshakeException: Received fatal alert: handshake_failure
     [java] 
     [java] ClientThread exiting...
     [java] 	at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:128)
     [java] ServerThread exiting...
     [java] 	at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:117)
     [java] 	at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:308)
     [java] 	at java.base/sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:279)
     [java] 	at java.base/sun.security.ssl.TransportContext.dispatch(TransportContext.java:181)
     [java] 	at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:164)
     [java] 	at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1152)
     [java] 	at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1063)
     [java] 	at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:402)
     [java] 	at tests.Test$ClientThread.run(Test.java:141)
     [java] javax.net.ssl.SSLHandshakeException: No available authentication scheme
     [java] 	at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:128)
     [java] 	at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:117)
     [java] 	at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:308)
     [java] 	at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:264)
     [java] 	at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:255)
     [java] 	at java.base/sun.security.ssl.CertificateMessage$T13CertificateProducer.onProduceCertificate(CertificateMessage.java:945)
     [java] 	at java.base/sun.security.ssl.CertificateMessage$T13CertificateProducer.produce(CertificateMessage.java:934)
     [java] 	at java.base/sun.security.ssl.SSLHandshake.produce(SSLHandshake.java:436)
     [java] 	at java.base/sun.security.ssl.ClientHello$T13ClientHelloConsumer.goServerHello(ClientHello.java:1189)
     [java] 	at java.base/sun.security.ssl.ClientHello$T13ClientHelloConsumer.consume(ClientHello.java:1125)
     [java] 	at java.base/sun.security.ssl.ClientHello$ClientHelloConsumer.onClientHello(ClientHello.java:831)
     [java] 	at java.base/sun.security.ssl.ClientHello$ClientHelloConsumer.consume(ClientHello.java:792)
     [java] 	at java.base/sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:392)
     [java] 	at java.base/sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:444)
     [java] 	at java.base/sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:421)
     [java] 	at java.base/sun.security.ssl.TransportContext.dispatch(TransportContext.java:178)
     [java] 	at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:164)
     [java] 	at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1152)
     [java] 	at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1063)
     [java] 	at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:402)
     [java] 	at java.base/sun.security.ssl.SSLSocketImpl.ensureNegotiated(SSLSocketImpl.java:716)
     [java] 	at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:799)
     [java] 	at tests.Test$ServerThread.run(Test.java:105)


---------- BEGIN SOURCE ----------
package tests;

import java.io.FileInputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.security.KeyStore;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Properties;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
import javax.net.SocketFactory;
import java.net.UnknownHostException;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import javax.net.ServerSocketFactory;
import javax.net.SocketFactory;
import javax.net.ssl.SSLServerSocket;
import javax.net.ssl.SSLServerSocketFactory;
import javax.net.ssl.SSLServerSocket;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicBoolean;

public class Test
{
  public static final String SSL_KEYSTORE = "javax.net.ssl.keyStore";
  public static final String KEYSTORE_LOCATION = "keystore/SSLTestServerKey.key";
  
  public static final String SSL_KEYSTORE_PASSWORD = "javax.net.ssl.keyStorePassword";
  public static final String KEYSTORE_PASSWORD = "qwerty";

  public static final String HOST_NAME = "localhost";
  public static final int PORT_NUMBER = 8246;

  public static final long SLEEP_TIME = 1000L;
  public static final int SOCKET_TIMEOUT = 20_000; // 20 seconds
  public static final long TRACE_WAIT = 2000L;
  public static final int MAX_CONNECTIONS = 1;

  private static final byte[] HELLO_WORLD = "Hello, world!".getBytes();
  public static final int READ_BUFFER_SIZE = HELLO_WORLD.length;

  private static AtomicBoolean _serverIsRunning = new AtomicBoolean();
  
  public static void main(String... args) throws Exception
  {
    String rawArg0 = args[0].toLowerCase();
    String rawArg1 = args[1].toLowerCase();

    println("Startup args = " + stringify(args));
    
    boolean startServer = rawArg0.equals("startserver");
    boolean startClient = rawArg1.equals("startclient");
    
    Thread serverThread = startServer ? new ServerThread() : null;
    Thread clientThread = startClient ? new ClientThread() : null;

    if (startServer)
    {
      serverThread.start();
      while(!_serverIsRunning.get()) { Thread.sleep(SLEEP_TIME); }
    }

    if (startClient)
    {
      clientThread.start();
      clientThread.join();
    }
    
    if (startServer) { serverThread.join(); }

    Thread.sleep(TRACE_WAIT);
  }

  public static class ServerThread extends Thread
  {
    public String name() { return "ServerThread"; }
    
    public void run()
    {
      println(name() + " starting...");

      try
      {
        SSLServerSocket serverSocket = createServerSocket();
        serverSocket.setSoTimeout(SOCKET_TIMEOUT);
        _serverIsRunning.set(true);

        for (int connectionCount = 0; connectionCount < MAX_CONNECTIONS; connectionCount++)
        {
          SSLSocket clientSocket = (SSLSocket) serverSocket.accept();
          println
            (
             name() + " successfully accepted connection request " + connectionCount
             );

          InputStream socketInputStream = clientSocket.getInputStream();
          byte[] readBuffer = new byte[READ_BUFFER_SIZE];
          int bytesRead = socketInputStream.read(readBuffer, 0, READ_BUFFER_SIZE);
          println(name() + " read '" + (new String(readBuffer)) + "' from the socket!");
        }
      }
      catch (Exception ex)
      {
        println("Oops! Socket failed!");
        ex.printStackTrace();
      }
      
      println(name() + " exiting...");
    }
  }

  public static class ClientThread extends Thread
  {
    public String name() { return "ClientThread"; }
    
    public void run()
    {
      println(name() + " starting...");

      try
      {
        Properties sslProperties = getSSLProperties();
        SSLSocket s1 = (SSLSocket) NaiveTrustManager
          .getSocketFactory(sslProperties)
          .createSocket(host(), PORT_NUMBER);

        String[] removeTwoProtocols = removeSSLv3andSSLv2Hello(s1.getEnabledProtocols());
        s1.setEnabledProtocols(removeTwoProtocols);

        Thread.sleep(SLEEP_TIME);

        println(name() + " initiating handshake...");
        s1.setSoTimeout(SOCKET_TIMEOUT);
        s1.startHandshake();
        println("Hooray! Successfully shook the server's hand!");

        s1.getOutputStream().write(HELLO_WORLD);
      }
      catch (Exception ex)
      {
        println("Oops! Handshake failed!");
        ex.printStackTrace();
      }
      
      println(name() + " exiting...");
    }
  }

  public static class NaiveTrustManager
    implements X509TrustManager
  {
    
    /**
     * We don't want more than one instence of this TrustManager
     */
    private NaiveTrustManager()
    {
    }

    static private TrustManager[] thisManager = null;

    /** 
     * Generate a socket factory with this trust manager. Derby
     * Utility routine which is not part of the X509TrustManager
     * interface.
     *
     * @param sslProperties Configuration settings for the socket factory
     *
     * @return a socket factory
     *
     * @throws java.security.NoSuchAlgorithmException on error
     * @throws java.security.KeyManagementException on error
     * @throws java.security.NoSuchProviderException on error
     * @throws java.security.KeyStoreException on error
     * @throws java.security.UnrecoverableKeyException on error
     * @throws java.security.cert.CertificateException on error
     * @throws java.io.IOException on error
     **/
    public static SocketFactory getSocketFactory(Properties sslProperties)
      throws java.security.NoSuchAlgorithmException,
             java.security.KeyManagementException,
             java.security.NoSuchProviderException,
             java.security.KeyStoreException,
             java.security.UnrecoverableKeyException,
             java.security.cert.CertificateException,
             java.io.IOException
    {
      if (thisManager == null)
      {
        thisManager = new TrustManager [] {new NaiveTrustManager()};
      }

      SSLContext ctx = SSLContext.getInstance("TLS");

      String keyStore = sslProperties.getProperty(SSL_KEYSTORE);
      String keyStorePassword = sslProperties.getProperty(SSL_KEYSTORE_PASSWORD);

      KeyStore ks = KeyStore.getInstance("JKS");
      ks.load(new FileInputStream(keyStore), keyStorePassword.toCharArray());
            
      KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509", "SunJSSE");
      kmf.init(ks, keyStorePassword.toCharArray());

      ctx.init(kmf.getKeyManagers(), thisManager, null); // Use default random source

      return ctx.getSocketFactory();
    }
    
    /** 
     * Checks whether the we trust the client. Since this trust manager
     * is just for the Derby clients, this routine is actually never
     * called, but need to be here when we implement X509TrustManager.
     * @param chain The client's certificate chain
     * @param authType authorization type (e.g. "RSA" or "DHE_DSS")
     **/
    public void checkClientTrusted(X509Certificate[] chain, 
                                   String authType)
      throws CertificateException
    {
      // Reject all attemtpts to trust a client. We should never end
      // up here.
      throw new CertificateException();
    }
    
    /** 
     * Checks wether the we trust the server, which we allways will.
     * @param chain The server's certificate chain
     * @param authType authorization type (e.g. "RSA" or "DHE_DSS")
     **/
    public void checkServerTrusted(X509Certificate[] chain, 
                                   String authType)
      throws CertificateException
    {
      // Do nothing. We trust everyone.
    }
    
    /**
     * Return an array of certificate authority certificates which are
     * trusted for authenticating peers. Not relevant for this trust
     * manager.
     */
    public X509Certificate[] getAcceptedIssuers()
    {
      return new X509Certificate[0];
    }
    
  }

  private static SSLServerSocket createServerSocket()
    throws IOException
  {
    SSLServerSocketFactory ssf =
      (SSLServerSocketFactory)SSLServerSocketFactory.getDefault();
    SSLServerSocket sss1= 
      (SSLServerSocket)ssf.createServerSocket(PORT_NUMBER, 0, host());

    String[] removeTwoProtocols = removeSSLv3andSSLv2Hello(sss1.getEnabledProtocols());
    sss1.setEnabledProtocols(removeTwoProtocols);
    
    return sss1;
  }
    
  private static Properties getSSLProperties()
  {
    Properties retval = new Properties();
        
    retval.setProperty(SSL_KEYSTORE, System.getProperty("user.dir") + "/" + KEYSTORE_LOCATION);
    retval.setProperty(SSL_KEYSTORE_PASSWORD, KEYSTORE_PASSWORD);

    return retval;
  }
  
  private static InetAddress host()
    throws UnknownHostException
  {
    InetAddress retval = InetAddress.getByName(HOST_NAME);

    println("Host address = " + retval);

    return retval;
  }
  
  //Remove SSLv3 and SSLv2Hello protocols from list of enabled protocols
  private static String[] removeSSLv3andSSLv2Hello(String[] enabledProtocols) {
    //If SSLv3 and SSLv2Hello are one of the enabled protocols, then 
    // remove them from the list of enabled protocols because of the 
    // possible security breach.
    String[] supportedProtocols = new String[enabledProtocols.length];
    int supportedProtocolsCount  = 0;
    for ( int i = 0; i < enabledProtocols.length; i++ )
    {
      if (!(enabledProtocols[i].toUpperCase().contains("SSLV3") ||
            enabledProtocols[i].toUpperCase().contains("SSLV2HELLO"))) {
        supportedProtocols[supportedProtocolsCount] = enabledProtocols[i];
        supportedProtocolsCount++;
      }
    }
    if(supportedProtocolsCount < enabledProtocols.length) {
      //We found SSLv3 and/or SSLv2Hello as one of the enabled 
      // protocols for this jvm. Following code will remove them from 
      // enabled list.
      String[] newEnabledProtocolsList = null;
      newEnabledProtocolsList = 
        new String[supportedProtocolsCount];
      System.arraycopy(supportedProtocols, 0, 
                       newEnabledProtocolsList, 0, 
                       supportedProtocolsCount);
      return(newEnabledProtocolsList);
    } else 
      return(enabledProtocols);
  }

  private static String stringify(String[] array) { return Arrays.toString(array); }

  private static void println(String text) { System.out.println(text); }
  
}

---------- END SOURCE ----------

CUSTOMER SUBMITTED WORKAROUND :
None.

FREQUENCY : always



Comments
While this can be a problem, DSA is old and no longer used for TLS1.3, Previous comments show the system property that can be updated to work around this. In general It is unlikely to be the only certificate available in one's keystore. If this becomes a significant problem out in the field where the property does not fix the problem this can be reopenned with justification
16-01-2019

To fix this is invasive and could hurt performance. It would result in keystore checks for certificate and key for each TLS connection. In the provided test only a DSA certificate is used; however, in TLS 1.3 DSA is not supported. One can argue this is a invalid configuration. This situation is exacerbated by the default keytool cert being DSA. The test succeeds when only configured with a RSA or EC certificate. As long as there is a TLS 1.3 supported certificate available in the keystore, it will work. This will not be addressed at this time, but left as an open bug to see if this is a common problem encountered. It is believed this will not be a problem for serious deployments where actual certificates are used. No CA issues a DSA certificate. Using the workaround previously stated "-Djdk.tls.[client|server].protocols" will resolve the problem.
29-10-2018

workaround is to set the protocol version system property: "-Djdk.tls.server.protocols=TLSv1.2" or "-Djdk.tls.client.protocols=TLSv1.2" As long as one of the sides has TLSv1.3 off, it will negotiate in a protocol where DSA is supported
23-10-2018

The test uses a DSA certificate which is not supported with TLS 1.3. This test works fine if the SSLContext.getInstance("TLS") is set to "TLSv1.2". The open question at the moment is if JSSE should know to use an older version of TLS to accommodate the weaker key. Given that the DSA cert was discovered in the middle of the handshake after the version was determined.
04-10-2018

It looks like a regression from JDK-8196584
03-10-2018

To reproduce the issue, run the attached test case : JDK 10.0.2 - Pass JDK 11-ea+19 - Pass JDK 11-ea+20 - Fail JDK 11+28 - Fail JDK 12-ea+13 - Fail Output in failed versions : runClientAndServer: [java] Startup args = [startServer, startClient] [java] ServerThread starting... [java] Host address = localhost/127.0.0.1 [java] ClientThread starting... [java] Host address = localhost/127.0.0.1 [java] ClientThread initiating handshake... [java] ServerThread successfully accepted connection request 0 [java] Oops! Socket failed! [java] ServerThread exiting... [java] Oops! Handshake failed! [java] ClientThread exiting... [java] javax.net.ssl.SSLHandshakeException: No available authentication scheme [java] at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:128) [java] at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:117) [java] at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:308) [java] at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:264) [java] at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:255) [java] at java.base/sun.security.ssl.CertificateMessage$T13CertificateProducer.onProduceCertificate(CertificateMessage.java:945) [java] at java.base/sun.security.ssl.CertificateMessage$T13CertificateProducer.produce(CertificateMessage.java:934) [java] at java.base/sun.security.ssl.SSLHandshake.produce(SSLHandshake.java:436) [java] at java.base/sun.security.ssl.ClientHello$T13ClientHelloConsumer.goServerHello(ClientHello.java:1189) [java] at java.base/sun.security.ssl.ClientHello$T13ClientHelloConsumer.consume(ClientHello.java:1125) [java] at java.base/sun.security.ssl.ClientHello$ClientHelloConsumer.onClientHello(ClientHello.java:831) [java] at java.base/sun.security.ssl.ClientHello$ClientHelloConsumer.consume(ClientHello.java:792) [java] at java.base/sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:392) [java] at java.base/sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:444) [java] at java.base/sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:421) [java] at java.base/sun.security.ssl.TransportContext.dispatch(TransportContext.java:178) [java] at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:164) [java] at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1152) [java] at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1063) [java] at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:402) [java] at java.base/sun.security.ssl.SSLSocketImpl.ensureNegotiated(SSLSocketImpl.java:716) [java] at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:799) [java] at tests.Test$ServerThread.run(Test.java:105) [java] javax.net.ssl.SSLHandshakeException: Received fatal alert: handshake_failure [java] at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:128) [java] at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:117) [java] at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:308) [java] at java.base/sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:279) [java] at java.base/sun.security.ssl.TransportContext.dispatch(TransportContext.java:181) [java] at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:164) [java] at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1152) [java] at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1063) [java] at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:402) [java] at tests.Test$ClientThread.run(Test.java:141)
03-10-2018