ADDITIONAL SYSTEM INFORMATION :
OS: any

$ java -version
openjdk version "11.0.1" 2018-10-16
OpenJDK Runtime Environment 18.9 (build 11.0.1+13)
OpenJDK 64-Bit Server VM 18.9 (build 11.0.1+13, mixed mode)

A DESCRIPTION OF THE PROBLEM :
The method SignatureScheme.getSignature() breaks third-party providers like Bouncy Castle (BC) because the order of invoking 
signer.initSign()
and
signer.setParameter()
is not properly aligned. The parameters are set AFTER the actual initialization was called.

This causes TLS connection with RSASSA-PSS signature schemes to fail. In contrast RSA-PKCS1 works fine, because no additional parameters are set (signAlgParameter is null).

The setParameter() call must be invoked before the actual initSign() or initVerify() initialization (SignatureScheme.java, line 480).

References:
[1] http://mail.openjdk.java.net/pipermail/security-dev/2018-September/018265.html
[2] http://bouncy-castle.1462172.n4.nabble.com/JDK11-amp-BC-amp-TLS-with-RSASSA-PSS-signature-tt4659518.html

REGRESSION : Last worked in version 10.0.2

STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
Run an SSL server and connect with Chrome or sample client

EXPECTED VERSUS ACTUAL BEHAVIOR :
EXPECTED -
Successful ECDH key exchange (as it does with --use-sun)
ACTUAL -
javax.net.ssl.SSLHandshakeException: Invalid ECDH ServerKeyExchange signature

---------- BEGIN SOURCE ----------
Server.java:

import java.math.BigInteger;
import java.net.InetSocketAddress;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.KeyStore;
import java.security.SecureRandom;
import java.security.Security;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Date;

import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLParameters;

import org.bouncycastle.asn1.x500.X500Name;
import org.bouncycastle.asn1.x509.AlgorithmIdentifier;
import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.cert.X509v3CertificateBuilder;
import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
import org.bouncycastle.crypto.params.AsymmetricKeyParameter;
import org.bouncycastle.crypto.util.PrivateKeyFactory;
import org.bouncycastle.operator.ContentSigner;
import org.bouncycastle.operator.DefaultDigestAlgorithmIdentifierFinder;
import org.bouncycastle.operator.DefaultSignatureAlgorithmIdentifierFinder;
import org.bouncycastle.operator.bc.BcRSAContentSignerBuilder;

import com.sun.net.httpserver.HttpsConfigurator;
import com.sun.net.httpserver.HttpsParameters;
import com.sun.net.httpserver.HttpsServer;


public class Server {
  private static final String keystorePass = "changeit";
  private static final String WEB_APP_CONTEXT = "/test";

  public static void main(final String[] args) throws Exception {
    if (args.length == 1 && args[0].equals("--use-sun")) {
      System.out.println("Using SunRsaSign security provider");
    } else {
      // enable bouncycastle
      Security.insertProviderAt(new org.bouncycastle.jce.provider.BouncyCastleProvider(), 2);
      System.out.println("Using bouncycastle provider");
    }

    startServer("localhost", 443);
  }

  private static HttpsServer startServer(final String host, final int port) throws Exception {
    final HttpsServer server = HttpsServer.create(new InetSocketAddress(host, port), 0);
    final Thread t = new Thread(() -> {
      try {
        final SSLContext sslctx = SSLContext.getInstance("TLSv1.2");
        final KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
        final KeyStore ks = KeyStore.getInstance("JKS");

        KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA");
        keyPairGenerator.initialize(2048);
        KeyPair keyPair = keyPairGenerator.generateKeyPair();
        Certificate[] chain = {generateCertificate("cn=Unknown", keyPair, "SHA256withRSA")};
        ks.load(null, null);
        ks.setKeyEntry("main", keyPair.getPrivate(), keystorePass.toCharArray(), chain);

        kmf.init(ks, keystorePass.toCharArray());

        sslctx.init(kmf.getKeyManagers(), null, new SecureRandom());
        final SSLParameters sslParameters = sslctx.getDefaultSSLParameters();
        sslParameters.setProtocols(new String[] { "TLSv1.2" });

        // use ECDHE specific ciphersuite
        sslParameters.setCipherSuites(new String[] {"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"});
        server.setHttpsConfigurator(new HttpsConfigurator(sslctx) {
          @Override
          public void configure(final HttpsParameters params) {
            params.setSSLParameters(sslParameters);
          }
        });

        server.createContext(WEB_APP_CONTEXT, (exchange)->
        exchange.sendResponseHeaders(200, -1));
        server.start();
        System.out.println("Started server at " + server.getAddress());
      } catch(Exception e) {
        throw new RuntimeException(e);
      }
    });
    t.start();
    return server;
  }

  public static X509Certificate generateCertificate(String dn, KeyPair keyPair, String algorithm) throws CertificateException {
    try {
      AlgorithmIdentifier sigAlgId = new DefaultSignatureAlgorithmIdentifierFinder().find(algorithm);
      AlgorithmIdentifier digAlgId = new DefaultDigestAlgorithmIdentifierFinder().find(sigAlgId);
      AsymmetricKeyParameter privateKeyAsymKeyParam = PrivateKeyFactory.createKey(keyPair.getPrivate().getEncoded());
      SubjectPublicKeyInfo subPubKeyInfo = SubjectPublicKeyInfo.getInstance(keyPair.getPublic().getEncoded());
      ContentSigner sigGen = new BcRSAContentSignerBuilder(sigAlgId, digAlgId).build(privateKeyAsymKeyParam);
      Date from = new Date();
      Date to = new Date(from.getTime() + 365 * 86400000L);
      X509v3CertificateBuilder v3CertGen = new X509v3CertificateBuilder(new X500Name(dn), new BigInteger("111"), from, to, new X500Name(dn), subPubKeyInfo);
      X509CertificateHolder certificateHolder = v3CertGen.build(sigGen);
      return new JcaX509CertificateConverter().getCertificate(certificateHolder);
    } catch (CertificateException ce) {
      throw ce;
    } catch (Exception e) {
      throw new CertificateException(e);
    }
  }

}


Client.java

import java.io.InputStream;
import java.net.URL;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;

import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;


public class Client {
  private static final String WEB_APP_CONTEXT = "/test";

  public static void main(final String[] args) throws Exception {       
        HttpsURLConnection.setDefaultHostnameVerifier((h, s) -> {return true;});
       
        final int port = 443;
        final URL targetURL = new URL("https://localhost:" + port + WEB_APP_CONTEXT);
        final HttpsURLConnection conn = (HttpsURLConnection) targetURL.openConnection();
       
        // use a SSLSocketFactory which "trusts all"
        final SSLContext sslctx = SSLContext.getInstance("TLSv1.2");
        sslctx.init(null, new TrustManager[] {new TrustAll()}, null);
        conn.setSSLSocketFactory(sslctx.getSocketFactory());

        // read
        try (final InputStream is = conn.getInputStream()) {
            is.read();
        }
        System.out.println("Received status code " + conn.getResponseCode());
    }

  private static class TrustAll implements X509TrustManager {

    @Override
    public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
      return;
    }

    @Override
    public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
      return;
    }

    @Override
    public X509Certificate[] getAcceptedIssuers() {
      return new X509Certificate[0];
    }
  }
}
---------- END SOURCE ----------

CUSTOMER SUBMITTED WORKAROUND :
disable RSASSA-PSS signature schemes and fallback to RSA-PKCS1

FREQUENCY : always