ADDITIONAL SYSTEM INFORMATION :
MacOS X 10.11, but also CentOS 7.
java version "11.0.1" 2018-10-16 LTS
Java(TM) SE Runtime Environment 18.9 (build 11.0.1+13-LTS)
Java HotSpot(TM) 64-Bit Server VM 18.9 (build 11.0.1+13-LTS, mixed mode)

A DESCRIPTION OF THE PROBLEM :
Using a PKCS11 module like NSS or SoftHSM to store keys, prevents TLSv1.3 clients to connect, while clients using TLSv1.2 succeed.

I noticed as a provider, PKCS11 does not expose RSASSA-PSS as SunRsaSign does, that is
	Alg.Alias.KeyPairGenerator.OID.1.2.840.113549.1.1.10	RSASSA-PSS
	Alg.Alias.Signature.OID.1.2.840.113549.1.1.10	RSASSA-PSS
	KeyFactory.RSASSA-PSS	sun.security.rsa.RSAKeyFactory$PSS
	Signature.RSASSA-PSS SupportedKeyClasses	java.security.interfaces.RSAPublicKey|java.security.interfaces.RSAPrivateKey
	Alg.Alias.AlgorithmParameters.1.2.840.113549.1.1.10	RSASSA-PSS
	Alg.Alias.KeyFactory.OID.1.2.840.113549.1.1.10	RSASSA-PSS
	Alg.Alias.KeyPairGenerator.1.2.840.113549.1.1.10	RSASSA-PSS
	KeyPairGenerator.RSASSA-PSS	sun.security.rsa.RSAKeyPairGenerator$PSS
	Signature.RSASSA-PSS	sun.security.rsa.RSAPSSSignature
	AlgorithmParameters.RSASSA-PSS	sun.security.rsa.PSSParameters
	Alg.Alias.AlgorithmParameters.OID.1.2.840.113549.1.1.10	RSASSA-PSS
	Alg.Alias.Signature.1.2.840.113549.1.1.10	RSASSA-PSS
	Alg.Alias.KeyFactory.1.2.840.113549.1.1.10	RSASSA-PSS
is missing from all SunPKCS11 providers

REGRESSION : Last worked in version 8u192

STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
To reproduce, I used a modified version of the HTTPSServer from
https://www.pixelstech.net/article/1445603357-A-HTTPS-client-and-HTTPS-server-demo-in-Java, to use PKCS11 instead of JKS as key store.


EXPECTED VERSUS ACTUAL BEHAVIOR :
EXPECTED -
Successfully established client session.
ACTUAL -
When a TLSv1.3 client tries to connect, the following stack trace is thrown server side:

javax.net.ssl.SSLHandshakeException: Cannot produce CertificateVerify signature
	at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:128)
	at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:321)
	at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:264)
	at java.base/sun.security.ssl.CertificateVerify$T13CertificateVerifyMessage.<init>(CertificateVerify.java:905)
	at java.base/sun.security.ssl.CertificateVerify$T13CertificateVerifyProducer.onProduceCertificateVerify(CertificateVerify.java:1077)
	at java.base/sun.security.ssl.CertificateVerify$T13CertificateVerifyProducer.produce(CertificateVerify.java:1070)
	at java.base/sun.security.ssl.SSLHandshake.produce(SSLHandshake.java:436)
	at java.base/sun.security.ssl.ClientHello$T13ClientHelloConsumer.goServerHello(ClientHello.java:1224)
	at java.base/sun.security.ssl.ClientHello$T13ClientHelloConsumer.consume(ClientHello.java:1160)
	at java.base/sun.security.ssl.ClientHello$ClientHelloConsumer.onClientHello(ClientHello.java:849)
	at java.base/sun.security.ssl.ClientHello$ClientHelloConsumer.consume(ClientHello.java:810)
	at java.base/sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:392)
	at java.base/sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:444)
	at java.base/sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:421)
	at java.base/sun.security.ssl.TransportContext.dispatch(TransportContext.java:178)
	at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:164)
	at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1152)
	at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1063)
	at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:402)
	at TLSServer$ServerThread.run(TLSServer.java:92)
Caused by: java.security.InvalidKeyException: No installed provider supports this key: sun.security.pkcs11.P11Key$P11PrivateKey
	at java.base/java.security.Signature$Delegate.chooseProvider(Signature.java:1163)
	at java.base/java.security.Signature$Delegate.engineInitSign(Signature.java:1204)
	at java.base/java.security.Signature.initSign(Signature.java:546)
	at java.base/sun.security.ssl.SignatureScheme.getSignature(SignatureScheme.java:473)
	at java.base/sun.security.ssl.CertificateVerify$T13CertificateVerifyMessage.<init>(CertificateVerify.java:895)
	... 16 more


---------- BEGIN SOURCE ----------
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.security.KeyStore;

import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLServerSocket;
import javax.net.ssl.SSLServerSocketFactory;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;

class TLSServer {
    private int port = 8080;
    private boolean isServerDone = false;

    public static void main(String[] args){
        TLSServer server = new TLSServer();
        server.run();
    }

    TLSServer(){
    }

    // Create the and initialize the SSLContext
    private SSLContext createSSLContext(){
        try{
            KeyStore keyStore = KeyStore.getInstance("PKCS11");
            keyStore.load(null,"password".toCharArray());
            System.out.println("Keystore provider : " + keyStore.getProvider());
            // Create key manager
            KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("SunX509");
            keyManagerFactory.init(keyStore, "password".toCharArray());
            KeyManager[] km = keyManagerFactory.getKeyManagers();

            // Create trust manager
            TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance("SunX509");
            trustManagerFactory.init(keyStore);
            TrustManager[] tm = trustManagerFactory.getTrustManagers();

            // Initialize SSLContext
            SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
            sslContext.init(km,  tm, null);

            return sslContext;
        } catch (Exception ex){
            ex.printStackTrace();
        }

        return null;
    }

    // Start to run the server
    public void run(){
        SSLContext sslContext = this.createSSLContext();

        try{
            // Create server socket factory
            SSLServerSocketFactory sslServerSocketFactory = sslContext.getServerSocketFactory();

            // Create server socket
            SSLServerSocket sslServerSocket = (SSLServerSocket) sslServerSocketFactory.createServerSocket(this.port);

            System.out.println("SSL server started");
            while(!isServerDone){
                SSLSocket sslSocket = (SSLSocket) sslServerSocket.accept();

                // Start the server thread
                new ServerThread(sslSocket).start();
            }
        } catch (Exception ex){
            ex.printStackTrace();
        }
    }

    // Thread handling the socket from client
    static class ServerThread extends Thread {
        private SSLSocket sslSocket = null;

        ServerThread(SSLSocket sslSocket){
            this.sslSocket = sslSocket;
        }

        public void run(){
            sslSocket.setEnabledCipherSuites(sslSocket.getSupportedCipherSuites());

            try{
                // Start handshake
                sslSocket.startHandshake();

                // Get session after the connection is established
                SSLSession sslSession = sslSocket.getSession();

                System.out.println("SSLSession :");
                System.out.println("\tProtocol : "+sslSession.getProtocol());
                System.out.println("\tCipher suite : "+sslSession.getCipherSuite());

                // Start handling application content
                InputStream inputStream = sslSocket.getInputStream();

                BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));

                String line = null;
                while((line = bufferedReader.readLine()) != null){
                    System.out.println("Input : "+line);

                    if(line.trim().isEmpty()){
                        break;
                    }
                }

                sslSocket.close();
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
    }
}

run with java -D -Djava.security.properties=/path/to/java-security-nss.conf,
which contains 
security.provider.14=SunPKCS11 /tmp/nss.conf

/tmp/nss.conf :
name = NSScrypto
nssLibraryDirectory = /tmp/nss-3.41/dist/Darwin15.6.0_cc_64_OPT.OBJ/lib
nssSecmodDirectory = sql:/tmp/nss-3.41/dist/Darwin15.6.0_cc_64_OPT.OBJ
---------- END SOURCE ----------

FREQUENCY : always