Steps to reproduce the applet unresponsiveness:

a. Run the applet below 
b. press "call dialog" button. This creates a modal dialog and a new browser window. 
c. click "OK" in the dialog. The dialog goes away
d. Try to press a button in the "NEW browser window" which was created in step "b"

Result: The button can not be pressed. The applet is frozen.

Reproducible with the latest 5.0u7 build. This doesn't happen on 1.4.2

TC05-01_Top.html
----------------
<APPLET CODE="SampleApplet.class" WIDTH=300 HEIGHT=220>
  <PARAM NAME="PAGE" VALUE="TC05-01_Top.html">
</APPLET>

SampleApplet.java
-----------------
import java.awt.*;
import java.awt.event.*;
import java.applet.*;
import javax.swing.*;
import java.net.URL;

public class SampleApplet extends JApplet{
    private JButton jButton1 = new JButton();

    public SampleApplet() {
    }

    public void init() {
        try {
            jbInit();
        }
        catch(Exception e) {
            e.printStackTrace();
        }
    }

    private void jbInit() throws Exception {
        jButton1.setBounds(new Rectangle(56, 98, 135, 38));
        jButton1.setText("Call Dialog");
        jButton1.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(ActionEvent e) {
                jButton1_actionPerformed(e);
            }
        });
        this.getContentPane().setLayout(null);
        this.setSize(new Dimension(400,300));
        this.getContentPane().add(jButton1, null);
    }

    public void start() {
    }

    public void stop() {
    }

    public void destroy() {
    }

    void jButton1_actionPerformed(ActionEvent e) {
        String strOption[] = new String[2];
        strOption[0] = "OK";
        strOption[1] = "Cancel";
		(new ThreadX(this)).start();        
       	
        JOptionPane.showOptionDialog(this, "Hello", "Hello", JOptionPane.DEFAULT_OPTION, 
JOptionPane.PLAIN_MESSAGE, null, strOption, strOption[0]);
    }

    class ThreadX extends Thread{
  
        SampleApplet jap = null;

        ThreadX(SampleApplet jap){
			this.jap = jap;
 		}

  		public void run(){
	        try {
            	Thread.sleep(1000);

   		        this.jap.getAppletContext().showDocument(new URL(this.jap.getCodeBase()+this.jap.getParameter("PAGE")), "_blank");
	        }
   		    catch (Exception ex) {
       		    ex.printStackTrace();
        	}
		}
	}
}