Please consider the following example. With either fastdebug or slowdebug options ON, new jdk8 builds (which are done with configure-make approach) fire AWT assert. This potentially may indicate some serious problem and is worth investigating.

import javax.swing.*;
import javax.swing.plaf.LayerUI;
import javax.swing.table.DefaultTableModel;

public class JLayerTableTest extends JFrame {
    public JLayerTableTest() {
        super("JLayer Table Test");
        setDefaultCloseOperation(EXIT_ON_CLOSE);
        DefaultTableModel model = new DefaultTableModel() {
            @Override
            public int getRowCount() {
                return 100;
            }

            @Override
            public int getColumnCount() {
                return 3;
            }

            @Override
            public Object getValueAt(int row, int column) {
                return "(" + row + "," + column + ")";
            }
        };
        JTable table = new JTable(model);
        LayerUI<JComponent> layerUI = new LayerUI<>();
        JLayer<JComponent> layer = new JLayer<>(table, layerUI);
        JScrollPane scrollPane = new JScrollPane(layer);
        add(scrollPane);
        pack();
        setLocationRelativeTo(null);
        setVisible(true);
    }

    public static void main(String[] args) {
        SwingUtilities.invokeLater(new Runnable() {
            @Override
            public void run() {
                new JLayerTableTest();
            }
        });
    }
}