Bug ID: JDK-6225605 Speed up indexing of Jar files and reduce compile time by 10%

Type: Bug
Component: tools
Sub-Component: javac
Affected Version: 6

Priority: P2
Status: Resolved
Resolution: Fixed
OS: generic
CPU: generic

Submitted: 2005-02-04
Updated: 2010-12-08
Resolved: 2005-03-19

JDK 6
6 b29Fixed

We can reduce compile time by approx 10% using an alternate
implementation of ClassReader.openArchive in combination
with a pure Java implementation for listing zip file entries.

###@###.### 2005-2-04 14:52:44 GMT

EVALUATION This should be fixed in mustang and back-ported to 5.0 once we have seen if it is robust. ###@###.### 2005-2-04 14:52:44 GMT The pure-java jar file reading has been dropped, as it might have negative performance impact on Solaris and is hard to maintenan. Furthermore, no significant speed up was seen; the new way of indexing jar files was really what reduced compile time. See 6186747 for how to verify the fix. ###@###.### 2005-03-07 02:01:11 GMT

04-02-2005

SUGGESTED FIX Index: src/share/classes/com/sun/tools/javac/jvm/ClassReader.java ================================================================= @@ -1418,11 +1418,22 @@ * that file. */ static class Archive { + public static final List<Entry> emptyEntryList = new List<Entry>(); + static class Entry { + Entry(String name, long size) { + this.name = name; + this.size = size; + } + String name; + long size; + } ZipFile zdir; - List<ZipEntry> entries; - Archive(ZipFile zdir, List<ZipEntry> entries) { + Jar jar; + Map<String,List<Entry>> map; + Archive(ZipFile zdir, Jar jar, Map<String,List<Entry>> map) { this.zdir = zdir; - this.entries = entries; + this.jar = jar; + this.map = map; } } @@ -1432,21 +1443,50 @@ /** Open a new zip file directory. */ + private static int lastIndexOf(byte[] bs, byte value) { + for (int i = bs.length-1; i > 0; i--) { + if (bs[i] == value) + return i; + } + return -1; + } Archive openArchive(String dirname) throws IOException { Archive archive = archives.get(dirname); if (archive == null) { ZipFile zdir = new ZipFile(dirname); - ListBuffer<ZipEntry> entries = new ListBuffer<ZipEntry>(); - for (java.util.Enumeration e = zdir.entries(); - e.hasMoreElements(); ) { - entries.append((ZipEntry)e.nextElement()); + Jar jar = new Jar(zdir); + final Map<String,List<Archive.Entry>> map + = new HashMap<String,List<Archive.Entry>>(); + jar.listJarFile(new Jar.Handler() + { + public void handleFileHeader(Jar jar) { + byte[] ename = jar.nameAsBytes(); + if (endsWith(ename, classBytes)) { + int i = lastIndexOf(ename, (byte)'/'); + try { + String dir = new String(ename, 0, i, "utf8"); + if (!dir.endsWith("/")) + dir += "/"; + String suffix + = new String(ename, i+1, ename.length-i-1, "utf8"); + List<Archive.Entry> list = map.get(dir); + if (list == null) + list = Archive.emptyEntryList; + map.put(dir, + list.prepend(new Archive.Entry(suffix, jar.cenlen()))); + } catch (UnsupportedEncodingException ex) { + throw new RuntimeException(ex); + } + } } - archive = new Archive(zdir, entries.toList()); + }); + archive = new Archive(zdir, null, map); archives.put(dirname, archive); } return archive; } + /** Close the ClassReader, releasing resources. */ public void close() { @@ -1556,7 +1596,7 @@ if (verbose) { printVerbose("loading", currentClassFileName); } - if (classfile.getName().endsWith(".class")) { + if (classfile instanceof FileEntry.Zipped || classfile.getName().endsWith(".class")) { filling = true; try { int size = (int)classfile.length(); @@ -1667,7 +1707,10 @@ String filename = file.getName(); int seen; int extlen; - if (filename.endsWith(".class")) { + if (file instanceof FileEntry.Zipped) { + seen = CLASS_SEEN; + extlen = 6; + } else if (filename.endsWith(".class")) { seen = CLASS_SEEN; extlen = 6; } else { @@ -1709,43 +1752,100 @@ classEntry; } + private static boolean startsWith(byte[] bs, byte[] prefix) { + if (prefix.length <= bs.length) { + int i; + for (i = 0; i < prefix.length && prefix[i] == bs[i]; i++) + ; + return i == prefix.length; + } else { + return false; + } + } + + private static boolean endsWith(byte[] bs, byte[] suffix) { + if (suffix.length <= bs.length) { + int i; + for (i = 1; i < suffix.length+1 && suffix[suffix.length-i] == bs[bs.length-i]; i++) + ; + return i == suffix.length+1; + } else { + return false; + } + } + + private static int indexOf(byte[] bs, int start, byte value) { + for (; start < bs.length; start++) { + if (bs[start] == value) + return start; + } + return -1; + } + /** Insert all files in subdirectory `name' of `pathname' * which end in one of the extensions in `extensions' into package. */ + private static final byte[] classBytes = {(byte)'.', + (byte)'c', + (byte)'l', + (byte)'a', + (byte)'s', + (byte)'s'}; private void list(String pathname, String name, - String[] extensions, PackageSymbol p) { + String[] extensions, final PackageSymbol p) { try { if (isZip(pathname)) { - // 4098712: don't look for .java files in .zip/.jar files - if (extensions == classOrJava) - extensions = classOnly; - else if (extensions == javaOnly) + if (extensions == javaOnly) return; - Archive archive = openArchive(pathname); + final Archive archive = openArchive(pathname); if (name.length() != 0) { name = name.replace('\\', '/'); if (!name.endsWith("/")) name = name + "/"; } - int namelen = name.length(); - for (List<ZipEntry> l = archive.entries; - l.nonEmpty(); - l = l.tail) - { - ZipEntry entry = l.head; - String ename = entry.getName(); - if (ename.startsWith(name)) { - if (endsWith(ename, extensions)) { - String suffix = ename.substring(namelen); - if (suffix.length() > 0 && - suffix.indexOf('/') < 0) { - includeClassFile( - p, - new FileEntry.Zipped(suffix, - archive.zdir, entry)); - } - } else extraZipFileActions(p, ename, name, pathname); + List<Archive.Entry> entries = archive.map.get(name); + if (entries == null) + return; + for (Archive.Entry entry : entries) { + if (entry.name.endsWith(".class")) { + ZipEntry ze = new ZipEntry(name + entry.name); + ze.setSize(entry.size); + includeClassFile(p, + new FileEntry.Zipped(entry.name, archive.zdir, ze)); } } + + // final byte[] nameBytes = name.getBytes("utf8"); + // final int namelen = nameBytes.length + classBytes.length; + // archive.jar.listJarFile(new Jar.Handler() { + // public void handleFileHeader(Jar jar) { + // byte[] ename = jar.nameAsBytes(); + // if (startsWith(ename, nameBytes)) { + // if (endsWith(ename, classBytes)) { + // if (indexOf(ename, nameBytes.length, (byte)'/') < 0) { + // String suffix; + // ZipEntry ze; + // try { + // suffix = new String(ename, + // nameBytes.length, + // ename.length-nameBytes.length, + // "utf8"); + // ze = new ZipEntry(new String(ename, "utf8")); + // ze.setSize(jar.cenlen()); + // } catch (UnsupportedEncodingException ex) { + // throw new RuntimeException(ex); + // } + // includeClassFile(p, + // new FileEntry.Zipped(suffix, + // archive.zdir, ze)); + // + // } + // } + // } + // } + // }); + + // if (endsWith(ename, extensions)) { + // } else extraZipFileActions(p, ename, name, pathname); } else { File f = name.length() != 0 ? new File(pathname, name) New file: src/share/classes/com/sun/tools/javac/util/Jar.java ============================================================= /* * @(#)Jar.java 1.1 05/02/04 * * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * SUN PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. * * Use and Distribution is subject to the Java Research License available * at <http://www.sun.com/software/communitysource/jrl.html>. */ package com.sun.tools.javac.util; import java.io.*; import java.nio.*; import java.nio.channels.*; import java.util.jar.*; import java.util.zip.*; /** * This file provides access to the central directory structure of a * jar/zip file. * * <p><b>This is NOT part of any API supported by Sun Microsystems. * If you write code that depends on this, you do so at your own * risk. This code and its internal interfaces are subject to change * or deletion without notice.</b></p> * * @author Martin Buchholz * @author Peter von der Ah\u00e9 */ public class Jar { private MappedByteBuffer b; final FileChannel ch; public static long MAX_COMMENT_SIZE = 1L<<16; public static long COMMENT_MAP_SIZE = ZipFile.ENDHDR + MAX_COMMENT_SIZE; public interface Handler { void handleFileHeader(Jar jar); } private static int asInt(long l) { if (l > Integer.MAX_VALUE) throw new RuntimeException("Integer overflow"); return (int)l; } public Jar(ZipFile zipfile) throws FileNotFoundException, IOException { ch = new FileInputStream(zipfile.getName()).getChannel(); // The Zip ends with a comment no longer than MAX_COMMENT_SIZE // preceeded by the "end of central dir record". That area is // mapped in. final long size = ch.size(); long mapStart; long mapLength; int mapPos; if (size < COMMENT_MAP_SIZE) { mapStart = 0; mapLength = size; } else { mapStart = size-COMMENT_MAP_SIZE; mapLength = COMMENT_MAP_SIZE; } mapPos = asInt(mapLength-ZipFile.ENDHDR); b = ch.map(FileChannel.MapMode.READ_ONLY, mapStart, mapLength); // Scan from (almost) the end of the file backward until // the "end of central dir record" signature is found. do { assert mapPos > -1; b.position(mapPos--); } while (getsig() != ZipFile.ENDSIG); assert getsig() == ZipFile.ENDSIG; // Get the size and offset for the "central dir record" final long censize = endsiz(); final long cenoff = endoff(); // Map the "central dir record" b = ch.map(FileChannel.MapMode.READ_ONLY, cenoff, censize); } public void listJarFile(Handler handler) { b.position(0); while (b.hasRemaining()) { if (getsig() != ZipFile.CENSIG) throw new RuntimeException("bad sig"); handler.handleFileHeader(this); nextEntry(); } } public byte[] nameAsBytes() { b.mark(); final int cennam = cennam(); b.position(b.position()+ZipFile.CENHDR); byte[] name = new byte[cennam]; b.get(name, 0, cennam); b.reset(); return name; } public long getsig() {return get32(0);} public long endsiz() {return get32(ZipFile.ENDSIZ);} public long endoff() {return get32(ZipFile.ENDOFF);} public int cenlen() {return get16(ZipFile.CENLEN);} public int censiz() {return get16(ZipFile.CENSIZ);} public long centim() {return get32(ZipFile.CENTIM);} public int cennam() {return get16(ZipFile.CENNAM);} public int cenext() {return get16(ZipFile.CENEXT);} public int cencom() {return get16(ZipFile.CENCOM);} public void nextEntry() { b.position(b.position() + ZipFile.CENHDR + cennam() + cenext() + cencom()); } public int get16(int off) { b.mark(); b.position(b.position()+off); final int res = get16(); b.reset(); return res; } public long get32(int off) { b.mark(); b.position(b.position() + off); final long res = get32(); b.reset(); return res; } public int get16() { final int b1 = b.get(); final int b2 = b.get(); return (b1 & 0xff) | ((b2 & 0xff) << 8); } public long get32() { final int s1 = get16(); final int s2 = get16(); return s1 | ((long)s2 << 16); } public static void main(String... args) throws Exception { FileOutputStream fdOut = new FileOutputStream(FileDescriptor.out); System.setOut(new PrintStream(new BufferedOutputStream(fdOut, 4096), false)); Handler handler = new Handler() { public void handleFileHeader(Jar jar) { try { System.out.write(jar.nameAsBytes()); System.out.println(); } catch (Exception e) { throw new RuntimeException(e); } } }; for (String file : args) { Jar jar = new Jar(new ZipFile(file)); jar.listJarFile(handler); } System.out.flush(); } } ###@###.### 2005-2-04 14:52:44 GMT

04-02-2005

Relates :	JDK-6186747 - javac performance regression
Relates :	JDK-6232676 - javac crashes with ArrayIndexOutOfBoundsException on ZIP merged jar files.