JDK-5052093 : URLConnection doesn't support large files
  • Type: Bug
  • Component: core-libs
  • Sub-Component: java.net
  • Affected Version: 5.0,6
  • Priority: P4
  • Status: Closed
  • Resolution: Fixed
  • OS: linux,windows_xp
  • CPU: x86
  • Submitted: 2004-05-24
  • Updated: 2017-05-16
  • Resolved: 2011-03-07
The Version table provides details related to the release that this issue/RFE will be addressed.

Unresolved : Release in which this issue/RFE will be addressed.
Resolved: Release in which this issue/RFE has been resolved.
Fixed : Release in which this issue/RFE has been fixed. The release containing this fix may be available for download as an Early Access Release or a General Availability Release.

To download the current JDK release, click here.
JDK 7
7 b22Fixed
Related Reports
Duplicate :  
Relates :  
Relates :  
Description

Name: gm110360			Date: 05/24/2004


FULL PRODUCT VERSION :
bug exists in JDK 1.4 and 1.5 as well

A DESCRIPTION OF THE PROBLEM :
At attempt to get bery big file (over 4GB) the exception happened

STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
Run attached source to download:
http://download.fedora.redhat.com/pub/fedora/linux/core/2/i386/iso/FC2-i386-DVD.iso

ERROR MESSAGES/STACK TRACES THAT OCCUR :
java.lang.NumberFormatException: For input string: "4370640896"
        at java.lang.NumberFormatException.forInputString(Unknown Source)
        at java.lang.Integer.parseInt(Unknown Source)
        at java.lang.Integer.parseInt(Unknown Source)
        at sun.net.www.protocol.ftp.FtpURLConnection.getInputStream(Unknown Source)
        at javaarchitect.examples.loader.Loader.resumeFileDownload(Loader.java:80)
        at javaarchitect.examples.loader.Loader.load(Loader.java:64)
        at javaarchitect.examples.loader.Loader.main(Loader.java:17)

REPRODUCIBILITY :
This bug can be reproduced always.

---------- BEGIN SOURCE ----------
package javaarchitect.examples.loader;
import java.io.*;
import java.net.*;
import sun.net.www.protocol.ftp.FtpURLConnection;
public  class Loader {
	static int BUF_SIZE = 100 * 1024; // 100 kb
	static int READ_TIMEOUT = 1000 * 60 * 45; // 45 mins

	public static void main(String[] args) {
		System.out.println("URL HTTP/FTP loader version 1.1  Copyright (c) 2000-2004 Dmitriy Rogatkin");
		for (int i=0; i < args.length; i++) {
			if (args[i].startsWith("@")) { // batch download
				try {
					BufferedReader r = new BufferedReader(new FileReader(args[i].substring(1)));
					String f;
					while ((f = r.readLine()) != null)
						load(f);
				} catch(Exception e) {
					System.err.println("Loader: Exception "+e+" at batch loading "+args[i]);
				}
			} else
				load(args[i]);
		}
		if (args.length == 0)
			System.out.println("Loader: usage [URL|@BATCH_URL] ...");
	}

	static  void load(String _url) {
		URL url;
		try {
			url = new URL(_url);
		} catch(Exception e) {
			System.err.println("Loader: Bad URL - "+_url);
			return;
		}
		boolean succ=false;
		File f = fromURL(url);
connLoop:
		do {
			try {
				URLConnection conn = url.openConnection();
				HttpURLConnection hconn = null;
				if (conn instanceof HttpURLConnection) {
					hconn = (HttpURLConnection)conn;
					
					if (hconn.getResponseCode() == HttpURLConnection.HTTP_OK) {
						succ = resumeFileDownload(conn, f, url);
					} else if (hconn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP) {
						// consider range 300..305
						String redirectPath = hconn.getHeaderField("Location");
						url = new URL(hconn.getURL(), redirectPath);
						hconn.disconnect();
						System.out.println("Loader: redirected to "+url);
						f = fromURL(url);
						continue connLoop;
					} else if (hconn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
						System.out.println("Loader: "+" Needs authorization.");
					} else {
						System.err.println("Loader: "+f+" can't be gotten from server. Error code:"+((HttpURLConnection)conn).getResponseCode());
						break;
					}
					hconn.disconnect();
				} else if (conn instanceof FtpURLConnection) {
					succ = resumeFileDownload(conn, f, url);
				}
				if (succ)
					System.out.println("Loader: "+f+" has been loaded successfully.");
			} catch(FileNotFoundException fnfe) {
				System.err.println("Loader:  file not found on remote host, skipped. "+fnfe);
				return;
			} catch(Exception e) {
				System.err.println("Loader: unsuccessful loading: "+e);
			}
		} while (!succ);
	}
	
	protected static boolean resumeFileDownload(URLConnection conn, File outFile, URL downloadUrl) throws IOException {
		RandomAccessFile o = null;
		int len = 0;
		final InputStream in = conn.getInputStream();
		len = conn.getContentLength();
		int count = BUF_SIZE;
		if (len < BUF_SIZE && len > 0)
			count = len;
		System.out.println("Loader: loading "+downloadUrl+" to "+outFile+'('+len+") stream class "+in.getClass().getName());
		final byte [] buf = new byte[count];
		for (int a=0; a<2; a++)
			try {
				o = new RandomAccessFile(outFile, "rw");
				break;
			} catch(Exception e1) {
				System.err.println("Loader: Can't create output file "+outFile+":"+e1);
				outFile = new File(downloadUrl.getHost()+'-'+Math.random()+".html");
			}
		if (o == null)
			throw new FileNotFoundException("Can't create destination file "+outFile+" from URL "+downloadUrl);
		final long[] ca = new long[1];
		long skip = o.length();
		if (skip == len) {
			System.out.println("Loader: file "+outFile+" ("+len+") is already there, skipped.");
			o.close();
			return true;
		}
		if (skip > 0) {
			System.out.println("Loader: skipping "+skip+" bytes.");
			ca[0] = skip;
			do {
				Thread rt = new Thread(new Runnable() {
					synchronized public void run() {
					try {
					long as = in.skip(ca[0]);
					if (as > 0)
					ca[0] -= as;
					else
					ca[0] = -1;
					System.out.print("s");
					} catch (IOException ioe2) {
					ca[0] = -1;
					}
					}
					}, "Skip thread");
				rt.start();
				try {
					rt.join(READ_TIMEOUT*10);
				} catch(InterruptedException ie) {
				}
				if (rt.isAlive() || rt.isInterrupted()) {
					rt.stop();
					throw new IOException ("Skipping timeout");
				}
				if (ca[0] >= 0)
					skip = ca[0];
			} while(ca[0] > 0);
			skip = o.length() - skip;
			System.err.println("Was skipped "+skip+" bytes.");
			o.seek(skip);
			if (len > 0)
				len -= skip;
		}
		try {
			do {
				// TODO: rewrite using on time created reading thread and
				// synchro using wait/notify
				Thread rt = new Thread(new Runnable() {
					synchronized public void run() {
					try {
					ca[0]/*count*/ =  in.read(buf);
					} catch (IOException ioe2) {
					ca[0] = -1;
					}
					}
					}, "Read thread");
				rt.start();
				try {
					rt.join(READ_TIMEOUT);
				} catch(InterruptedException ie) {
				}
				if (rt.isAlive() || rt.isInterrupted()) {
					rt.stop();
					//rt.destroy();
					throw new IOException ("Reading timeout");
				}
				count = (int)ca[0];
				if (count > 0) {
					o.write(buf, 0, count);
					if (len > 0)
						len -= count;
				} else if (len < 0)
					len = 0;
				System.out.print('.');
			} while( count > 0 );
		} catch(IOException ioe1) {
			System.err.println("IO exception "+ioe1);
		}
		o.close();
		return len == 0;
	}
	
	protected static File fromURL(URL url) {
		String name = url.getFile();
		int p = name.lastIndexOf('/');
		if (p >= 0 && p < (name.length()-1))
			name = name.substring(p+1);
		if (name.equals("/") || name.length()==0)
			name = url.getHost()+'-'+Math.random()+".html";
		name = name.replace('?', '_');
		name = name.replace('&', '_');
		name = name.replace('=', '-');
		File f = new File(name);
		return f;
	}
}

---------- END SOURCE ----------

CUSTOMER SUBMITTED WORKAROUND :
Just ignore the exception
(Incident Review ID: 270592) 
======================================================================

Comments
EVALUATION Yes, most of the URLConnection code (including FtpURLConnection) uses ints to deal with content-length. This is obviously an issue when dealing with filed >4GB. We should fix this as soon as possible. Will consider for an update release as it may not require an API change. ###@###.### 2004-05-25
25-05-2004