Java array with more than 4gb elements

Question

I have a big file, it's expected to be around 12 GB. I want to load it all into memory on a beefy 64-bit machine with 16 GB RAM, but I think Java does not support byte arrays that big:

File f = new File(file); long size = f.length(); byte data[] = new byte[size]; // <- does not compile, not even on 64bit JVM 

Is it possible with Java?

The compile error from the Eclipse compiler is:

Type mismatch: cannot convert from long to int 

javac gives:

possible loss of precision found   : long required: int          byte data[] = new byte[size]; 

Answer 1

Java array indices are of type int (4 bytes or 32 bits), so I'm afraid you're limited to 2³¹ − 1 or 2147483647 slots in your array. I'd read the data into another data structure, like a 2D array.

Answer 2

package com.deans.rtl.util;  import java.io.FileInputStream; import java.io.IOException;  /**  *   * @author [email protected]  *  * Written to work with byte arrays requiring address space larger than 32 bits.   *   */  public class ByteArray64 {      private final long CHUNK_SIZE = 1024*1024*1024; //1GiB      long size;     byte [][] data;      public ByteArray64( long size ) {         this.size = size;         if( size == 0 ) {             data = null;         } else {             int chunks = (int)(size/CHUNK_SIZE);             int remainder = (int)(size - ((long)chunks)*CHUNK_SIZE);             data = new byte[chunks+(remainder==0?0:1)][];             for( int idx=chunks; --idx>=0; ) {                 data[idx] = new byte[(int)CHUNK_SIZE];             }             if( remainder != 0 ) {                 data[chunks] = new byte[remainder];             }         }     }     public byte get( long index ) {         if( index<0 || index>=size ) {             throw new IndexOutOfBoundsException("Error attempting to access data element "+index+".  Array is "+size+" elements long.");         }         int chunk = (int)(index/CHUNK_SIZE);         int offset = (int)(index - (((long)chunk)*CHUNK_SIZE));         return data[chunk][offset];     }     public void set( long index, byte b ) {         if( index<0 || index>=size ) {             throw new IndexOutOfBoundsException("Error attempting to access data element "+index+".  Array is "+size+" elements long.");         }         int chunk = (int)(index/CHUNK_SIZE);         int offset = (int)(index - (((long)chunk)*CHUNK_SIZE));         data[chunk][offset] = b;     }     /**      * Simulates a single read which fills the entire array via several smaller reads.      *       * @param fileInputStream      * @throws IOException      */     public void read( FileInputStream fileInputStream ) throws IOException {         if( size == 0 ) {             return;         }         for( int idx=0; idx<data.length; idx++ ) {             if( fileInputStream.read( data[idx] ) != data[idx].length ) {                 throw new IOException("short read");             }         }     }     public long size() {         return size;     } } }