How to upload a file more that 32Mb to google cloud storage using Java API

Question

I am working on to create an application that will upload flies from local machine to google cloud storage using java API. But the problem that I am facing is that the API doesn't allow me to upload a file more than 32 Mb and the file I want to upload are over 100 Mb or may be 200 Mb. I would really appreciate help on this and would like to know what are the best practices or API to upload files to Google Cloud Storage.

I am using the following code.

UploadFileServlet.java

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.nio.channels.Channels;

import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.FileItemStream;
import org.apache.commons.fileupload.servlet.ServletFileUpload;

import com.google.appengine.api.blobstore.BlobKey;
import com.google.appengine.api.users.User;
import com.google.appengine.api.users.UserService;
import com.google.appengine.api.users.UserServiceFactory;
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsOutputChannel;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.igt.service.StorageService;

public class UploadFileServlet extends HttpServlet {

private static final long serialVersionUID = 1L;
private StorageService storage = new StorageService();
private static int BUFFER_SIZE = 1024 * 1024 * 10;
public static final String BUCKET_NAME = "my-bucket-test";
private final GcsService gcsService =  GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());

@Override
public void doPost(HttpServletRequest req, HttpServletResponse resp)
        throws IOException {

    UserService userService = UserServiceFactory.getUserService();
    User user = userService.getCurrentUser();

    if (user != null) {
        resp.setContentType("text/plain");
        resp.getWriter().println("Now see here your file content, that you have uploaded on storage..");

        ServletFileUpload upload = new ServletFileUpload();
        FileItemIterator iter;
        try {
            iter = upload.getItemIterator(req);
            while (iter.hasNext()) {
                FileItemStream item = iter.next();
                String fileName = item.getName();
                String mime = item.getContentType();

                GcsFileOptions options = new GcsFileOptions.Builder()
                .acl("public_read")
                .mimeType(mime)
                .build();

                GcsFilename filename = new GcsFilename(BUCKET_NAME, fileName);

                GcsOutputChannel outputChannel = gcsService.createOrReplace(filename, options);                 

                // Writing the file to input stream
                InputStream is = new BufferedInputStream(item.openStream());

                // Copying InputStream to GcsOutputChannel
                try {
                    copy(is, Channels.newOutputStream(outputChannel));
                } finally {
                    outputChannel.close();
                    is.close();
                }                                           

                resp.getWriter().println("File uploading done");
                System.out.println("File uploading done");

                // resp.getWriter().println("READ:" +
                // storage.readTextFileOnly(fileName));
                BlobKey key = storage.getBlobkey(fileName);
                if (key != null) {
                    resp.sendRedirect("/serve?blob-key=" + key.getKeyString());
                } else {
                    resp.sendRedirect("/login");
                }
                resp.sendRedirect("/login");
            }
        } catch (Exception e) {
            e.printStackTrace(resp.getWriter());
            System.out.println("Exception::" + e.getMessage());
            e.printStackTrace();
        }
    } else {
        resp.sendRedirect(userService.createLoginURL(req.getRequestURI()));
    }
}

private void copy(InputStream input, OutputStream output) throws IOException {
    byte[] buffer = new byte[BUFFER_SIZE];
    int bytesRead = input.read(buffer);
    while (bytesRead != -1) {
        output.write(buffer, 0, bytesRead);
        bytesRead = input.read(buffer);
    }
}

}

StorageService.java

import java.io.BufferedOutputStream;
import java.nio.channels.Channels;
import java.util.logging.Logger;

import com.google.appengine.api.blobstore.BlobKey;
import com.google.appengine.api.blobstore.BlobstoreService;
import com.google.appengine.api.blobstore.BlobstoreServiceFactory;
import com.google.appengine.api.files.AppEngineFile;
import com.google.appengine.api.files.FileService;
import com.google.appengine.api.files.FileServiceFactory;
import com.google.appengine.api.files.FileWriteChannel;
import com.google.appengine.api.files.GSFileOptions.GSFileOptionsBuilder;

@SuppressWarnings("deprecation")

public class StorageService {

private static final Logger log = Logger.getLogger(StorageService.class.getName());
private BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService();  


public BlobKey getBlobkey(String filename) {
    BlobKey bk = blobstoreService.createGsBlobKey("/gs/sample-bucket/"  + filename);
    return bk;
}

}

login.jsp

 <form action="/upload" method="post" enctype="multipart/form-data">

    <div><input name = "file" type="file" value="Upload" /></div>    
    <div><input type="submit" value="Upload File" /></div>
  </form>

Answer 1

A single AppEngine connection from a user cannot transfer more than 32 MB. Instead, you should have your users upload the file directly to Google Cloud Storage. Google Cloud Storage supports standard HTML form uploads. Complete documentation of this is available here: https://developers.google.com/storage/docs/reference-methods#postobject

Have the user's page submit a form post that contains all of the appropriate form fields specifying the bucket and object name. The bucket will need to be publicly writable or you'll need to include an appropriate signature field.

Set the "success_action_redirect" field to an AppEngine URL. When the upload successfully completes, the browser will be redirected to this URL.