Unable to upload huge file on google drive using python

Question

I am trying to upload files to Google drive by Google API using the following code

import httplib2
from apiclient import discovery
from httplib2 import Http
from oauth2client import file, client, tools
try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
    flags = None

SCOPES =['https://www.googleapis.com/auth/drive','https://www.googleapis.com/auth/drive.file','https://www.googleapis.com/auth/drive.appdata', 'https://www.googleapis.com/auth/drive.apps.readonly']
store = file.Storage('scope.json')
creds = store.get()
if not creds or creds.invalid:
    flow = client.flow_from_clientsecrets('client_secret.json', SCOPES)
    creds = tools.run_flow(flow, store, flags) if flags else tools.run(flow, store)
    DRIVE = discovery.build('drive', 'v3', http=creds.authorize(Http()))
else:
    credentials = creds
    http = credentials.authorize(httplib2.Http())
    DRIVE = discovery.build('drive', 'v3', http=http)

FILES = (
    ('/home/vkm/mayur/Demo_Google_API.zip', 'application/vmd.google-apps.document'),
)

for filename, mimeType in FILES:
    metadata = {'name': filename}
    if mimeType:
        metadata['mimeType'] = mimeType
    res = DRIVE.files().create(body=metadata, media_body=filename).execute()
    if res:
        print('Uploaded "%s" (%s)' % (filename, res['mimeType']))

I am able to upload the small files but when I am trying with 8GB of the file, it is giving MemorryErro.Please find the error message that I am getting.

Traceback (most recent call last):
  File "demo.py", line 46, in <module>
    res = DRIVE.files().create(body=metadata, media_body=filename).execute()
  File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 853, in method
    payload = media_upload.getbytes(0, media_upload.size())
  File "/usr/local/lib/python2.7/dist-packages/googleapiclient/http.py", line 482, in getbytes
    return self._fd.read(length)
MemoryError

Answer 1

Vikram's comment revealed a problem in mhawke's answer: next_chunk needs to be called upon the return value of:

request = DRIVE.files().create(body=metadata, media_body=media)

not on the return value of request.execute().

Here is a snippet of Python code I verified as working on files up to 10MB to my Google Drive account:

# Upload some file that just happens to be binary (we
# don't care about metadata, just upload it without
# translation):
the_file_to_upload = 'some_binary_file'
metadata = {'name': the_file_to_upload}
# Note the chunksize restrictions given in
# https://developers.google.com/api-client-library/python/guide/media_upload
media = MediaFileUpload(the_file_to_upload,
                        chunksize=1024 * 1024,
                        # Not sure whether or not this mimetypes is necessary:
                        mimetype='text/plain',
                        resumable=True)
request = drive_service.files().create(body=metadata, media_body=media)
response = None
while response is None:
    status, response = request.next_chunk()
    if status:
        print("Uploaded %d%%." % int(status.progress() * 100))
print("Upload of {} is complete.".format(the_file_to_upload))

Here is a snippet of Python code that downloads the same file, but to a different file, so that I can use sha1sum to verify file has not been altered by Google Drive going in and out.

# Verify downloading works without translation:
request = drive_service.files().get_media(fileId=response['id'])
# Use io.FileIO. Refer to:
# https://google.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaIoBaseDownload-class.html
out_filename = the_file_to_upload + ".out"
fh = io.FileIO(out_filename, mode='wb')
downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)
done = False
while done is False:
    status, done = downloader.next_chunk()
    if status:
        print("Download %d%%." % int(status.progress() * 100))
print("Download Complete!")