Python 3.4 - Multipart Post Using Standard Library

Question

Does anyone have an example on how to do a multipart post in Python 3.4 without using a 3rd party library like requests?

I am having issues porting my old Python 2 code to Python 3.4.

Here is the python 2 encoding code:

def _encode_multipart_formdata(self, fields, files):
    boundary = mimetools.choose_boundary()
    buf = StringIO()
    for (key, value) in fields.iteritems():
        buf.write('--%s\r\n' % boundary)
        buf.write('Content-Disposition: form-data; name="%s"' % key)
        buf.write('\r\n\r\n' + self._tostr(value) + '\r\n')
    for (key, filepath, filename) in files:
        if os.path.isfile(filepath):
            buf.write('--%s\r\n' % boundary)
            buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
            buf.write('Content-Type: %s\r\n' % (self._get_content_type3(filename)))
            file = open(filepath, "rb")
            try:
                buf.write('\r\n' + file.read() + '\r\n')
            finally:
                file.close()
    buf.write('--' + boundary + '--\r\n\r\n')
    buf = buf.getvalue()
    content_type = 'multipart/form-data; boundary=%s' % boundary
    return content_type, buf

The I figured out I can replace the mimetools.choose_boundary() with the following:

import email.generator
print (email.generator._make_boundary())

For the _get_content_type3() method, I am doing the following:

def _get_content_type(self, filename):
        return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

When I change the StringIO to BytesIO in using Python3.4, the data never seems to be put into the POST method.

Any suggestions?

Answer 1

Yes, email.generator._make_boundary() would work:

import email.generator
import io
import shutil

def _encode_multipart_formdata(self, fields, files):
    boundary = email.generator._make_boundary()
    buf = io.BytesIO()
    textwriter = io.TextIOWrapper(
        buf, 'utf8', newline='', write_through=True)

    for (key, value) in fields.items():
        textwriter.write(
            '--{boundary}\r\n'
            'Content-Disposition: form-data; name="{key}"\r\n\r\n'
            '{value}\r\n'.format(
                boundary=boundary, key=key, value=value))

    for (key, filepath, filename) in files:
        if os.path.isfile(filepath):
            textwriter.write(
                '--{boundary}\r\n'
                'Content-Disposition: form-data; name="{key}"; '
                'filename="{filename}"\r\n'
                'Content-Type: {content_type}\r\n\r\n'.format(
                    boundary=boundary, key=key, filename=filename,
                    content_type=self._get_content_type3(filename)))
            with open(filepath, "rb") as f:
                shutil.copyfileobj(f, buf)
            textwriter.write('\r\n')

    textwriter.write('--{}--\r\n\r\n'.format(boundary))
    content_type = 'multipart/form-data; boundary={}'.format(boundary)
    return content_type, buf.getvalue()

This uses a io.TextIOWrapper() object to make header formatting and encoding easier (bytes objects don't support formatting operations; you'll have to wait for Python 3.5 which adds % support).

If you insist on using the email package for the whole job, take into account that you'll need twice the memory; once to hold the email.mime objects, and again to hold the written result:

from email.mime import multipart, nonmultipart, text
from email.generator import BytesGenerator
from email import policy
from io import BytesIO

def _encode_multipart_formdata(self, fields, files):
    msg = multipart.MIMEMultipart('form-data')

    for (key, value) in fields.items():
        part = text.MIMEText(value)
        part['Content-Disposition'] = 'form-data; name="{}"'.format(key)
        msg.attach(part)

    for (key, filepath, filename) in files:
        if os.path.isfile(filepath):
            ct = self._get_content_type3(filename)
            part = nonmultipart.MIMENonMultipart(*ct.split('/'))
            part['Content-Disposition'] = (
                'form-data; name="{}"; filename="{}"'.format(
                    key, filename))
            with open(filepath, "rb") as f:
                part.set_payload(f.read())
            msg.attach(part)

    body = BytesIO()
    generator = BytesGenerator(
        body, mangle_from_=False, policy=policy.HTTP)
    generator.flatten(msg)
    return msg['content-type'], body.getvalue().partition(b'\r\n\r\n')[-1]

The result is otherwise basically the same, with the addition of some MIME-Version and Content-Transfer-Encoding headers.