how to upload file saved in memory by multer to another API

Question

I have 2 separate NodeJS APIs that uses multer to save a file in memory.

My express middleware looks like this

import multer from 'multer';
const storage = multer.memoryStorage();

export default multer({ storage }).single('image');

I am able to receive the file which is saved in memory successfully so my req.file.image looks like this

{ 
  fieldname: 'image',
  originalname: 'image-2017-08-28-13-47-31-218.png',
  encoding: '7bit',   mimetype: 'image/png',
  buffer: <Buffer 89 50 4e 47 0d 0a 1a 0 ... >, 
  size: 493181
}

After receiving the file, on the first API, I need to send it to the second API that also uses multer & express

function secondApiReceiveImage(req, res) {
  console.log(req.file)
  console.log(req.files)
  console.log(req.body)
  res.send('ok');
}

I tried sending using the following implementations

Via https://github.com/request/request#multipartform-data-multipart-form-uploads

import request from 'request';

function firstApiReceiveImage(req, res) {
  const options = {
    url:'http://SECOND_API/api/image',
    formData: { image: req.file.buffer }
  };
  request.post(options, (err, httpResponse, body) => {
    console.log('err', err);
    console.log('body', body);
  });
}

In this case, logs of req.file, req.files and req.body are all undefined on secondApiReceiveImage API handler function

My next try was with https://github.com/form-data/form-data

import multer from 'multer';
const storage = multer.memoryStorage();

export default multer({ storage }).single('image');

function firstApiReceiveImage(req, res) {
  const CRLF = '\r\n';
  const form = new FormData();
  const opts = {
    header: `${CRLF} + '--' + ${form.getBoundary()} + ${CRLF} + 'X-Custom-Header: 123' + ${CRLF} + ${CRLF}`,
    knownLength: 1
  };

  form.append('image', req.file.image.buffer, opts);
  form.submit('http://SECOND_API/api/image', (err, res) => {
    console.log('err', err);
    console.log('res', res);
  });
}

I got the same result, undefined for req.file, req.files & req.body on the second API

These are my middlewares for both APIs aside from multer BTW

app.use(compression());
app.use(helmet());
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));

I could have had an easier life if I can persist the file on the first API, but we are not allowed to save to disk in this case :(

Any advice for me?

Answer 1

I've stumbled upon the same issue and here is what I've managed to come up with.

First of all I used the form-data package to mimic the FormData data structure on the server side and here is the util file that does that.

    const FormData = require('form-data')
    const formData = new FormData()
    const config = {
       headers: {
         'Content-Type': `multipart/form-data; boundary=${formData.getBoundary()}`
       }
    }
    export default { data: formData, config }

and here is my node.js api file

import formSet from '../utils/formData'

const multer = require('multer')
const upload = multer()

const { Router } = require('express')
const router = Router()

......

router.post('/api/images', upload.single('image'), async (req, res) => {
  formSet.data.append('image', req.file.buffer, { filename: req.file.originalname })
  formSet.data.append('other_data', 'value')
  const response = await axios.post('/images', formSet.data, formSet.config)

  .......

})

Answer 2

The team was able to get through this by converting the buffer to Stream first and send form data differently.

import stream from 'stream';
import rq from 'request-promise';

const { Duplex } = stream;

function bufferToStream(buffer) {
  const duplexStream = new Duplex();
  duplexStream.push(buffer);
  duplexStream.push(null);
  return duplexStream;
}

async function store({
  originalname, mimetype, size, buffer,
}) {
  logger.debug(`Saving image name:${originalname} type:${mimetype} size:${size}`);

  const formData = {
    image: {
      value: bufferToStream(buffer),
      options: {
        filename: originalname,
        contentType: mimetype,
        knownLength: size,
      },
    },
  };

  const options = Object.assign({}, IMAGE_SAVE_ENDPOINT, { formData });
  const response = await rq(options);
  return response.id;
}

Answer 3

Using Request, formdata should be set to the buffer:

formData: { image: req.file.buffer }