Forwarding an uploaded file stream to another server with Node.js

Question

I'm building an Angular / Node.js application connected to a DMS (Alfresco). The server side Node.js / Express layer acts as a proxy to hide the complexity of Alfresco from the client:

Angular client <--> Node backend <--> Alfresco

This question is only about the Node.js backend.

When uploading a file I would like to forward the incoming file directly to Alfresco without temporarily storing it on the disk. With temporary disk storage this works as expected:

const fileUpload = require('express-fileupload');
const FormData = require('form-data');

// app is the express object
app.use(fileUpload({ createParentPath: true }));

app.post('/file', async (req, res) => {

    // filedata is the FormData field of the client containing the actual file
    let file = req.files.filedata;
    let tmpPath = __dirname + '/tmp/' + file.name;

    // save the file temporarily on the server
    file.mv(tmpPath, async function(err) {

        // create a new FormData Object for the upload to the DMS
        const formData = new FormData();
        formData.append('name', name);
        // creates an fs.ReadStream object which is inherited from stream.Readable
        formData.append('filedata', fs.createReadStream(tmpPath));
    
        // upload the file to the DMS
        let response = await axios.post('/files/...', formData, { headers: formData.getHeaders() });
        
        // remove the temporary file
        fs.rm(tmpPath, () => {

            // return the answer of the DMS to the client
            res.send(response.data);
        });
    });
});

Now I would like to avoid the disk access and forward the file directly to the DMS. Taking into consideration Converting a Buffer into a ReadableStream in Node.js I tried the following three alternatives.

const { Readable } = require('stream');

app.post('/file', async (req, res) => {
    let file = req.files.fileData;

    // create a new FormData Object for the upload to the DMS
    const formData = new FormData();
    formData.append('name', name);

    /* alternatives starting here */
    // Source: https://stackoverflow.com/questions/13230487/
    
    // #1
    const readable = new Readable();
    readable._read = () => {};
    readable.push(file.data);
    readable.push(null);
    
    // #2
    const readable = new Readable();
    readable._read = () => {};
    const buffer = new Buffer(file.data, 'utf-8');
    readable.push(buffer);  
    readable.push(null);
    
    // #3
    const readable = Readable.from(file.data);

    /* alternatives ending here */

    // put the Readable into the FormData object
    formData.append('filedata', readable);

    // upload the file to the DMS
    let response = await axios.post('/files/...', formData, { headers: formData.getHeaders() });

    // return the answer of the DMS to the client
    res.send(response.data);
});

Whatever alternative I try, Alfresco always complains, required fields would be missing. Nonetheless, all required fields are provided, since the example with storing the file temporarily works fine. I think, Alfresco cannot handle the stream I provide and that I have a problem to completely understand how streams work in this situation. What should I do differently?

Please note, that all error handling as well as Alfresco request configuration / API URL is omitted for the sake of readability.

Answer 1

try providing file related information such as filename, knownLength etc.

let file = req.files.fileData;

const formData = new FormData();
// buffer with file related info
formData.append(name, file.data, {
    filename: file.name,
    contentType: file.mimetype,
    knownLength: file.size
});

// upload the file to the DMS
let response = await axios.post('/files/...', formData, {headers: formData.getHeaders() });

// return the answer of the DMS to the client
res.send(response.data);