In NodeJS, how to download files from S3

Question

In ExpressJS, I would like to download files previously uploaded to an Amazon S3 bucket.

Here is my current route:

const express = require('express');
const AWS = require('aws-sdk');
const mammoth = require('mammoth');
const fs = require('fs').promises
const path = require('path')
const router = express.Router();

router.put('/:id/download', async (req, res, next) => {
  console.log('hitting download route')

  var id = req.params.id;
  let upload = await Upload.query().findById( id ).eager('user');

  console.log("file to download is: ", upload.name)

  AWS.config.update({
    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
  });

  const s3 = new AWS.S3();


  // var fileStream = fs.createWriteStream('/tmp/file.docx');
  // var s3Stream = s3.getObject(params).createReadStream();

  const downloadFromS3 = async () => {
    const params = {
      Bucket: process.env.AWS_BUCKET, 
      Key: upload.file_url.split("com/").reverse()[0]
    };
    const { Body } = await s3.getObject(params).promise()
    await fs.writeFile(`${ __dirname }/download.docx`, Body)

    return Body
  }


  // mammoth.convertToHtml({ path: '/Users/dariusgoore/Downloads/1585930968750.docx' })
  //   .then(async function(result) {
  //     await Upload.query().findById( id )
  //       .patch({
  //          html: result.value,
  //          conversion_messages: result.messages
  //       })  
  //     res.json(result);
  //   })

    // .done();

  res.send(downloadFromS3)
});

I get no errors, but the file is not created, or if I manually create the file, it remains empty.

Answer 1

If I've understood you correctly the issue is that you're not waiting for the file to be written to the local file system, you're returning it in the response via express.

Give this code a go.

const express = require('express')
const AWS = require('aws-sdk')
const mammoth = require('mammoth')

const fs = require('fs').promises
const path = require('path')

const router = express.Router()

const s3 = new AWS.S3()

AWS.config.update({
    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
})

const downloadFromS3 = async (key, location) => {
    const params = {
        Bucket: process.env.AWS_BUCKET,
        Key: key,
    }

    const { Body } = await s3.getObject(params).promise()
    await fs.writeFile(location, Body)

    return true
}

router.put('/:id/download', async (req, res, next) => {
    console.log('hitting download route')

    const upload = await Upload.query()
        .findById(req.params.id)
        .eager('user')

    console.log('file to download is: ', upload.name)

    const key = upload.file_url.split('com/').reverse()[0]

    const location = `${__dirname}/${key}.docx`

    await downloadFromS3(key, location)

    res.send({ key, location })
})

Answer 2

import { S3 } from 'aws-sdk';
import fs from 'fs';

export default class S3Service {
    s3: S3;
    constructor() {
        this.s3 = new S3({
            apiVersion: *****,
            region: ********
        });
    }
    
    //Download File
    async download(bucketName: string, keyName: string, localDest?: string): Promise<any> {
        if (typeof localDest == 'undefined') {
            localDest = keyName;
        }
        const params = {
            Bucket: bucketName,
            Key: keyName
        };
        console.log("params: ", params);

        let writeStream = fs.createWriteStream(localDest);

        return new Promise<any>((resolve, reject) => {
            const readStream = this.s3.getObject(params).createReadStream();

            // Error handling in read stream
            readStream.on("error", (e) => {
                console.error(e);
                reject(e);
            });

            // Resolve only if we are done writing
            writeStream.once('finish', () => {
                resolve(keyName);
            });

            // pipe will automatically finish the write stream once done
            readStream.pipe(writeStream);
        });
     }
  }