How do I host static files on AWS S3 using next?

Question

I'm coming from Django background where static files are mostly stored on S3, I'm trying to understand how it works on NodeJS since I'd like to migrate an app from Django/React to NodeJS/NextJS/ExpressJS/React.

I don't get how & where to store my static files (client side js, css, images) on a production environment? I think to know how to upload to s3 and manage dynamic files since the work is done by the users through the express api, but I'm searching for something where I can batch upload all the public files to s3 on deploy (is this even the right way to do with express?).

Since I'd like to deploy to Heroku I know that they have a policy of not keeping those static files, (In Django I use "collectstatic" command to batch upload all my static files to S3 on each deploy), how & from where do you serve those files in here?

Any advice would help.

Answer 1

next.config.js, set assetPrefix to your aws s3 link in production.

const isDev = process.env.NODE_ENV !== 'production';
const version = require('./package.json').version;

assetPrefix: isDev ? '' : `https://${process.env.AWS_REGION}.amazonaws.com/${process.env.AWS_S3_BUCKET_NAME}/${version}`,

collectstatic.js, run on postbuild release.

require('dotenv').config();
const fs = require('fs');
const readDir = require('recursive-readdir');
const path = require('path');
const AWS = require('aws-sdk');
const mime = require('mime-types');
const version = require('./package.json').version;

// You will run this script from your CI/Pipeline after build has completed.
// It will read the content of the build directory and upload to S3 (live assets bucket)
// Every deployment is immutable. Cache will be invalidated every time you deploy.
AWS.config.update({
  region: process.env.AWS_S3_REGION,
  accessKeyId: process.env.AWS_S3_ACCESS_KEY,
  secretAccessKey: process.env.AWS_S3_SECRET_KEY,
  maxRetries: 3
});

// Retrive al the files path in the build directory
const getDirectoryFilesRecursive = (dir, ignores = []) => {
  return new Promise((resolve, reject) => {
    readDir(dir, ignores, (err, files) => (err ? reject(err) : resolve(files)));
  });
};

// The Key will look like this: _next/public/<buildid>/pages/index.js
// the <buildid> is exposed by nextJS and it's unique per deployment.
// See: https://nextjs.org/blog/next-7/#static-cdn-support
const generateFileKey = (fileName, toReplace, replaced) => {
  // I'm interested in only the last part of the file: '/some/path/.next/build-manifest.json',
  const S3objectPath = fileName.split(toReplace)[1];
  return version + replaced + S3objectPath;
};

const s3 = new AWS.S3();

const uploadToS3 = async (fileArray, toReplace, replaced) => {
  try {
    fileArray.map(file => {
      // Configuring parameters for S3 Object
      const S3params = {
        Bucket: process.env.AWS_S3_BUCKET_NAME,
        Body: fs.createReadStream(file),
        Key: generateFileKey(file, toReplace, replaced),
        ACL: 'public-read',
        ContentType: String(mime.lookup(file)),
        ContentEncoding: 'utf-8',
        CacheControl: 'immutable,max-age=31536000,public'
      };

      s3.upload(S3params, function(err, data) {
        if (err) {
          // Set the exit code while letting
          // the process exit gracefully.
          console.error(err);
          process.exitCode = 1;
        } else {
          console.log(`Assets uploaded to S3:`, data.key);
        }
      });
    });
  } catch (error) {
    console.error(error);
  }
};

// Start function
// getDirectoryFilesRecursive(path, ignore);
const start = async function(dict) {
  for (var i = 0; i < dict.length; i++) {
    const files = await getDirectoryFilesRecursive(path.resolve(__dirname, dict[i].filePath), ['.DS_Store', 'BUILD_ID']);
    uploadToS3(files, dict[i].toReplace, dict[i].replaced);
  }
}

// Call start
start([
  {
    filePath: '.next',
    toReplace: '.next/',
    replaced: '/_next/'
  }
]);

Run node collectstatic.js to upload all your assets to S3.