How to execute server side file handling operations in Meteor?

Question

I'm storing Word(.docx) files using GridFS on the server. I'd like to be able to merge the documents into one Word file by using the docx-builder NPM package.

Here's how I am uploading the files:

Meteor.methods({
    uploadFiles: function (files) {
      check(files, [Object]);

      if (files.length < 1)
        throw new Meteor.Error("invalid-files", "No files were uploaded");

      var documentPaths = [];

      _.each(files, function (file) {
        ActivityFiles.insert(file, function (error, fileObj) {
          if (error) {
            console.log("Could not upload file");
          } else {
            documentPaths.push("/cfs/files/activities/" + fileObj._id);
          }
        });
      });

      return documentPaths;
    }
})

How can I go about doing this on the server side ? I can only do this server side because the package I'm using requires the fs package which cannot be executed client side.

Here's how I am trying to work on this at the moment. From the client, I'm calling the following method (which is declared as a Meteor.method):

print: function(programId) {
  // Get the program by ID.
  var program = Programs.findOne(programId);
  // Create a new document.
  var docx = new docxbuilder.Document();
  // Go through all activities in the program.
  program.activityIds.forEach(function(activityId) {
    // Create a temporary server side folder to store activity files.
    const tempDir = fs.mkdtempSync('/tmp/');
    // Get the activity by ID.
    var activity = Activities.findOne(activityId);
    // Get the document by ID.
    var document = ActivityFiles.findOne(activity.documents.pop()._id);
    // Declare file path to where file will be read.
    const filePath = tempDir + sep + document.name();
    // Create stream to write to path.
    const fileStream = fs.createWriteStream(filePath);
    // Read from document, write to file.
    document.createReadStream().pipe(fileStream);
    // Insert into final document when finished writinf to file.
    fileStream.on('finish', () => {
      docx.insertDocxSync(filePath);
      // Delete file when operation is completed.
      fs.unlinkSync(filePath);
    });
  });
  // Save the merged document.
  docx.save('/tmp' + sep + 'output.docx', function (error) {
    if (error) {
      console.log(error);
    }
    // Insert into Collection so client can access merged document.
    Fiber = Npm.require('fibers');
    Fiber(function() {
      ProgramFiles.insert('/tmp' + sep + 'output.docx');
    }).run();
  });
}

However, when I'm downloading the final document from the ProgramFiles collection on the client side, the document is an empty Word document.

What's going wrong here ?

---

I've incorporated @FrederickStark's answer into my code. Just stuck on this part now.

---

Here's another try:

'click .merge-icon': (e) => {
    var programId = Router.current().url.split('/').pop();
    var programObj = Programs.findOne(programId);
    var insertedDocuments = [];
    programObj.activityIds.forEach(function(activityId) {
      var activityObj = Activities.findOne(activityId);
      var documentObj = ActivityFiles.findOne(activityObj.documents.pop()._id);
      JSZipUtils.getBinaryContent(documentObj.url(), callback);
      function callback(error, content) {
        var zip = new JSZip(content);
        var doc = new Docxtemplater().loadZip(zip);
        var xml = zip.files[doc.fileTypeConfig.textPath].asText();
        xml = xml.substring(xml.indexOf("<w:body>") + 8);
        xml = xml.substring(0, xml.indexOf("</w:body>"));
        xml = xml.substring(0, xml.indexOf("<w:sectPr"));
        insertedDocuments.push(xml);
      }
    });
    JSZipUtils.getBinaryContent('/assets/template.docx', callback);
    function callback(error, content) {
      var zip = new JSZip(content);
      var doc = new Docxtemplater().loadZip(zip);
      console.log(doc);
      setData(doc);
    }


    function setData(doc) {
      doc.setData({
        // Insert blank line between contents.
        inserted_docs_formatted: insertedDocuments.join('<w:br/><w:br/>')
        // The template file must use a `{@inserted_docs_formatted}` placeholder
        // that will be replaced by the above value.
      });

      doc.render();

      useResult(doc);
    }

    function useResult(doc) {
      var out = doc.getZip().generate({
        type: 'blob',
        mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
      });
      saveAs(out, 'output.docx');
    }

Answer 1

Looking at the docs for docx-builder, it only supports reading docx files from the file system. The issue with calling document.url() is that it gives you a url that you can access over http, not a path on the file system.

So to use GridFS you will first need to write the files out to a temporary folder before docx-builder can read them.

import fs from 'fs';
import { sep } from 'path';
const tempDir = fs.mkdtempSync('/tmp/' + sep);

program.activityIds.forEach(function(activityId) {
  var activity = Activities.findOne(activityId);
  console.log(activity);
  var document = ActivityFiles.findOne(activity.documents.pop()._id);
  documents.push(document);

  // Build a file path in the temp folder
  const filePath = tempDir + sep + document.name();

  // Write the document to the file system using streams
  const fileStream = fs.createWriteStream(filePath);
  document.createReadStream().pipe(fileStream);

  // When the stream has finished writing the file, add it to your docx
  fileStream.on('finish', () => {
    console.log(filePath);
    docx.insertDocxSync(filePath);
    // Delete the file after you're done
    fs.unlinkSync(filePath);
  });

});

I suspect you can do this synchronously using fs.writeFileSync(filePath, document.data) but wasn't sure, so didn't use it in the example.

Alternatively you could look for a docx package that can support reading from a stream or buffer, and then you won't need temp files.

Answer 2

I can only do this server side because the package I'm using requires the fs package which cannot be executed client side.

Whereas it is true that the docx-builder library that you currently use depends on fs (hence on a Node environment), it is possible to use directly its dependency docxtemplater and to use it only on Client (browser) side.

Very much like docx-builder works, you start with a "standard" template file and you can merge local docx files into it, then generate the resulting docx file and save it locally or send it to your server.

The essential steps of achieving what you are trying to do (i.e. merging docx files) but on Client side would be:

1. Retrieve files on Client side if they already exist in the network, or let the user choose files from their local File System through File type input and HTML5 File API:

<input type="file" id="fileInput" />

2. Read the file content and extract its body, as done in docx-builder:

var insertedDocsFormatted = [];

// If the file is locally provided through File type input:
document.getElementById('fileInput').addEventListener('change', function (evt) {
  var file = evt.currentTarget.files[0],
      fr = new FileReader();

  fr.onload = function () {
    insertDocContent(fr.result);
  };
  fr.readAsArrayBuffer(file);
});

// Or if the file already exists on a server:
JSZipUtils.getBinaryContent(url, function (err, content) {
  insertDocContent(content);
});

function insertDocContent(content) {
  var zip = new JSZip(content),
      doc = new Docxtemplater().loadZip(zip);

  var xml = zip.files[doc.fileTypeConfig.textPath].asText();

  // Inspired from https://github.com/raulbojalil/docx-builder
  xml = xml.substring(xml.indexOf("<w:body>") + 8);
  xml = xml.substring(0, xml.indexOf("</w:body>"));
  xml = xml.substring(0, xml.indexOf("<w:sectPr"));

  // Keep for later use.
  insertedDocsFormatted.push(xml);
}

3. Once all files to be merged are processed, load the starter template file:

// 'template.docx' is a static file on your server (e.g. in `public/` folder)
// But it could even be replaced by a user locally provided file,
// as done in step 2 above.
JSZipUtils.getBinaryContent('template.docx', callback);

function callback(err, content) {
  var zip = new JSZip(content);
  var doc = new Docxtemplater().loadZip(zip);

  setData(doc);
}

4. Join the contents and define a data key with formatting, so that it is inserted into the template file, then render the document:

function setData(doc) {
  doc.setData({
    // Insert blank line between contents.
    inserted_docs_formatted: insertedDocsFormatted.join('<w:br/><w:br/>')
    // The template file must use a `{@inserted_docs_formatted}` placeholder
    // that will be replaced by the above value.
  });

  doc.render();

  useResult(doc);
}

5. Either prompt a "Save As" dialog or send the File (blob) to your server.

function useResult(doc) {
  var out = doc.getZip().generate({
    type: 'blob',
    mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
  });
  saveAs(out, 'output.docx');
}

Online demo: https://ghybs.github.io/docx-builder-demo/ (no Server side processing at all, pure Client side logic)

Source code: https://github.com/ghybs/docx-builder-demo

Libraries:

• JSZip to unzip docx files
• JSZipUtils to read a file as binary (that can be fed into JSZip)
• docxtemplater to manipulate a docx file
• FileSaver.js for the "Save As" dialog (optional)