Apigee Usergrid: Mass delete option missing

Question

I am using usergrid to store data for a customer project. It's got two collections carShowrooms and cars. So far I am good. But I have a scenario where I have refresh the masterdata of the collection cars. Everytime I do this, I have to delete all the existing data in cars and replace it with incoming cars data from the master inventory system.

Now, with the docu in https://www.npmjs.org/package/usergrid, I see that I can only destroy one car at a time.

car.destroy(function(err){
    if (err){
        //error - car not deleted
        //winston log - tbd
    } else {
        //success - car deleted
    }
});

This is ok for smaller showrooms, but bigger multibrand showrooms have variety of cars - sometimes even upto 50 different varieties (8 car brands * approx. 8 different options).

Is there a mass delete option? can someone please point me to a docu if I am missing something here.

P.S. I am new to usergrid, if this is a repeated question, please mark so and point me to the right url

Answer 1

If you're so inclined, I've written a Node.js bulk deleter that runs delete requests in parallel. It takes approximately 3 minutes to delete 1000 entities.

Here's an always up-to-date gist, and a copy for SO:

// Installation 

// 1. Install Node.js http://nodejs.org/download/
// 2. In Terminal, cd (navigate) to the directory where you saved this file
// 3. Run 'npm install request async'
// 4. Edit the script config below with your token, org, app, and collection name.
// 5. To run the script, at the Terminal prompt, run 'node api_baas_deleter.js'

// Config

var access_token = "{token}";
var as_basepath = "http://api.usergrid.com/{org}/{app}/"; // You need the trailing slash!
var collection = "{collection_name}";

// End Config

var request = require('request');
var async = require('async');

var authstring = "access_token=" + access_token;

var total = 0;
var startTime = Date.now();

function deleteRecords(callback) {
    request.get({
        url: as_basepath + collection + "?" + authstring,
        json: true
    }, function(e, r, body) {
        if (body.count === undefined) {
            var err = "Error: invalid endpoint. Check your basepath and collection name.";
            console.log(err);
            if (typeof(callback) === 'function') {
                callback(err)
            }
        } else {
            // console.log("Found " + body.count + " entities");
            if (body.count > 0) {
                var deletes = [];
                for (var i = 0; i < body.count; i++) {
                    deletes.push({
                        url: as_basepath + collection + "/" + body.entities[i].uuid + "?" + authstring,
                        json: true
                    });
                    console.log("Deleting " + body.entities[i].uuid)
                }
                async.each(deletes, function(options, callback) {
                    request.del(options, function(e, r, body) {
                        if (r.statusCode === 200) {
                            total++;
                        }
                        callback(e);
                    });
                }, function(err) {
                    setTimeout(function() {
                        deleteRecords(collection, function(e) {
                            callback(e);
                        });
                    }, 600); // Mandatory, since it seems to not retrieve entities if you make a request in < 600ms
                });
            } else {
                var timeInMinutes = minutesFromMs(Date.now() - startTime);
                console.log("Deleted " + total + " entities in " + timeInMinutes + " minute" + ((timeInMinutes > 1 || timeInMinutes < 1) ? "s" : ""));
                if (typeof(callback) === 'function') {
                    callback()
                }
            }
        }
    });
}

function minutesFromMs(time) {
    return Math.round(((time % 86400000) % 3600000) / 60000).toString();
}

deleteRecords();

Answer 2

There currently isn't a mass delete function in the Usergrid Node SDK, but you can create one. This is how I added a monkey-patched delete-by-query function into the Node SDK:

Usergrid.client.prototype.delete = function(opts, callback) {
  if (_.isFunction(opts)) { callback = opts; opts = undefined; }

  if (!opts.qs.q) { opts.qs.q = '*'; }

  var options = {
    method: 'DELETE',
    endpoint: opts.type,
    qs: opts.qs
  };
  var self = this;
  this.request(options, function (err, data) {
    if (err && self.logging) {
      console.log('entities could not be deleted');
    }
    if (typeof(callback) === 'function') {
      callback(err, data);
    }
  });
};

Hope that helps! Scott