How to get the CloudWatch Agent and Metric Filters to Report Dimensions

Question

Setup

CloudWatch Agent running on an EC2 instance reports audit logs to CloudWatch. Metric Filter in CloudWatch creates metrics for successful logins, failed logins, ect... when logs are reported.

Problem

Metrics created through the Metric Filter does not assign dimensions so I cant query CloudWatch to get a set of metric statistics by InstanceId. This would be extremely useful because I want to know the audit metrics per machine not per log group.

Comments

Attaching dimensions is pretty easy using the put-metric-data command. I am able to tag the metrics with the dimension for InstanceId and then retrieve only those metrics using get-metric-statistics. Is this kind of functionality not possible using the Metric Filters + CloudWatch Agent setup? What would be a possible workaround?

Answer 1

So what you need to do is create a lambda that has an event source set to the log group that you are wanting to create metrics for. I created metric objects that would check each log for certain patterns. The code below is the gist of what you will need to do. I had to rip out some stuff that wasnt applicable. If anyone tries it let me know if it has problems.

I created a lambda (notice the metrics array):

var zlib = require('zlib');
var Q = require('q');
var cloudwatchAgent = require('cloudwatchAgent');

var metrics = [
{
    "name": "SuccessfulLogins",
    "patterns": ["session opened for user", "successful logon"],
    "dimensionName":"HostName",
    "namespace":"UserStats"
}];

exports.handler = function (event, context) {
var payload = new Buffer(event.awslogs.data, 'base64');

zlib.gunzip(payload, function (e, result) {
if (e) {
  context.fail(e);
} else {
  result = JSON.parse(result.toString('utf-8'));
  console.log('Decoded payload: ', JSON.stringify(result));

metrics.forEach(function (metric) {
        promises.push(cloudwatchAgent.processMetric(metric, result.logStream, result.logEvents));
  });

Q.all(promises).fail(function (error) {
        context.fail('Error processing log events: ' + error);
  }).done(function () {
        context.succeed('Successfully processed log events.');
  });
  }
 });
};

That lambda calls this function:

var sdk = require('aws-sdk');
var Q = require('q');
var cloudwatch = new sdk.CloudWatch();

function processMetric(metric, streamName, logs) {
console.log('Checking metric ' + metric.name + ' against these patterns:' + metric.patterns.toString());
var deferred = Q.defer();
var value = 0;

logs.forEach(function (logEvent) {
metric.patterns.forEach(function(pattern) {
    if(logEvent.message.toLowerCase().indexOf(pattern) > -1) {
    value++;
    }
});
});

var params = {
  MetricData: [{
      MetricName: metric.name,
      Dimensions: [{
          Name: metric.dimensionName,
          Value: streamName
        }],
      Timestamp: new Date(),
      Value: value
    }],
  Namespace: metric.namespace
};

cloudwatch.putMetricData(params, function (err, data) {
  if (err) {
    console.log(err, err.stack);
    deferred.reject(err);
  } else {
    console.log('Successfully created metric for ' + metric.name + ' with value ' + value);
    deferred.resolve(data);
  }
});

return deferred.promise;
}

module.exports.processMetric = processMetric;