Error calling AWS Fargate task from AWS Lambda

Question

I'm trying to call a Fargate (ECS) task from a lambda and am seeing an error pop-up. I've tried looking through the source code but since it's coming back as a response it's not clear what's going on. I'd appreciate any suggestions. The error message and my code are pasted below.

The main message is: com.amazonaws.services.ecs.model.InvalidParameterException: name cannot be blank

{                                                                                                                                                                         
  "errorMessage": "name cannot be blank. (Service: AmazonECS; Status Code: 400; Error Code: InvalidParameterException; Request ID: 15746fff-35e7-11e8-90bf-fb7a32bec470)",
  "errorType": "com.amazonaws.services.ecs.model.InvalidParameterException",                                                                                              
  "stackTrace": [                                                                                                                                                         
    "com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1630)",                                                                
    "com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1302)",                                                                  
    "com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1056)",                                                                      
    "com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)",                                                                           
    "com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)",                                                                    
    "com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)",                                                                             
    "com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)",                                                                          
    "com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)",                                                                 
    "com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)",                                                                                             
    "com.amazonaws.services.ecs.AmazonECSClient.doInvoke(AmazonECSClient.java:2742)",                                                                                     
    "com.amazonaws.services.ecs.AmazonECSClient.invoke(AmazonECSClient.java:2718)",                                                                                       
    "com.amazonaws.services.ecs.AmazonECSClient.executeRunTask(AmazonECSClient.java:2042)",                                                                               
    "com.amazonaws.services.ecs.AmazonECSClient.runTask(AmazonECSClient.java:2017)",                                                                                      
    "Lambda.triggerLoad(Lambda.scala:79)",                                                                                                                                
    "Lambda.$anonfun$handleRequest$2(Lambda.scala:27)",                                                                                                                   
    "Lambda.$anonfun$handleRequest$2$adapted(Lambda.scala:27)",                                                                                                           
    "scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:59)",                                                                                           
    "scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:52)",                                                                                          
    "scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)",                                                                                                 
    "Lambda.handleRequest(Lambda.scala:27)",                                                                                                                              
    "sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)",                                                                                                        
    "sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)",                                                                                      
    "sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)",                                                                              
    "java.lang.reflect.Method.invoke(Method.java:498)"                                                                                                                    
  ]                                                                                                                                                                       
}            

Here is the code:

import com.amazonaws.ClientConfiguration
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain
import com.amazonaws.regions.{Region, Regions}
import com.amazonaws.services.ecs.{AmazonECSClient, AmazonECSClientBuilder}
import com.amazonaws.services.ecs.model._
import com.amazonaws.services.lambda.runtime.events.S3Event
import com.amazonaws.services.lambda.runtime.{Context, RequestHandler}

import scala.collection.JavaConverters._

class Lambda extends RequestHandler[S3Event, Unit] {

  val subnets = Seq(
    "subnet-xx",
    "subnet-xx",
    "subnet-xx"
  )

  val envOverrideStrings = Seq[(String, String)](
    ("SPARK_LOCAL_IP", "127.0.0.1"),
    ("AWS_ACCESS_KEY_ID", "xx"),
    ("AWS_SECRET_ACCESS_KEY", "xx")
  )

  override def handleRequest(event: S3Event, context: Context): Unit ={
    val files = event.getRecords.asScala.map(s3 => (s3.getS3.getBucket.getName, s3.getS3.getObject.getKey))

    files.foreach(tuple => triggerLoad(tuple._1, tuple._2))
  }

  def triggerLoad(bucket: String, key: String): Unit ={

    val s3aFile = s"s3a://$bucket/$key"

    println(s3aFile)

    val vpcConfig = new AwsVpcConfiguration()
      .withAssignPublicIp("DISABLED")
      .withSubnets(subnets.toList.asJava)

    val networkConfig = new NetworkConfiguration()
      .withAwsvpcConfiguration(vpcConfig)

    val envOverrideKeyPairs = envOverrideStrings.map(pair => {
      val key = pair._1
      val value = pair._2

      val keyPair = new KeyValuePair()
        .withName(key)
        .withValue(value)

      keyPair
    })

    val overrides = new ContainerOverride()
      .withEnvironment(envOverrideKeyPairs.asJava)
      .withCommand(s3aFile)

    val containerOverrides = List(
      overrides
    ).asJava

    val taskOverride = new TaskOverride()
      .withContainerOverrides(containerOverrides)
      .withExecutionRoleArn("arn:aws:iam::xx")
      .withTaskRoleArn("arn:aws:iam::xx")

    val taskRequest = new RunTaskRequest()
      .withNetworkConfiguration(networkConfig)
      .withLaunchType("FARGATE")
      .withTaskDefinition("task-xx")
      .withCluster("default")
      .withOverrides(taskOverride)
      .withGroup("task-xx")

    val creds = new DefaultAWSCredentialsProviderChain()

    val client = AmazonECSClientBuilder.standard().withCredentials(creds).build()

    client.runTask(taskRequest)
  }

}                                                                                                                                                             

Answer 1

Ahh... of course, it was the name of the container.

I've added the last line here to the above code and it's working:

val overrides = new ContainerOverride()
  .withEnvironment(envOverrideKeyPairs.asJava)
  .withCommand(s3aFile)
  .withName("container-name")