AWS Lambda in VPC doesn't have internet access behind NAT

Question

My problem is that a Lambda function that I run behind NAT inside a VPC with an IGW doesn't have access to anything on the Internet.

What I'm trying to do is creating a VPC that has:

• Internet Gateway;
• 2 private subnets (PrivateA and PrivateB) in availability zones A and B respectively;
• 1 public subnet (PublicA) in availability zone A
• NAT Gateway in PublicA subnet
• PrivateA and PrivateB have a route table that routes 0.0.0.0/0 to the NAT Gateway.
• PublicA has a route table that routes 0.0.0.0/0 to the Internet Gateway.
• Private subnets, as well as the public subnet have Access Control Lists allowing all Ingress and Egress traffic.

That part kind of works.

Next, I want to create a Lambda function inside the VPC. I put it into PrivateA and PrivateB and assign to it a Security Group that allows all Egress and Ingress traffic.

Below is a self-contained example (the whole template) that reproduces the issue. I've read all the possible docs and articles on the Internet so I would very much appreciate it if anybody could point me in the right direction.

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Resources": {

    "Vpc": {
      "Type": "AWS::EC2::VPC",
      "Properties": {
        "CidrBlock": "10.0.0.0/16",
        "EnableDnsSupport": true,
        "EnableDnsHostnames": true,
        "InstanceTenancy": "default"
      }
    },

    "InternetGateway": {
      "Type": "AWS::EC2::InternetGateway"
    },

    "VpcGatewayAttachment": {
      "Type": "AWS::EC2::VPCGatewayAttachment",
      "Properties": {
        "VpcId": { "Ref": "Vpc" },
        "InternetGatewayId": { "Ref": "InternetGateway" }
      }
    },

    "ElasticIP":{
      "Type": "AWS::EC2::EIP",
      "Properties": {
        "Domain": "vpc"
      }
    },

    "NatGateway": {
      "Type": "AWS::EC2::NatGateway",
      "DependsOn": [ "VpcGatewayAttachment" ],
      "Properties": {
        "AllocationId": { "Fn::GetAtt": [ "ElasticIP", "AllocationId" ] },
        "SubnetId": { "Ref": "SubnetAPublic" }
      }
    },

    "SubnetAPublic": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": { "Fn::Select" : [ "0", { "Fn::GetAZs" : "" } ] },
        "CidrBlock": "10.0.0.0/19",
        "MapPublicIpOnLaunch": true,
        "VpcId": { "Ref": "Vpc" }
      }
    },

    "SubnetAPrivate": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": { "Fn::Select" : [ "0", { "Fn::GetAZs" : "" } ] },
        "CidrBlock": "10.0.64.0/19",
        "VpcId": { "Ref": "Vpc" }
      }
    },

    "SubnetBPrivate": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": { "Fn::Select" : [ "1", { "Fn::GetAZs" : "" } ] },
        "CidrBlock": "10.0.96.0/19",
        "VpcId": { "Ref": "Vpc" }
      }
    },

    "RouteTablePublic": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "VpcId": { "Ref": "Vpc" }
      }
    },

    "RouteTablePrivate": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "VpcId": { "Ref": "Vpc" }
      }
    },

    "RouteTableAssociationAPublic": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "SubnetId": { "Ref": "SubnetAPublic" },
        "RouteTableId": { "Ref": "RouteTablePublic" }
      }
    },

    "RouteTableAssociationAPrivate": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "SubnetId": { "Ref": "SubnetAPrivate" },
        "RouteTableId": { "Ref": "RouteTablePrivate" }
      }
    },

    "RouteTableAssociationBPrivate": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "SubnetId": { "Ref": "SubnetBPrivate" },
        "RouteTableId": { "Ref": "RouteTablePrivate" }
      }
    },

    "RouteTablePrivateInternetRoute": {
      "Type": "AWS::EC2::Route",
      "DependsOn": [ "VpcGatewayAttachment" ],
      "Properties": {
        "RouteTableId": { "Ref": "RouteTablePrivate" },
        "DestinationCidrBlock": "0.0.0.0/0",
        "NatGatewayId": { "Ref": "NatGateway" }
      }
    },

    "RouteTablePublicInternetRoute": {
      "Type": "AWS::EC2::Route",
      "DependsOn": [ "VpcGatewayAttachment" ],
      "Properties": {
        "RouteTableId": { "Ref": "RouteTablePublic" },
        "DestinationCidrBlock": "0.0.0.0/0",
        "GatewayId": { "Ref": "InternetGateway" }
      }
    },

    "NetworkAclPublic": {
      "Type": "AWS::EC2::NetworkAcl",
      "Properties": {
        "VpcId": { "Ref": "Vpc" }
      }
    },

    "NetworkAclPrivate": {
      "Type": "AWS::EC2::NetworkAcl",
      "Properties": {
        "VpcId": { "Ref": "Vpc" }
      }
    },

    "SubnetNetworkAclAssociationAPublic": {
      "Type": "AWS::EC2::SubnetNetworkAclAssociation",
      "Properties":{
        "SubnetId": { "Ref": "SubnetAPublic" },
        "NetworkAclId": { "Ref": "NetworkAclPublic" }
      }
    },

    "SubnetNetworkAclAssociationAPrivate": {
      "Type": "AWS::EC2::SubnetNetworkAclAssociation",
      "Properties":{
        "SubnetId": { "Ref": "SubnetAPrivate" },
        "NetworkAclId": { "Ref": "NetworkAclPrivate" }
      }
    },

    "SubnetNetworkAclAssociationBPrivate": {
      "Type": "AWS::EC2::SubnetNetworkAclAssociation",
      "Properties": {
        "SubnetId": { "Ref": "SubnetBPrivate" },
        "NetworkAclId": { "Ref": "NetworkAclPrivate" }
      }
    },

    "NetworkAclEntryInPublicAllowAll": {
      "Type": "AWS::EC2::NetworkAclEntry",
      "Properties": {
        "NetworkAclId": { "Ref": "NetworkAclPublic" },
        "RuleNumber": 99,
        "Protocol": -1,
        "RuleAction": "allow",
        "Egress": false,
        "CidrBlock": "0.0.0.0/0"
      }
    },

    "NetworkAclEntryOutPublicAllowAll": {
      "Type": "AWS::EC2::NetworkAclEntry",
      "Properties": {
        "NetworkAclId": { "Ref": "NetworkAclPublic" },
        "RuleNumber": 99,
        "Protocol": -1,
        "RuleAction": "allow",
        "Egress": true,
        "CidrBlock": "0.0.0.0/0"
      }
    },

    "NetworkAclEntryInPrivateAllowVpc": {
      "Type": "AWS::EC2::NetworkAclEntry",
      "Properties": {
        "NetworkAclId": { "Ref": "NetworkAclPrivate" },
        "RuleNumber": 99,
        "Protocol": -1,
        "RuleAction": "allow",
        "Egress": false,
        "CidrBlock": "0.0.0.0/16"
      }
    },

    "NetworkAclEntryOutPrivateAllowVpc": {
      "Type": "AWS::EC2::NetworkAclEntry",
      "Properties": {
        "NetworkAclId": { "Ref": "NetworkAclPrivate" },
        "RuleNumber": 99,
        "Protocol": -1,
        "RuleAction": "allow",
        "Egress": true,
        "CidrBlock": "0.0.0.0/0"
      }
    },

    "LambdasSecurityGroup": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Lambdas security group",
        "SecurityGroupEgress": [
          { "CidrIp": "0.0.0.0/0", "IpProtocol": "-1" }
        ],
        "SecurityGroupIngress": [
          { "CidrIp": "0.0.0.0/0", "IpProtocol": "-1" }
        ],
        "VpcId": { "Ref": "Vpc" }
      }
    },

    "LambdaFunctionExecutionRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Principal": { "Service": "lambda.amazonaws.com" },
              "Action": "sts:AssumeRole"
            }
          ]
        },
        "ManagedPolicyArns": [
          "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
          "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
        ]
      }
    },

    "LambdaFunction": {
      "Type": "AWS::Lambda::Function",
      "Properties": {
        "Handler": "index.lambda_handler",
        "Runtime": "python2.7",
        "Role": {
          "Fn::GetAtt": ["LambdaFunctionExecutionRole", "Arn"]
        },
        "Code": {
          "ZipFile": {
            "Fn::Join": ["\n", [
              "import urllib2",
              "def lambda_handler(event, context):",
              "\tresponse = urllib2.urlopen('http://python.org/')",
              "\treturn response.read()"
            ]]
          }
        },
        "VpcConfig": {
          "SecurityGroupIds": [
            { "Fn::GetAtt": [ "LambdasSecurityGroup", "GroupId"] }
          ],
          "SubnetIds": [
            { "Ref": "SubnetAPrivate" },
            { "Ref": "SubnetBPrivate" }
          ]
        }
      }
    }
  }
}

Answer 1

The cause of the failed connectivity lies within your ACL config for "NetworkAclEntryInPrivateAllowVpc" and "NetworkAclEntryOutPrivateAllowVpc".

If you open that CIDR block from "0.0.0.0/16" to "0.0.0.0/0", Lambda can access the internet.

I'm not that knowledgeable about NAT, but it seems that the NAT traffic is blocked by that ACL rule.