AWS Glue connections from AWS secret manager

Question

While creating JDBC connections in AWS Glue, is there any way to get the password from AWS secret manager instead of hardcoding it manually?

Answer 1

I had to do this in my current project to connect to a Cassandra DB and here's how I did it.. I pass in the actual secrets_key as a job param --SECRETS_KEY my/secrets/key

// here's method to pull from secrets manager
def retrieveSecrets(secrets_key: String) :Map[String,String] = {
    val awsSecretsClient = AWSSecretsManagerClientBuilder.defaultClient()
    val getSecretValueRequest = new GetSecretValueRequest().withSecretId(secrets_key)   
    val secretValue = awsSecretsClient.getSecretValue(getSecretValueRequest)
    val secretJson = secretValue.getSecretString()
    val result = JSON.parseFull(secretJson)
    val jsonMap:Map[String,String] = result.get.asInstanceOf[Map[String, String]]
    return jsonMap
    }

// main script where I invoke this call BEFORE the glue stuff is setup..
def main(sysArgs: Array[String]) {
  println("***** RETRIEVING Secrets from SecretsManager ****** ")
  // pull secrets from map
  val secretsMap = retrieveSecrets(secrets_key)
  val host_names = secretsMap.get("HOST_NAMES").get
  val user_id = secretsMap.get("USER_ID").get
  val password = secretsMap.get("PASSWORD").get

  // I needed to configure this BEFORE we create the spark Context
  val conf = setupCassandraConfiguration(host_names, user_id, password)    
  val sparkContext: SparkContext = new SparkContext(conf)         
  // now create the glue context using the spark context created with the cassandra options attached
  val glueContext: GlueContext = new GlueContext(sparkContext)
  Job.init(args("JOB_NAME"), glueContext, args.asJava)
  val spark = glueContext.getSparkSession

  println("***** ALL contexts created - starting loading tables ****** ")
  import spark.implicits._
  val myDF =  spark.read.format("org.apache.spark.sql.cassandra").options(Map( "table" -> "my_table", "keyspace" -> "my_keyspace" )).load()
}