Authentication to Kubernetes API of GKE cluster within Google Cloud Function

Question

What is the best way to authenticate to the Kubernetes API of a GKE cluster in the context of a Google Cloud Function? After digging into the source code of google-auth-library and @kubernetes/client-node, I came up with the solution below by using some undocumented APIs. It works, but I wonder if this is the right way to do it, and if there is something ready-to-use out there.

It is particularly strange that https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.zones.clusters#MasterAuth also returns clientCertificate and clientKey. Using those for opts.cert and opts.key instead of the access token lead to the following error:

Error from server (Forbidden): namespaces "footest" is forbidden: User "client" cannot delete namespaces in the namespace "footest": Unknown user "client"

const { auth } = require('google-auth-library');
const k8s = require('@kubernetes/client-node');

const CLUSTER_ID = 'cluster';
const ZONE = 'us-central1';

async function deleteNamespace(namespace) {
  const cluster = await getCluster(ZONE, CLUSTER_ID);
  const token = await auth.getAccessToken();
  const k8sApi = new k8s.Core_v1Api('https://' + cluster.endpoint);
  k8sApi.setDefaultAuthentication({
    applyToRequest: (opts) => {
      opts.ca = Buffer.from(cluster.masterAuth.clusterCaCertificate, 'base64');
      if (!opts.headers) {
        opts.headers = [];
      }
      opts.headers.Authorization = 'Bearer ' + token;
    },
  });

  await k8sApi.deleteNamespace(namespace, {});
}

async function getCluster(zone, clusterId) {
  const googleApi = await getGoogleApi();
  const projectId = googleApi.projectId;
  const res = await googleApi.client.request({
    url: `https://container.googleapis.com/v1/projects/${projectId}/zones/${zone}/clusters/${clusterId}`,
  });
  return res.data;
}

async function getGoogleApi() {
  const res = await auth.getApplicationDefault();
  const client = res.credential;

  // The createScopedRequired method returns true when running on GAE or a local developer
  // machine. In that case, the desired scopes must be passed in manually. When the code is
  // running in GCE or a Managed VM, the scopes are pulled from the GCE metadata server.
  // See https://cloud.google.com/compute/docs/authentication for more information.
  if (client.createScopedRequired && client.createScopedRequired()) {
    // Scopes can be specified either as an array or as a single, space-delimited string.
    const scopes = ['https://www.googleapis.com/auth/cloud-platform'];
    client = client.createScoped(scopes);
  }

  return {
    client: client,
    projectId: res.projectId,
  };
}

Answer 1

The error above seems to be due to either the user not being valid or not having the correct permissions.

You can use the @google-cloud/container library to fetch details about your cluster from GKE and use the returned information to set up the configuration in the client for @kubernetes/client-node.

In order to be able to fetch cluster information using the @google-cloud/container library, you should have a valid GCP Service Account JsonKey file pointed to by an environment variable: GOOGLE_APPLICATION_CREDENTIALS .

The following is a code snippet that fetches cluster credentials from GKE and subsequently uses the @kubernetes/client-node to interact with the K8s API.

This sample provided here follows the explanation provided in these two comments. (1 and 2)

const googleContainer = require('@google-cloud/container');
const k8s = require('@kubernetes/client-node');

// Create the Cluster Manager Client
const client = new googleContainer.v1.ClusterManagerClient();

/**
 * The following function is equivalent to the 'get-credentials' call using
 * gcloud. The client assumes that the 'GOOGLE_APPLICATION_CREDENTIALS'
 * environment variable is set to the json key file associated to your GCP
 * service account (https://cloud.google.com/docs/authentication/production#create_service_account).
 *
 * The return values of this method are the credentials that are used to update
 * the k8s config file (~/.kube/config) to add a new context when
 * 'get-credentials' is invoked by the 'gcloud' CLI
 */
async function getCredentials(cluster, zone) {
  const projectId = await client.getProjectId();
  const accessToken = await client.auth.getAccessToken();
  const request = {
    projectId: projectId,
    zone: zone,
    clusterId: cluster
  };

  const [response] = await client.getCluster(request);
  // the following are the parameters added when a new k8s context is created
  return {
    // the endpoint set as 'cluster.server'
    endpoint: response.endpoint,
    // the certificate set as 'cluster.certificate-authority-data'
    certificateAuthority: response.masterAuth.clusterCaCertificate,
    // the accessToken set as 'user.auth-provider.config.access-token'
    accessToken: accessToken
  }
}

async function listServices(cluster, zone) {
  const k8sCredentials = await getCredentials(cluster, zone);
  const k8sClientConfig = new k8s.KubeConfig();
  k8sClientConfig.loadFromOptions({
    clusters: [{
      name: `my-gke-cluster_${cluster}`,            // any name can be used here
      caData: k8sCredentials.certificateAuthority,  // <-- this is from getCredentials call
      server: `https://${k8sCredentials.endpoint}`, // <-- this is from getCredentials call
    }],
    users: [{
      name: `my-gke-cluster_${cluster}`,
      authProvider: 'gcp',                          // the is not a required field
      token: k8sCredentials.accessToken             // <-- this is from getCredentials call
    }],
    contexts: [{
      name: `my-gke-cluster_${cluster}`,
      user: `my-gke-cluster_${cluster}`,
      cluster: `my-gke-cluster_${cluster}`
    }],
    currentContext: `my-gke-cluster_${cluster}`,
  });
  const k8sApi = k8sClientConfig.makeApiClient(k8s.CoreV1Api);
  k8sApi.listNamespacedService('default').then((res) => {
    printServices(res.body.items);
  });
}

function printServices(services) {
  services.forEach(svc => {
    const name = svc.metadata.name;
    const type = svc.spec.type;
    const clusterIp = svc.spec.clusterIP;
    const externalIP = svc.spec.externalIPs || "<none>";
    let ports = "";
    svc.spec.ports.forEach((p) => ports += `${p.port}/${p.protocol},`)
    ports = ports || "<none>"
    console.log(name + "\t" + type + "\t" + clusterIp + "\t" + externalIP + "\t" + ports);
  });
}

// list k8s services in the given cluster and zone
listServices('<CLUSTER>', '<ZONE>');