I need to create a Kubernetes clientset using a token extracted from JSON service account key file.
I explicitly provide this token inside the config, however it still looks for Google Application-Default credentials, and crashes because it cannot find them.
Below is my code:
package main
import (
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
gke "google.golang.org/api/container/v1"
"google.golang.org/api/option"
"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/clientcmd/api"
)
const (
projectID = "my_project_id"
clusterName = "my_cluster_name"
scope = "https://www.googleapis.com/auth/cloud-platform"
)
func main() {
ctx := context.Background()
// Read JSON key and extract the token
data, err := ioutil.ReadFile("sa_key.json")
if err != nil {
panic(err)
}
creds, err := google.CredentialsFromJSON(ctx, data, scope)
if err != nil {
panic(err)
}
token, err := creds.TokenSource.Token()
if err != nil {
panic(err)
}
fmt.Println("token", token.AccessToken)
// Create GKE client
tokenSource := oauth2.StaticTokenSource(token)
gkeClient, err := gke.NewService(ctx, option.WithTokenSource(tokenSource))
if err != nil {
panic(err)
}
// Create a dynamic kube config
inMemKubeConfig, err := createInMemKubeConfig(ctx, gkeClient, token, projectID)
if err != nil {
panic(err)
}
// Use it to create a rest.Config
config, err := clientcmd.NewNonInteractiveClientConfig(*inMemKubeConfig, clusterName, &clientcmd.ConfigOverrides{CurrentContext: clusterName}, nil).ClientConfig()
if err != nil {
panic(err)
}
// Create the clientset
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
panic(err) // this where the code crashes because it can't find the Google ADCs
}
fmt.Printf("clientset %+v\n", clientset)
}
func createInMemKubeConfig(ctx context.Context, client *gke.Service, token *oauth2.Token, projectID string) (*api.Config, error) {
k8sConf := api.Config{
APIVersion: "v1",
Kind: "Config",
Clusters: map[string]*api.Cluster{},
AuthInfos: map[string]*api.AuthInfo{},
Contexts: map[string]*api.Context{},
}
// List all clusters in project with id projectID across all zones ("-")
resp, err := client.Projects.Zones.Clusters.List(projectID, "-").Context(ctx).Do()
if err != nil {
return nil, err
}
for _, f := range resp.Clusters {
name := fmt.Sprintf("gke_%s_%s_%s", projectID, f.Zone, f.Name) // My custom naming convention
cert, err := base64.StdEncoding.DecodeString(f.MasterAuth.ClusterCaCertificate)
if err != nil {
return nil, err
}
k8sConf.Clusters[name] = &api.Cluster{
CertificateAuthorityData: cert,
Server: "https://" + f.Endpoint,
}
k8sConf.Contexts[name] = &api.Context{
Cluster: name,
AuthInfo: name,
}
k8sConf.AuthInfos[name] = &api.AuthInfo{
Token: token.AccessToken,
AuthProvider: &api.AuthProviderConfig{
Name: "gcp",
Config: map[string]string{
"scopes": scope,
},
},
}
}
return &k8sConf, nil
}
and here is the error message:
panic: cannot construct google default token source: google: could not find default credentials. See https://developers.google.com/accounts/docs/application-default-credentials for more information.
Here's what worked for me.
It is based on this gist
and it's exactly what I was looking for. It uses an oauth2.TokenSource object which can be fed with a variety of token types so it's quite flexible.
It took me a long time to find this solution so I hope this helps somebody!
package main
import (
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"log"
"net/http"
gke "google.golang.org/api/container/v1"
"google.golang.org/api/option"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)
const (
googleAuthPlugin = "gcp"
projectID = "my_project"
clusterName = "my_cluster"
zone = "my_cluster_zone"
scope = "https://www.googleapis.com/auth/cloud-platform"
)
type googleAuthProvider struct {
tokenSource oauth2.TokenSource
}
// These funcitons are needed even if we don't utilize them
// So that googleAuthProvider is an rest.AuthProvider interface
func (g *googleAuthProvider) WrapTransport(rt http.RoundTripper) http.RoundTripper {
return &oauth2.Transport{
Base: rt,
Source: g.tokenSource,
}
}
func (g *googleAuthProvider) Login() error { return nil }
func main() {
ctx := context.Background()
// Extract a token from the JSON SA key
data, err := ioutil.ReadFile("sa_key.json")
if err != nil {
panic(err)
}
creds, err := google.CredentialsFromJSON(ctx, data, scope)
if err != nil {
panic(err)
}
token, err := creds.TokenSource.Token()
if err != nil {
panic(err)
}
tokenSource := oauth2.StaticTokenSource(token)
// Authenticate with the token
// If it's nil use Google ADC
if err := rest.RegisterAuthProviderPlugin(googleAuthPlugin,
func(clusterAddress string, config map[string]string, persister rest.AuthProviderConfigPersister) (rest.AuthProvider, error) {
var err error
if tokenSource == nil {
tokenSource, err = google.DefaultTokenSource(ctx, scope)
if err != nil {
return nil, fmt.Errorf("failed to create google token source: %+v", err)
}
}
return &googleAuthProvider{tokenSource: tokenSource}, nil
}); err != nil {
log.Fatalf("Failed to register %s auth plugin: %v", googleAuthPlugin, err)
}
gkeClient, err := gke.NewService(ctx, option.WithTokenSource(tokenSource))
if err != nil {
panic(err)
}
clientset, err := getClientSet(ctx, gkeClient, projectID, org, env)
if err != nil {
panic(err)
}
// Demo to make sure it works
pods, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{})
if err != nil {
panic(err)
}
log.Printf("There are %d pods in the cluster", len(pods.Items))
for _, pod := range pods.Items {
fmt.Println(pod.Name)
}
}
func getClientSet(ctx context.Context, client *gke.Service, projectID, name string) (*kubernetes.Clientset, error) {
// Get cluster info
cluster, err := client.Projects.Zones.Clusters.Get(projectID, zone, name).Context(ctx).Do()
if err != nil {
panic(err)
}
// Decode cluster CA certificate
cert, err := base64.StdEncoding.DecodeString(cluster.MasterAuth.ClusterCaCertificate)
if err != nil {
return nil, err
}
// Build a config using the cluster info
config := &rest.Config{
TLSClientConfig: rest.TLSClientConfig{
CAData: cert,
},
Host: "https://" + cluster.Endpoint,
AuthProvider: &clientcmdapi.AuthProviderConfig{Name: googleAuthPlugin},
}
return kubernetes.NewForConfig(config)
}