深入探索 Kubernetes 工作负载标识 – 应用

过去两周我们介绍了 Kubernetes 工作负载标识的原理、以及如何配置和部署 Kubernetes 工作负载标识。今天这篇我们将介绍如何将 Kubernetes 工作负载标识应用到实际业务环境中。

过往客户使用 VMSS 节点集用户托管标识，其具有较为宽泛的管理权限。随着业务需求的变化，客户需要更细颗粒度的权限管控，因此，我们建议客户启用服务级工作负载标识。

我们用 python 或者 Golang 写一个简单的代码示例，pod 绑定对应的工作负载标识，通过授权用户托管标识对 blob 的写入权限，最终把 pod 元数据 pod name 和 ip 地址，写入包含 pod 名字的随机文件，并上传 blob，最后进行相关压力测试。因为要写入 blob, 我们授权用户托管标识 Storage Blob Data Contributor 权限。

🔗  https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/storage#storage-blob-data-contributor

通过以下方法获取对应 Azure 资源的访问权限：

◉ 虚拟机内可以访问 Azure Instance Metadata Service (IMDS)获取访问令牌,令牌时效1-24小时不等，默认通常1个小时。

🔗  https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/storage#storage-blob-data-contributorhttps://learn.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service?tabs=windows

Python 示例代码两种验证方式都包含，并注释了 MSAL, Golang 代码主要用的第一种方式 azidentity SDK，第二种方式参考官网 key vault 示例 msal-go

🔗 https://github.com/Azure/azure-workload-identity/tree/main/examples/msal-go

参考以下项目代码，注意代码黄色高亮环境变量，测试中并没有赋值，实际可以通过工作负载关联的Mutating Admission Webhook自动注入，示例代码Repo: 

🔗 https://github.com/zhangchl007/workload-identity-demo

import os
import uuid
import time
import logging


from token_credential import MyClientAssertionCredential
from azure.identity import ManagedIdentityCredential
from azure.storage.blob import BlobClient
from azure.storage.blob import BlobServiceClient
from datetime import datetime


def main():
    # get environment variables to authenticate to the key vault
    azure_client_id = os.getenv('AZURE_CLIENT_ID', '')
    azure_tenant_id = os.getenv('AZURE_TENANT_ID', '')
    azure_authority_host = os.getenv('AZURE_AUTHORITY_HOST', '')
    azure_federated_token_file = os.getenv('AZURE_FEDERATED_TOKEN_FILE', '')
    
    storage_account_name = os.getenv('STORAGE_ACCOUNT_NAME', '')
    storage_container_name = os.getenv('STORAGE_CONTAINER_NAME', '')
    pod_name = os.getenv('POD_NAME', '')
    pod_ip = os.getenv('POD_IP', '')
    
    # create a token credential object, which has a get_token method that returns a token
    credential = ManagedIdentityCredential()
    #token_credential = MyClientAssertionCredential(azure_client_id, azure_tenant_id, azure_authority_host, azure_federated_token_file)
 
    # create a blob service client
    #blob_service_client = BlobServiceClient(account_url=f"https://{storage_account_name}.blob.core.windows.net/", credential=token_credential)
    #blob_service_client = BlobServiceClient(account_url=f"https://{storage_account_name}.blob.core.windows.net/", credential=credential)
    
    # create a container client
    storage_url = f"https://{storage_account_name}.blob.core.windows.net/"
    # Create the client object using the storage URL and the credential
    blob_client = BlobClient(
        storage_url,
        container_name=storage_container_name,
        blob_name=pod_name + '.txt',
        credential=credential,
    )
    
    while True:  # This creates an infinite loop
        with open('/tmp/sample-source.txt', 'a') as f:
            f.write(f"Timestamp: {datetime.now()}\n") 
            f.write(f"POD Name: {pod_name}\n")
            f.write(f"POD IP: {pod_ip}\n")
            f.write('hello ,I am writing \n')
          


        
        # upload the file to the blob and overwrite the existing file
        with open("/tmp/sample-source.txt", "rb") as data:
            blob_client.upload_blob(data, overwrite=True)
            print(f"Uploaded sample-source.txt to {blob_client.url}")
        # sleep for 5 seconds
        time.sleep(10)
        
            
if __name__ == '__main__':
    main()

// workload identity with Azure AD MSAL to access Azure Storage Account
// This example demonstrates how to use Azure AD Workload Identity with MSAL to access an Azure Storage Account
// The Azure AD Workload Identity webhook will inject the following environment variables:
package main


import (
  "context"
  "os"
  "time"


  "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
  "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
  "k8s.io/klog"
)


func main() {


  // Azure Container


  azure_storage_account_Name := os.Getenv("STORAGE_ACCOUNT_NAME")


  if azure_storage_account_Name == "" {
    klog.Fatal("AZURE_STORAGE_ACCOUNT_NAME environment variable is not set")
  }


  azure_storage_container_Name := os.Getenv("STORAGE_CONTAINER_NAME")


  if azure_storage_container_Name == "" {
    klog.Fatal("AZURE_STORAGE_CONTAINER_NAME environment variable is not set")
  }


  pod_name := os.Getenv("POD_NAME")


  if pod_name == "" {
    klog.Fatal("POD_NAME environment variable is not set")
  }


  pod_ip := os.Getenv("POD_IP")


  if pod_ip == "" {
    klog.Fatal("POD_IP environment variable is not set")


  }


  // Azure AD Workload Identity webhook will inject the following env vars
  //   AZURE_CLIENT_ID with the clientID set in the service account annotation
  //   AZURE_TENANT_ID with the tenantID set in the service account annotation. If not defined, then
  //   the tenantID provided via azure-wi-webhook-config for the webhook will be used.
  //   AZURE_FEDERATED_TOKEN_FILE is the service account token path
  //   AZURE_AUTHORITY_HOST is the AAD authority hostname
  clientID := os.Getenv("AZURE_CLIENT_ID")
  tenantID := os.Getenv("AZURE_TENANT_ID")
  tokenFilePath := os.Getenv("AZURE_FEDERATED_TOKEN_FILE")
  //authorityHost := os.Getenv("AZURE_AUTHORITY_HOST")


  // ...


  if clientID == "" {
    klog.Fatal("AZURE_CLIENT_ID environment variable is not set")
  }
  if tenantID == "" {
    klog.Fatal("AZURE_TENANT_ID environment variable is not set")
  }


  // newClientAssertionCredential
  cred, err := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{
    ClientID:      clientID,
    TenantID:      tenantID,
    TokenFilePath: tokenFilePath,
  })


  if err != nil {
    klog.Fatal(err)
  }


  storageaccountURL := "https://" + azure_storage_account_Name + ".blob.core.windows.net"
  containerName := azure_storage_container_Name
  blobName := pod_name + ".txt"
  ctx := context.Background()


  // Create a new container client


  client, err := azblob.NewClient(storageaccountURL, cred, nil)


  if err != nil {
    klog.Fatal(err)


  }


  ticker := time.NewTicker(10 * time.Second)
  defer ticker.Stop()


  for range ticker.C {
    // data to write to blob
    mydata := time.Now().Format(time.RFC3339) + pod_name + "Hello, World!" + "\n"


    // Open the file
    file, err := os.OpenFile("/tmp/data.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
    if err != nil {
      klog.Fatal(err)
    }


    // append data to file
    if _, err := file.WriteString(mydata); err != nil {
      klog.Fatal(err)
    }


    // Close the file
    err = file.Close()
    if err != nil {
      klog.Fatal(err)
    }


    newfile, err := os.OpenFile("/tmp/data.txt", os.O_RDONLY, 0)
    if err != nil {
      klog.Fatal(err)
    }
    defer file.Close()


    // upload the file to a blob
    _, err = client.UploadFile(ctx, containerName, blobName, newfile, &azblob.UploadFileOptions{})
    if err != nil {
      klog.Fatal(err)
    }


    klog.Infof("Blob uploaded: %s", blobName)


  }


}

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    azure.workload.identity/use: "true"
    app: workload-golang
  name: workload-golang
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: workload-golang
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      labels:
        azure.workload.identity/use: "true"
        app: workload-golang
    spec:
      containers:
      - image: quay.io/zhangchl007/workloaddemo:v2
        imagePullPolicy: Always
        name: oidc
        env:
        - name: STORAGE_ACCOUNT_NAME
          value: "mit000"
        - name: STORAGE_CONTAINER_NAME
          value: "mit"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
      serviceAccountName: workidentity-sa
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - preference:
              matchExpressions:
              - key: kubernetes.azure.com/mode
                operator: In
                values:
                - user
            weight: 100
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.azure.com/cluster
                operator: Exists
              - key: type
                operator: NotIn
                values:
                - virtual-kubelet
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - workload-golang
              topologyKey: topology.kubernetes.io/zone
            weight: 100

kubectl get pods |grep workload |awk '{print "kubectl logs "$1" >"$1".txt"}' |sh
cd test
grep -v ^Token * | wc -l


$ more workload-golang-78c84f4884-rjnsw.txt
I0326 08:32:28.087126       1 main.go:85] Token: eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1……


I0326 08:32:38.140046       1 main.go:137] Blob uploaded: workload-golang-78c84f4884-rjnsw.txt
I0326 08:32:48.096208       1 main.go:137] Blob uploaded: workload-golang-78c84f4884-rjnsw.txt
I0326 08:32:58.098256       1 main.go:137] Blob uploaded: workload-golang-78c84f4884-rjnsw.txt

*间隔10秒完成一次上传。

az storage blob list \
    --account-name mit000 \
    --container-name mit \
    --output table


Name                                Blob Type    Blob Tier    Length    Content Type              Last Modified              Snapshot
----------------------------------  -----------  -----------  --------  ------------------------  -------------------------  ----------
workload-demo-67c6cf68c8-226cn.txt  BlockBlob    Hot          6475      application/octet-stream  2024-03-21T08:41:08+00:00
workload-demo-67c6cf68c8-24485.txt  BlockBlob    Hot          6481      application/octet-stream  2024-03-21T08:41:03+00:00
workload-demo-67c6cf68c8-27mxs.txt  BlockBlob    Hot          6593      application/octet-stream  2024-03-21T08:41:06+00:00