从源码看imageCredentialProvider

背景

以前k8s pod在拉取镜像时候使用的是长期存在的imagepullsecret且往往没做工作负载级别的认证信息区分,存在安全隐患

简单总结

1.33起,添加了imageCredentialProvider(在节点上的一个二进制程序),kubelet在拉取镜像前会把pod以及sa的相关信息作为参数调用这个imageCredentialProvider二进制,这个imageCredentialProvider二进制返回对应的认证信息,然后kubelet用于镜像拉取

源码

注册插件

pkg/kubelet/kuberuntime/kuberuntime_manager.go

构建kubelet manager
func NewKubeGenericRuntimeManager(
    recorder record.EventRecorder,
    livenessManager proberesults.Manager,
    readinessManager proberesults.Manager,
    startupManager proberesults.Manager,
    rootDirectory string,
    podLogsDirectory string,
    machineInfo *cadvisorapi.MachineInfo,
    podStateProvider podStateProvider,
    osInterface kubecontainer.OSInterface,
    runtimeHelper kubecontainer.RuntimeHelper,
    insecureContainerLifecycleHTTPClient types.HTTPDoer,
    imageBackOff *flowcontrol.Backoff,
    serializeImagePulls bool,
    maxParallelImagePulls *int32,
    imagePullQPS float32,
    imagePullBurst int,
    imagePullsCredentialVerificationPolicy string,
    preloadedImagesCredentialVerificationWhitelist []string,
    imageCredentialProviderConfigFile string,
    imageCredentialProviderBinDir string,
    singleProcessOOMKill *bool,
    cpuCFSQuota bool,
    cpuCFSQuotaPeriod metav1.Duration,
    runtimeService internalapi.RuntimeService,
    imageService internalapi.ImageManagerService,
    containerManager cm.ContainerManager,
    logManager logs.ContainerLogManager,
    runtimeClassManager *runtimeclass.Manager,
    allocationManager allocation.Manager,
    seccompDefault bool,
    memorySwapBehavior string,
    getNodeAllocatable func() v1.ResourceList,
    memoryThrottlingFactor float64,
    podPullingTimeRecorder images.ImagePodPullingTimeRecorder,
    tracerProvider trace.TracerProvider,
    tokenManager *token.Manager,
    getServiceAccount plugin.GetServiceAccountFunc,
) (KubeGenericRuntime, []images.PostImageGCHook, error) {
    ...
    注册CredentialProvider
    if imageCredentialProviderConfigFile != "" || imageCredentialProviderBinDir != "" {
        if err := plugin.RegisterCredentialProviderPlugins(imageCredentialProviderConfigFile, imageCredentialProviderBinDir, tokenManager.GetServiceAccountToken, getServiceAccount); err != nil {
            klog.ErrorS(err, "Failed to register CRI auth plugins")
            os.Exit(1)
        }
    }

    ...
}

pkg/credentialprovider/plugin/plugin.go中

注册插件
func RegisterCredentialProviderPlugins(pluginConfigFile, pluginBinDir string,
    getServiceAccountToken getServiceAccountTokenFunc,
    getServiceAccount GetServiceAccountFunc,
) error {
    ...
    读取配置
    credentialProviderConfig, err := readCredentialProviderConfigFile(pluginConfigFile)
    if err != nil {
        return err
    }
    遍历插件配置
    for _, provider := range credentialProviderConfig.Providers {
        ...
        实际注册插件
        registerCredentialProviderPlugin(provider.Name, plugin)
        ...
    }
}

使用插件

func (m *imageManager) EnsureImageExists(ctx context.Context, objRef *v1.ObjectReference, pod *v1.Pod, requestedImage string, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, podRuntimeHandler string, pullPolicy v1.PullPolicy) (imageRef, message string, err error) {
    ...
    构建外部CredentialProvider
    externalCredentialProviderKeyring := credentialproviderplugin.NewExternalCredentialProviderDockerKeyring(
        podNamespace,
        podName,
        podUID,
        pod.Spec.ServiceAccountName)
    联合imagepullsecret,nodekeyring(使用docker auth config),以及外部CredentialProvider
    keyring, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, credentialprovider.UnionDockerKeyring{m.nodeKeyring, externalCredentialProviderKeyring})
    if err != nil {
        return "", err.Error(), err
    }
    ...
    pullCredentials, _ := keyring.Lookup(repoToPull)
    ...
}

pkg/credentialprovider/keyring.go中

联合的keyring
type UnionDockerKeyring []DockerKeyring

func (k UnionDockerKeyring) Lookup(image string) ([]TrackedAuthConfig, bool) {
    authConfigs := []TrackedAuthConfig{}
    for _, subKeyring := range k {
        if subKeyring == nil {
            continue
        }

        currAuthResults, _ := subKeyring.Lookup(image)
        authConfigs = append(authConfigs, currAuthResults...)
    }

    return authConfigs, (len(authConfigs) > 0)
}

pkg/credentialprovider/plugin/plugins.go中

externalkeyring
type externalCredentialProviderKeyring struct {
    providers []credentialprovider.DockerConfigProvider
}

func (k *externalCredentialProviderKeyring) Lookup(image string) ([]credentialprovider.TrackedAuthConfig, bool) {
    keyring := &credentialprovider.BasicDockerKeyring{}

    for _, p := range k.providers {
        // TODO: modify the credentialprovider.CredentialSource to contain the SA/pod information
        keyring.Add(nil, p.Provide(image))
    }

    return keyring.Lookup(image)
}

pkg/credentialprovider/plugin/plugin.go中

external provider
type perPodPluginProvider struct {
    name string

    provider *pluginProvider

    podNamespace string
    podName      string
    podUID       types.UID

    serviceAccountName string
}


func (p *perPodPluginProvider) Provide(image string) credentialprovider.DockerConfig {
    return p.provider.provide(image, p.podNamespace, p.podName, p.podUID, p.serviceAccountName)
}

func (p *pluginProvider) provide(image, podNamespace, podName string, podUID types.UID, serviceAccountName string) credentialprovider.DockerConfig {
    ...
    查找serviceAccountToken
            if serviceAccountToken, err = p.serviceAccountProvider.getServiceAccountToken(podNamespace, podName, serviceAccountName, podUID); err != nil {
                klog.Errorf("Error getting service account token %s/%s: %v", podNamespace, serviceAccountName, err)
                return credentialprovider.DockerConfig{}
            }
    ...
    传入image,serviceAccountToken,saAnnotations调用插件
    res, err, _ := p.group.Do(singleFlightKey, func() (interface{}, error) {
        return p.plugin.ExecPlugin(context.Background(), image, serviceAccountToken, saAnnotations)
    })
    ...
    response, ok := res.(*credentialproviderapi.CredentialProviderResponse)
    if !ok {
        klog.Errorf("Invalid response type returned by external credential provider")
        return credentialprovider.DockerConfig{}
    }
    ...
    构建响应
    dockerConfig := make(credentialprovider.DockerConfig, len(response.Auth))
    for matchImage, authConfig := range response.Auth {
        dockerConfig[matchImage] = credentialprovider.DockerConfigEntry{
            Username: authConfig.Username,
            Password: authConfig.Password,
        }
    }
    ...
    return dockerConfig

}

pkg/credentialprovider/plugin/plugin.go中

执行插件
func (e *execPlugin) ExecPlugin(ctx context.Context, image, serviceAccountToken string, serviceAccountAnnotations map[string]string) (*credentialproviderapi.CredentialProviderResponse, error) {
    ...
    构建请求对象并且序列化
    authRequest := &credentialproviderapi.CredentialProviderRequest{Image: image, ServiceAccountToken: serviceAccountToken, ServiceAccountAnnotations: serviceAccountAnnotations}
    data, err := e.encodeRequest(authRequest)
    if err != nil {
        return nil, fmt.Errorf("failed to encode auth request: %w", err)
    }
    ...
    构建命令并且调用
    cmd := exec.CommandContext(ctx, filepath.Join(e.pluginBinDir, e.name), e.args...)
    cmd.Stdout, cmd.Stderr, cmd.Stdin = stdout, stderr, stdin

    var configEnvVars []string
    for _, v := range e.envVars {
        configEnvVars = append(configEnvVars, fmt.Sprintf("%s=%s", v.Name, v.Value))
    }


    cmd.Env = mergeEnvVars(e.environ(), configEnvVars)

    if err = e.runPlugin(ctx, cmd, image); err != nil {
        return nil, fmt.Errorf("%w: %s", err, stderr.String())
    }
    ...
    反序列化响应
    response, err := e.decodeResponse(data)
    if err != nil {
        // err is explicitly not wrapped since it may contain credentials in the response.
        return nil, errors.New("error decoding credential provider plugin response from stdout")
    }
    return response, nil

}