..
Docker 仓库由公开转私有：保障 K8s Pod 无影响
背景
背景情况是，当初为了方便，将 Docker 仓库全部设置为公开，不需要认证就能拉代码，但是随着公司规模的增长，很多供应商也能访问我们的内网，这可能导致我们的镜像泄漏，我们的 Docker 镜像里面的内容可能包含一些敏感信息，所以我们需要将 Docker 仓库设置为私有。
难点
所有的 Pod 都没有添加 ImagePullSecrets，如果我们直接修改镜像仓库为私有，那么Pod重新调度到新的节点上，就会拉取失败。或者添加了新的 node, 那么 DaemonSet 就会应为无法拉镜像导致部署失败。
如果直接编辑线上环境的 deployment 和 daemonset，会导致 Pod 重启, 并且会导致业务中断，这样的变更对业务来说是不可接受的。
解决方案
每一个 pod 都会关联一个 ServiceAccount 关联的 ImagePullSerect 就是用来拉取镜像的,如果 Deployment 没有配置 imagePullSecrets，那么就会使用默认的 ServiceAccount 的 ImagePullSerect。如果不特殊设置的情况下，Deployment 都会关联当前命名空间下的 default ServiceAccount。
apiVersion: v1
kind: ServiceAccount
metadata:
creationTimestamp: "2022-01-20T09:23:16Z"
name: default
namespace: default
secrets:
- name: default-token-cvrmd
我们可以通过修改 default serviceaccount 的 ImagePullSerect 来达到修改镜像仓库的目的。执行如下命令即可
kubectl create secret docker-registry docker --docker-username=$username --docker-password=$password --docker-server=docker.jidudev.com -n $namespace
kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "docker"}]}' -n $namespace
然后我们再看 service 就被添加了 ImagePullSerect
apiVersion: v1
imagePullSecrets:
- name: docker
kind: ServiceAccount
metadata:
creationTimestamp: "2022-01-20T09:23:16Z"
name: default
namespace: default
secrets:
- name: default-token-cvrmd
集群里面有非常多的 serviceaccount，如果一个一个修改的话，工作量会非常大。并且有的服务部署不是关联的默认 serviceaccount，所以我们需要写一个脚本自动修改。我使用 golang 编写了这个脚本，自动修改所有的 serviceaccount 的 ImagePullSerect。
package main
import (
"context"
"encoding/base64"
"encoding/json"
"flag"
"fmt"
"log"
"path/filepath"
"strings"
"slices"
"github.com/schollz/progressbar/v3"
coreV1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/client-go/util/homedir"
)
var (
config *clientcmdapi.Config
kubeconfig *string
ignore *string
username *string
password *string
cluster *string
server *string
)
func init() {
if home := homedir.HomeDir(); home != "" {
kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
} else {
kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
}
username = flag.String("username", "", "(optional) username for docker registry")
password = flag.String("password", "", "(optional) password for docker registry")
cluster = flag.String("cluster", "", "(optional) specify the cluster name,splice with ','")
ignore = flag.String("ignore", "", "(optional) ignore the namespace,splice with ','")
server = flag.String("server", "", "(optional) server for docker registry")
flag.Parse()
if server == nil || *server == "" {
log.Fatalf("server is required")
}
if username == nil || *username == "" {
log.Fatalf("username is required")
}
if password == nil || *password == "" {
log.Fatalf("password is required")
}
c, err := clientcmd.LoadFromFile(*kubeconfig)
if err != nil {
panic(err.Error())
}
config = c
}
func main() {
for contextName, _ := range config.Contexts {
if (*cluster != "") && (!slices.Contains(strings.Split(*cluster, ","), contextName)) {
continue
}
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
loadingRules.ExplicitPath = *kubeconfig
loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig
// 根据集群上下文创建REST配置
restConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
loadingRules,
&clientcmd.ConfigOverrides{
CurrentContext: contextName,
}).ClientConfig()
if err != nil {
log.Fatalf("创建REST配置失败：%v\n", err)
}
log.Printf("正在处理集群：%s\n", contextName)
// 创建集群客户端
clientset, err := kubernetes.NewForConfig(restConfig)
if err != nil {
log.Fatalf("创建客户端失败：%v\n", err)
}
handleK8sCluster(clientset, contextName)
}
}
func handleK8sCluster(clinet *kubernetes.Clientset, contextName string) {
namespaceList, err := getK8SNameSpaceList(clinet)
if err != nil {
log.Fatalf("get k8s namespace list error: %v", err)
}
bar := progressbar.Default(int64(len(namespaceList.Items)), fmt.Sprintf("%-20s", contextName))
var secretName = "docker"
for _, namespace := range namespaceList.Items {
bar.Add(1)
if (*ignore != "") && slices.Contains(strings.Split(*ignore, ","), namespace.Name) {
log.Printf("ignore namespace: %s\n", namespace.Name)
continue
}
if !existsSecret(clinet, namespace.Name, secretName) {
dockerConfigJSONContent, err := handleDockerCfgJSONContent(*username, *password, *server)
if err != nil {
log.Fatalf("handle docker config json content error: %v", err)
}
secret := &coreV1.Secret{
Type: coreV1.SecretTypeDockerConfigJson,
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
},
Data: map[string][]byte{
".dockerconfigjson": []byte(dockerConfigJSONContent),
},
}
if _, err := clinet.CoreV1().Secrets(namespace.Name).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil {
if !errors.IsAlreadyExists(err) {
log.Fatalf("create secret error: %v", err)
}
}
}
serviceAccountList, err := getAllServiceAccount(clinet, namespace.Name)
if err != nil {
log.Fatalf("get service account list error: %v", err)
}
for _, sa := range serviceAccountList.Items {
var alreadyExists bool
for _, secret := range sa.ImagePullSecrets {
if secret.Name == secretName {
alreadyExists = true
break
}
}
if !alreadyExists {
sa.ImagePullSecrets = append(sa.ImagePullSecrets, coreV1.LocalObjectReference{Name: secretName})
if _, err := clinet.CoreV1().ServiceAccounts(sa.Namespace).Update(context.TODO(), &sa, metav1.UpdateOptions{}); err != nil {
log.Fatalf("update service account error: %v", err)
}
}
}
}
}
// 使用k8s api 获取namespace列表
func getK8SNameSpaceList(clinet *kubernetes.Clientset) (*coreV1.NamespaceList, error) {
return clinet.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{})
}
func existsSecret(clinet *kubernetes.Clientset, namespace, name string) bool {
secret, err := clinet.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return false
}
if secret == nil {
return false
}
return true
}
func getAllServiceAccount(clinet *kubernetes.Clientset, namespace string) (*coreV1.ServiceAccountList, error) {
return clinet.CoreV1().ServiceAccounts(namespace).List(context.TODO(), metav1.ListOptions{})
}
func handleDockerCfgJSONContent(username, password, server string) ([]byte, error) {
type DockerConfigEntry struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty" datapolicy:"password"`
Email string `json:"email,omitempty"`
Auth string `json:"auth,omitempty" datapolicy:"token"`
}
type DockerConfig map[string]DockerConfigEntry
type DockerConfigJSON struct {
Auths DockerConfig `json:"auths" datapolicy:"token"`
// +optional
HttpHeaders map[string]string `json:"HttpHeaders,omitempty" datapolicy:"token"`
}
dockerConfigAuth := DockerConfigEntry{
Username: username,
Password: password,
Auth: encodeDockerConfigFieldAuth(username, password),
}
dockerConfigJSON := DockerConfigJSON{
Auths: map[string]DockerConfigEntry{server: dockerConfigAuth},
}
return json.Marshal(dockerConfigJSON)
}
func encodeDockerConfigFieldAuth(username, password string) string {
fieldValue := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(fieldValue))
}
使用方式:
Usage of ./check-serviceaccount:
-cluster string
(optional) specify the cluster name,splice with ','
-ignore string
(optional) ignore the namespace,splice with ','
-kubeconfig string
(optional) absolute path to the kubeconfig file (default "/Users/licong/.kube/config")
-password string
(optional) password for docker registry
-server string
(optional) server for docker registry
-username string
(optional) username for docker registry
使用效果,直接批量就执行完了，这样 pod 重新调度就会使用 ServiceAccount 关联的 ImagePullSecrets 拉镜像了
2024/11/21 11:54:56 正在处理集群：sre
sre 100% |██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| (93/93, 3 it/s)