简介：
kubernetes 提供了多种安全认证机制，
其中对于集群通讯间可采用 TLS(https) 双向认证机制，也可采用基于 Token 或用户名密码的单向 tls 认证。
kubernetes 一般在内网部署，采用私有 IP 地址进行通讯，权威 CA 貌似只能签署域名证书，所以我们采用自建CA。
目标：
在kubernetes集群中，自建CA并配置各组件间的双向TLS认证
环境：
k8s-master：192.168.12.177
k8s-node01：192.168.12.178
软件版本：
k8s：v1.3.4
实施步骤：
1.master主机
1.1使用自建的CA签署证书
#新建CA [root@master ~]# cd key/ [root@master key]# openssl genrsa -out ca-key.pem 2048 [root@master key]# openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" [root@master key]# ll total 8 -rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem -rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem#新建openssl.conf文件 [root@master key]# vi openssl.cnf [req] req_extensions = v3_req distinguished_name = req_distinguished_name [req_distinguished_name] [ v3_req ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment subjectAltName = @alt_names [alt_names] DNS.1 = kubernetes DNS.2 = kubernetes.default DNS.3 = kubernetes.default.svc DNS.4 = kubernetes.default.svc.test01.com IP.1 = 10.254.0.1 IP.2 = 192.168.12.177#制作apiserver key [root@master key]# openssl genrsa -out apiserver-key.pem 2048 [root@master key]# openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf#签署apiserver证书 [root@master key]# openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf [root@master key]# ll total 28 -rw-r--r-- 1 root root 1115 Sep 21 11:11 apiserver.csr -rw-r--r-- 1 root root 1679 Sep 21 11:11 apiserver-key.pem -rw-r--r-- 1 root root 1184 Sep 21 11:12 apiserver.pem -rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem -rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem -rw-r--r-- 1 root root 17 Sep 21 11:12 ca.srl -rw-r--r-- 1 root root 398 Sep 21 11:10 openssl.cnf#新建node01的openssl.conf [root@master key]# vi worker-openssl-001.cnf [req] req_extensions = v3_req distinguished_name = req_distinguished_name [req_distinguished_name] [ v3_req ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment subjectAltName = @alt_names [alt_names] IP.1 = 192.168.12.178#制作node01 key [root@master key]# openssl genrsa -out k8s-node001-worker-key.pem 2048 [root@master key]# openssl req -new -key k8s-node001-worker-key.pem -out k8s-node001-worker.csr -subj "/CN=k8s-node001" -config worker-openssl-001.cnf#签署node01证书 [root@master key]# openssl x509 -req -in k8s-node001-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out k8s-node001-worker.pem -days 365 -extensions v3_req -extfile worker-openssl-001.cnf [root@master key]# ll total 44 -rw-r--r-- 1 root root 1115 Sep 21 11:11 apiserver.csr -rw-r--r-- 1 root root 1679 Sep 21 11:11 apiserver-key.pem -rw-r--r-- 1 root root 1184 Sep 21 11:12 apiserver.pem -rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem -rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem -rw-r--r-- 1 root root 17 Sep 21 11:15 ca.srl -rw-r--r-- 1 root root 972 Sep 21 11:14 k8s-node001-worker.csr -rw-r--r-- 1 root root 1679 Sep 21 11:14 k8s-node001-worker-key.pem -rw-r--r-- 1 root root 1046 Sep 21 11:15 k8s-node001-worker.pem -rw-r--r-- 1 root root 398 Sep 21 11:10 openssl.cnf -rw-r--r-- 1 root root 262 Sep 21 11:13 worker-openssl-001.cnf
1.2master主机修改相关配置
#复制apiserver的pem文件到ssl目录 [root@master key]# mkdir -p /etc/kubernetes/ssl [root@master key]# cd /etc/kubernetes/ssl [root@master ssl]# cp /root/key/ca.pem . [root@master ssl]# cp /root/key/apiserver.pem . [root@master ssl]# cp /root/key/apiserver-key.pem .#新建kubeconfig文件 [root@master ~]# vi /etc/kubernetes/cm-kubeconfig.yaml apiVersion: v1 kind: Config clusters: - name: localcluster:certificate-authority: /etc/kubernetes/ssl/ca.pem users: - name: controllermanageruser:client-certificate: /etc/kubernetes/ssl/apiserver.pemclient-key: /etc/kubernetes/ssl/apiserver-key.pem contexts: - context:cluster: localuser: controllermanagername: kubelet-context current-context: kubelet-context#按下面配置，修改各组件 #因controllmanager、scheduler与apiserver在同一台主机上，也可以使用非安全端口8080） [root@master ~]# grep -v ^# /etc/kubernetes/config KUBE_LOGTOSTDERR="--logtostderr=true" KUBE_LOG_LEVEL="--v=0" KUBE_ALLOW_PRIV="--allow-privileged=false" KUBE_MASTER="--master=https://192.168.12.177:6443" [root@master ~]# grep -v ^# /etc/kubernetes/apiserver KUBE_API_ADDRESS="--bind-address=0.0.0.0 --insecure-bind-address=127.0.0.1" KUBE_API_PORT="--secure-port=6443 --insecure-port=8080" KUBE_ETCD_SERVERS="--etcd-servers=http://192.168.12.177:2379" KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16" KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota" KUBE_API_ARGS="--client-ca-file=/etc/kubernetes/ssl/ca.pem \--tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \--tls-cert-file=/etc/kubernetes/ssl/apiserver.pem \--service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem" [root@master ~]# grep -v ^# /etc/kubernetes/controller-manager KUBE_CONTROLLER_MANAGER_ARGS="--service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \--root-ca-file=/etc/kubernetes/ssl/ca.pem \--master=http://127.0.0.1:8080 \--kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml" [root@master ~]# grep -v ^# /etc/kubernetes/scheduler KUBE_SCHEDULER_ARGS="--master=http://127.0.0.1:8080 \--kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml"#重启各组件 [root@master kubernetes]# systemctl restart kube-controller-manager [root@master kubernetes]# systemctl restart kube-scheduler [root@master kubernetes]# systemctl status kube-apiserver#查看状态 [root@master kubernetes]# systemctl status kube-controller-manager -l [root@master kubernetes]# systemctl status kube-scheduler -l [root@master kubernetes]# systemctl status kube-apiserver -l#验证证书 [root@master kubernetes]# curl https://192.168.12.177:6443/api/v1/nodes --cert /etc/kubernetes/ssl/apiserver.pem --key /etc/kubernetes/ssl/apiserver-key.pem --cacert /etc/kubernetes/ssl/ca.pem
2.node01主机
#将master主机生成的node01证书复制到ssl目录 [root@node01 ~]# mkdir -p /etc/kubernetes/ssl/ [root@node01 ~]# cd /etc/kubernetes/ssl/ [root@node01 ssl]# ll total 12 -rw-r--r-- 1 root root 1090 Sep 21 11:49 ca.pem -rw-r--r-- 1 root root 1679 Sep 21 11:49 k8s-node001-worker-key.pem -rw-r--r-- 1 root root 1046 Sep 21 11:50 k8s-node001-worker.pem#做一个超链接 [root@node01 ssl]# ln -s k8s-node001-worker.pem worker.pem [root@node01 ssl]# ln -s k8s-node001-worker-key.pem worker-key.pem#新建kubeconfig [root@node01 ssl]# vi /etc/kubernetes/worker-kubeconfig.yaml apiVersion: v1 kind: Config clusters: - name: localcluster:certificate-authority: /etc/kubernetes/ssl/ca.pem users: - name: kubeletuser:client-certificate: /etc/kubernetes/ssl/worker.pemclient-key: /etc/kubernetes/ssl/worker-key.pem contexts: - context:cluster: localuser: kubeletname: kubelet-context current-context: kubelet-context#按照如下配置，修改各配置文件 [root@node01 ssl]# grep -v ^# /etc/kubernetes/config KUBE_LOGTOSTDERR="--logtostderr=true" KUBE_LOG_LEVEL="--v=0" KUBE_ALLOW_PRIV="--allow-privileged=false" KUBE_MASTER="--master=https://192.168.12.177:6443" [root@node01 ssl]# grep -v ^# /etc/kubernetes/kubelet KUBELET_ADDRESS="--address=0.0.0.0" KUBELET_PORT="--port=10250" KUBELET_HOSTNAME="--hostname-override=node01.test01.com" KUBELET_API_SERVER="--api-servers=https://192.168.12.177:6443" KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest" KUBELET_ARGS="--cluster-dns=192.168.12.177 \--cluster-domain=test01.com \--tls-cert-file=/etc/kubernetes/ssl/worker.pem \--tls-private-key-file=/etc/kubernetes/ssl/worker-key.pem \--kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml" [root@node01 ssl]# grep -v ^# /etc/kubernetes/proxy KUBE_PROXY_ARGS="--kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"#重启服务 [root@node01 ssl]# systemctl restart docker kubelet kube-proxy#查看 [root@node01 ssl]# systemctl status docker kubelet kube-proxy -l#验证证书 [root@node01 ssl]# curl https://192.168.12.177:6443/api/v1/nodes --cert /etc/kubernetes/ssl/worker.pem --key /etc/kubernetes/ssl/worker-key.pem --cacert /etc/kubernetes/ssl/ca.pem
3.新建svc进行测试
#新建svc [root@master template]# cd registry/ [root@master registry]# kubectl create -f registry-rc.yaml [root@master registry]# kubectl create -f registry-svc.yaml#查看 [root@master registry]# kubectl get pods --namespace=kube-system NAME READY STATUS RESTARTS AGE kube-registry-v0-jypg1 1/1 Running 0 38s [root@master registry]# kubectl get rc --namespace=kube-system NAME DESIRED CURRENT AGE kube-registry-v0 1 1 38s [root@master registry]# kubectl get svc --namespace=kube-system NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE kube-registry 10.254.80.217 <nodes> 5000/TCP 38s#上传镜像至私有仓库 [root@master ssl]# docker pull alpine [root@master ssl]# docker images [root@master ssl]# docker tag docker.io/alpine 192.168.12.178:30099/alpine [root@master ssl]# docker push 192.168.12.178:30099/alpine The push refers to a repository [192.168.12.178:30099/alpine] 3fc666989c1d: Pushed latest: digest: sha256:02eb5cfe4b721495135728ab4aea87418fd4edbfbf83612130a81191f0b2aae3 size: 506
4.遗留问题-dashboard部署(已经解决)
4.1目前双向TLS配置后，dashboard报错如下（已经解决）
#错误提示 [root@master ui]# kubectl get pods --namespace=kube-system NAME READY STATUS RESTARTS AGE kube-registry-v0-jypg1 1/1 Running 2 2h kubernetes-dashboard-v1.1.0-wd9g3 0/1 CrashLoopBackOff 5 4m [root@master ui]# kubectl logs -f pods/kubernetes-dashboard-v1.1.0-wd9g3 --namespace=kube-system Starting HTTP server on port 9090 Creating API server client for https://192.168.12.177:6443 Error while initializing connection to Kubernetes apiserver. This most likely means that the cluster is misconfigured (e.g., it has invalid apiserver certificates or service accounts configuration) or the --apiserver-host param points to a server that does not exist. Reason: Get https://192.168.12.177:6443/version: x509: failed to load system roots and no roots provided
上述问题，是因为认证问题，我们需要修改一下yaml文件。
#完整的yaml文件如下： [root@master ui]# vi dashboard-controller.yaml apiVersion: v1 kind: ReplicationController metadata:name: kubernetes-dashboard-v1.1.0namespace: kube-systemlabels:k8s-app: kubernetes-dashboardversion: v1.1.0kubernetes.io/cluster-service: "true" spec:replicas: 1selector:k8s-app: kubernetes-dashboardtemplate:metadata:labels:k8s-app: kubernetes-dashboardversion: v1.1.0kubernetes.io/cluster-service: "true"spec:containers:- name: kubernetes-dashboardimage: index.tenxcloud.com/google_containers/kubernetes-dashboard-amd64:v1.1.0resources:# keep request = limit to keep this container in guaranteed classlimits:cpu: 100mmemory: 50Mirequests:cpu: 100mmemory: 50Miports:- containerPort: 9090livenessProbe:httpGet:path: /port: 9090initialDelaySeconds: 30timeoutSeconds: 30volumeMounts:- mountPath: /etc/kubernetes/sslname: ssl-certs-kubernetesreadOnly: true- mountPath: /etc/ssl/certsname: ssl-certs-hostreadOnly: truevolumes:- hostPath:path: /etc/kubernetes/sslname: ssl-certs-kubernetes- hostPath:path: /etc/pki/tls/certsname: ssl-certs-host#创建 [root@master ui]# kubectl create -f dashboard-controller.yaml replicationcontroller "kubernetes-dashboard-v1.1.0" created#查看 [root@master ui]# kubectl get pods --namespace=kube-system NAME READY STATUS RESTARTS AGE kubernetes-dashboard-v1.1.0-dt16g 1/1 Running 0 2s
可以看到，相比原来yaml文件，增加了ssl认证部分。
4.2自建CA，masterIP只能https访问了，那么windows如何访问dashboard（已经解决）
默认dashboard是通过master的ip和端口进行访问的，但是在windows下因无法加载自建ca证书到时无法访问dashboard。
解决办法是修改yaml，添加nodeport。
#完整的yaml文件，如下 [root@master ui]# vi dashboard-service.yaml apiVersion: v1 kind: Service metadata:name: kubernetes-dashboardnamespace: kube-systemlabels:k8s-app: kubernetes-dashboardkubernetes.io/cluster-service: "true" spec:selector:k8s-app: kubernetes-dashboardtype: NodePortports:- port: 80nodePort: 30080targetPort: 9090#创建 [root@master ui]# kubectl create -f dashboard-service.yaml You have exposed your service on an external port on all nodes in your cluster. If you want to expose this service to the external internet, you may need to set up firewall rules for the service port(s) (tcp:30080) to serve traffic.See http://releases.k8s.io/release-1.3/docs/user-guide/services-firewalls.md for more details. service "kubernetes-dashboard" created#查看 [root@master ui]# kubectl get svc --namespace=kube-system NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes-dashboard 10.254.0.253 <nodes> 80/TCP 1m
windows下浏览器打开
http://192.168.12.178:30080/
Kubernetes双向TLS配置-Centos7
本文：Kubernetes双向TLS配置-Centos7