kubernetes pod + harbor
文章目录
- kubernetes pod + harbor
- pod
-
- pod容器分类
- 镜像拉取策略(image PullPolicy)
- 私有仓库harbor
pod
pod资源管理
特点:
最小部署单元
一组容器的集合
一个pod中的容器共享网络命名空间
pod是短暂的
pod容器分类
1:infrastructure container 基础容器
//维护整个Pod网络空间
//node节点操作
//查看容器的网络
[root@node1 ~]# cat /opt/kubernetes/cfg/kubeletKUBELET_OPTS="--logtostderr=true \ --v=4 \ --hostname-override=192.168.20.20 \ --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \ --bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \ --config=/opt/kubernetes/cfg/kubelet.config \ --cert-dir=/opt/kubernetes/ssl \ --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0" //基础容器
//每次创建Pod时候就会创建,与Pod对应的,对于用户是透明的
[root@node1 ~]# docker ps -a
registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0 "/pause"
2:initcontainers 初始化容器
//先于业务容器开始执行,原先Pod中容器是并行开启,现在进行了改进
3:container 业务容器
//并行启动
官方网站
https://kubernetes.io/docs/concepts/workloads/pods/init-containers/
镜像拉取策略(image PullPolicy)
IfNotPresent:默认值,镜像在宿主机上不存在时才拉取
Always:每次创建Pod都会重新拉取一次镜像
Never:Pod永远不会主动拉取这个镜像
https://kubernetes.io/docs/concepts/containers/images
//master1操作
[root@master ~]# kubectl edit deployment/nginx
# Please edit the object below. Lines beginning with a '#' will be ignored,
# and an empty file will abort the edit. If an error occurs while saving this file will be
# reopened with the relevant failures.
# apiVersion: extensions/v1beta1
kind: Deployment
metadata:annotations:deployment.kubernetes.io/revision: "1"creationTimestamp: 2020-10-11T11:40:34Zgeneration: 1labels:run: nginxname: nginxnamespace: defaultresourceVersion: "80812"selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/nginxuid: 935beca5-0bb6-11eb-aede-000c29959657
spec:progressDeadlineSeconds: 600replicas: 3revisionHistoryLimit: 2selector:matchLabels:run: nginxstrategy:rollingUpdate:maxSurge: 25%maxUnavailable: 25%type: RollingUpdatetemplate:metadata:creationTimestamp: nulllabels:run: nginxspec:containers:- image: nginx:latestimagePullPolicy: Always //IfNotPresent Always Nevername: nginxports:- containerPort: 80protocol: TCPresources: {
}terminationMessagePath: /dev/termination-logterminationMessagePolicy: FilednsPolicy: ClusterFirstrestartPolicy: AlwaysschedulerName: default-schedulersecurityContext: {
}terminationGracePeriodSeconds: 30
status:availableReplicas: 3conditions:- lastTransitionTime: 2020-10-11T11:40:34ZlastUpdateTime: 2020-10-11T11:40:38Zmessage: ReplicaSet "nginx-7697996758" has successfully progressed.reason: NewReplicaSetAvailablestatus: "True"type: Progressing- lastTransitionTime: 2020-10-12T08:30:33ZlastUpdateTime: 2020-10-12T08:30:33Zmessage: Deployment has minimum availability.reason: MinimumReplicasAvailablestatus: "True"type: AvailableobservedGeneration: 1readyReplicas: 3replicas: 3updatedReplicas: 3
[root@master ~]# cd demo/
[root@master demo]# vim pod1.yaml
apiVersion: v1
kind: Pod
metadata:name: mypod
spec:containers:- name: nginximage: nginximagePullPolicy: Alwayscommand: [ "echo", "SUCCESS" ][root@master demo]# kubectl create -f pod1.yaml
pod/mypod created[root@master demo]# kubectl get pods
NAME READY STATUS RESTARTS AGE
mypod 0/1 Completed 0 25s
[root@master demo]# kubectl get pods
NAME READY STATUS RESTARTS AGE
mypod 0/1 CrashLoopBackOff 1 47s
//失败的状态的原因是因为命令启动冲突
删除 command: [ “echo”, “SUCCESS” ]
//同时更改一下版本
image: nginx:1.14
//删除原有的资源
[root@master demo]# kubectl delete -f pod1.yaml
pod "mypod" deleted
//更新资源
[root@master demo]# kubectl apply -f pod1.yaml
pod/mypod created
[root@master demo]# kubectl get pods
NAME READY STATUS RESTARTS AGE
mypod 0/1 ContainerCreating 0 7s
[root@master demo]# kubectl get pods
NAME READY STATUS RESTARTS AGE
mypod 1/1 Running 0 18s
//查看分配节点
[root@master demo]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
mypod 1/1 Running 0 88s 172.17.59.5 192.168.20.30 <none>
//在任意node节点使用curl 查看头部信息
//node节点操作
[root@node2 ~]# curl -I 172.17.59.5
HTTP/1.1 200 OK
Server: nginx/1.14.2
Date: Mon, 12 Oct 2020 09:01:56 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Tue, 04 Dec 2018 14:44:49 GMT
Connection: keep-alive
ETag: "5c0692e1-264"
Accept-Ranges: bytes
//如果节点查看时出现 拒绝连接
[root@node1 ~]# curl -I 172.17.59.5
curl: (7) Failed connect to 172.17.59.5:80; 拒绝连接
[root@node1 ~]# ip a //查看flanneld和docker地址
[root@node1 ~]# systemctl status flanneld.service //如果地址没问题,查看flanneld状态
● flanneld.service - Flanneld overlay address etcd agentLoaded: loaded (/usr/lib/systemd/system/flanneld.service; enabled; vendor preset: disabled)Active: inactive (dead) since 六 2020-10-10 15:44:42 CST; 2 days agoMain PID: 79092 (code=exited, status=0/SUCCESS)
解决,所有节点都重启flanneld和docker
[root@node1 ~]# systemctl start flanneld.service
[root@node1 ~]# systemctl restart docker
[root@node2 ~]# systemctl start flanneld.service
[root@node2 ~]# systemctl restart docker
注意:这时分配的节点会发生变动,需要重新查看
[root@master demo]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
mypod 1/1 Running 1 14m 172.17.93.5 192.168.20.30 <none>
[root@node1 ~]# curl -I 172.17.93.5
HTTP/1.1 200 OK
Server: nginx/1.14.2
Date: Mon, 12 Oct 2020 09:13:48 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Tue, 04 Dec 2018 14:44:49 GMT
Connection: keep-alive
ETag: "5c0692e1-264"
Accept-Ranges: bytes
[root@node2 ~]# curl -I 172.17.93.5
HTTP/1.1 200 OK
Server: nginx/1.14.2
Date: Mon, 12 Oct 2020 09:22:52 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Tue, 04 Dec 2018 14:44:49 GMT
Connection: keep-alive
ETag: "5c0692e1-264"
Accept-Ranges: bytes
私有仓库harbor
master1:192.168.20.10 kube-apiserver kube-controller-manager kube-scheduler etcd
master2:192.168.20.40
node1:192.168.20.20 kubelet kube-proxy docker flannel etcd
node2:192.168.20.30 kubelet kube-proxy docker flannel etcd
漂移地址:192.168.20.111
负载均衡LoadBalance
lb01:master 192.168.20.50
lb02:backup 192.168.20.60
Harbor私有仓库:192.168.20.70
[root@localhost ~]# hostnamectl set-hostname harbor
[root@localhost ~]# su
//安装docker环境
//部署harbor创建私有项目
//查看 Docker-Compose 版本判断安装是否成功
//拖入docker-compose
[root@harbor ~]# cp docker-compose /usr/local/bin/
[root@harbor ~]# chmod +x /usr/local/bin/docker-compose
[root@harbor ~]# docker-compose -v
docker-compose version 1.21.1, build 5a3f1a3
//下载 Harbor 安装程序
[root@harbor ~]# wget http:// harbor.orientsoft.cn/harbor-1.2.2/harbor-offline-installer-v1.2.2.tgz
[root@harbor ~]# tar zxvf harbor-offline-installer-v1.2.2.tgz -C /usr/local/[root@harbor ~]# vim /usr/local/harbor/harbor.cfg5 hostname = 192.168.20.70
//启动 Harbor
[root@harbor ~]# sh /usr/local/harbor/install.sh
//查看 Harbor 启动镜像
[root@harbor ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
vmware/harbor-log v1.2.2 36ef78ae27df 2 years ago 200MB
vmware/harbor-jobservice v1.2.2 e2af366cba44 2 years ago 164MB
vmware/harbor-ui v1.2.2 39efb472c253 2 years ago 178MB
vmware/harbor-adminserver v1.2.2 c75963ec543f 2 years ago 142MB
vmware/harbor-db v1.2.2 ee7b9fa37c5d 2 years ago 329MB
vmware/nginx-photon 1.11.13 6cc5c831fc7f 3 years ago 144MB
vmware/registry 2.6.2-photon 5d9100e4350e 3 years ago 173MB
vmware/postgresql 9.6.4-photon c562762cbd12 3 years ago 225MB
vmware/clair v2.0.1-photon f04966b4af6c 3 years ago 297MB
vmware/harbor-notary-db mariadb-10.1.10 64ed814665c6 3 years ago 324MB
vmware/notary-photon signer-0.5.0 b1eda7d10640 3 years ago 156MB
vmware/notary-photon server-0.5.0 6e2646682e3c 3 years ago 157MB
photon 1.0 e6e4e4a2ba1b 4 years ago 128MB
//查看容器
[root@harbor ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
5d2c732fb68a vmware/harbor-jobservice:v1.2.2 "/harbor/harbor_jobs…" 42 minutes ago Up 42 minutes harbor-jobservice
3afa1c9fa978 vmware/nginx-photon:1.11.13 "nginx -g 'daemon of…" 42 minutes ago Up 42 minutes 0.0.0.0:80->80/tcp, 0.0.0.0:443->443/tcp, 0.0.0.0:4443->4443/tcp nginx
13329c20fb73 vmware/harbor-ui:v1.2.2 "/harbor/harbor_ui" 42 minutes ago Up 42 minutes harbor-ui
c19d427d1cce vmware/registry:2.6.2-photon "/entrypoint.sh serv…" 42 minutes ago Up 42 minutes 5000/tcp registry
d241785fbfd6 vmware/harbor-db:v1.2.2 "docker-entrypoint.s…" 42 minutes ago Up 42 minutes 3306/tcp harbor-db
998d134ee01d vmware/harbor-adminserver:v1.2.2 "/harbor/harbor_admi…" 42 minutes ago Up 42 minutes harbor-adminserver
a6265f2e0279 vmware/harbor-log:v1.2.2 "/bin/sh -c 'crond &…" 42 minutes ago Up 42 minutes 127.0.0.1:1514->514/tcp harbor-log[root@harbor ~]# cd /usr/local/harbor/
[root@harbor harbor]# docker-compose psName Command State Ports
-------------------------------------------------------------------------------------------------------------------
harbor-adminserver /harbor/harbor_adminserver Up
harbor-db docker-entrypoint.sh mysqld Up 3306/tcp
harbor-jobservice /harbor/harbor_jobservice Up
harbor-log /bin/sh -c crond && rm -f ... Up 127.0.0.1:1514->514/tcp
harbor-ui /harbor/harbor_ui Up
nginx nginx -g daemon off; Up 0.0.0.0:443->443/tcp, 0.0.0.0:4443->4443/tcp, 0.0.0.0:80->80/tcp
registry /entrypoint.sh serve /etc/ ... Up 5000/tcp [root@harbor harbor]# setenforce 0
[root@harbor harbor]# iptables -F
如果一切都正常,应该可以打开浏览器访问 http://192.168.20.70 的管理页面,默认的管理员用户名和密码是 admin/Harbor12345
//所有node节点配置连接私有仓库(注意后面的逗号要添加,不然服务起不起来)
[root@node1 ~]# vim /etc/docker/daemon.json
{
"registry-mirrors": ["https://yzku4lgz.mirror.aliyuncs.com"],"insecure-registries":["192.168.20.70"]
}
//登录harbor私有仓库
[root@node1 ~]# systemctl restart docker
[root@node1 ~]# docker login 192.168.20.70
Username: admin
Password: //输入密码Harbor12345
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-storeLogin Succeeded
//下载Tomcat镜像进行推送
[root@node1 ~]# docker pull tomcat
Using default tag: latest
latest: Pulling from library/tomcat
57df1a1f1ad8: Pull complete
71e126169501: Pull complete
1af28a55c3f3: Pull complete
03f1c9932170: Pull complete
881ad7aafb13: Pull complete
9c0ffd4062f3: Pull complete
bd62e479351a: Pull complete
48ee8bc64dbc: Pull complete
07cb85cca4f0: Pull complete
6a78fac8d191: Pull complete
Digest: sha256:1bab37d5d97bd8c74a474b2c1a62bbf1f1b4b62f151c8dcc472c7d577eb3479d
Status: Downloaded newer image for tomcat:latest
docker.io/library/tomcat:latest
//推送格式
docker tag SOURCE_IMAGE[:TAG] 192.168.195.80/project/IMAGE[:TAG]
//打标签
[root@node1 ~]# docker tag tomcat 192.168.20.70/project/tomcat
//推送成功
[root@node1 ~]# docker push 192.168.20.70/project/tomcat
The push refers to repository [192.168.20.70/project/tomcat]
b654a29de9ee: Pushed
1485ce09f585: Pushed
eb6e8fe5c6dc: Pushed
8b185d674aef: Pushed
4f17d163126f: Pushed
df95ed2a791d: Pushed
17bdf5e22660: Pushed
d37096232ed8: Pushed
6add0d2b5482: Pushed
4ef54afed780: Pushed
latest: digest: sha256:99c20ba4ab117d182a0aa2266123b2cfb425777495fd62e2ba37f489c3e2f808 size: 2421
//进行镜像下载问题就会出现,需要登录才能下载
//问题点:缺少仓库的凭据
[root@node2 ~]# docker pull 192.168.20.70/project/tomcat
Using default tag: latest
Error response from daemon: pull access denied for 192.168.195.80/project/tomcat, repository does not exist or may require 'docker login': denied: requested access to the resource is denied
//node节点下载tomcat镜像
docker pull tomcat:8.0.52
[root@master demo]# vim tomcat-deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:name: my-tomcat
spec:replicas: 2template:metadata:labels:app: my-tomcatspec:containers:- name: my-tomcatimage: docker.io/tomcat:8.0.52ports:- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:name: my-tomcat
spec:type: NodePortports:- port: 8080targetPort: 8080nodePort: 31111selector:app: my-tomcat[root@master demo]# kubectl create -f tomcat-deployment.yaml
deployment.extensions/my-tomcat created
service/my-tomcat created[root@master demo]# kubectl get pods,deploy,svc
NAME READY STATUS RESTARTS AGE
pod/my-tomcat-57667b9d9-7z584 1/1 Running 0 41s
pod/my-tomcat-57667b9d9-bzhzz 1/1 Running 0 41s
pod/mypod 1/1 Running 2 112m
pod/nginx-7697996758-24gnd 1/1 Running 2 23h
pod/nginx-7697996758-69hsc 1/1 Running 2 23h
pod/nginx-7697996758-lf7k6 1/1 Running 2 23h
pod/nginx-deployment-d55b94fd-gzsvv 1/1 Running 2 23h
pod/nginx-deployment-d55b94fd-rhp67 1/1 Running 2 23h
pod/nginx-deployment-d55b94fd-tkdjs 1/1 Running 2 23hNAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
deployment.extensions/my-tomcat 2 2 2 2 41s
deployment.extensions/nginx 3 3 3 3 23h
deployment.extensions/nginx-deployment 3 3 3 3 23hNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.0.0.1 <none> 443/TCP 3d21h
service/my-tomcat NodePort 10.0.0.27 <none> 8080:31111/TCP 41s
service/nginx-service NodePort 10.0.0.217 <none> 80:32302/TCP 23h
//问题处理
//如果遇到处于Terminating状态的无法删除的资源如何处理
[root@master demo]# kubectl get pods
NAME READY STATUS RESTARTS AGE
my-tomcat-57667b9d9-nklvj 1/1 Terminating 0 10h
my-tomcat-57667b9d9-wllnp 1/1 Terminating 0 10h
//这种情况下可以使用强制删除命令:
kubectl delete pod [pod name] --force --grace-period=0 -n [namespace][root@master demo]# kubectl delete pod my-tomcat-57667b9d9-nklvj --force --grace-period=0 -n default
warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
pod "my-tomcat-57667b9d9-nklvj" force deleted[root@master demo]# kubectl delete pod my-tomcat-57667b9d9-wllnp --force --grace-period=0 -n default
warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
pod "my-tomcat-57667b9d9-wllnp" force deleted[root@master demo]# kubectl get pods
NAME READY STATUS RESTARTS AGE
pod/mypod 1/1 Running 2 112m
pod/nginx-7697996758-24gnd 1/1 Running 2 23h
pod/nginx-7697996758-69hsc 1/1 Running 2 23h
pod/nginx-7697996758-lf7k6 1/1 Running 2 23h
pod/nginx-deployment-d55b94fd-gzsvv 1/1 Running 2 23h
pod/nginx-deployment-d55b94fd-rhp67 1/1 Running 2 23h
pod/nginx-deployment-d55b94fd-tkdjs 1/1 Running 2 23h
//node01上操作(之前登陆过harbor仓库的节点)
//镜像打标签
[root@node1 ~]# docker tag tomcat:8.0.52 192.168.195.80/project/tomcat
//上传镜像到harbor
[root@node1 ~]# docker tag tomcat:8.0.52 192.168.20.70/project/tomcat
[root@node1 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
nginx latest 992e3b7be046 6 days ago 133MB
tomcat latest f796d3d2c195 3 weeks ago 647MB
192.168.20.70/project/tomcat <none> f796d3d2c195 3 weeks ago 647MB
centos 7 7e6257c9f8d8 2 months ago 203MB
nginx 1.14 295c7be07902 18 months ago 109MB
nginx 1.15.4 bc26f1ed35cf 2 years ago 109MB
192.168.20.70/project/tomcat latest b4b762737ed4 2 years ago 356MB
tomcat 8.0.52 b4b762737ed4 2 years ago 356MB
siriuszg/kubernetes-dashboard-amd64 v1.8.3 784cf2722f44 2 years ago 102MB
registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64 3.0 99e59f495ffa 4 years ago 747kB
//查看登陆凭据
[root@node1 ~]# ls -a
. .bash_profile .dbus .ICEauthority .pki .viminfo 图片
.. .bashrc .docker initial-setup-ks.cfg proxy.sh .Xauthority 文档
anaconda-ks.cfg .cache .esd_auth kubelet.sh README.md 公共 下载
.bash_history .config flannel.sh .local .ssh 模板 音乐
.bash_logout .cshrc flannel-v0.10.0-linux-amd64.tar.gz node.zip .tcshrc 视频 桌面
//.docker/config.json 凭证,登陆后出现
[root@node1 ~]# cat .docker/config.json |base64 -w 0 //-w, --wrap=字符数 在指定的字符数后自动换行(默认为76),0 为禁用自动换行
ewoJImF1dGhzIjogewoJCSIxOTIuMTY4LjIwLjcwIjogewoJCQkiYXV0aCI6ICJZV1J0YVc0NlNHRnlZbTl5TVRJek5EVT0iCgkJfQoJfSwKCSJIdHRwSGVhZGVycyI6IHsKCQkiVXNlci1BZ2VudCI6ICJEb2NrZXItQ2xpZW50LzE5LjAzLjEzIChsaW51eCkiCgl9Cn0=
[root@master demo]# vim registry-pull-secret.yaml
apiVersion: v1
kind: Secret
metadata:name: registry-pull-secret
data:.dockerconfigjson: ewoJImF1dGhzIjogewoJCSIxOTIuMTY4LjIwLjcwIjogewoJCQkiYXV0aCI6ICJZV1J0YVc0NlNHRnlZbTl5TVRJek5EVT0iCgkJfQoJfSwKCSJIdHRwSGVhZGVycyI6IHsKCQkiVXNlci1BZ2VudCI6ICJEb2NrZXItQ2xpZW50LzE5LjAzLjEzIChsaW51eCkiCgl9Cn0=
type: kubernetes.io/dockerconfigjson
//创建secret资源
[root@master demo]# kubectl create -f registry-pull-secret.yaml
secret/registry-pull-secret created
//查看secret资源
[root@master demo]# kubectl get secret
NAME TYPE DATA AGE
default-token-kfkw4 kubernetes.io/service-account-token 3 3d21h
registry-pull-secret kubernetes.io/dockerconfigjson 1 44s
//删除原来的
[root@master demo]# kubectl delete -f tomcat-deployment.yaml
deployment.extensions "my-tomcat" deleted
service "my-tomcat" deleted
//创建资源从harbor中下载镜像
[root@master demo]# vim tomcat-deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:name: my-tomcat
spec:replicas: 2template:metadata:labels:app: my-tomcatspec:imagePullSecrets:- name: registry-pull-secretcontainers:- name: my-tomcatimage: 192.168.20.70/project/tomcatports:- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:name: my-tomcat
spec:type: NodePortports:- port: 8080targetPort: 8080nodePort: 31111selector:app: my-tomcat[root@master demo]# kubectl create -f tomcat-deployment.yaml
deployment.extensions/my-tomcat created
service/my-tomcat created
//私有仓库中的镜像被下载了2次
[root@master demo]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
my-tomcat-86cb68c9c4-lx7lm 1/1 Running 0 5m53s 172.17.87.6 192.168.20.20 <none>
my-tomcat-86cb68c9c4-wpq6z 1/1 Running 0 5m53s 172.17.93.6 192.168.20.30 <none>
//查看详细信息
[root@master demo]# kubectl describe deployment/my-tomcat
Name: my-tomcat
Namespace: default
CreationTimestamp: Mon, 12 Oct 2020 19:16:52 +0800
Labels: app=my-tomcat
Annotations: deployment.kubernetes.io/revision: 1
Selector: app=my-tomcat
Replicas: 2 desired | 2 updated | 2 total | 2 available | 0 unavailable
StrategyType: RollingUpdate
MinReadySeconds: 0
RollingUpdateStrategy: 1 max unavailable, 1 max surge
Pod Template:Labels: app=my-tomcatContainers:my-tomcat:Image: 192.168.20.70/project/tomcat Port: 80/TCPHost Port: 0/TCPEnvironment: <none>Mounts: <none>Volumes: <none>
Conditions:Type Status Reason---- ------ ------Available True MinimumReplicasAvailable
OldReplicaSets: <none>
NewReplicaSet: my-tomcat-86cb68c9c4 (2/2 replicas created)
Events:Type Reason Age From Message---- ------ ---- ---- -------Normal ScalingReplicaSet 9m17s deployment-controller Scaled up replica set my-tomcat-86cb68c9c4 to 2