FROM openjdk:8
WORKDIR /usr/local
ADD apache-zookeeper-3.8.0-bin.tar.gz /usr/local/
# 修改docker時區為東八區，規避應用程序和北京時間相差8小時問題
ENV TZ=Asia/Shanghai
# 下一步可以刪除，在pod中設置command來操作也可以
CMD /usr/local/apache-zookeeper-3.8.0-bin/bin/zkServer.sh start-foreground
Docker

[root@node01 ~]# ls -rlt
總用量 12888
-rw-------. 1 root root     1698 3月 28 16:37 anaconda-ks.cfg
-rw-r--r--  1 root root 13185104 4月 27 14:34 apache-zookeeper-3.8.0-bin.tar.gz
-rw-r--r--  1 root root      226 4月 27 18:05 dockerfile
# 構建鏡像名為myzk:1.0的鏡像
[root@node01 ~]# docker build -f dockerfile -t myzk:1.0 .
Bash

# 在master上安裝nfs服務，設置開機自啟動
[root@nfs ~]# yum install nfs-utils -y
[root@nfs ~]# systemctl restart nfs
[root@nfs ~]# systemctl enable nfs

# 在node上安裝nfs服務，注意不需要啟動
[root@k8s-master01 ~]# yum install nfs-utils -y
Bash

# 創建共享目錄
for x in $(seq 1 3);
> do 
> mkdir -p /data/pv/zk${x}
> done

# 將共享目錄暴露，暴露給指定主機將“192.168.158.0/24”改成主機ip即可
[root@master ~]# vim /etc/exports
/data/pv/zk1 192.168.13.0/24(rw,no_root_squash)
/data/pv/zk2 192.168.13.0/24(rw,no_root_squash)
/data/pv/zk3 192.168.13.0/24(rw,no_root_squash)
Bash

apiVersion: v1
kind: PersistentVolume
metadata:
 name: k8s-pv-zk1
 namespace: zookeeper
spec:
 capacity:
   storage: 2Gi
 accessModes:
   - ReadWriteOnce
 storageClassName: "zookeeper"
 nfs:
   path: /data/pv/zk1
   server: 192.168.13.129
 persistentVolumeReclaimPolicy: Recycle
---
apiVersion: v1
kind: PersistentVolume
metadata:
 name: k8s-pv-zk2
 namespace: zookeeper
spec:
 capacity:
   storage: 2Gi
 accessModes:
   - ReadWriteOnce
 storageClassName: "zookeeper"
 nfs:
   path: /data/pv/zk2
   server: 192.168.13.129
 persistentVolumeReclaimPolicy: Recycle
---
apiVersion: v1
kind: PersistentVolume
metadata:
 name: k8s-pv-zk3
 namespace: zookeeper
spec:
 capacity:
   storage: 2Gi
 accessModes:
   - ReadWriteOnce
 storageClassName: "zookeeper"
 nfs:
   path: /data/pv/zk3
   server: 192.168.13.129
 persistentVolumeReclaimPolicy: Recycle
YAML

[root@master ~]# kubectl apply -f pv-zk.yaml
# 創建成功可以通過kubectl命令看到下列3個pv
[root@master ~]# kubectl get pv -n zookeeper
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
k8s-pv-zk1 2Gi RWO Recycle Bound zookeeper/data-zookeeper-cluster-1   zookeeper 17m
k8s-pv-zk2 2Gi RWO Recycle Bound zookeeper/data-zookeeper-cluster-0   zookeeper 17m
k8s-pv-zk3 2Gi RWO Recycle Bound zookeeper/data-zookeeper-cluster-2   zookeeper 17m
Bash

apiVersion: v1
kind: Service # 該svc用于zk集群內部通信
metadata:
 name: zk-hs
 namespace: zookeeper
 labels:
   app: zk
spec:
 ports:
 - port: 2888
   name: server
 - port: 3888
   name: leader-election
 clusterIP: None
 selector:
   app: zk
---
apiVersion: v1
kind: Service # 該svc用于與zk集群外部通信
metadata:
 name: zk-cs
 namespace: zookeeper
 labels:
   app: zk
spec:
 type: ClusterIP # 僅允許k8s集群內部機器訪問，若需要外部機器或瀏覽器訪問，需要將該類型改為NodePort
 ports:
 - port: 2181
   name: client
 selector:
   app: zk
---
apiVersion: v1
kind: ConfigMap # 用于保存zk配置文件
metadata:
 name: zoo-conf
 namespace: zookeeper
data:
 zoo.cfg: |+
 
   # The number of milliseconds of each tick
   tickTime=2000
   # The number of ticks that the initial
   # synchronization phase can take
   initLimit=10
   # The number of ticks that can pass between
   # sending a request and getting an acknowledgement
   syncLimit=5
   # the directory where the snapshot is stored.
   # do not use /tmp for storage, /tmp here is just
   # example sakes.
   dataDir=/usr/local/apache-zookeeper-3.8.0-bin/data
   dataLogDir=/usr/local/apache-zookeeper-3.8.0-bin/data/log
   # the port at which the clients will connect
   clientPort=2181
   # the maximum number of client connections.
   # increase this if you need to handle more clients
   #maxClientCnxns=60
   #
   # Be sure to read the maintenance section of the
   # administrator guide before turning on autopurge.
   #
   # https://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
   #
   # The number of snapshots to retain in dataDir
   #autopurge.snapRetainCount=3
   # Purge task interval in hours
   # Set to "0" to disable auto purge feature
   #autopurge.purgeInterval=1
   
   ## Metrics Providers
   #
   # https://prometheus.io Metrics Exporter
   #metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider
   #metricsProvider.httpHost=0.0.0.0
   #metricsProvider.httpPort=7000
   #metricsProvider.exportJvmInfo=true
   # 由于pod重啟后容器ip會改變，將ip轉換為DNS域名 格式"pod名.內部通信的服務名.命名空間名.svc.cluster.local"
   server.1=zookeeper-cluster-0.zk-hs.zookeeper.svc.cluster.local:2888:3888 
   server.2=zookeeper-cluster-1.zk-hs.zookeeper.svc.cluster.local:2888:3888
   server.3=zookeeper-cluster-2.zk-hs.zookeeper.svc.cluster.local:2888:3888
---
apiVersion: apps/v1
kind: StatefulSet # 由于每次pod重啟容器ip會改變，所以需要使用StatefulSet控制器保證pod名不變，從而可以使用DNS域名替代ip
metadata:
 name: zookeeper-cluster
 namespace: zookeeper
spec:
 selector:
   matchLabels:
     app: zk
 serviceName: zk-hs
 replicas: 3
 template:
   metadata:
     labels:
       app: zk
   spec:
     containers:
     - name: zookeeper
       imagePullPolicy: IfNotPresent
       image: "myzk:1.0"
       ports:
       - containerPort: 2181
         name: client
       - containerPort: 2888
         name: server
       - containerPort: 3888
         name: leader-election
       volumeMounts:
       - name: conf
         mountPath: /usr/local/apache-zookeeper-3.8.0-bin/conf
       - name: data
         mountPath: /usr/local/apache-zookeeper-3.8.0-bin/data
       command: [ "/usr/local/apache-zookeeper-3.8.0-bin/bin/zkServer.sh","start-foreground" ] # 這一步可以在dockerfile中用CMD實現
     volumes:
     - name: conf
       configMap:
         name: zoo-conf
         defaultMode: 0755
 volumeClaimTemplates:
 - metadata:
     name: data
   spec:
     accessModes: [ "ReadWriteOnce" ]
     resources:
       requests:
         storage: 2Gi
     storageClassName: "zookeeper"
YAML

[root@master ~]# kubectl apply -f zk-cluster.yaml
# 查看
[root@master ~]# kubectl get pods -n zookeeper
NAME READY STATUS RESTARTS AGE
zookeeper-cluster-0   0/1     CrashLoopBackOff 1          17s
Bash

[root@master ~]# kubectl logs zookeeper-cluster-0 -n zookeeper
21:20:18.807 [main] ERROR org.apache.zookeeper.server.quorum.QuorumPeerMain - Invalid config, exiting abnormally
org.apache.zookeeper.server.quorum.QuorumPeerConfig$ConfigException: Error processing /usr/local/apache-zookeeper-3.8.0-bin/bin/../conf/zoo.cfg
       at org.apache.zookeeper.server.quorum.QuorumPeerConfig.parse(QuorumPeerConfig.java:198)
       at org.apache.zookeeper.server.quorum.QuorumPeerMain.initializeAndRun(QuorumPeerMain.java:125)
       at org.apache.zookeeper.server.quorum.QuorumPeerMain.main(QuorumPeerMain.java:91)
Caused by: java.lang.IllegalArgumentException: myid file is missing
       at org.apache.zookeeper.server.quorum.QuorumPeerConfig.checkValidity(QuorumPeerConfig.java:792)
       at org.apache.zookeeper.server.quorum.QuorumPeerConfig.setupQuorumPeerConfig(QuorumPeerConfig.java:663)
       at org.apache.zookeeper.server.quorum.QuorumPeerConfig.parseProperties(QuorumPeerConfig.java:487)
       at org.apache.zookeeper.server.quorum.QuorumPeerConfig.parse(QuorumPeerConfig.java:194)
       ... 2 common frames omitted
YAML

[root@master ~]# kubectl get pv -n zookeeper 
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
k8s-pv-zk1 2Gi RWO Recycle Bound zookeeper/data-zookeeper-cluster-1   zookeeper 38m
k8s-pv-zk2 2Gi RWO Recycle Bound zookeeper/data-zookeeper-cluster-0   zookeeper 38m
k8s-pv-zk3 2Gi RWO Recycle Bound zookeeper/data-zookeeper-cluster-2   zookeeper 38m
[root@master zk1]# echo 2 > /data/pv/zk1/myid
[root@master zk1]# echo 1 > /data/pv/zk1/myid
[root@master zk1]# echo 3 > /data/pv/zk1/myid
Bash

[root@master ~]# kubectl get pod -n zookeeper
NAME READY STATUS RESTARTS AGE
zookeeper-cluster-0   1/1     Running 4          41m
zookeeper-cluster-1   1/1     Running 4          40m
zookeeper-cluster-2   1/1     Running 0          39m
Bash

[root@master ~]# kubectl exec -it -n zookeeper zookeeper-cluster-0 -- /usr/local/apache-zookeeper-3.8.0-bin/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/apache-zookeeper-3.8.0-bin/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower

[root@master ~]# kubectl exec -it -n zookeeper zookeeper-cluster-1 -- /usr/local/apache-zookeeper-3.8.0-bin/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/apache-zookeeper-3.8.0-bin/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: leader

[root@master ~]# kubectl exec -it -n zookeeper zookeeper-cluster-2 -- /usr/local/apache-zookeeper-3.8.0-bin/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/apache-zookeeper-3.8.0-bin/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower
Bash