Kubernetes部署Nacos集群
一、k8s架构
1master
2node
二、安装nfs
安装nfs-utils和rpcbind
nfs客户端和服务端都安装nfs-utils包
yum install nfs-utils rpcbind
创建共享目录
mkdir -p /nfsdata
chmod 777 /nfsdata
编辑/etc/exports文件添加如下内容
vi /etc/exports/nfsdata *(rw,sync,no_root_squash)
启动服务
systemctl enable rpcbind.service --nowsystemctl enable nfs.service --now
启动顺序一定是rpcbind->nfs，否则有可能出现错误
三、部署自动分配PV的相关程序
创建StorageClass
因为StorageClass可以实现自动配置，所以使用StorageClass之前，我们需要先安装存储驱动的自动配置程序，而这个配置程序必须拥有一定的权限去访问我们的kubernetes集群(类似dashboard一样，必须有权限访问各种api，才能实现管理)。
创建rbac(Role-Based Access Control:基于角色的访问控制)：
apiVersion: v1
kind: ServiceAccount
metadata:name: nfs-client-provisionernamespace: elk
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: nfs-client-provisioner-runnernamespace: elk
rules:- apiGroups: [""]resources: ["persistentvolumes"]verbs: ["get", "list", "watch", "create", "delete"]- apiGroups: [""]resources: ["persistentvolumeclaims"]verbs: ["get", "list", "watch", "update"]- apiGroups: ["storage.k8s.io"]resources: ["storageclasses"]verbs: ["get", "list", "watch"]- apiGroups: [""]resources: ["events"]verbs: ["create", "update", "patch"]###此处需要注意的是，如果你的名称空间是default，可以不加下面这个授权###- apiGroups: [""]resources: ["endpoints"]verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: run-nfs-client-provisioner
subjects:- kind: ServiceAccountname: nfs-client-provisionernamespace: elk
roleRef:kind: ClusterRolename: nfs-client-provisioner-runnerapiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: leader-locking-nfs-client-provisioner# replace with namespace where provisioner is deployednamespace: elk
rules:- apiGroups: [""]resources: ["endpoints"]verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: leader-locking-nfs-client-provisioner
subjects:- kind: ServiceAccountname: nfs-client-provisioner# replace with namespace where provisioner is deployednamespace: elk
roleRef:kind: Rolename: leader-locking-nfs-client-provisionerapiGroup: rbac.authorization.k8s.io
创建StorageClass
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: master-nfs-storage
provisioner: master-nfs-storage #这里的名称要和下面的provisioner配置文件中的环境变量PROVISIONER_NAME保持一致
parameters: archiveOnDelete: "false"
创建自动配置程序 - NFS客户端
apiVersion: apps/v1
kind: Deployment
metadata:name: nfs-client-provisionerlabels:app: nfs-client-provisionernamespace: elk
spec:replicas: 1selector:matchLabels:app: nfs-client-provisionerstrategy:type: Recreatetemplate:metadata:labels:app: nfs-client-provisionerspec:serviceAccountName: nfs-client-provisionercontainers:- name: nfs-client-provisionerimage: quay.io/external_storage/nfs-client-provisioner:latestvolumeMounts:- name: nfs-client-rootmountPath: /persistentvolumesenv:- name: PROVISIONER_NAMEvalue: master-nfs-storage- name: NFS_SERVERvalue: 11.0.1.5- name: NFS_PATHvalue: /nfsdatavolumes:- name: nfs-client-rootnfs:server: 11.0.1.5path: /nfsdata
四、部署MySQL
[root@master1 mysql]# cat mysql.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:name: mysqlnamespace: devopsannotations:volume.beta.kubernetes.io/storage-provisioner: master-nfs-storage
spec:accessModes:- ReadWriteOnceresources:requests:storage: 1GistorageClassName: master-nfs-storage
---
apiVersion: v1
kind: ReplicationController
metadata:name: mysqlnamespace: devopslabels:name: mysql
spec:replicas: 1selector:name: mysqltemplate:metadata:labels:name: mysqlspec:containers:- name: mysqlimage: nacos/nacos-mysql:5.7 ports:- containerPort: 3306env:- name: MYSQL_ROOT_PASSWORDvalue: "root"- name: MYSQL_DATABASEvalue: "nacos_devtest"- name: MYSQL_USERvalue: "nacos"- name: MYSQL_PASSWORDvalue: "nacos"volumeMounts:- name: mysql-datamountPath: /var/lib/mysqlvolumes:- name: mysql-datapersistentVolumeClaim:claimName: mysql
---
apiVersion: v1
kind: Service
metadata:name: mysqlnamespace: devopslabels:name: mysql
spec:type: NodePortports:- port: 3306targetPort: 3306nodePort: 30001selector:name: mysql
五、部署Nacos集群
[root@master1 nacos]# cat nacos-tmp.yaml
# 请阅读Wiki文章
# https://github.com/nacos-group/nacos-k8s/wiki/%E4%BD%BF%E7%94%A8peerfinder%E6%89%A9%E5%AE%B9%E6%8F%92%E4%BB%B6
---
apiVersion: v1
kind: Service
metadata:name: nacos-headlessnamespace: devopslabels:app: nacos
spec:publishNotReadyAddresses: true ports:- port: 8848name: servertargetPort: 8848- port: 9848name: client-rpctargetPort: 9848- port: 9849name: raft-rpctargetPort: 9849## 兼容1.4.x版本的选举端口- port: 7848name: old-raft-rpctargetPort: 7848clusterIP: Noneselector:app: nacos
---
apiVersion: v1
kind: ConfigMap
metadata:name: nacos-cmnamespace: devops
data:mysql.host: "mysql"mysql.db.name: "nacos_devtest"mysql.port: "3306"mysql.user: "nacos"mysql.password: "nacos"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:name: nacosnamespace: devops
spec:podManagementPolicy: ParallelserviceName: nacos-headlessreplicas: 3template:metadata:labels:app: nacosannotations:pod.alpha.kubernetes.io/initialized: "true"spec:affinity:podAntiAffinity:requiredDuringSchedulingIgnoredDuringExecution:- labelSelector:matchExpressions:- key: "app"operator: Invalues:- nacostopologyKey: "kubernetes.io/hostname"serviceAccountName: nfs-client-provisionerinitContainers:- name: peer-finder-plugin-installimage: nacos/nacos-peer-finder-plugin:1.1imagePullPolicy: IfNotPresentvolumeMounts:- mountPath: /home/nacos/plugins/peer-findername: datasubPath: peer-findercontainers:- name: nacosimagePullPolicy: IfNotPresentimage: nacos/nacos-server:v2.2.0resources:requests:memory: "2Gi"cpu: "500m"ports:- containerPort: 8848name: client-port- containerPort: 9848name: client-rpc- containerPort: 9849name: raft-rpc- containerPort: 7848name: old-raft-rpcenv:- name: NACOS_REPLICASvalue: "3"- name: SERVICE_NAMEvalue: "nacos-headless"- name: DOMAIN_NAMEvalue: "cluster.local"- name: POD_NAMESPACEvalueFrom:fieldRef:apiVersion: v1fieldPath: metadata.namespace- name: MYSQL_SERVICE_HOSTvalueFrom:configMapKeyRef:name: nacos-cmkey: mysql.host- name: MYSQL_SERVICE_DB_NAMEvalueFrom:configMapKeyRef:name: nacos-cmkey: mysql.db.name- name: MYSQL_SERVICE_PORTvalueFrom:configMapKeyRef:name: nacos-cmkey: mysql.port- name: MYSQL_SERVICE_USERvalueFrom:configMapKeyRef:name: nacos-cmkey: mysql.user- name: MYSQL_SERVICE_PASSWORDvalueFrom:configMapKeyRef:name: nacos-cmkey: mysql.password- name: SPRING_DATASOURCE_PLATFORMvalue: "mysql"- name: NACOS_SERVER_PORTvalue: "8848"- name: NACOS_APPLICATION_PORTvalue: "8848"- name: PREFER_HOST_MODEvalue: "hostname"volumeMounts:- name: datamountPath: /home/nacos/plugins/peer-findersubPath: peer-finder- name: datamountPath: /home/nacos/datasubPath: data- name: datamountPath: /home/nacos/logssubPath: logsvolumeClaimTemplates:- metadata:name: dataannotations:volume.beta.kubernetes.io/storage-class: "master-nfs-storage"spec:accessModes: [ "ReadWriteMany" ]resources:requests:storage: 1Giselector:matchLabels:app: nacos
六、创建ingress
获取ingress-nginx，本次案例使用的是0.30版本
wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.30.0/deploy/static/mandatory.yaml
wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.30.0/deploy/static/provider/baremetal/service-nodeport.yaml
修改mandatory.yaml文件中的仓库
修改quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0
为quay-mirror.qiniu.com/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0
创建ingress-nginx
[root@k8s-master01 ingress-controller]# kubectl apply -f ./
查看ingress-nginx
[root@k8s-master01 ingress-controller]# kubectl get pod -n ingress-nginx
NAME READY STATUS RESTARTS AGE
pod/nginx-ingress-controller-fbf967dd5-4qpbp 1/1 Running 0 12h
查看service
[root@k8s-master01 ingress-controller]# kubectl get svc -n ingress-nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ingress-nginx NodePort 10.98.75.163 <none> 80:32240/TCP,443:31335/TCP 11h
创建nacos的ingress
[root@master1 nacos]# cat nacos-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:name: nacos-ingressnamespace: devops
spec:rules:- host: www.mynacos.com http:paths:- path: /nacosbackend:serviceName: nacos-headless servicePort: 8848
访问测试。