一、k8s架构
1master
2node
二、安装nfs
安装nfs-utils和rpcbind
nfs客户端和服务端都安装nfs-utils包
yum install nfs-utils rpcbind
创建共享目录
mkdir -p /nfsdata
chmod 777 /nfsdata
编辑/etc/exports文件添加如下内容
vi /etc/exports
/nfsdata *(rw,sync,no_root_squash)
启动服务
systemctl enable rpcbind.service --now
systemctl enable nfs.service --now
启动顺序一定是rpcbind->nfs，否则有可能出现错误
三、部署自动分配PV的相关程序
创建StorageClass
因为StorageClass可以实现自动配置，所以使用StorageClass之前，我们需要先安装存储驱动的自动配置程序，而这个配置程序必须拥有一定的权限去访问我们的kubernetes集群(类似dashboard一样，必须有权限访问各种api，才能实现管理)。
创建rbac(Role-Based Access Control:基于角色的访问控制)：
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
namespace: elk
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
namespace: elk
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
###此处需要注意的是，如果你的名称空间是default，可以不加下面这个授权###
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: elk
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: elk
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: elk
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
创建StorageClass
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: master-nfs-storage
provisioner: master-nfs-storage #这里的名称要和下面的provisioner配置文件中的环境变量PROVISIONER_NAME保持一致
parameters:
archiveOnDelete: "false"
创建自动配置程序 - NFS客户端
apiVersion: apps/v1
kind: Deployment
metadata:
name: nfs-client-provisioner
labels:
app: nfs-client-provisioner
namespace: elk
spec:
replicas: 1
selector:
matchLabels:
app: nfs-client-provisioner
strategy:
type: Recreate
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
image: quay.io/external_storage/nfs-client-provisioner:latest
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: master-nfs-storage
- name: NFS_SERVER
value: 11.0.1.5
- name: NFS_PATH
value: /nfsdata
volumes:
- name: nfs-client-root
nfs:
server: 11.0.1.5
path: /nfsdata
四、部署MySQL
[root@master1 mysql]# cat mysql.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: mysql
namespace: devops
annotations:
volume.beta.kubernetes.io/storage-provisioner: master-nfs-storage
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: master-nfs-storage
---
apiVersion: v1
kind: ReplicationController
metadata:
name: mysql
namespace: devops
labels:
name: mysql
spec:
replicas: 1
selector:
name: mysql
template:
metadata:
labels:
name: mysql
spec:
containers:
- name: mysql
image: nacos/nacos-mysql:5.7
ports:
- containerPort: 3306
env:
- name: MYSQL_ROOT_PASSWORD
value: "root"
- name: MYSQL_DATABASE
value: "nacos_devtest"
- name: MYSQL_USER
value: "nacos"
- name: MYSQL_PASSWORD
value: "nacos"
volumeMounts:
- name: mysql-data
mountPath: /var/lib/mysql
volumes:
- name: mysql-data
persistentVolumeClaim:
claimName: mysql
---
apiVersion: v1
kind: Service
metadata:
name: mysql
namespace: devops
labels:
name: mysql
spec:
type: NodePort
ports:
- port: 3306
targetPort: 3306
nodePort: 30001
selector:
name: mysql
五、部署Nacos集群
[root@master1 nacos]# cat nacos-tmp.yaml
# 请阅读Wiki文章
# https://github.com/nacos-group/nacos-k8s/wiki/%E4%BD%BF%E7%94%A8peerfinder%E6%89%A9%E5%AE%B9%E6%8F%92%E4%BB%B6
---
apiVersion: v1
kind: Service
metadata:
name: nacos-headless
namespace: devops
labels:
app: nacos
spec:
publishNotReadyAddresses: true
ports:
- port: 8848
name: server
targetPort: 8848
- port: 9848
name: client-rpc
targetPort: 9848
- port: 9849
name: raft-rpc
targetPort: 9849
## 兼容1.4.x版本的选举端口
- port: 7848
name: old-raft-rpc
targetPort: 7848
clusterIP: None
selector:
app: nacos
---
apiVersion: v1
kind: ConfigMap
metadata:
name: nacos-cm
namespace: devops
data:
mysql.host: "mysql"
mysql.db.name: "nacos_devtest"
mysql.port: "3306"
mysql.user: "nacos"
mysql.password: "nacos"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: nacos
namespace: devops
spec:
podManagementPolicy: Parallel
serviceName: nacos-headless
replicas: 3
template:
metadata:
labels:
app: nacos
annotations:
pod.alpha.kubernetes.io/initialized: "true"
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- nacos
topologyKey: "kubernetes.io/hostname"
serviceAccountName: nfs-client-provisioner
initContainers:
- name: peer-finder-plugin-install
image: nacos/nacos-peer-finder-plugin:1.1
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /home/nacos/plugins/peer-finder
name: data
subPath: peer-finder
containers:
- name: nacos
imagePullPolicy: IfNotPresent
image: nacos/nacos-server:v2.2.0
resources:
requests:
memory: "2Gi"
cpu: "500m"
ports:
- containerPort: 8848
name: client-port
- containerPort: 9848
name: client-rpc
- containerPort: 9849
name: raft-rpc
- containerPort: 7848
name: old-raft-rpc
env:
- name: NACOS_REPLICAS
value: "3"
- name: SERVICE_NAME
value: "nacos-headless"
- name: DOMAIN_NAME
value: "cluster.local"
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: MYSQL_SERVICE_HOST
valueFrom:
configMapKeyRef:
name: nacos-cm
key: mysql.host
- name: MYSQL_SERVICE_DB_NAME
valueFrom:
configMapKeyRef:
name: nacos-cm
key: mysql.db.name
- name: MYSQL_SERVICE_PORT
valueFrom:
configMapKeyRef:
name: nacos-cm
key: mysql.port
- name: MYSQL_SERVICE_USER
valueFrom:
configMapKeyRef:
name: nacos-cm
key: mysql.user
- name: MYSQL_SERVICE_PASSWORD
valueFrom:
configMapKeyRef:
name: nacos-cm
key: mysql.password
- name: SPRING_DATASOURCE_PLATFORM
value: "mysql"
- name: NACOS_SERVER_PORT
value: "8848"
- name: NACOS_APPLICATION_PORT
value: "8848"
- name: PREFER_HOST_MODE
value: "hostname"
volumeMounts:
- name: data
mountPath: /home/nacos/plugins/peer-finder
subPath: peer-finder
- name: data
mountPath: /home/nacos/data
subPath: data
- name: data
mountPath: /home/nacos/logs
subPath: logs
volumeClaimTemplates:
- metadata:
name: data
annotations:
volume.beta.kubernetes.io/storage-class: "master-nfs-storage"
spec:
accessModes: [ "ReadWriteMany" ]
resources:
requests:
storage: 1Gi
selector:
matchLabels:
app: nacos
六、创建ingress
获取ingress-nginx，本次案例使用的是0.30版本
wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.30.0/deploy/static/mandatory.yaml
wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.30.0/deploy/static/provider/baremetal/service-nodeport.yaml
修改mandatory.yaml文件中的仓库
修改quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0
为quay-mirror.qiniu.com/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0
创建ingress-nginx
[root@k8s-master01 ingress-controller]# kubectl apply -f ./
查看ingress-nginx
[root@k8s-master01 ingress-controller]# kubectl get pod -n ingress-nginx
NAME READY STATUS RESTARTS AGE
pod/nginx-ingress-controller-fbf967dd5-4qpbp 1/1 Running 0 12h
查看service
[root@k8s-master01 ingress-controller]# kubectl get svc -n ingress-nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ingress-nginx NodePort 10.98.75.163 <none> 80:32240/TCP,443:31335/TCP 11h
创建nacos的ingress
[root@master1 nacos]# cat nacos-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: nacos-ingress
namespace: devops
spec:
rules:
- host: www.mynacos.com
http:
paths:
- path: /nacos
backend:
serviceName: nacos-headless
servicePort: 8848
访问测试。