一、任务目标
本次要在K8S集群中配置一个3节点的NACOS集群，并使用NFS服务做持久化
NFS服务器地址：192.168.1.150
NFS目录：/nfs
Mysql数据库：192.168.1.185
数据库名：nacos
nacos集群命名空间：dc-prod-ns
NACOS镜像版本号：2.3.2
二、配置NFS Provisioner
2.1 配置ServiceAccount
$ vi nfs-provisioner-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-sa
namespace: dc-prod-ns
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: nfs-provisioner-runner
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events","endpoints", "configmaps", "leases"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: run-nfs-provisioner
namespace: dc-prod-ns
subjects:
- kind: ServiceAccount
name: nfs-sa
namespace: dc-prod-ns
roleRef:
kind: ClusterRole
name: nfs-provisioner-runner
apiGroup: rbac.authorization.k8s.io
$ kubectl apply -f nfs-provisioner-rbac.yaml
2.2 创建NFS 的 动态卷供应器
# vi nfs-client-provisioner.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nfs-client-provisioner
labels:
app: nfs-client-provisioner
namespace: dc-prod-ns
spec:
replicas: 2
strategy:
type: Recreate
selector:
matchLabels:
app: nfs-client-provisioner
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-sa
containers:
- name: nfs-client-provisioner
image: registry.k8s.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: dc-prod-ns/dc-nfs-provisioner
- name: NFS_SERVER
value: 192.168.1.150
- name: NFS_PATH
value: /nfs
volumes:
- name: nfs-client-root
nfs:
server: 192.168.1.150
path: /nfs
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: dc-nfs-storage
provisioner: dc-prod-ns/dc-nfs-provisioner
volumeBindingMode: WaitForFirstConsumer
parameters:
archiveOnDelete: "false" # 删除PVC时是否归档数据，默认为true
$ kubectl apply -f nfs-client-provisioner.yaml
上个这个应用在用户创建 PVC 时，它会自动在 NFS 服务器上创建子目录，并生成对应的 PV
现在测试一下
# cat test-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: test-claim
namespace: dc-prod-ns
spec:
storageClassName: dc-nfs-storage # 这个storageClass的名称就是上面创建的StorageClass
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Mi
$ kf test-pvc.yaml &&
$ kg pvc -n dc-prod-ns # 查看这个pvc声明是否能正常申请到磁盘
三、配置数据库
3.1 创建数据库
# 先登录数据库，再执行如下命令
CREATE DATABASE nacos CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
CREATE USER 'nacos'@'%' IDENTIFIED BY 'xxxx';
GRANT ALL PRIVILEGES ON nacos.* TO 'nacos'@'%';
FLUSH PRIVILEGES;
3.2 执行数据库初始化脚本
# cat mysql-schema.sql
/*
* Copyright 1999-2018 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/******************************************/
/* 表名称 = config_info */
/******************************************/
CREATE TABLE `config_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id',
`data_id` varchar(255) NOT NULL COMMENT 'data_id',
`group_id` varchar(128) DEFAULT NULL COMMENT 'group_id',
`content` longtext NOT NULL COMMENT 'content',
`md5` varchar(32) DEFAULT NULL COMMENT 'md5',
`gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
`src_user` text COMMENT 'source user',
`src_ip` varchar(50) DEFAULT NULL COMMENT 'source ip',
`app_name` varchar(128) DEFAULT NULL COMMENT 'app_name',
`tenant_id` varchar(128) DEFAULT '' COMMENT '租户字段',
`c_desc` varchar(256) DEFAULT NULL COMMENT 'configuration description',
`c_use` varchar(64) DEFAULT NULL COMMENT 'configuration usage',
`effect` varchar(64) DEFAULT NULL COMMENT '配置生效的描述',
`type` varchar(64) DEFAULT NULL COMMENT '配置的类型',
`c_schema` text COMMENT '配置的模式',
`encrypted_data_key` varchar(1024) NOT NULL DEFAULT '' COMMENT '密钥',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_configinfo_datagrouptenant` (`data_id`,`group_id`,`tenant_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_info';
/******************************************/
/* 表名称 = config_info since 2.5.0 */
/******************************************/
CREATE TABLE `config_info_gray` (
`id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
`data_id` varchar(255) NOT NULL COMMENT 'data_id',
`group_id` varchar(128) NOT NULL COMMENT 'group_id',
`content` longtext NOT NULL COMMENT 'content',
`md5` varchar(32) DEFAULT NULL COMMENT 'md5',
`src_user` text COMMENT 'src_user',
`src_ip` varchar(100) DEFAULT NULL COMMENT 'src_ip',
`gmt_create` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT 'gmt_create',
`gmt_modified` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT 'gmt_modified',
`app_name` varchar(128) DEFAULT NULL COMMENT 'app_name',
`tenant_id` varchar(128) DEFAULT '' COMMENT 'tenant_id',
`gray_name` varchar(128) NOT NULL COMMENT 'gray_name',
`gray_rule` text NOT NULL COMMENT 'gray_rule',
`encrypted_data_key` varchar(256) NOT NULL DEFAULT '' COMMENT 'encrypted_data_key',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_configinfogray_datagrouptenantgray` (`data_id`,`group_id`,`tenant_id`,`gray_name`),
KEY `idx_dataid_gmt_modified` (`data_id`,`gmt_modified`),
KEY `idx_gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='config_info_gray';
/******************************************/
/* 表名称 = config_tags_relation */
/******************************************/
CREATE TABLE `config_tags_relation` (
`id` bigint(20) NOT NULL COMMENT 'id',
`tag_name` varchar(128) NOT NULL COMMENT 'tag_name',
`tag_type` varchar(64) DEFAULT NULL COMMENT 'tag_type',
`data_id` varchar(255) NOT NULL COMMENT 'data_id',
`group_id` varchar(128) NOT NULL COMMENT 'group_id',
`tenant_id` varchar(128) DEFAULT '' COMMENT 'tenant_id',
`nid` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'nid, 自增长标识',
PRIMARY KEY (`nid`),
UNIQUE KEY `uk_configtagrelation_configidtag` (`id`,`tag_name`,`tag_type`),
KEY `idx_tenant_id` (`tenant_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_tag_relation';
/******************************************/
/* 表名称 = group_capacity */
/******************************************/
CREATE TABLE `group_capacity` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键ID',
`group_id` varchar(128) NOT NULL DEFAULT '' COMMENT 'Group ID，空字符表示整个集群',
`quota` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '配额，0表示使用默认值',
`usage` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '使用量',
`max_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个配置大小上限，单位为字节，0表示使用默认值',
`max_aggr_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '聚合子配置最大个数，，0表示使用默认值',
`max_aggr_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个聚合数据的子配置大小上限，单位为字节，0表示使用默认值',
`max_history_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '最大变更历史数量',
`gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_group_id` (`group_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='集群、各Group容量信息表';
/******************************************/
/* 表名称 = his_config_info */
/******************************************/
CREATE TABLE `his_config_info` (
`id` bigint(20) unsigned NOT NULL COMMENT 'id',
`nid` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'nid, 自增标识',
`data_id` varchar(255) NOT NULL COMMENT 'data_id',
`group_id` varchar(128) NOT NULL COMMENT 'group_id',
`app_name` varchar(128) DEFAULT NULL COMMENT 'app_name',
`content` longtext NOT NULL COMMENT 'content',
`md5` varchar(32) DEFAULT NULL COMMENT 'md5',
`gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
`src_user` text COMMENT 'source user',
`src_ip` varchar(50) DEFAULT NULL COMMENT 'source ip',
`op_type` char(10) DEFAULT NULL COMMENT 'operation type',
`tenant_id` varchar(128) DEFAULT '' COMMENT '租户字段',
`encrypted_data_key` varchar(1024) NOT NULL DEFAULT '' COMMENT '密钥',
`publish_type` varchar(50) DEFAULT 'formal' COMMENT 'publish type gray or formal',
`gray_name` varchar(50) DEFAULT NULL COMMENT 'gray name',
`ext_info` longtext DEFAULT NULL COMMENT 'ext info',
PRIMARY KEY (`nid`),
KEY `idx_gmt_create` (`gmt_create`),
KEY `idx_gmt_modified` (`gmt_modified`),
KEY `idx_did` (`data_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='多租户改造';
/******************************************/
/* 表名称 = tenant_capacity */
/******************************************/
CREATE TABLE `tenant_capacity` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键ID',
`tenant_id` varchar(128) NOT NULL DEFAULT '' COMMENT 'Tenant ID',
`quota` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '配额，0表示使用默认值',
`usage` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '使用量',
`max_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个配置大小上限，单位为字节，0表示使用默认值',
`max_aggr_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '聚合子配置最大个数',
`max_aggr_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个聚合数据的子配置大小上限，单位为字节，0表示使用默认值',
`max_history_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '最大变更历史数量',
`gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_tenant_id` (`tenant_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='租户容量信息表';
CREATE TABLE `tenant_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id',
`kp` varchar(128) NOT NULL COMMENT 'kp',
`tenant_id` varchar(128) default '' COMMENT 'tenant_id',
`tenant_name` varchar(128) default '' COMMENT 'tenant_name',
`tenant_desc` varchar(256) DEFAULT NULL COMMENT 'tenant_desc',
`create_source` varchar(32) DEFAULT NULL COMMENT 'create_source',
`gmt_create` bigint(20) NOT NULL COMMENT '创建时间',
`gmt_modified` bigint(20) NOT NULL COMMENT '修改时间',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_tenant_info_kptenantid` (`kp`,`tenant_id`),
KEY `idx_tenant_id` (`tenant_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='tenant_info';
CREATE TABLE `users` (
`username` varchar(50) NOT NULL PRIMARY KEY COMMENT 'username',
`password` varchar(500) NOT NULL COMMENT 'password',
`enabled` boolean NOT NULL COMMENT 'enabled'
);
CREATE TABLE `roles` (
`username` varchar(50) NOT NULL COMMENT 'username',
`role` varchar(50) NOT NULL COMMENT 'role',
UNIQUE INDEX `idx_user_role` (`username` ASC, `role` ASC) USING BTREE
);
CREATE TABLE `permissions` (
`role` varchar(50) NOT NULL COMMENT 'role',
`resource` varchar(128) NOT NULL COMMENT 'resource',
`action` varchar(8) NOT NULL COMMENT 'action',
UNIQUE INDEX `uk_role_permission` (`role`,`resource`,`action`) USING BTREE
);
# mysql -unacos -p nacos < mysql-schema.sql
四、创建nacos集群
4.1 创建Nacos配置文件
# vi nacos-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: nacos-config
namespace: dc-prod-ns
data:
application.properties: |
spring.datasource.platform=mysql
db.num=1
db.url.0=jdbc:mysql://192.168.1.185:3306/nacos?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true
db.user=nacos
db.password=xxxx
nacos.mode=cluster
server.port=8848
nacos.inetutils.preferred-networks=0.0.0.0/0
nacos.discovery.cluster-name=cluster-1
nacos.discovery.server-faList=nacos-0.nacos-headless.dc-prod-ns.svc.cluster.local:8848,nacos-1.nacos-headless.dc-prod-ns.svc.cluster.local:8848,nacos-2.nacos-headless.dc-prod-ns.svc.cluster.local:8848
## 开启客户端访问鉴权，默认为关闭，可选
nacos.core.auth.enabled=true
## 开启控制台访问鉴权，默认为开启
nacos.core.auth.console.enabled=true
nacos.core.auth.system.type=nacos
# 可选，建议设置(生成命令：openssl rand -base64 32)
nacos.core.auth.plugin.nacos.token.secret.key=7tJ7/FfQd3F/VPiJpL7pW86ioUFa5TTiweR/dzB/d9k=
nacos.core.auth.server.identity.key=dddd
nacos.core.auth.server.identity.value=dddd
$ kubectl apply -f nacos-config.yaml
说明：将其中的MySQL连接地址、用户名、密码等信息修改为你实际的MySQL服务器配置。nacos.discovery.server-faList 中的地址是Nacos集群中各个实例的访问地址，格式为 nacos-<index>.nacos-headless.<namespace>.svc.cluster.local:8848，其中 <index> 是从0开始的实例序号，<namespace> 是Nacos命名空间的名称。
4.2 创建Nacos StatefulSet
$ vi nacos-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: nacos
namespace: dc-prod-ns
spec:
serviceName: nacos-headless
replicas: 3
template:
metadata:
labels:
app: nacos
annotations:
pod.alpha.kubernetes.io/initialized: "true"
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- nacos
topologyKey: "kubernetes.io/hostname"
containers:
- name: nacos
imagePullPolicy: IfNotPresent
image: nacos/nacos-server:2.3.2
resources:
requests:
memory: "2Gi"
cpu: "500m"
ports:
- containerPort: 8848
name: client
- containerPort: 9848
name: client-rpc
- containerPort: 9849
name: raft-rpc
- containerPort: 7848
name: old-raft-rpc
volumeMounts:
- name: data
mountPath: /home/nacos/data
- name: app-properties
mountPath: /home/nacos/conf/application.properties
subPath: application.properties # 挂载到文件，不至于覆盖该目录下其他文件
env:
- name: NACOS_REPLICAS
value: "3" # 这个数目要与上面的副本数目相当
- name: PREFER_HOST_MODE
value: "hostname"
- name: NACOS_SERVERS
value: "nacos-0.nacos-headless.dc-prod-ns.svc.cluster.local:8848 nacos-1.nacos-headless.dc-prod-ns.svc.cluster.local:8848 nacos-2.nacos-headless.dc-prod-ns.svc.cluster.local:8848"
volumes:
- name: app-properties
configMap:
name: nacos-config
items:
- key: application.properties
path: application.properties
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: "dc-nfs-storage" # 这是上面准备好的storageClass
resources:
requests:
storage: 5Gi
selector:
matchLabels:
app: nacos
$ kubectl apply -f nacos-statefulset.yaml
说明：以上NACOS_SERVERS的值定义时，也要注意替换命名空间
4.3 创建Nacos服务
这里创建两个服务，一个是pod之间互相调用用到的，一个是暴露给用户使用的
# vi nacos-service.yaml
apiVersion: v1
kind: Service
metadata:
name: nacos-outside
namespace: dc-prod-ns
spec:
type: NodePort
ports:
- port: 8848
targetPort: 8848
nodePort: 30088 # 使用nodePort方式
protocol: TCP
name: http
selector:
app: nacos
---
apiVersion: v1
kind: Service
metadata:
name: nacos-headless
namespace: dc-prod-ns
labels:
app: nacos-headless
spec:
type: ClusterIP
clusterIP: None
ports:
- port: 8848
name: server
targetPort: 8848
- port: 9848
name: client-rpc
targetPort: 9848
- port: 9849
name: raft-rpc
targetPort: 9849
## 兼容1.4.x版本的选举端口
- port: 7848
name: old-raft-rpc
targetPort: 7848
selector:
app: nacos
$ kubectl apply -f nacos-service.yaml
五、登录管理界面
以上资源全部部署成功后，在浏览器访问K8S群集任一主机IP:30088/nacos
以初始用户：nacos/nacos登录后，修改密码
再查看实例
image.png
网友评论