kubeadm高可用集群安装

系统初始化

运行以下脚本init.sh，初始化环境，主要修改版本号与主机名

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
#!/bin/bash

# docker版本，k8s版本
DockerV='-18.09.9'
K8sV='-1.19.9'

#yum源

yum install -y yum-utils device-mapper-persistent-data lvm2 epel-release vim screen bash-completion mtr lrzsz wget telnet zip unzip sysstat  ntpdate libcurl openssl bridge-utils nethogs dos2unix iptables-services

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

wget http://mirrors.aliyun.com/repo/epel-7.repo -O /etc/yum.repos.d/epel.repo

cat >>/etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

#安装K8S组件
yum install -y kubelet${K8sV} kubeadm${K8sV} kubectl${K8sV} docker-ce${DockerV}

systemctl enable kubelet

mkdir -p /etc/docker

tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://docker.mirrors.sjtug.sjtu.edu.cn"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-opts": {
    "max-size": "100m",
    "max-file": "10"
    }
}
EOF

systemctl restart docker && systemctl enable docker

#禁用防火墙与selinux

setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
sed -i 's/SELINUX=permissive/SELINUX=disabled/g' /etc/selinux/config

service firewalld stop
systemctl disable firewalld.service
service iptables stop
systemctl disable iptables.service

service postfix stop
systemctl disable postfix.service

wget http://mirrors.aliyun.com/repo/epel-7.repo -O /etc/yum.repos.d/epel.repo

echo '/etc/security/limits.conf 参数调优，需重启系统后生效'

cp -rf /etc/security/limits.conf /etc/security/limits.conf.back

cat > /etc/security/limits.conf << EOF
* soft nofile 655350
* hard nofile 655350
* soft nproc unlimited
* hard nproc unlimited
* soft core unlimited
* hard core unlimited
root soft nofile 655350
root hard nofile 655350
root soft nproc unlimited
root hard nproc unlimited
root soft core unlimited
root hard core unlimited
EOF

echo '/etc/sysctl.conf 文件调优'

cp -rf /etc/sysctl.conf /etc/sysctl.conf.back
cat > /etc/sysctl.conf << EOF

vm.swappiness = 0
net.ipv4.neigh.default.gc_stale_time = 120

# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2

# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets = 20000
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2

net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1

kernel.sysrq = 1
kernel.pid_max=1000000

net.netfilter.nf_conntrack_max=2048576
net.netfilter.nf_conntrack_tcp_timeout_established=3600
EOF

sysctl -p

echo "ipvs模块开启"
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF

echo "1">/proc/sys/net/bridge/bridge-nf-call-iptables

chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules

lsmod | grep -e ip_vs -e nf_conntrack_ipv4

echo "禁用swap"
swapoff -a
sed -i '/swap/d' /etc/fstab

集群高可用脚本

如果需要安装高可用集群，运行以下脚本，install.sh，该脚本安装keepalived与nginx，变量根据情况修改，如果单节点，可以不执行该脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
#!/bin/bash

# 定义变量
master0="192.168.40.100:6443"	#k8s master ip
master1="192.168.40.101:6443"
master2="192.168.40.102:6443"
vip="192.168.40.201"	# keepalived vip地址
role=MASTER				# keepalived的角色，MASTER或BACKUP
network=ens33			# 本机网卡名
router_id=LVS_1			# keepalived本机名，一般是主机名，也可以自己定义
priority=100			# keepalived权重
port=7443				# nginx代理监听端口
nginx_dir=/etc			# nginx安装目录

#安装依赖
yum -y install gcc gcc-c++ pcre pcre-devel zlib zlib-devel openssl openssl-devel wget

#下载nginx
wget http://nginx.org/download/nginx-1.14.2.tar.gz

#解压并安装
tar xf ./nginx-1.14.2.tar.gz
cd nginx-1.14.2/
./configure --prefix=$nginx_dir/nginx --user=root --group=root --with-http_ssl_module --with-http_stub_status_module --with-http_gzip_static_module --with-http_realip_module --with-stream
make && make install
[ ! -d /var/log/nginx ] && mkdir /var/log/nginx

cat >/etc/nginx/conf/nginx.conf <<EOF
user root;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

events {
  worker_connections 1024;
}
stream {
  log_format proxy '\$remote_addr \$remote_port - [\$time_local] \$status \$protocol '
  '"\$upstream_addr" "\$upstream_bytes_sent" "\$upstream_connect_time"' ;
  
  access_log /var/log/nginx/nginx-proxy.log proxy;
  
  upstream kubernetes_lb{
    server $master0 weight=5 max_fails=3 fail_timeout=30s;
    server $master1 weight=5 max_fails=3 fail_timeout=30s;
    server $master2 weight=5 max_fails=3 fail_timeout=30s;
  }
  
  server {
    listen $port;
    proxy_connect_timeout 30s;
    proxy_timeout 30s;
    proxy_pass kubernetes_lb;
  }
}
EOF

/etc/nginx/sbin/nginx

yum -y install keepalived
systemctl start keepalived && systemctl enable keepalived

cat >/etc/keepalived/keepalived.conf <<EOF
global_defs {
  router_id $router_id
}
vrrp_instance VI_1 {
  state $role
  interface $network  #网卡设备名称，根据自己网卡信息进行更改
  lvs_sync_daemon_inteface $network 
  virtual_router_id 100
  advert_int 1
  priority $priority
  authentication {
    auth_type PASS
    auth_pass 1111
}
  virtual_ipaddress {
    $vip  # 这就就是虚拟IP地址
  }
}
EOF

systemctl restart keepalived

初始化kubeadm-init.yaml

创建：kubeadm-init.yaml，添加以下内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: v1.19.9
apiServer:
controlPlaneEndpoint: "192.168.40.201:7443"
networking:
  dnsDomain: cluster.local
  podSubnet: "10.244.0.0/16"
  serviceSubnet: 10.96.0.0/12
imageRepository: registry.aliyuncs.com/google_containers
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: "ipvs"
ipvs:
  excludeCIDRs: null
  minSyncPeriod: 0s
  scheduler: "rr"
  strictARP: false
  syncPeriod: 30s

执行初始化k8s集群命令

1
2
3
4
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> ~/.bash_profile
source ~/.bash_profile

kubeadm init --config kubeadm-init.yaml

如果失败可以以执行重置

1
kubeadm reset

执行成功会出现添加集群的命令，需要记下，以方便添加新master和node。

安装flannel网络插件

创建文件：kube-flannel.yml，添加以下内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
cat > kube-flannel.yml << EOF
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: psp.flannel.unprivileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
  privileged: false
  volumes:
  - configMap
  - secret
  - emptyDir
  - hostPath
  allowedHostPaths:
  - pathPrefix: "/etc/cni/net.d"
  - pathPrefix: "/etc/kube-flannel"
  - pathPrefix: "/run/flannel"
  readOnlyRootFilesystem: false
  # Users and groups
  runAsUser:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  fsGroup:
    rule: RunAsAny
  # Privilege Escalation
  allowPrivilegeEscalation: false
  defaultAllowPrivilegeEscalation: false
  # Capabilities
  allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
  defaultAddCapabilities: []
  requiredDropCapabilities: []
  # Host namespaces
  hostPID: false
  hostIPC: false
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  # SELinux
  seLinux:
    # SELinux is unused in CaaSP
    rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
rules:
- apiGroups: ['extensions']
  resources: ['podsecuritypolicies']
  verbs: ['use']
  resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: jmgao1983/flannel
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: jmgao1983/flannel
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg
EOF

执行命令

1
kubectl apply -f kube-flannel.yml

执行以下命令，解决组件健康检查报错问题

1
2
3
sed -i '/--port=0/d' /etc/kubernetes/manifests/kube-controller-manager.yaml
sed -i '/--port=0/d' /etc/kubernetes/manifests/kube-scheduler.yaml
systemctl restart kubelet

新增节点

生成token命令

1
kubeadm token create --print-join-command

新增node

1
kubeadm join 10.0.1.33:6443 --token x9z4w5.5b82ogtxoenw602m     --discovery-token-ca-cert-hash sha256:b5607480f65e5feefac87ee6abfc55fbcad9bfa2e4aeae133f3b4231a5111d15

新增master

1
kubeadm join 10.0.1.50:6443 --token s8l0qd.ygzywq5cfmdri2oq     --discovery-token-ca-cert-hash sha256:2e6748819909eeb4fee505d9dfd0e7ec7d93d5210c574888eff49cffee6bd455  --control-plane   --certificate-key 90d8fb966e66d2a1b5712564da13dde0148109df93fa0246aef996fbf6c88154

添加web管理界面

1
2
3
4
5
# 创建kuboard
kubectl apply -f https://kuboard.cn/install-script/kuboard.yaml
kubectl apply -f https://addons.kuboard.cn/metrics-server/0.3.7/metrics-server.yaml
# 获取token
echo $(kubectl -n kube-system get secret $(kubectl -n kube-system get secret | grep kuboard-user | awk '{print $1}') -o go-template='{{.data.token}}' | base64 -d)

命令补全

1
2
3
4
yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

证书续签

1
2
3
4
5
6
# 查看证书时间
kubeadm alpha certs check-expiration
# 更新证书
kubeadm alpha certs renew all
# 重启组件
docker ps |grep -E 'k8s_kube-apiserver|k8s_kube-controller-manager|k8s_kube-scheduler|k8s_etcd_etcd' | awk -F ' ' '{print $1}' |xargs docker restart

dns解析

修改coredns的配置

hosts字段为自定义域名解析，修改后不需要重启coredns服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
.:53 {
    errors
    health {
       lameduck 5s
    }
    hosts {
       10.0.0.10 huang.com
       10.0.0.10 xiao.com
       10.0.0.10 qi.com
       fallthrough
    }
    ready
    kubernetes cluster.local in-addr.arpa ip6.arpa {
       pods insecure
       fallthrough in-addr.arpa ip6.arpa
       ttl 30
    }
    prometheus :9153
    forward . /etc/resolv.conf {
       max_concurrent 1000
    }
    cache 30
    loop
    reload
    loadbalance
}
kubernetes
k8s

本博客所有文章除特别声明外，均采用 CC BY-SA 4.0 协议 ，转载请注明出处！