# 更新系统
yum update -y

# 设置正确的时区和时间
yum install -y ntpdate
timedatectl set-timezone Asia/Shanghai
ntpdate cn.ntp.org.cn

# 关闭防火墙
systemctl disable firewalld.service
systemctl stop firewalld.service

# 关闭swap分区
swapoff -a
sed -i 's#/dev/mapper/cl-swap#\# /dev/mapper/cl-swap#' /etc/fstab

# 关闭selinux
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

# 设置各个节点的主机名
## 192.168.112.128
hostnamectl set-hostname master

## 192.168.112.129
hostnamectl set-hostname node01

## 192.168.112.130
hostnamectl set-hostname node02

# 配置主机名和IP的映射
cat <<EOF >> /etc/hosts

# For Kubernetes Cluster
192.168.112.128 master
192.168.112.129 node01
192.168.112.130 node02
EOF

# 修改内核参数
cat <<EOF > /etc/sysctl.d/kubernetes.conf
net.bridge.bridge-nf-call-iptables = 1

net.ipv4.ip_forward = 1

net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_tw_recycle = 1 

net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_tw_resue = 1
EOF

sysctl --system

# 修改ulimit限制
cat <<EOF >  /etc/security/limits.d/kubernetes.conf
* hard nofile 65535
* soft nofile 65535
EOF

# 配置yum源，然后安装Docker
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum makecache fast
yum install -y docker-ce-18.09.9-3.el7 docker-ce-cli-18.09.9-3.el7

# 启动Docker，并将其设置为开机启动
systemctl daemon-reload
systemctl enable docker.service
systemctl start docker.service

# 确认Docker启动是否正常
systemctl status docker.service
docker info
docker version

# 方法一：检查iptables的forward链的默认策略
iptables -nL
。。。
Chain FORWARD (policy ACCEPT)
。。。

# 方法二：检查iptables的forward链的默认策略
iptables-save -t filter
。。。
# Generated by iptables-save v1.4.21 on Thu Oct  3 12:28:24 2019
*filter
:INPUT ACCEPT [2117:366255]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [2188:436727]
。。。

# 设置docker daemon的cgroup driver为systemd
cat <<EOF > /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver":"json-file",
  "log-opts": {
     "max-size": "10m",
     "max-file": "3"
  }
}
EOF

systemctl daemon-reload
systemctl restart docker.service
systemctl status docker.service

# 验证docker daemon的cgroup driver是否为systemd
docker info
。。。
Cgroup Driver: systemd
。。。

# 192.168.112.128 （Master上执行）
tar -zxvf etcd-v3.4.3-linux-amd64.tar.gz
tar -zxvf kubernetes-server-linux-amd64.tar.gz

cp etcd-v3.4.3-linux-amd64/etcd /usr/bin/
cp etcd-v3.4.3-linux-amd64/etcdctl /usr/bin/
cp kubernetes/server/bin/kube-apiserver /usr/bin/
cp kubernetes/server/bin/kube-controller-manager /usr/bin/
cp kubernetes/server/bin/kube-scheduler /usr/bin/
cp kubernetes/server/bin/kubectl /usr/bin/


# 192.168.112.129 和 192.168.112.130 （Node上执行）
tar -zxvf kubernetes-server-linux-amd64.tar.gz

cp kubernetes/server/bin/kubelet /usr/bin/
cp kubernetes/server/bin/kube-proxy /usr/bin/

# 创建证书和配置文件的存放目录
mkdir -p /etc/kubernetes/pki/etcd/

# 生成etcd的相关证书
cd /etc/kubernetes/pki/etcd/
openssl genrsa -out ca.key 2048
openssl req -x509 -new -nodes -key ca.key -subj "/CN=etcd-ca" -days 5000 -out ca.crt

cat <<EOF > server_ssl.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name

[req_distinguished_name]
[v3_req]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation,digitalSignature,keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = master
DNS.2 = localhost
IP.1 = 192.168.112.128
IP.2 = 127.0.0.1
EOF
openssl genrsa -out server.key 2048
openssl req -new -key server.key -subj "/CN=master" -config server_ssl.cnf -out server.csr
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 5000 -extensions v3_req -extfile server_ssl.cnf -out server.crt

cat <<EOF > peer_ssl.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name

[req_distinguished_name]
[v3_req]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation,digitalSignature,keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = master
DNS.2 = localhost
IP.1 = 192.168.112.128
IP.2 = 127.0.0.1
EOF
openssl genrsa -out peer.key 2048
openssl req -new -key peer.key -subj "/CN=master" -config peer_ssl.cnf -out peer.csr
openssl x509 -req -in peer.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 5000 -extensions v3_req -extfile peer_ssl.cnf -out peer.crt

openssl genrsa -out healthcheck-client.key 2048
openssl req -new -key healthcheck-client.key -subj "/O=system:masters/CN=kube-etcd-healthcheck-client" -out healthcheck-client.csr
openssl x509 -req -in healthcheck-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out healthcheck-client.crt -days 5000

cd /etc/kubernetes/pki/
openssl genrsa -out apiserver-etcd-client.key 2048
openssl req -new -key apiserver-etcd-client.key -subj "/O=system:masters/CN=kube-apiserver-etcd-client" -out apiserver-etcd-client.csr
openssl x509 -req -in apiserver-etcd-client.csr -CA /etc/kubernetes/pki/etcd/ca.crt -CAkey /etc/kubernetes/pki/etcd/ca.key -CAcreateserial -out apiserver-etcd-client.crt -days 5000


# 生成kubernetes组件的相关证书
# 生成rsa的公钥和私钥
openssl genrsa -out sa.key 2048
openssl rsa -in sa.key -pubout -out sa.pub

# 生成根证书
openssl genrsa -out ca.key 2048
openssl req -x509 -new -nodes -key ca.key -subj "/CN=kubernetes" -days 5000 -out ca.crt

openssl genrsa -out front-proxy-ca.key 2048
openssl req -x509 -new -nodes -key front-proxy-ca.key -subj "/CN=front-proxy-ca" -days 5000 -out front-proxy-ca.crt

# 为kube-apiserver生成相关的证书和配置文件
cat <<EOF > master_ssl.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name

[req_distinguished_name]
[v3_req]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation,digitalSignature,keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = master
DNS.2 = kubernetes
DNS.3 = kubernetes.default
DNS.4 = kubernetes.default.svc
DNS.5 = kubernetes.default.svc.cluster.local
IP.1 = 10.96.0.1
IP.2 = 192.168.112.128
EOF

openssl genrsa -out apiserver.key 2048
openssl req -new -key apiserver.key -subj "/CN=kube-apiserver" -config master_ssl.cnf -out apiserver.csr
openssl x509 -req -in apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 5000 -extensions v3_req -extfile master_ssl.cnf -out apiserver.crt

openssl genrsa -out front-proxy-client.key 2048
openssl req -new -key front-proxy-client.key -subj "/CN=front-proxy-client" -out front-proxy-client.csr
openssl x509 -req -in front-proxy-client.csr -CA front-proxy-ca.crt -CAkey front-proxy-ca.key -CAcreateserial -out front-proxy-client.crt -days 5000

openssl genrsa -out apiserver-kubelet-client.key 2048
openssl req -new -key apiserver-kubelet-client.key -subj "/O=system:masters/CN=kube-apiserver-kubelet-client" -out apiserver-kubelet-client.csr
openssl x509 -req -in apiserver-kubelet-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out apiserver-kubelet-client.crt -days 5000

# 为kube-controller-manager生成相关的证书和配置文件
openssl genrsa -out controller-manager.key 2048
openssl req -new -key controller-manager.key -subj "/CN=system:kube-controller-manager" -out controller-manager.csr
openssl x509 -req -in controller-manager.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out controller-manager.crt -days 5000

export KUBECONFIG=/etc/kubernetes/controller-manager.conf
kubectl config set-cluster kubernetes --server=https://192.168.112.128:6443 --certificate-authority=/etc/kubernetes/pki/ca.crt --embed-certs=true
kubectl config set-credentials system:kube-controller-manager --client-certificate=/etc/kubernetes/pki/controller-manager.crt --client-key=/etc/kubernetes/pki/controller-manager.key --embed-certs=true
kubectl config set-context system:kube-controller-manager@kubernetes --cluster=kubernetes --user=system:kube-controller-manager
kubectl config use-context system:kube-controller-manager@kubernetes
unset KUBECONFIG

# 为kube-scheduler生成相关的证书和配置文件
openssl genrsa -out scheduler.key 2048
openssl req -new -key scheduler.key -subj "/CN=system:kube-scheduler" -out scheduler.csr
openssl x509 -req -in scheduler.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out scheduler.crt -days 5000

export KUBECONFIG=/etc/kubernetes/scheduler.conf
kubectl config set-cluster kubernetes --server=https://192.168.112.128:6443 --certificate-authority=/etc/kubernetes/pki/ca.crt --embed-certs=true
kubectl config set-credentials system:kube-scheduler --client-certificate=/etc/kubernetes/pki/scheduler.crt --client-key=/etc/kubernetes/pki/scheduler.key --embed-certs=true
kubectl config set-context system:kube-scheduler@kubernetes --cluster=kubernetes --user=system:kube-scheduler
kubectl config use-context system:kube-scheduler@kubernetes
unset KUBECONFIG

# 为kubectl生成相关的证书和配置文件
openssl genrsa -out kubectl.key 2048
openssl req -new -key kubectl.key -subj "/O=system:masters/CN=kubernetes-admin" -out kubectl.csr
openssl x509 -req -in kubectl.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out kubectl.crt -days 5000

export KUBECONFIG=/etc/kubernetes/admin.conf
kubectl config set-cluster kubernetes --server=https://192.168.112.128:6443 --certificate-authority=/etc/kubernetes/pki/ca.crt --embed-certs=true
kubectl config set-credentials kubernetes-admin --client-certificate=/etc/kubernetes/pki/kubectl.crt --client-key=/etc/kubernetes/pki/kubectl.key --embed-certs=true
kubectl config set-context kubernetes-admin@kubernetes --cluster=kubernetes --user=kubernetes-admin
kubectl config use-context kubernetes-admin@kubernetes
unset KUBECONFIG

# 注意：因为kubelet使用了tls bootstrap的方式，所以下面这部分内容没有必要了，需把其注释掉
# 为kubelet生成相关的证书和配置文件
# openssl genrsa -out kubelet.key 2048
# openssl req -new -key kubelet.key -subj "/O=system:nodes/CN=system:node:node01" -out kubelet.csr
# openssl x509 -req -in kubelet.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out kubelet.crt -days 5000
# 
# export KUBECONFIG=/etc/kubernetes/kubelet.conf
# kubectl config set-cluster kubernetes --server=https://192.168.112.128:6443 --certificate-authority=/etc/kubernetes/pki/ca.crt # --embed-certs=true
# kubectl config set-credentials system:node:node01 --client-certificate=/etc/kubernetes/pki/kubelet.crt --client-key=/etc/kubernetes/pki/# kubelet.key --embed-certs=true
# kubectl config set-context system:node:node01@kubernetes --cluster=kubernetes --user=system:node:node01
# kubectl config use-context system:node:node01@kubernetes
# unset KUBECONFIG

# 为kube-proxy生成相关的证书和配置文件
# kubernetes内置的为kube-proxy而生的clusterrole，可以使用kubectl get clusterrole system:node-proxier -o yaml进行查看
# kubernetes内置的为kube-proxy而生的clusterrolebinding，绑定到了用户system:kube-proxy，可以使用kubectl get clusterrolebinding system:node-proxier -o yaml进行查看
openssl genrsa -out proxy.key 2048
openssl req -new -key proxy.key -subj "/CN=system:kube-proxy" -out proxy.csr
openssl x509 -req -in proxy.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out proxy.crt -days 5000

export KUBECONFIG=/etc/kubernetes/proxy.conf
kubectl config set-cluster kubernetes --server=https://192.168.112.128:6443 --certificate-authority=/etc/kubernetes/pki/ca.crt --embed-certs=true
kubectl config set-credentials system:kube-proxy --client-certificate=/etc/kubernetes/pki/proxy.crt --client-key=/etc/kubernetes/pki/proxy.key --embed-certs=true
kubectl config set-context system:kube-proxy@kubernetes --cluster=kubernetes --user=system:kube-proxy
kubectl config use-context system:kube-proxy@kubernetes
unset KUBECONFIG

# 配置和启动etcd服务
mkdir -p /etc/etcd/
mkdir -p /var/lib/etcd/

cat <<EOF > /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server 
After=network.target

[Service]
Type=simple
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.env
ExecStart=/usr/bin/etcd \$ETCD_ARGS

[Install]
WantedBy=multi-user.target
EOF

cat <<EOF > /etc/etcd/etcd.env
ETCD_ARGS="--advertise-client-urls=https://192.168.112.128:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --initial-advertise-peer-urls=https://192.168.112.128:2380 --initial-cluster=master=https://192.168.112.128:2380 --key-file=/etc/kubernetes/pki/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://192.168.112.128:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://192.168.112.128:2380 --name=master --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt"
EOF

systemctl daemon-reload
systemctl enable etcd.service
systemctl start etcd.service
systemctl status etcd.service

# 配置和启动kube-apiserver服务
cat <<EOF > /usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service

[Service]
EnvironmentFile=-/etc/kubernetes/kube-apiserver.env
ExecStart=/usr/bin/kube-apiserver \$KUBE_API_ARGS
Restart=on-failure
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

cat <<EOF > /etc/kubernetes/kube-apiserver.env
KUBE_API_ARGS=" --advertise-address=192.168.112.128 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --insecure-port=0 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-cluster-ip-range=10.96.0.0/16 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key"
EOF

systemctl daemon-reload
systemctl enable kube-apiserver.service
systemctl start kube-apiserver.service
systemctl status kube-apiserver.service

export KUBECONFIG=/etc/kubernetes/admin.conf

# 为各个系统组件涉及的用户绑定cluster-admin角色
# export KUBECONFIG=/etc/kubernetes/admin.conf
# kubectl create clusterrolebinding kube-apiserver --clusterrole=cluster-admin --user=kube-apiserver
# kubectl create clusterrolebinding front-proxy-client --clusterrole=cluster-admin --user=front-proxy-client
# kubectl create clusterrolebinding system:kube-controller-manager --clusterrole=cluster-admin --user=system:kube-controller-manager 
# kubectl create clusterrolebinding system:kube-scheduler --clusterrole=cluster-admin --user=system:kube-scheduler 
# kubectl create clusterrolebinding system:kubelet --clusterrole=cluster-admin --user=system:kubelet 
# kubectl create clusterrolebinding system:kube-proxy --clusterrole=cluster-admin --user=system:kube-proxy

# 创建Bootstrap Token的相关配置 注意：expiration必须要在当前日期以后，否则会出现token创建后，kubernetes就会自动删除
cat <<EOF > /etc/kubernetes/bootstrap-token-abcdef.yaml
apiVersion: v1
kind: Secret
metadata:
  name: bootstrap-token-abcdef
  namespace: kube-system
type: bootstrap.kubernetes.io/token
stringData:
  auth-extra-groups: system:bootstrappers:default-node-token
  expiration: 2020-01-31T00:00:00+08:00
  token-id: abcdef
  token-secret: 0123456789abcdef
  usage-bootstrap-authentication: "true"
  usage-bootstrap-signing: "true"
EOF
kubectl create -f /etc/kubernetes/bootstrap-token-abcdef.yaml

cat <<EOF > /etc/kubernetes/create-csrs-for-bootstrapping.yaml
# enable bootstrapping nodes to create CSR
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: create-csrs-for-bootstrapping
subjects:
- kind: Group
  name: system:bootstrappers
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:node-bootstrapper
  apiGroup: rbac.authorization.k8s.io
EOF
kubectl create -f /etc/kubernetes/create-csrs-for-bootstrapping.yaml

cat <<EOF > /etc/kubernetes/auto-approve-csrs-for-group.yaml
# Approve all CSRs for the group "system:bootstrappers"
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: auto-approve-csrs-for-group
subjects:
- kind: Group
  name: system:bootstrappers
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
  apiGroup: rbac.authorization.k8s.io
EOF
kubectl create -f /etc/kubernetes/auto-approve-csrs-for-group.yaml

cat <<EOF > /etc/kubernetes/auto-approve-renewals-for-nodes.yaml
# Approve renewal CSRs for the group "system:nodes"
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: auto-approve-renewals-for-nodes
subjects:
- kind: Group
  name: system:nodes
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
  apiGroup: rbac.authorization.k8s.io
EOF
kubectl create -f /etc/kubernetes/auto-approve-renewals-for-nodes.yaml

export KUBECONFIG=/etc/kubernetes/bootstrap-kubelet.conf
kubectl config set-cluster kubernetes --server=https://192.168.112.128:6443 --certificate-authority=/etc/kubernetes/pki/ca.crt --embed-certs=true
kubectl config set-credentials system:bootstrap:abcdef --token=abcdef.0123456789abcdef
kubectl config set-context system:bootstrap:abcdef@kubernetes --cluster=kubernetes --user=system:bootstrap:abcdef
kubectl config use-context system:bootstrap:abcdef@kubernetes
unset KUBECONFIG

# 配置和启动kube-controller-manager服务
cat <<EOF > /usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=kube-apiserver.service
Requires=kube-apiserver.service

[Service]
EnvironmentFile=-/etc/kubernetes/kube-controller-manager.env
ExecStart=/usr/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

cat <<EOF > /etc/kubernetes/kube-controller-manager.env
KUBE_CONTROLLER_MANAGER_ARGS="--allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf --bind-address=127.0.0.1 --client-ca-file=/etc/kubernetes/pki/ca.crt --cluster-cidr=10.211.0.0/16 --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt --cluster-signing-key-file=/etc/kubernetes/pki/ca.key --controllers=*,bootstrapsigner,tokencleaner --kubeconfig=/etc/kubernetes/controller-manager.conf --leader-elect=true --node-cidr-mask-size=24 --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/16 --use-service-account-credentials=true"
EOF

systemctl daemon-reload
systemctl enable kube-controller-manager.service
systemctl start kube-controller-manager.service
systemctl status kube-controller-manager.service

# 配置和启动kube-scheduler服务
cat <<EOF > /usr/lib/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=kube-apiserver.service
Requires=kube-apiserver.service

[Service]
EnvironmentFile=-/etc/kubernetes/kube-scheduler.env
ExecStart=/usr/bin/kube-scheduler \$KUBE_SCHEDULER_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

cat <<EOF > /etc/kubernetes/kube-scheduler.env
KUBE_SCHEDULER_ARGS=" --authentication-kubeconfig=/etc/kubernetes/scheduler.conf --authorization-kubeconfig=/etc/kubernetes/scheduler.conf --bind-address=127.0.0.1 --kubeconfig=/etc/kubernetes/scheduler.conf --leader-elect=true"
EOF

systemctl daemon-reload
systemctl enable kube-scheduler.service
systemctl start kube-scheduler.service
systemctl status kube-scheduler.service

# 如果master节点需要具备node节点的功能，那么请参考5中的步骤，先在master上完成kubelet和kube-proxy的安装后，再给master节点打上下面的标签和污点
kubectl label node master node-role.kubernetes.io/master=
kubectl taint node master node-role.kubernetes.io/master=:NoSchedule

# 创建配置目录和工作目录
mkdir -p /etc/kubernetes/manifests
mkdir -p /etc/kubernetes/pki/
mkdir -p /var/lib/kubelet/
mkdir -p /var/lib/kube-proxy/

# 传输相关配置文件到当前节点上
scp root@192.168.112.128:/etc/kubernetes/bootstrap-kubelet.conf /etc/kubernetes/
scp root@192.168.112.128:/etc/kubernetes/pki/ca.crt /etc/kubernetes/pki/
# scp root@192.168.112.128:/etc/kubernetes/kubelet.conf /etc/kubernetes/ # 使用了tls bootstrap的方式，故这步没有必要，需要注释掉
scp root@192.168.112.128:/etc/kubernetes/proxy.conf /etc/kubernetes/

# 创建kubelet的配置文件
cat <<EOF > /var/lib/kubelet/config.yaml
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: true
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
EOF

# 配置和启动kubelet服务
cat <<EOF > /usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
EnvironmentFile=-/etc/kubernetes/kubelet.env
ExecStart=/usr/bin/kubelet \$KUBELET_ARGS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

cat <<EOF > /etc/kubernetes/kubelet.env
KUBELET_ARGS="--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --cgroup-driver=systemd --network-plugin=cni --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1"
EOF

systemctl daemon-reload
systemctl enable kubelet.service
systemctl start kubelet.service
systemctl status kubelet.service


# 创建kube-proxy的配置文件
cat <<EOF > /var/lib/kube-proxy/config.conf
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
  acceptContentTypes: ""
  burst: 0
  contentType: ""
  kubeconfig: /etc/kubernetes/proxy.conf
  qps: 0
clusterCIDR: 10.211.0.0/16
configSyncPeriod: 0s
conntrack:
  maxPerCore: null
  min: null
  tcpCloseWaitTimeout: null
  tcpEstablishedTimeout: null
enableProfiling: false
healthzBindAddress: ""
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: null
  minSyncPeriod: 0s
  syncPeriod: 0s
ipvs:
  excludeCIDRs: null
  minSyncPeriod: 0s
  scheduler: ""
  strictARP: false
  syncPeriod: 0s
kind: KubeProxyConfiguration
metricsBindAddress: ""
mode: ""
nodePortAddresses: null
oomScoreAdj: null
portRange: ""
udpIdleTimeout: 0s
winkernel:
  enableDSR: false
  networkName: ""
  sourceVip: ""
EOF

# 配置和启动kube-proxy服务
cat <<EOF > /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
Requires=network.service

[Service]
EnvironmentFile=-/etc/kubernetes/kube-proxy.env
ExecStart=/usr/bin/kube-proxy \$KUBE_PROXY_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

cat <<EOF > /etc/kubernetes/kube-proxy.env
KUBE_PROXY_ARGS="--config=/var/lib/kube-proxy/config.conf --hostname-override=node01"
EOF

yum install -y conntrack

systemctl daemon-reload
systemctl enable kube-proxy.service
systemctl start kube-proxy.service
systemctl status kube-proxy.service

mkdir -p calico/

cat <<EOF > calico/calico.yaml
---
# Source: calico/templates/calico-config.yaml
# This ConfigMap is used to configure a self-hosted Calico installation.
kind: ConfigMap
apiVersion: v1
metadata:
  name: calico-config
  namespace: kube-system
data:
  # Typha is disabled.
  typha_service_name: "none"
  # Configure the backend to use.
  calico_backend: "bird"

  # Configure the MTU to use
  veth_mtu: "1440"

  # The CNI network configuration to install on each node.  The special
  # values in this config will be automatically populated.
  cni_network_config: |-
    {
      "name": "k8s-pod-network",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "calico",
          "log_level": "info",
          "datastore_type": "kubernetes",
          "nodename": "__KUBERNETES_NODE_NAME__",
          "mtu": __CNI_MTU__,
          "ipam": {
              "type": "calico-ipam"
          },
          "policy": {
              "type": "k8s"
          },
          "kubernetes": {
              "kubeconfig": "__KUBECONFIG_FILEPATH__"
          }
        },
        {
          "type": "portmap",
          "snat": true,
          "capabilities": {"portMappings": true}
        }
      ]
    }

---
# Source: calico/templates/kdd-crds.yaml
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: felixconfigurations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: FelixConfiguration
    plural: felixconfigurations
    singular: felixconfiguration
---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ipamblocks.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPAMBlock
    plural: ipamblocks
    singular: ipamblock

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: blockaffinities.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: BlockAffinity
    plural: blockaffinities
    singular: blockaffinity

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ipamhandles.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPAMHandle
    plural: ipamhandles
    singular: ipamhandle

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ipamconfigs.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPAMConfig
    plural: ipamconfigs
    singular: ipamconfig

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: bgppeers.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: BGPPeer
    plural: bgppeers
    singular: bgppeer

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: bgpconfigurations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: BGPConfiguration
    plural: bgpconfigurations
    singular: bgpconfiguration

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ippools.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPPool
    plural: ippools
    singular: ippool

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: hostendpoints.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: HostEndpoint
    plural: hostendpoints
    singular: hostendpoint

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: clusterinformations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: ClusterInformation
    plural: clusterinformations
    singular: clusterinformation

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: globalnetworkpolicies.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: GlobalNetworkPolicy
    plural: globalnetworkpolicies
    singular: globalnetworkpolicy

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: globalnetworksets.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: GlobalNetworkSet
    plural: globalnetworksets
    singular: globalnetworkset

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: networkpolicies.crd.projectcalico.org
spec:
  scope: Namespaced
  group: crd.projectcalico.org
  version: v1
  names:
    kind: NetworkPolicy
    plural: networkpolicies
    singular: networkpolicy

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: networksets.crd.projectcalico.org
spec:
  scope: Namespaced
  group: crd.projectcalico.org
  version: v1
  names:
    kind: NetworkSet
    plural: networksets
    singular: networkset
---
# Source: calico/templates/rbac.yaml

# Include a clusterrole for the kube-controllers component,
# and bind it to the calico-kube-controllers serviceaccount.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: calico-kube-controllers
rules:
  # Nodes are watched to monitor for deletions.
  - apiGroups: [""]
    resources:
      - nodes
    verbs:
      - watch
      - list
      - get
  # Pods are queried to check for existence.
  - apiGroups: [""]
    resources:
      - pods
    verbs:
      - get
  # IPAM resources are manipulated when nodes are deleted.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - ippools
    verbs:
      - list
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - blockaffinities
      - ipamblocks
      - ipamhandles
    verbs:
      - get
      - list
      - create
      - update
      - delete
  # Needs access to update clusterinformations.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - clusterinformations
    verbs:
      - get
      - create
      - update
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: calico-kube-controllers
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: calico-kube-controllers
subjects:
- kind: ServiceAccount
  name: calico-kube-controllers
  namespace: kube-system
---
# Include a clusterrole for the calico-node DaemonSet,
# and bind it to the calico-node serviceaccount.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: calico-node
rules:
  # The CNI plugin needs to get pods, nodes, and namespaces.
  - apiGroups: [""]
    resources:
      - pods
      - nodes
      - namespaces
    verbs:
      - get
  - apiGroups: [""]
    resources:
      - endpoints
      - services
    verbs:
      # Used to discover service IPs for advertisement.
      - watch
      - list
      # Used to discover Typhas.
      - get
  - apiGroups: [""]
    resources:
      - nodes/status
    verbs:
      # Needed for clearing NodeNetworkUnavailable flag.
      - patch
      # Calico stores some configuration information in node annotations.
      - update
  # Watch for changes to Kubernetes NetworkPolicies.
  - apiGroups: ["networking.k8s.io"]
    resources:
      - networkpolicies
    verbs:
      - watch
      - list
  # Used by Calico for policy information.
  - apiGroups: [""]
    resources:
      - pods
      - namespaces
      - serviceaccounts
    verbs:
      - list
      - watch
  # The CNI plugin patches pods/status.
  - apiGroups: [""]
    resources:
      - pods/status
    verbs:
      - patch
  # Calico monitors various CRDs for config.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - globalfelixconfigs
      - felixconfigurations
      - bgppeers
      - globalbgpconfigs
      - bgpconfigurations
      - ippools
      - ipamblocks
      - globalnetworkpolicies
      - globalnetworksets
      - networkpolicies
      - networksets
      - clusterinformations
      - hostendpoints
      - blockaffinities
    verbs:
      - get
      - list
      - watch
  # Calico must create and update some CRDs on startup.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - ippools
      - felixconfigurations
      - clusterinformations
    verbs:
      - create
      - update
  # Calico stores some configuration information on the node.
  - apiGroups: [""]
    resources:
      - nodes
    verbs:
      - get
      - list
      - watch
  # These permissions are only requried for upgrade from v2.6, and can
  # be removed after upgrade or on fresh installations.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - bgpconfigurations
      - bgppeers
    verbs:
      - create
      - update
  # These permissions are required for Calico CNI to perform IPAM allocations.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - blockaffinities
      - ipamblocks
      - ipamhandles
    verbs:
      - get
      - list
      - create
      - update
      - delete
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - ipamconfigs
    verbs:
      - get
  # Block affinities must also be watchable by confd for route aggregation.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - blockaffinities
    verbs:
      - watch
  # The Calico IPAM migration needs to get daemonsets. These permissions can be
  # removed if not upgrading from an installation using host-local IPAM.
  - apiGroups: ["apps"]
    resources:
      - daemonsets
    verbs:
      - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: calico-node
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: calico-node
subjects:
- kind: ServiceAccount
  name: calico-node
  namespace: kube-system

---
# Source: calico/templates/calico-node.yaml
# This manifest installs the calico-node container, as well
# as the CNI plugins and network config on
# each master and worker node in a Kubernetes cluster.
kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: calico-node
  namespace: kube-system
  labels:
    k8s-app: calico-node
spec:
  selector:
    matchLabels:
      k8s-app: calico-node
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        k8s-app: calico-node
      annotations:
        # This, along with the CriticalAddonsOnly toleration below,
        # marks the pod as a critical add-on, ensuring it gets
        # priority scheduling and that its resources are reserved
        # if it ever gets evicted.
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      nodeSelector:
        beta.kubernetes.io/os: linux
      hostNetwork: true
      tolerations:
        # Make sure calico-node gets scheduled on all nodes.
        - effect: NoSchedule
          operator: Exists
        # Mark the pod as a critical add-on for rescheduling.
        - key: CriticalAddonsOnly
          operator: Exists
        - effect: NoExecute
          operator: Exists
      serviceAccountName: calico-node
      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
      terminationGracePeriodSeconds: 0
      priorityClassName: system-node-critical
      initContainers:
        # This container performs upgrade from host-local IPAM to calico-ipam.
        # It can be deleted if this is a fresh installation, or if you have already
        # upgraded to use calico-ipam.
        - name: upgrade-ipam
          image: calico/cni:v3.11.1
          command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
          env:
            - name: KUBERNETES_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: CALICO_NETWORKING_BACKEND
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: calico_backend
          volumeMounts:
            - mountPath: /var/lib/cni/networks
              name: host-local-net-dir
            - mountPath: /host/opt/cni/bin
              name: cni-bin-dir
          securityContext:
            privileged: true
        # This container installs the CNI binaries
        # and CNI network config file on each node.
        - name: install-cni
          image: calico/cni:v3.11.1
          command: ["/install-cni.sh"]
          env:
            # Name of the CNI config file to create.
            - name: CNI_CONF_NAME
              value: "10-calico.conflist"
            # The CNI network config to install on each node.
            - name: CNI_NETWORK_CONFIG
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: cni_network_config
            # Set the hostname based on the k8s node name.
            - name: KUBERNETES_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            # CNI MTU Config variable
            - name: CNI_MTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
            # Prevents the container from sleeping forever.
            - name: SLEEP
              value: "false"
          volumeMounts:
            - mountPath: /host/opt/cni/bin
              name: cni-bin-dir
            - mountPath: /host/etc/cni/net.d
              name: cni-net-dir
          securityContext:
            privileged: true
        # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
        # to communicate with Felix over the Policy Sync API.
        - name: flexvol-driver
          image: calico/pod2daemon-flexvol:v3.11.1
          volumeMounts:
          - name: flexvol-driver-host
            mountPath: /host/driver
          securityContext:
            privileged: true
      containers:
        # Runs calico-node container on each Kubernetes node.  This
        # container programs network policy and routes on each
        # host.
        - name: calico-node
          image: calico/node:v3.11.1
          env:
            # Use Kubernetes API as the backing datastore.
            - name: DATASTORE_TYPE
              value: "kubernetes"
            # Wait for the datastore.
            - name: WAIT_FOR_DATASTORE
              value: "true"
            # Set based on the k8s node name.
            - name: NODENAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            # Choose the backend to use.
            - name: CALICO_NETWORKING_BACKEND
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: calico_backend
            # Cluster type to identify the deployment type
            - name: CLUSTER_TYPE
              value: "k8s,bgp"
            # Auto-detect the BGP IP address.
            - name: IP
              value: "autodetect"
            # Enable IPIP
            - name: CALICO_IPV4POOL_IPIP
              value: "CrossSubnet"
            # Set MTU for tunnel device used if ipip is enabled
            - name: FELIX_IPINIPMTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
            # The default IPv4 pool to create on startup if none exists. Pod IPs will be
            # chosen from this range. Changing this value after installation will have
            # no effect. This should fall within \`--cluster-cidr\`.
            - name: CALICO_IPV4POOL_CIDR
              value: "10.211.0.0/16"
            # Disable file logging so \`kubectl logs\` works.
            - name: CALICO_DISABLE_FILE_LOGGING
              value: "true"
            # Set Felix endpoint to host default action to ACCEPT.
            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
              value: "ACCEPT"
            # Disable IPv6 on Kubernetes.
            - name: FELIX_IPV6SUPPORT
              value: "false"
            # Set Felix logging to "info"
            - name: FELIX_LOGSEVERITYSCREEN
              value: "info"
            - name: FELIX_HEALTHENABLED
              value: "true"
          securityContext:
            privileged: true
          resources:
            requests:
              cpu: 250m
          livenessProbe:
            exec:
              command:
              - /bin/calico-node
              - -felix-live
              - -bird-live
            periodSeconds: 10
            initialDelaySeconds: 10
            failureThreshold: 6
          readinessProbe:
            exec:
              command:
              - /bin/calico-node
              - -felix-ready
              - -bird-ready
            periodSeconds: 10
          volumeMounts:
            - mountPath: /lib/modules
              name: lib-modules
              readOnly: true
            - mountPath: /run/xtables.lock
              name: xtables-lock
              readOnly: false
            - mountPath: /var/run/calico
              name: var-run-calico
              readOnly: false
            - mountPath: /var/lib/calico
              name: var-lib-calico
              readOnly: false
            - name: policysync
              mountPath: /var/run/nodeagent
      volumes:
        # Used by calico-node.
        - name: lib-modules
          hostPath:
            path: /lib/modules
        - name: var-run-calico
          hostPath:
            path: /var/run/calico
        - name: var-lib-calico
          hostPath:
            path: /var/lib/calico
        - name: xtables-lock
          hostPath:
            path: /run/xtables.lock
            type: FileOrCreate
        # Used to install CNI.
        - name: cni-bin-dir
          hostPath:
            path: /opt/cni/bin
        - name: cni-net-dir
          hostPath:
            path: /etc/cni/net.d
        # Mount in the directory for host-local IPAM allocations. This is
        # used when upgrading from host-local to calico-ipam, and can be removed
        # if not using the upgrade-ipam init container.
        - name: host-local-net-dir
          hostPath:
            path: /var/lib/cni/networks
        # Used to create per-pod Unix Domain Sockets
        - name: policysync
          hostPath:
            type: DirectoryOrCreate
            path: /var/run/nodeagent
        # Used to install Flex Volume Driver
        - name: flexvol-driver-host
          hostPath:
            type: DirectoryOrCreate
            path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds
---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: calico-node
  namespace: kube-system

---
# Source: calico/templates/calico-kube-controllers.yaml

# See https://github.com/projectcalico/kube-controllers
apiVersion: apps/v1
kind: Deployment
metadata:
  name: calico-kube-controllers
  namespace: kube-system
  labels:
    k8s-app: calico-kube-controllers
spec:
  # The controllers can only have a single active instance.
  replicas: 1
  selector:
    matchLabels:
      k8s-app: calico-kube-controllers
  strategy:
    type: Recreate
  template:
    metadata:
      name: calico-kube-controllers
      namespace: kube-system
      labels:
        k8s-app: calico-kube-controllers
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      nodeSelector:
        beta.kubernetes.io/os: linux
      tolerations:
        # Mark the pod as a critical add-on for rescheduling.
        - key: CriticalAddonsOnly
          operator: Exists
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
      serviceAccountName: calico-kube-controllers
      priorityClassName: system-cluster-critical
      containers:
        - name: calico-kube-controllers
          image: calico/kube-controllers:v3.11.1
          env:
            # Choose which controllers to run.
            - name: ENABLED_CONTROLLERS
              value: node
            - name: DATASTORE_TYPE
              value: kubernetes
          readinessProbe:
            exec:
              command:
              - /usr/bin/check-status
              - -r

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: calico-kube-controllers
  namespace: kube-system
---
# Source: calico/templates/calico-etcd-secrets.yaml

---
# Source: calico/templates/calico-typha.yaml

---
# Source: calico/templates/configure-canal.yaml

EOF

kubectl create -f calico/

mkdir -p kube-dns/

cat <<EOF > kube-dns/configmap.yaml
apiVersion: v1
data:
  Corefile: |
    .:53 {
        errors
        health {
           lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
           pods insecure
           fallthrough in-addr.arpa ip6.arpa
           ttl 30
        }
        prometheus :9153
        forward . /etc/resolv.conf
        cache 30
        loop
        reload
        loadbalance
    }
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
EOF

cat <<EOF > kube-dns/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: coredns
  namespace: kube-system
EOF

cat <<EOF > kube-dns/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:coredns
rules:
- apiGroups:
  - ""
  resources:
  - endpoints
  - services
  - pods
  - namespaces
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
EOF

cat <<EOF > kube-dns/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:coredns
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:coredns
subjects:
- kind: ServiceAccount
  name: coredns
  namespace: kube-system
EOF

cat <<EOF > kube-dns/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    deployment.kubernetes.io/revision: "1"
  labels:
    k8s-app: kube-dns
  name: coredns
  namespace: kube-system
spec:
  replicas: 2
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: kube-dns
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        k8s-app: kube-dns
    spec:
      containers:
      - args:
        - -conf
        - /etc/coredns/Corefile
        image: registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.6.5
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 5
          httpGet:
            path: /health
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        name: coredns
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9153
          name: metrics
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /ready
            port: 8181
            scheme: HTTP
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        resources:
          limits:
            memory: 170Mi
          requests:
            cpu: 100m
            memory: 70Mi
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            add:
            - NET_BIND_SERVICE
            drop:
            - all
          readOnlyRootFilesystem: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/coredns
          name: config-volume
          readOnly: true
      dnsPolicy: Default
      nodeSelector:
        beta.kubernetes.io/os: linux
      priorityClassName: system-cluster-critical
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: coredns
      serviceAccountName: coredns
      terminationGracePeriodSeconds: 30
      tolerations:
      - key: CriticalAddonsOnly
        operator: Exists
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
      volumes:
      - configMap:
          defaultMode: 420
          items:
          - key: Corefile
            path: Corefile
          name: coredns
        name: config-volume
EOF

cat <<EOF > kube-dns/service.yaml
apiVersion: v1
kind: Service
metadata:
  annotations:
    prometheus.io/port: "9153"
    prometheus.io/scrape: "true"
  labels:
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: KubeDNS
  name: kube-dns
  namespace: kube-system
spec:
  clusterIP: 10.96.0.10
  ports:
  - name: dns
    port: 53
    protocol: UDP
    targetPort: 53
  - name: dns-tcp
    port: 53
    protocol: TCP
    targetPort: 53
  - name: metrics
    port: 9153
    protocol: TCP
    targetPort: 9153
  selector:
    k8s-app: kube-dns
  sessionAffinity: None
  type: ClusterIP
EOF

kubectl create -f kube-dns/

mkdir -p nginx/

cat <<EOF > nginx/01-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.7.9
        ports:
        - containerPort: 80
EOF

cat <<EOF > nginx/02-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: nginx
spec:
  selector:
    app: nginx
  ports:
    - nodePort: 31073
      protocol: TCP
      port: 80
      targetPort: 80
  type: NodePort
EOF

kubectl create -f nginx/

# 有响应视为正常，否则视为不正常
curl -XGET http://192.168.112.129:31073
curl -XGET http://192.168.112.130:31073

# 在node01节点和node02节点上分别操作
mkdir -p network/
cd network/

cat <<EOF > Dockerfile
FROM alpine:3.8

MAINTAINER wangxin_0611@126.com

RUN apk add --no-cache ca-certificates bind-tools iputils iproute2 net-tools tcpdump
EOF

docker build -t wangx/alpine:3.8-network .

# 在master节点上操作
mkdir -p network/

cat <<EOF > network/network.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: network
  namespace: default
spec:
  selector:
    matchLabels:
      app: network
  template:
    metadata:
      labels:
        app: network
    spec:
      containers:
      - name: network
        image: wangx/alpine:3.8-network
        imagePullPolicy: IfNotPresent
        command:
        - sleep
        - "3600"
      restartPolicy: Always
EOF

kubectl create -f network/

[root@master ~]# kubectl get pod -o wide
NAME            READY     STATUS    RESTARTS   AGE       IP           NODE
network-bl7jx   1/1       Running   0          6m        10.211.1.4   node02
network-m2vp6   1/1       Running   0          6m        10.211.0.4   node01

# 在node01上的pod中验证
[root@master ~]# kubectl exec -it network-m2vp6 /bin/sh
/ # cat /etc/resolv.conf
nameserver 10.96.0.10
search default.svc.cluster.local svc.cluster.local cluster.local
options ndots:5
/ # ip address
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1
    link/ipip 0.0.0.0 brd 0.0.0.0
4: eth0@if11: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 72:01:49:fa:fb:f3 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 10.211.0.4/32 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::7001:49ff:fefa:fbf3/64 scope link
       valid_lft forever preferred_lft forever
/ # ping -c 4 10.211.1.4
PING 10.211.1.4 (10.211.1.4) 56(84) bytes of data.
64 bytes from 10.211.1.4: icmp_seq=1 ttl=62 time=0.314 ms
64 bytes from 10.211.1.4: icmp_seq=2 ttl=62 time=0.490 ms
64 bytes from 10.211.1.4: icmp_seq=3 ttl=62 time=0.415 ms
64 bytes from 10.211.1.4: icmp_seq=4 ttl=62 time=0.491 ms

--- 10.211.1.4 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 3000ms
rtt min/avg/max/mdev = 0.314/0.427/0.491/0.075 ms
/ # nslookup kubernetes.default
Server:   10.96.0.10
Address:  10.96.0.10#53

Name: kubernetes.default.svc.cluster.local
Address: 10.96.0.1

/ # nslookup kubernetes
Server:   10.96.0.10
Address:  10.96.0.10#53

Name: kubernetes.default.svc.cluster.local
Address: 10.96.0.1

/ # exit

# 在node02上的pod中验证
[root@master ~]# kubectl get pod -o wide
NAME            READY     STATUS    RESTARTS   AGE       IP           NODE
network-bl7jx   1/1       Running   0          9m        10.211.1.4   node02
network-m2vp6   1/1       Running   0          9m        10.211.0.4   node01
[root@master ~]# kubectl exec -it network-bl7jx /bin/sh
/ # cat /etc/resolv.conf
nameserver 10.96.0.10
search default.svc.cluster.local svc.cluster.local cluster.local
options ndots:5
/ # ip address
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1
    link/ipip 0.0.0.0 brd 0.0.0.0
4: eth0@if9: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 5a:a6:51:22:9d:2b brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 10.211.1.4/32 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::58a6:51ff:fe22:9d2b/64 scope link
       valid_lft forever preferred_lft forever
/ # ping -c 4 10.211.0.4
PING 10.211.0.4 (10.211.0.4) 56(84) bytes of data.
64 bytes from 10.211.0.4: icmp_seq=1 ttl=62 time=0.450 ms
64 bytes from 10.211.0.4: icmp_seq=2 ttl=62 time=0.685 ms
64 bytes from 10.211.0.4: icmp_seq=3 ttl=62 time=0.726 ms
64 bytes from 10.211.0.4: icmp_seq=4 ttl=62 time=0.707 ms

--- 10.211.0.4 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 3003ms
rtt min/avg/max/mdev = 0.450/0.642/0.726/0.111 ms
/ # nslookup kubernetes.default
Server:   10.96.0.10
Address:  10.96.0.10#53

Name: kubernetes.default.svc.cluster.local
Address: 10.96.0.1

/ # nslookup kubernetes
Server:   10.96.0.10
Address:  10.96.0.10#53

Name: kubernetes.default.svc.cluster.local
Address: 10.96.0.1

/ # exit

docker pull goharbor/clair-adapter-photon:v1.0.1-v1.10.1
docker pull goharbor/clair-photon:v2.1.1-v1.10.1
docker pull goharbor/notary-server-photon:v0.6.1-v1.10.1
docker pull goharbor/harbor-core:v1.10.1
docker pull goharbor/harbor-portal:v1.10.1
docker pull goharbor/chartmuseum-photon:v0.9.0-v1.10.1
docker pull goharbor/redis-photon:v1.10.1
docker pull goharbor/notary-signer-photon:v0.6.1-v1.10.1
docker pull goharbor/harbor-registryctl:v1.10.1
docker pull goharbor/registry-photon:v2.7.1-patch-2819-2553-v1.10.1
docker pull goharbor/harbor-jobservice:v1.10.1
docker pull goharbor/harbor-db:v1.10.1

# curl -o helm-v3.1.0-linux-amd64.tar.gz https://get.helm.sh/helm-v3.1.0-linux-amd64.tar.gz
# tar -zxvf helm-v3.1.0-linux-amd64.tar.gz
# cd linux-amd64/
# cp helm /usr/local/bin/

# helm version
version.BuildInfo{Version:"v3.1.0", GitCommit:"b29d20baf09943e134c2fa5e1e1cab3bf93315fa", GitTreeState:"clean", GoVersion:"go1.13.7"}

## 本文以安装了NFS Client Provisioner为例，下面是我环境中的相关资源对象展示，请依据实际环境进行配置。我环境中的 StorageClass 对象叫 managed-nfs-storage 
# kubectl get pod -n storage -o wide
NAME                                      READY   STATUS    RESTARTS   AGE   IP               NODE     NOMINATED NODE   READINESS GATES
nfs-client-provisioner-6c45d64447-hrjhq   1/1     Running   2          21h   10.211.196.155   node01   <none>           <none>

# kubectl get storageclass managed-nfs-storage -o yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  creationTimestamp: "2020-03-03T12:10:27Z"
  name: managed-nfs-storage
  resourceVersion: "27152"
  selfLink: /apis/storage.k8s.io/v1/storageclasses/managed-nfs-storage
  uid: 05c23ff9-5f9a-4a6b-89dd-ed99013cc344
mountOptions:
- vers=4
parameters:
  archiveOnDelete: "false"
provisioner: fuseim.pri/ifs
reclaimPolicy: Delete
volumeBindingMode: Immediate

# kubectl get storageclass -o wide
NAME                  PROVISIONER      RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
managed-nfs-storage   fuseim.pri/ifs   Delete          Immediate           false                  21h