使用Heketi工具管理GlusterFS集群

使用Heketi管理GlusterFS Cluster

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
# 创建集群
[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 --json=true cluster create {"id":"0f0e5d914e3d9a17bef670dd6e295512","nodes":[],"volumes":[]}

# 查看集群
[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 --json=true cluster list {"clusters":["0f0e5d914e3d9a17bef670dd6e295512"]}

# 添加节点
[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 --json=true node add --cluster=0f0e5d914e3d9a17bef670dd6e295512 --management-host-name=192.168.86.135 --storage-host-name=192.168.86.135 --zone=1
{"zone":1,"hostnames":{"manage":["192.168.86.135"],"storage":["192.168.86.135"]},"cluster":"0f0e5d914e3d9a17bef670dd6e295512","i

[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 --json=true node add --cluster=0f0e5d914e3d9a17bef670dd6e295512 --management-host-name=192.168.86.136 --storage-host-name=192.168.86.136 --zone=1
{"zone":1,"hostnames":{"manage":["192.168.86.136"],"storage":["192.168.86.136"]},"cluster":"0f0e5d914e3d9a17bef670dd6e295512","i

[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 --json=true node add --cluster=0f0e5d914e3d9a17bef670dd6e295512 --management-host-name=192.168.86.137 --storage-host-name=192.168.86.137 --zone=1
{"zone":1,"hostnames":{"manage":["192.168.86.137"],"storage":["192.168.86.137"]},"cluster":"0f0e5d914e3d9a17bef670dd6e295512","i

# 查看节点
[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 --json=true node list Id:a2c00d537210d42d68679bfa240db8b5 Cluster:0f0e5d914e3d9a17bef670dd6e295512
Id:a73f8bdd464f02a4c7e0dd30c4ed5afa Cluster:0f0e5d914e3d9a17bef670dd6e295512
Id:aba2f2cfe93c0f3e4164aa05b8d6ddb2 Cluster:0f0e5d914e3d9a17bef670dd6e295512

# 给节点添加设备(裸硬盘)
[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 --json=true device add --name=/dev/sdb --node=a2c00d537210d42d68679bfa240db8b5
Device added successfully
[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 --json=true device add --name=/dev/sdb --node=a73f8bdd464f02a4c7e0dd30c4ed5afa
Device added successfully
[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 --json=true device add --name=/dev/sdb --node=aba2f2cfe93c0f3e4164aa05b8d6ddb2
Device added successfully

# 创建volume
[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 volume create --size=10 --replica=2 
Name: vol_d1fd574462c162f42fb61ef55c3e4e6e
Size: 10
Volume Id: d1fd574462c162f42fb61ef55c3e4e6e
Cluster Id: 0f0e5d914e3d9a17bef670dd6e295512
Mount: 192.168.86.135:vol_d1fd574462c162f42fb61ef55c3e4e6e
Mount Options: backup-volfile-servers=192.168.86.136,192.168.86.137
Durability Type: replicate
Distributed+Replica: 2

# 查看volume
[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 volume list Id:d1fd574462c162f42fb61ef55c3e4e6e Cluster:0f0e5d914e3d9a17bef670dd6e295512 Name:vol_d1fd574462c162f42fb61ef55c3e4e6e
# 删除volume
[root@server07 ~]# heketi-cli --server http://192.168.86.135:8080 --user admin --secret 12345678 volume delete d1fd574462c162f42fb61ef55c3e4e6e
Volume d1fd574462c162f42fb61ef55c3e4e6e deleted

# 使用topology.json初始化GlusterFS Cluster
cat <<EOF > /etc/heketi/topology.json
{
  "clusters": [
    {
      "nodes": [
        {
          "node": {
            "hostnames": {
              "manage": [
                "server07"
              ],
              "storage": [
                "172.16.170.134"
              ]
            },
            "zone": 1
          },
          "devices": [
            "/dev/sdb"
          ]
        },
        {
          "node": {
            "hostnames": {
              "manage": [
                "server08"
              ],
              "storage": [
                "172.16.170.135"
              ]
            },
            "zone": 1
          },
          "devices": [
            "/dev/sdb"
          ]
        },
        {
          "node": {
            "hostnames": {
              "manage": [
                "server09"
              ],
              "storage": [
                "172.16.170.136"
              ]
            },
            "zone": 1
          },
          "devices": [
            "/dev/sdb"
          ]
        }
      ]
    }
  ]
}
EOF

heketi-cli --user admin --secret 12345678 topology load --json=/etc/heketi/topology.json
Creating cluster ... ID: 4e01ca44d2e6f0077ee3abe9c6783183
	Allowing file volumes on cluster.
	Allowing block volumes on cluster.
	Creating node server07 ... ID: 2e8b83add06cde8713d56ecb7d424033
		Adding device /dev/sdb ... OK
	Creating node server08 ... ID: 56a23b0579e7f79511843e046e69008f
		Adding device /dev/sdb ... OK
	Creating node server09 ... ID: 9fb13189ba78ef302046fe4414f633bf
		Adding device /dev/sdb ... OK

heketi-cli --user admin --secret 12345678 topology info

Cluster Id: 4e01ca44d2e6f0077ee3abe9c6783183

    File:  true
    Block: true

    Volumes:

    Nodes:

	Node Id: 2e8b83add06cde8713d56ecb7d424033
	State: online
	Cluster Id: 4e01ca44d2e6f0077ee3abe9c6783183
	Zone: 1
	Management Hostnames: server07
	Storage Hostnames: 172.16.170.134
	Devices:
		Id:4995671aaef70cb1f640d0f411e94d2f   Name:/dev/sdb            State:online    Size (GiB):19      Used (GiB):0       Free (GiB):19
			Bricks:

	Node Id: 56a23b0579e7f79511843e046e69008f
	State: online
	Cluster Id: 4e01ca44d2e6f0077ee3abe9c6783183
	Zone: 1
	Management Hostnames: server08
	Storage Hostnames: 172.16.170.135
	Devices:
		Id:c4a56208419516d4fbc437d9dc3b265e   Name:/dev/sdb            State:online    Size (GiB):19      Used (GiB):0       Free (GiB):19
			Bricks:

	Node Id: 9fb13189ba78ef302046fe4414f633bf
	State: online
	Cluster Id: 4e01ca44d2e6f0077ee3abe9c6783183
	Zone: 1
	Management Hostnames: server09
	Storage Hostnames: 172.16.170.136
	Devices:
		Id:be2a9558f0634233be72f0c55d051898   Name:/dev/sdb            State:online    Size (GiB):19      Used (GiB):0       Free (GiB):19
			Bricks:
容器云技术

Kubernetes对接GlusterFS（把GlusterFS部署在Kubernetes集群外）

一、环境版本信息

1. 版本信息

CentOS 7.6.1810
Kubernetes v1.11.0
GlusterFS 5.3-1.el7 @centos-gluster5
Heketi 8.0.0-1.el7 @centos-gluster5

2. 服务器信息

server01 <-> 172.16.170.128 <-> kubernetes master
server02 <-> 172.16.170.129 <-> kubernetes node
server03 <-> 172.16.170.130 <-> kubernetes node

server07 <-> 172.16.170.134 <-> glusterfs node
server08 <-> 172.16.170.135 <-> glusterfs node
server09 <-> 172.16.170.136 <-> glusterfs node

二、相关注意事项

1. Heketi所在Node到GlusterFS Cluster的所有Node需要配置root用户的SSH免密登录；

2. /etc/heketi/heketi.json 中需要修改配置，不能不做任何修改就拿来直接使用；

（1）安全的做法是需要开启jwt认证，两个key字段分别用来设置admin用户和user用户的访问密码：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
...
  "use_auth": true,

  "_jwt": "Private keys for access",
  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "key": "12345678"
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "key": "12345678"
    }
  },
...

（2）需要把executor修改为ssh，不能使用默认的mock：

1
2
3
4
5
6
7
8
...
    "executor": "ssh",

    "sshexec": {
      "keyfile": "/root/.ssh/id_rsa",
      "user": "root"
    },
...

（3）json文件中有些配置想使用默认值，需要注释掉或者删除掉，不然无法启动Heketi，因为这些字段的value部分写得是字段的使用说明而非默认值。

3. /lib/systemd/system/heketi.service 中的User值应该为root，不修改的话默认为heketi，无法成功启动。

1
2
3
...
User=root
...

三、完整的实现过程记录

1. Kubernetes集群的所有节点上（既包括Master也包括Node）上都执行：

1
2
3
4
# 安装GlusterFS的yum源
yum install -y centos-release-gluster
# 安装挂载GlusterFS Volume所需要的驱动
yum install -y glusterfs glusterfs-fuse

2. 在Kubernetes集群外部安装GlusterFS集群：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
# 在server07、server08和server09三台服务器上执行
yum update -y
setenforce 0
systemctl disable firewalld.service
systemctl stop firewalld.service

systemctl disable NetworkManager.service
systemctl stop NetworkManager.service

cat <<EOF >> /etc/hosts

# For GlusterFS Cluster
172.16.170.134 server07
172.16.170.135 server08
172.16.170.136 server09

EOF

yum install -y centos-release-gluster
yum install -y glusterfs glusterfs-server glusterfs-fuse glusterfs-rdma glusterfs-geo-replication glusterfs-devel
mkdir /opt/glusterd
sed -i 's/var\/lib/opt/g' /etc/glusterfs/glusterd.vol
systemctl enable glusterd.service
systemctl start glusterd.service
systemctl status glusterd.service

# 在server07服务器上执行
yum install -y heketi heketi-client
ssh-keygen
ssh-copy-id root@server07
ssh-copy-id root@server08
ssh-copy-id root@server09

cat <<EOF > /etc/heketi/heketi.json
{
  "_port_comment": "Heketi Server Port Number",
  "port": "8080",

  "_use_auth": "Enable JWT authorization. Please enable for deployment",
  "use_auth": true,

  "_jwt": "Private keys for access",
  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "key": "12345678"
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "key": "12345678"
    }
  },

  "_glusterfs_comment": "GlusterFS Configuration",
  "glusterfs": {
    "_executor_comment": [
      "Execute plugin. Possible choices: mock, ssh",
      "mock: This setting is used for testing and development.",
      "      It will not send commands to any node.",
      "ssh:  This setting will notify Heketi to ssh to the nodes.",
      "      It will need the values in sshexec to be configured.",
      "kubernetes: Communicate with GlusterFS containers over",
      "            Kubernetes exec api."
    ],
    "executor": "ssh",

    "sshexec": {
      "keyfile": "/root/.ssh/id_rsa",
      "user": "root"
    },

    "_kubeexec_comment": "Kubernetes configuration",
    "kubeexec": {
      "host" :"https://kubernetes.host:8443",
      "cert" : "/path/to/crt.file",
      "insecure": false,
      "user": "kubernetes username",
      "password": "password for kubernetes user",
      "namespace": "OpenShift project or Kubernetes namespace",
      "fstab": "Optional: Specify fstab file on node.  Default is /etc/fstab"
    },

    "_db_comment": "Database file name",
    "db": "/var/lib/heketi/heketi.db",

    "_loglevel_comment": [
      "Set log level. Choices are:",
      "  none, critical, error, warning, info, debug",
      "Default is warning"
    ],
    "loglevel" : "debug"
  }
}
EOF

cat <<EOF > /lib/systemd/system/heketi.service
[Unit]
Description=Heketi Server

[Service]
Type=simple
WorkingDirectory=/var/lib/heketi
User=root
ExecStart=/usr/bin/heketi --config=/etc/heketi/heketi.json
Restart=on-failure
StandardOutput=syslog
StandardError=syslog

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable heketi.service
systemctl start heketi.service
systemctl status heketi.service

curl -XGET http://localhost:8080/hello
Hello from Heketi

3. 使用Heketi初始化GlusterFS集群：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
cat <<EOF > /etc/heketi/topology.json
{
  "clusters": [
    {
      "nodes": [
        {
          "node": {
            "hostnames": {
              "manage": [
                "server07"
              ],
              "storage": [
                "172.16.170.134"
              ]
            },
            "zone": 1
          },
          "devices": [
            "/dev/sdb"
          ]
        },
        {
          "node": {
            "hostnames": {
              "manage": [
                "server08"
              ],
              "storage": [
                "172.16.170.135"
              ]
            },
            "zone": 1
          },
          "devices": [
            "/dev/sdb"
          ]
        },
        {
          "node": {
            "hostnames": {
              "manage": [
                "server09"
              ],
              "storage": [
                "172.16.170.136"
              ]
            },
            "zone": 1
          },
          "devices": [
            "/dev/sdb"
          ]
        }
      ]
    }
  ]
}
EOF

heketi-cli --user admin --secret 12345678 topology load --json=/etc/heketi/topology.json
Creating cluster ... ID: 4e01ca44d2e6f0077ee3abe9c6783183
	Allowing file volumes on cluster.
	Allowing block volumes on cluster.
	Creating node server07 ... ID: 2e8b83add06cde8713d56ecb7d424033
		Adding device /dev/sdb ... OK
	Creating node server08 ... ID: 56a23b0579e7f79511843e046e69008f
		Adding device /dev/sdb ... OK
	Creating node server09 ... ID: 9fb13189ba78ef302046fe4414f633bf
		Adding device /dev/sdb ... OK

heketi-cli --user admin --secret 12345678 cluster list
Clusters:
Id:4e01ca44d2e6f0077ee3abe9c6783183 [file][block]

heketi-cli --user admin --secret 12345678 --json=true node list
Id:2e8b83add06cde8713d56ecb7d424033	Cluster:4e01ca44d2e6f0077ee3abe9c6783183
Id:56a23b0579e7f79511843e046e69008f	Cluster:4e01ca44d2e6f0077ee3abe9c6783183
Id:9fb13189ba78ef302046fe4414f633bf	Cluster:4e01ca44d2e6f0077ee3abe9c6783183


heketi-cli --user admin --secret 12345678 topology info

Cluster Id: 4e01ca44d2e6f0077ee3abe9c6783183

    File:  true
    Block: true

    Volumes:

    Nodes:

	Node Id: 2e8b83add06cde8713d56ecb7d424033
	State: online
	Cluster Id: 4e01ca44d2e6f0077ee3abe9c6783183
	Zone: 1
	Management Hostnames: server07
	Storage Hostnames: 172.16.170.134
	Devices:
		Id:4995671aaef70cb1f640d0f411e94d2f   Name:/dev/sdb            State:online    Size (GiB):19      Used (GiB):0       Free (GiB):19
			Bricks:

	Node Id: 56a23b0579e7f79511843e046e69008f
	State: online
	Cluster Id: 4e01ca44d2e6f0077ee3abe9c6783183
	Zone: 1
	Management Hostnames: server08
	Storage Hostnames: 172.16.170.135
	Devices:
		Id:c4a56208419516d4fbc437d9dc3b265e   Name:/dev/sdb            State:online    Size (GiB):19      Used (GiB):0       Free (GiB):19
			Bricks:

	Node Id: 9fb13189ba78ef302046fe4414f633bf
	State: online
	Cluster Id: 4e01ca44d2e6f0077ee3abe9c6783183
	Zone: 1
	Management Hostnames: server09
	Storage Hostnames: 172.16.170.136
	Devices:
		Id:be2a9558f0634233be72f0c55d051898   Name:/dev/sdb            State:online    Size (GiB):19      Used (GiB):0       Free (GiB):19
			Bricks:

4. 编写验证Heketi和GlusterFS集群对接Kubernetes集群的相关YAML：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
mkdir -p glusterfs/
cd glusterfs/

cat <<EOF > 01-heketi-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: heketi-secret
  namespace: default
data:
  # base64 encoded password. E.g.: echo -n "mypassword" | base64
  key: MTIzNDU2Nzg=
type: kubernetes.io/glusterfs
EOF

cat <<EOF > 02-glusterfs-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: glusterfs
provisioner: kubernetes.io/glusterfs
parameters:
  resturl: "http://172.16.170.134:8080"
  clusterid: "4e01ca44d2e6f0077ee3abe9c6783183"
  restauthenabled: "true"
  restuser: "admin"
  secretNamespace: "default"
  secretName: "heketi-secret"
  volumetype: "replicate:3"
EOF

cat <<EOF > 03-persistentvolumeclaim.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: myclaim
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: glusterfs
EOF

cat <<EOF > 04-pod.yaml
apiVersion: v1
kind: Pod
metadata:
  name: alpine
spec:
  containers:
  - image: alpine:3.8
    command:
    - sleep
    - "3600"
    imagePullPolicy: IfNotPresent
    name: alpine
    volumeMounts:
    - name: myvolume
      mountPath: "/data"
  restartPolicy: Always
  volumes:
  - name: myvolume
    persistentVolumeClaim:
      claimName: myclaim
EOF

5. 验证Heketi和GlusterFS集群对接Kubernetes集群：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
# 创建StorageClass对接到GlusterFS
kubectl create -f 01-heketi-secret.yaml
secret/heketi-secret created

kubectl create -f 02-glusterfs-storageclass.yaml
storageclass.storage.k8s.io/glusterfs created

# 创建PVC使用上面的StorageClass
kubectl create -f 03-persistentvolumeclaim.yaml
persistentvolumeclaim/myclaim created

# 验证PVC动态供应PV
kubectl get pvc -o wide
NAME      STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
myclaim   Bound     pvc-9d5a1b14-2456-11e9-a588-000c2921eba0   1Gi        RWX            glusterfs      40s

kubectl get pv -o wide
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS    CLAIM             STORAGECLASS   REASON    AGE
pvc-9d5a1b14-2456-11e9-a588-000c2921eba0   1Gi        RWX            Delete           Bound     default/myclaim   glusterfs                40s

# 创建测试Pod使用上面的PVC
kubectl create -f 04-pod.yaml
pod/alpine created

kubectl get pod -o wide
NAME      READY     STATUS    RESTARTS   AGE       IP           NODE
alpine    1/1       Running   0          17s       10.211.1.2   server02

# 验证PVC是否挂载到Pod中
kubectl exec -it alpine /bin/sh
/ # df -h
Filesystem                Size      Used Available Use% Mounted on
overlay                  17.0G      2.2G     14.8G  13% /
tmpfs                   911.8M         0    911.8M   0% /dev
tmpfs                   911.8M         0    911.8M   0% /sys/fs/cgroup
172.16.170.134:vol_b46e22b816b18672ecb5a8c3289b0193
                       1014.0M     42.7M    971.3M   4% /data
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /dev/termination-log
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /etc/resolv.conf
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /etc/hostname
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /etc/hosts
shm                      64.0M         0     64.0M   0% /dev/shm
tmpfs                   911.8M     12.0K    911.8M   0% /run/secrets/kubernetes.io/serviceaccount
tmpfs                   911.8M         0    911.8M   0% /proc/kcore
tmpfs                   911.8M         0    911.8M   0% /proc/timer_list
tmpfs                   911.8M         0    911.8M   0% /proc/timer_stats
tmpfs                   911.8M         0    911.8M   0% /proc/sched_debug
tmpfs                   911.8M         0    911.8M   0% /sys/firmware

# 创建测试文件，验证存储容量的限制
/data # fallocate -l 971M onebox.disk
/data # df -h
Filesystem                Size      Used Available Use% Mounted on
overlay                  17.0G      2.2G     14.8G  13% /
tmpfs                   911.8M         0    911.8M   0% /dev
tmpfs                   911.8M         0    911.8M   0% /sys/fs/cgroup
172.16.170.134:vol_b46e22b816b18672ecb5a8c3289b0193
                       1014.0M   1013.7M    348.0K 100% /data
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /dev/termination-log
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /etc/resolv.conf
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /etc/hostname
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /etc/hosts
shm                      64.0M         0     64.0M   0% /dev/shm
tmpfs                   911.8M     12.0K    911.8M   0% /run/secrets/kubernetes.io/serviceaccount
tmpfs                   911.8M         0    911.8M   0% /proc/kcore
tmpfs                   911.8M         0    911.8M   0% /proc/timer_list
tmpfs                   911.8M         0    911.8M   0% /proc/timer_stats
tmpfs                   911.8M         0    911.8M   0% /proc/sched_debug
tmpfs                   911.8M         0    911.8M   0% /sys/firmware
/data # fallocate -l 971M onebox.disk
fallocate: fallocate 'onebox.disk': No space left on device

# 测试文件删除后，验证容量是否恢复
/data # rm -rf onebox.disk
/data # df -h
Filesystem                Size      Used Available Use% Mounted on
overlay                  17.0G      2.2G     14.8G  13% /
tmpfs                   911.8M         0    911.8M   0% /dev
tmpfs                   911.8M         0    911.8M   0% /sys/fs/cgroup
172.16.170.134:vol_b46e22b816b18672ecb5a8c3289b0193
                       1014.0M     42.7M    971.3M   4% /data
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /dev/termination-log
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /etc/resolv.conf
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /etc/hostname
/dev/mapper/cl-root      17.0G      2.2G     14.8G  13% /etc/hosts
shm                      64.0M         0     64.0M   0% /dev/shm
tmpfs                   911.8M     12.0K    911.8M   0% /run/secrets/kubernetes.io/serviceaccount
tmpfs                   911.8M         0    911.8M   0% /proc/kcore
tmpfs                   911.8M         0    911.8M   0% /proc/timer_list
tmpfs                   911.8M         0    911.8M   0% /proc/timer_stats
tmpfs                   911.8M         0    911.8M   0% /proc/sched_debug
tmpfs                   911.8M         0    911.8M   0% /sys/firmware
容器云技术

Kubernetes对接GlusterFS（把GlusterFS部署在Kubernetes集群内）

一、环境版本信息

1. 版本信息

CentOS 7.6.1810
Kubernetes v1.11.0
GlusterFS gluster3u13_centos7
Heketi 4

2. 服务器信息

server03 <-> 172.16.170.130 <-> kubernetes master
server07 <-> 172.16.170.134 <-> kubernetes node
server08 <-> 172.16.170.135 <-> kubernetes node
server09 <-> 172.16.170.136 <-> kubernetes node

二、相关注意事项

1. 不要使用链接 https://github.com/gluster/gluster-kubernetes/tree/v1.1 中的部署脚本；

笔者最开始用了，这个过程中还参考了网上很多的帖子，可谓是吃尽了苦头。。。。。。

2. Heketi只有ServiceAccount是不够用的，需要给这个ServiceAccount创建ClusterRoleBinding对象，为其分配比较高的API访问权限；

比如说笔者这里就简单处理，为其分配cluster-admin权限。

1
2
3
4
5
6
7
8
9
10
11
12
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: heketi
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: heketi
  namespace: storage

3. 所有宿主机都要执行modprobe dm_thin_pool，否则创建volume会报错；

1
modprobe dm_thin_pool

4. GlusterFS和Heketi的Docker镜像不要像社区代码中那样使用latest和dev这样镜像内容会随时更新的标签，建议选择内容固定的标签对应的镜像。

笔者选择的Docker镜像如下：

1
2
gluster/gluster-centos:gluster3u13_centos7
heketi/heketi:4

三、完整的实现过程记录

1. Kubernetes集群的所有节点上（既包括Master也包括Node）上都执行：

1
2
3
4
5
6
# 安装GlusterFS的yum源
yum install -y centos-release-gluster
# 安装挂载GlusterFS Volume所需要的驱动
yum install -y glusterfs glusterfs-fuse
# 开启Linux内核的 device-mapper target(s) 支持
modprobe dm_thin_pool

2. 参考社区的YAML，编写部署GlusterFS的YAML：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
mkdir -p gluster-kubernetes/deploy/gluster-kubernetes/
cd gluster-kubernetes/deploy/gluster-kubernetes/

cat <<EOF > 01-common-namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: storage
EOF

cat <<EOF > 02-glusterfs-daemonset.yaml
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: glusterfs
  namespace: storage
  labels:
    glusterfs: daemonset
  annotations:
    description: GlusterFS DaemonSet
    tags: glusterfs
spec:
  template:
    metadata:
      name: glusterfs
      labels:
        glusterfs-node: pod
    spec:
      nodeSelector:
        storagenode: glusterfs
      hostNetwork: true
      containers:
      - image: gluster/gluster-centos:gluster3u13_centos7
        imagePullPolicy: IfNotPresent
        name: glusterfs
        volumeMounts:
        - name: glusterfs-heketi
          mountPath: "/var/lib/heketi"
        - name: glusterfs-run
          mountPath: "/run"
        - name: glusterfs-lvm
          mountPath: "/run/lvm"
        - name: glusterfs-etc
          mountPath: "/etc/glusterfs"
        - name: glusterfs-logs
          mountPath: "/var/log/glusterfs"
        - name: glusterfs-config
          mountPath: "/var/lib/glusterd"
        - name: glusterfs-dev
          mountPath: "/dev"
        - name: glusterfs-misc
          mountPath: "/var/lib/misc/glusterfsd"
        - name: glusterfs-cgroup
          mountPath: "/sys/fs/cgroup"
          readOnly: true
        - name: glusterfs-ssl
          mountPath: "/etc/ssl"
          readOnly: true
        securityContext:
          capabilities: {}
          privileged: true
        readinessProbe:
          timeoutSeconds: 3
          initialDelaySeconds: 60
          exec:
            command:
            - "/bin/bash"
            - "-c"
            - systemctl status glusterd.service
        livenessProbe:
          timeoutSeconds: 3
          initialDelaySeconds: 60
          exec:
            command:
            - "/bin/bash"
            - "-c"
            - systemctl status glusterd.service
      volumes:
      - name: glusterfs-heketi
        hostPath:
          path: "/var/lib/heketi"
      - name: glusterfs-run
      - name: glusterfs-lvm
        hostPath:
          path: "/run/lvm"
      - name: glusterfs-etc
        hostPath:
          path: "/etc/glusterfs"
      - name: glusterfs-logs
        hostPath:
          path: "/var/log/glusterfs"
      - name: glusterfs-config
        hostPath:
          path: "/var/lib/glusterd"
      - name: glusterfs-dev
        hostPath:
          path: "/dev"
      - name: glusterfs-misc
        hostPath:
          path: "/var/lib/misc/glusterfsd"
      - name: glusterfs-cgroup
        hostPath:
          path: "/sys/fs/cgroup"
      - name: glusterfs-ssl
        hostPath:
          path: "/etc/ssl"
EOF

cat <<EOF > 03-heketi-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: heketi
  namespace: storage
EOF

cat <<EOF > 04-heketi-clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: heketi
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: heketi
  namespace: storage
EOF

cat <<EOF > 05-heketi-deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: heketi
  namespace: storage
  labels:
    glusterfs: heketi-deployment
    deploy-heketi: heketi-deployment
  annotations:
    description: Defines how to deploy Heketi
spec:
  replicas: 1
  template:
    metadata:
      name: heketi
      labels:
        name: deploy-heketi
        glusterfs: heketi-pod
    spec:
      serviceAccountName: heketi
      containers:
      - image: heketi/heketi:4
        imagePullPolicy: IfNotPresent
        name: heketi
        env:
        - name: HEKETI_EXECUTOR
          value: kubernetes
        - name: HEKETI_FSTAB
          value: "/var/lib/heketi/fstab"
        - name: HEKETI_SNAPSHOT_LIMIT
          value: '14'
        - name: HEKETI_KUBE_GLUSTER_DAEMONSET
          value: "y"
        - name: HEKETI_CLI_SERVER
          value: "http://127.0.0.1:8080"
        ports:
        - containerPort: 8080
        volumeMounts:
        - name: db
          mountPath: "/var/lib/heketi"
        readinessProbe:
          timeoutSeconds: 3
          initialDelaySeconds: 3
          httpGet:
            path: "/hello"
            port: 8080
        livenessProbe:
          timeoutSeconds: 3
          initialDelaySeconds: 30
          httpGet:
            path: "/hello"
            port: 8080
      volumes:
      - name: db
EOF

cat <<EOF > 06-heketi-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: heketi
  namespace: storage
  labels:
    glusterfs: heketi-service
    deploy-heketi: support
  annotations:
    description: Exposes Heketi Service
spec:
  selector:
    name: deploy-heketi
  ports:
  - name: deploy-heketi
    port: 8080
    targetPort: 8080
EOF

3. 在Kubernetes集群上创建Heketi和GlusterFS集群：

1
2
3
4
5
6
7
8
9
cd ../../
kubectl create -f deploy/gluster-kubernetes/

kubectl get pod -n storage -o wide
NAME                      READY     STATUS    RESTARTS   AGE       IP               NODE
glusterfs-6shxs           1/1       Running   0          1m        172.16.170.135   server08
glusterfs-9vpjg           1/1       Running   0          1m        172.16.170.136   server09
glusterfs-pmq2n           1/1       Running   0          1m        172.16.170.134   server07
heketi-5bdc8f9db5-nmjrw   1/1       Running   0          1m        10.211.1.12      server07

4. 使用Heketi初始化GlusterFS集群：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
cat <<EOF > topology.json
{
  "clusters": [
    {
      "nodes": [
        {
          "node": {
            "hostnames": {
              "manage": [
                "server07"
              ],
              "storage": [
                "172.16.170.134"
              ]
            },
            "zone": 1
          },
          "devices": [
            "/dev/sdb"
          ]
        },
        {
          "node": {
            "hostnames": {
              "manage": [
                "server08"
              ],
              "storage": [
                "172.16.170.135"
              ]
            },
            "zone": 1
          },
          "devices": [
            "/dev/sdb"
          ]
        },
        {
          "node": {
            "hostnames": {
              "manage": [
                "server09"
              ],
              "storage": [
                "172.16.170.136"
              ]
            },
            "zone": 1
          },
          "devices": [
            "/dev/sdb"
          ]
        }
      ]
    }
  ]
}
EOF

kubectl cp topology.json heketi-5bdc8f9db5-nmjrw:/ -n storage

kubectl exec -it heketi-5bdc8f9db5-nmjrw /bin/bash -n storage
[root@heketi-5bdc8f9db5-nmjrw /]# heketi-cli topology load --json=topology.json
Creating cluster ... ID: dde6f33a6cb852e04cb0685e210d3959
	Creating node server07 ... ID: 2818eb53b31234c2cc852b339538d856
		Adding device /dev/sdb ... OK
	Creating node server08 ... ID: d5773e74abd6f3041d07b095a2f4ae99
		Adding device /dev/sdb ... OK
	Creating node server09 ... ID: 4caafd29101b77da4a6cd265dad65140
		Adding device /dev/sdb ... OK

heketi-cli topology info

Cluster Id: dde6f33a6cb852e04cb0685e210d3959

    Volumes:

    Nodes:

	Node Id: 2818eb53b31234c2cc852b339538d856
	State: online
	Cluster Id: dde6f33a6cb852e04cb0685e210d3959
	Zone: 1
	Management Hostname: server07
	Storage Hostname: 172.16.170.134
	Devices:
		Id:25bbec25514b711d48d0abef98809310   Name:/dev/sdb            State:online    Size (GiB):19      Used (GiB):0       Free (GiB):19
			Bricks:

	Node Id: 4caafd29101b77da4a6cd265dad65140
	State: online
	Cluster Id: dde6f33a6cb852e04cb0685e210d3959
	Zone: 1
	Management Hostname: server09
	Storage Hostname: 172.16.170.136
	Devices:
		Id:78385691f170d5ae6aae78448ab7d710   Name:/dev/sdb            State:online    Size (GiB):19      Used (GiB):0       Free (GiB):19
			Bricks:

	Node Id: d5773e74abd6f3041d07b095a2f4ae99
	State: online
	Cluster Id: dde6f33a6cb852e04cb0685e210d3959
	Zone: 1
	Management Hostname: server08
	Storage Hostname: 172.16.170.135
	Devices:
		Id:2678d65a18f0c21b38288d218d209a3f   Name:/dev/sdb            State:online    Size (GiB):19      Used (GiB):0       Free (GiB):19
			Bricks:

5. 编写验证Heketi和GlusterFS集群对接Kubernetes集群的相关YAML：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
mkdir -p example/
cd example/

kubectl get service -n storage
NAME      TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)    AGE
heketi    ClusterIP   10.96.162.3   <none>        8080/TCP   1h

cat <<EOF > 01-glusterfs-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: glusterfs
provisioner: kubernetes.io/glusterfs
parameters:
  resturl: "http://10.96.162.3:8080"
  restauthenabled: "false"
  volumetype: "replicate:3"
EOF

cat <<EOF > 02-persistentvolumeclaim.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: myclaim
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: glusterfs
EOF

cat <<EOF > 03-pod.yaml
apiVersion: v1
kind: Pod
metadata:
  name: alpine
spec:
  containers:
  - image: alpine:3.8
    command:
    - sleep
    - "3600"
    imagePullPolicy: IfNotPresent
    name: alpine
    volumeMounts:
    - name: myvolume
      mountPath: "/data"
  restartPolicy: Always
  volumes:
  - name: myvolume
    persistentVolumeClaim:
      claimName: myclaim
EOF

6. 验证Heketi和GlusterFS集群对接Kubernetes集群：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# 创建StorageClass对接到GlusterFS
kubectl create -f 01-glusterfs-storageclass.yaml
storageclass.storage.k8s.io/glusterfs created

# 创建PVC使用上面的StorageClass
kubectl create -f 02-persistentvolumeclaim.yaml
persistentvolumeclaim/myclaim created

# 验证PVC动态供应PV
kubectl get pvc -o wide
NAME      STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
myclaim   Bound     pvc-f910f4b2-239d-11e9-b3b6-000c295a4c4c   1Gi        RWX            glusterfs      1m

kubectl get pv -o wide
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS    CLAIM             STORAGECLASS   REASON    AGE
pvc-f910f4b2-239d-11e9-b3b6-000c295a4c4c   1Gi        RWX            Delete           Bound     default/myclaim   glusterfs                1m

# 创建测试Pod使用上面的PVC
kubectl create -f 03-pod.yaml
pod/alpine created

kubectl get pod -o wide
NAME      READY     STATUS    RESTARTS   AGE       IP            NODE
alpine    1/1       Running   0          5s        10.211.2.10   server08

# 验证PVC是否挂载到Pod中
kubectl exec -it alpine /bin/sh
/ # df -h
Filesystem                Size      Used Available Use% Mounted on
overlay                  17.0G      4.3G     12.7G  25% /
tmpfs                   909.8M         0    909.8M   0% /dev
tmpfs                   909.8M         0    909.8M   0% /sys/fs/cgroup
172.16.170.134:vol_7f5ea2a6035e2f7885687db4ef46d2e3
                       1015.3M     42.8M    982.6M   4% /data
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /dev/termination-log
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /etc/resolv.conf
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /etc/hostname
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /etc/hosts
shm                      64.0M         0     64.0M   0% /dev/shm
tmpfs                   909.8M     12.0K    909.8M   0% /run/secrets/kubernetes.io/serviceaccount
tmpfs                   909.8M         0    909.8M   0% /proc/kcore
tmpfs                   909.8M         0    909.8M   0% /proc/timer_list
tmpfs                   909.8M         0    909.8M   0% /proc/timer_stats
tmpfs                   909.8M         0    909.8M   0% /proc/sched_debug
tmpfs                   909.8M         0    909.8M   0% /sys/firmware

# 创建测试文件，验证存储容量的限制
/data # fallocate -l 982M onebox.disk
/data # df -h
Filesystem                Size      Used Available Use% Mounted on
overlay                  17.0G      4.3G     12.7G  25% /
tmpfs                   909.8M         0    909.8M   0% /dev
tmpfs                   909.8M         0    909.8M   0% /sys/fs/cgroup
172.16.170.134:vol_7f5ea2a6035e2f7885687db4ef46d2e3
                       1015.3M      1.0G    572.0K 100% /data
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /dev/termination-log
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /etc/resolv.conf
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /etc/hostname
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /etc/hosts
shm                      64.0M         0     64.0M   0% /dev/shm
tmpfs                   909.8M     12.0K    909.8M   0% /run/secrets/kubernetes.io/serviceaccount
tmpfs                   909.8M         0    909.8M   0% /proc/kcore
tmpfs                   909.8M         0    909.8M   0% /proc/timer_list
tmpfs                   909.8M         0    909.8M   0% /proc/timer_stats
tmpfs                   909.8M         0    909.8M   0% /proc/sched_debug
tmpfs                   909.8M         0    909.8M   0% /sys/firmware
/data # fallocate -l 982M onebox.disk
fallocate: fallocate 'onebox.disk': No space left on device

# 测试文件删除后，验证容量是否恢复
/data # rm -rf onebox.disk
/data # df -h
Filesystem                Size      Used Available Use% Mounted on
overlay                  17.0G      4.3G     12.7G  25% /
tmpfs                   909.8M         0    909.8M   0% /dev
tmpfs                   909.8M         0    909.8M   0% /sys/fs/cgroup
172.16.170.134:vol_7f5ea2a6035e2f7885687db4ef46d2e3
                       1015.3M     42.8M    982.6M   4% /data
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /dev/termination-log
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /etc/resolv.conf
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /etc/hostname
/dev/mapper/cl-root      17.0G      4.3G     12.7G  25% /etc/hosts
shm                      64.0M         0     64.0M   0% /dev/shm
tmpfs                   909.8M     12.0K    909.8M   0% /run/secrets/kubernetes.io/serviceaccount
tmpfs                   909.8M         0    909.8M   0% /proc/kcore
tmpfs                   909.8M         0    909.8M   0% /proc/timer_list
tmpfs                   909.8M         0    909.8M   0% /proc/timer_stats
tmpfs                   909.8M         0    909.8M   0% /proc/sched_debug
tmpfs                   909.8M         0    909.8M   0% /sys/firmware

四、参考资料

https://github.com/gluster/gluster-kubernetes/tree/v1.1
https://hub.docker.com/r/gluster/gluster-centos/tags
https://blog.csdn.net/q1403539144/article/details/86566796
http://www.cnitblog.com/xijia0524/archive/2014/04/19/89466.html

容器云技术

Calico IPIP（Always和CrossSubnet）模式和BGP模式的区别

一、分析网路配置的区别

1. IPIP Always 模式

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# ip link
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
    link/ether 00:0c:29:21:eb:a0 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
    link/ether 02:42:2f:64:85:3d brd ff:ff:ff:ff:ff:ff
8: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
9: cali436bd393c1c@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
10: cali09539dd2c0c@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
# ip address
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:21:eb:a0 brd ff:ff:ff:ff:ff:ff
    inet 172.16.170.128/24 brd 172.16.170.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe21:eba0/64 scope link
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
    link/ether 02:42:2f:64:85:3d brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 scope global docker0
       valid_lft forever preferred_lft forever
8: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
    inet 10.211.0.1/32 brd 10.211.0.1 scope global tunl0
       valid_lft forever preferred_lft forever
9: cali436bd393c1c@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet6 fe80::ecee:eeff:feee:eeee/64 scope link
       valid_lft forever preferred_lft forever
10: cali09539dd2c0c@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
    inet6 fe80::ecee:eeff:feee:eeee/64 scope link
       valid_lft forever preferred_lft forever
# ip route
default via 172.16.170.2 dev ens33
blackhole 10.211.0.0/24 proto bird
10.211.0.16 dev cali436bd393c1c scope link
10.211.0.17 dev cali09539dd2c0c scope link
10.211.1.0/24 via 172.16.170.129 dev tunl0 proto bird onlink
10.211.2.0/24 via 172.16.170.130 dev tunl0 proto bird onlink
169.254.0.0/16 dev ens33 scope link metric 1002
172.16.170.0/24 dev ens33 proto kernel scope link src 172.16.170.128
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1

2. IPIP CrossSubnet 模式

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# ip link
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
    link/ether 00:0c:29:21:eb:a0 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
    link/ether 02:42:74:e2:55:16 brd ff:ff:ff:ff:ff:ff
4: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
5: califb703006b7d@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
6: cali3c997f406a6@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
# ip address
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:21:eb:a0 brd ff:ff:ff:ff:ff:ff
    inet 172.16.170.128/24 brd 172.16.170.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe21:eba0/64 scope link
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
    link/ether 02:42:74:e2:55:16 brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 scope global docker0
       valid_lft forever preferred_lft forever
4: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
    inet 10.211.0.1/32 brd 10.211.0.1 scope global tunl0
       valid_lft forever preferred_lft forever
5: califb703006b7d@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet6 fe80::ecee:eeff:feee:eeee/64 scope link
       valid_lft forever preferred_lft forever
6: cali3c997f406a6@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
    inet6 fe80::ecee:eeff:feee:eeee/64 scope link
       valid_lft forever preferred_lft forever
# ip route
default via 172.16.170.2 dev ens33
blackhole 10.211.0.0/24 proto bird
10.211.0.18 dev califb703006b7d scope link
10.211.0.19 dev cali3c997f406a6 scope link
10.211.1.0/24 via 172.16.170.129 dev ens33 proto bird
10.211.2.0/24 via 172.16.170.130 dev ens33 proto bird
169.254.0.0/16 dev ens33 scope link metric 1002
172.16.170.0/24 dev ens33 proto kernel scope link src 172.16.170.128
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1

3. BGP 模式

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
# ip link
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
    link/ether 00:0c:29:21:eb:a0 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
    link/ether 02:42:b7:6f:cc:b2 brd ff:ff:ff:ff:ff:ff
6: cali72590f2ff6e@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
7: cali439e67ff763@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
# ip address
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:21:eb:a0 brd ff:ff:ff:ff:ff:ff
    inet 172.16.170.128/24 brd 172.16.170.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe21:eba0/64 scope link
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
    link/ether 02:42:b7:6f:cc:b2 brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 scope global docker0
       valid_lft forever preferred_lft forever
6: cali72590f2ff6e@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet6 fe80::ecee:eeff:feee:eeee/64 scope link
       valid_lft forever preferred_lft forever
7: cali439e67ff763@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
    inet6 fe80::ecee:eeff:feee:eeee/64 scope link
       valid_lft forever preferred_lft forever
# ip route
default via 172.16.170.2 dev ens33
blackhole 10.211.0.0/24 proto bird
10.211.0.22 dev cali72590f2ff6e scope link
10.211.0.23 dev cali439e67ff763 scope link
10.211.1.0/24 via 172.16.170.129 dev ens33 proto bird
10.211.2.0/24 via 172.16.170.130 dev ens33 proto bird
169.254.0.0/16 dev ens33 scope link metric 1002
172.16.170.0/24 dev ens33 proto kernel scope link src 172.16.170.128
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1

二、区别总结

Calico的IP Pool包括IPIP模式和BGP模式，其中IPIP模式又包括Always和CrossSubnet。IPIP Always简单说是指，Calico网路的路由的分发始终通过Node上的tunl0隧道实现；IPIP CrossSubnet简单说是指，当两个Pod所在的Node的地址在同一网段时，Calico网路的路由的分发则通过各个Node上的主机网卡实现。当两个Pod所在的Node的地址不在同一网段时，Calico网路的路由的分发才通过Node上的tunl0隧道实现。这种模式是IPIP Always和BGP模式的合体实现。

三、参考资料

Calico开启BGP模式：
http://www.cnblogs.com/jinxj/p/9414830.html

Calico原理解读：
https://blog.csdn.net/ccy19910925/article/details/82423452

Calico基本原理和模拟：
http://ju.outofmemory.cn/entry/367749

calico/node配置文档：
https://docs.projectcalico.org/v3.1/reference/node/configuration

Calico IP Pool介绍：
https://www.jianshu.com/p/dcad6d74e526
http://www.361way.com/linux-tunnel/5199.html
https://blog.csdn.net/kkdelta/article/details/39611061

Calico 跨网段问题：
https://blog.csdn.net/mailjoin/article/details/79695463
https://www.lijiaocn.com/项目/2017/09/25/calico-ipip.html

静态路由配置示例：
https://jingyan.baidu.com/article/6dad5075f7c67aa123e36eb9.html
http://blog.51cto.com/11101034/1906726

容器云技术

如何使用Kubernetes源代码做编译构建

一、操作系统版本

CentOS 7.6.1810
Kubernetes v1.11.0

二、准备工作

1. 直接使用yum安装rsync和gcc

1
yum install -y git rsync gcc

2. 源码安装golang 1.10.2

详见官方文档

3. 克隆源代码到GOPATH的src目录下

1
2
3
4
5
6
cd $GOPATH/src/
mkdir -p k8s.io/
cd k8s.io/
git clone https://github.com/kubernetes/kubernetes.git
cd kubernetes/
git checkout -b v1.11.0 v1.11.0

三、构建过程记录

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
[root@server01 kubernetes]# make clean
+++ [0107 11:01:28] Verifying Prerequisites....
+++ [0107 11:01:28] Removing _output directory
Removing pkg/generated/openapi/zz_generated.openapi.go ..
Removing pkg/generated/bindata.go ..
Removing test/e2e/generated/bindata.go ..
[root@server01 kubernetes]# KUBE_GIT_MAJOR="1" KUBE_GIT_MINOR="11" KUBE_GIT_VERSION="v1.11.0" KUBE_GIT_COMMIT="91e7b4fd31fcd3d5f436da26c980becec37ceefe" KUBE_GIT_TREE_STATE="clean" make all
+++ [0107 11:01:57] Building go targets for linux/amd64:
    ./vendor/k8s.io/code-generator/cmd/deepcopy-gen
+++ [0107 11:02:05] Building go targets for linux/amd64:
    ./vendor/k8s.io/code-generator/cmd/defaulter-gen
+++ [0107 11:02:09] Building go targets for linux/amd64:
    ./vendor/k8s.io/code-generator/cmd/conversion-gen
+++ [0107 11:02:13] Building go targets for linux/amd64:
    ./vendor/k8s.io/code-generator/cmd/openapi-gen
+++ [0107 11:02:19] Building go targets for linux/amd64:
    ./vendor/github.com/jteeuwen/go-bindata/go-bindata
+++ [0107 11:02:20] Building go targets for linux/amd64:
    cmd/kube-proxy
    cmd/kube-apiserver
    cmd/kube-controller-manager
    cmd/cloud-controller-manager
    cmd/kubelet
    cmd/kubeadm
    cmd/hyperkube
    cmd/kube-scheduler
    vendor/k8s.io/kube-aggregator
    vendor/k8s.io/apiextensions-apiserver
    cluster/gce/gci/mounter
    cmd/kubectl
    cmd/gendocs
    cmd/genkubedocs
    cmd/genman
    cmd/genyaml
    cmd/genswaggertypedocs
    cmd/linkcheck
    vendor/github.com/onsi/ginkgo/ginkgo
    test/e2e/e2e.test
    cmd/kubemark
    vendor/github.com/onsi/ginkgo/ginkgo
    test/e2e_node/e2e_node.test
[root@server01 kubernetes]# ls -la _output/bin/
总用量 2323024
drwxr-xr-x. 2 root root      4096 1月   7 11:08 .
drwxr-xr-x. 3 root root        19 1月   7 11:01 ..
-rwxr-xr-x. 1 root root  59300539 1月   7 11:08 apiextensions-apiserver
-rwxr-xr-x. 1 root root 138056772 1月   7 11:08 cloud-controller-manager
-rwxr-xr-x. 1 root root   7691655 1月   7 11:02 conversion-gen
-rwxr-xr-x. 1 root root   7687498 1月   7 11:01 deepcopy-gen
-rwxr-xr-x. 1 root root   7665142 1月   7 11:02 defaulter-gen
-rwxr-xr-x. 1 root root 209978368 1月   7 11:08 e2e_node.test
-rwxr-xr-x. 1 root root 173425408 1月   7 11:08 e2e.test
-rwxr-xr-x. 1 root root  54135796 1月   7 11:08 gendocs
-rwxr-xr-x. 1 root root 226087352 1月   7 11:08 genkubedocs
-rwxr-xr-x. 1 root root 232056512 1月   7 11:08 genman
-rwxr-xr-x. 1 root root   5477486 1月   7 11:08 genswaggertypedocs
-rwxr-xr-x. 1 root root  54084772 1月   7 11:08 genyaml
-rwxr-xr-x. 1 root root  10641201 1月   7 11:08 ginkgo
-rwxr-xr-x. 1 root root   2831370 1月   7 11:02 go-bindata
-rwxr-xr-x. 1 root root 227335584 1月   7 11:08 hyperkube
-rwxr-xr-x. 1 root root  57251126 1月   7 11:08 kubeadm
-rwxr-xr-x. 1 root root  57912503 1月   7 11:08 kube-aggregator
-rwxr-xr-x. 1 root root 185160079 1月   7 11:08 kube-apiserver
-rwxr-xr-x. 1 root root 153806893 1月   7 11:08 kube-controller-manager
-rwxr-xr-x. 1 root root  55277901 1月   7 11:08 kubectl
-rwxr-xr-x. 1 root root 162729192 1月   7 11:08 kubelet
-rwxr-xr-x. 1 root root 160066432 1月   7 11:08 kubemark
-rwxr-xr-x. 1 root root  51920351 1月   7 11:08 kube-proxy
-rwxr-xr-x. 1 root root  55479582 1月   7 11:08 kube-scheduler
-rwxr-xr-x. 1 root root   6698678 1月   7 11:08 linkcheck
-rwxr-xr-x. 1 root root   2330265 1月   7 11:08 mounter
-rwxr-xr-x. 1 root root  13634378 1月   7 11:02 openapi-gen
容器云技术

Kubernetes对接Ceph RBD关键点记录

Ceph RBD External Provisioner版本

https://github.com/kubernetes-incubator/external-storage/blob/v5.1.0/ceph/rbd/

对Linux操作系统的内核版本有要求，经初步测试需要版本为4.x

CentOS 7.5.1804如何升级操作系统内核？
https://www.howtoforge.com/tutorial/how-to-upgrade-kernel-in-centos-7-server/
https://zhuanlan.zhihu.com/p/29617407

Kubernetes各个Node上需要安装匹配版本的ceph-common

这里以宿主机操作系统为CentOS 7.5.1804，ceph版本为mimic为例

1
2
rpm -Uvh https://download.ceph.com/rpm-mimic/el7/noarch/ceph-release-1-1.el7.noarch.rpm
yum install -y --nogpgcheck ceph-common

参考资料：
https://github.com/kubernetes-incubator/external-storage/blob/v5.1.0/ceph/rbd/Dockerfile

StorageClass和Ceph RBD External Provisioner的正常工作依赖于Kubernetes内部的DNS解析

  1. 配置DNS记录，让Kubernetes集群内部可以把ceph monitor解析到对应的外部地址上；

    1
    2
    3
    4
    5
    6
    7
    8
    kind: Service
    apiVersion: v1
    metadata:
      name: ceph-mon
      namespace: kube-system
    spec:
      type: ExternalName
      externalName: 172.16.170.134.xip.io # ceph monitor的地址

  2. 配置StorageClass，让monitors参数使用1中配置的DNS记录。

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    kind: StorageClass
    apiVersion: storage.k8s.io/v1
    metadata:
      name: managed-cephrbd-storage
    provisioner: yonghui.cn/cephrbd
    parameters:
      monitors: ceph-mon.kube-system.svc.cluster.local:6789 # 这里需要使用Kubernetes内部的DNS配置ceph monitor的地址
      pool: kube
      adminId: admin
      adminSecretNamespace: kube-system
      adminSecretName: cephrbd-admin-secret
      userId: kube
      userSecretNamespace: kube-system
      userSecretName: cephrbd-secret
      imageFormat: "2"
      imageFeatures: layering

推荐参考资料：
http://blog.51cto.com/ygqygq2/2163656
https://segmentfault.com/q/1010000011440882

Ceph RBD的简单操作记录

1
2
3
4
5
# rbd ls -p kube
kubernetes-dynamic-pvc-4392eab0-fc4c-11e8-a9f7-ee2ca7a031f1
# rbd rm kube/kubernetes-dynamic-pvc-4392eab0-fc4c-11e8-a9f7-ee2ca7a031f1
Removing image: 100% complete...done.
# rbd ls -p kube
容器云技术

Fluentd不使用运行时的时区向Elasticsearch写入日志记录，怎么解决？

一、环境版本信息

Fluentd 1.1.0
Elasticsearch 6.3.0
fluent-plugin-elasticsearch 2.4.1

二、报错现象描述

以时区设置为Asia/Shanghai为例：
虽然Fluentd所在的运行时已经设置正确，但是上报到Elasticsearch对应索引中的日志记录，与实际相比还是相差八个小时。更直接的现象是，你发现明明应该归档在今天对应的索引中的日志，却归档在了昨天对应的索引中。再有就是你会发现，无论你怎么修改运行时的时区，Fluentd向Elasticsearch写入日志记录都始终使用UTC时区。

三、解决方法

把utc_index的值设置为false，允许Fluentd使用本地运行时设置的时区。

官方文档的原文描述如下：（详见参考资料部分的链接）
By default, the records inserted into index logstash-YYMMDD with UTC (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.

配置修改示例如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
<match **>
  @id elasticsearch
  @type elasticsearch
  @log_level info
  include_tag_key true
  host elasticsearch-ingest.logging
  port 9200
  logstash_format true
  utc_index false
  slow_flush_log_threshold 60.0
  <buffer>
    @type file
    path /var/log/fluentd-buffers/kubernetes.system.buffer
    flush_mode interval
    retry_type exponential_backoff
    flush_thread_count 2
    flush_interval 5s
    retry_forever
    retry_max_interval 30
    chunk_limit_size 6M
    queue_limit_length 256
    overflow_action block
  </buffer>
</match>

四、参考资料

https://github.com/uken/fluent-plugin-elasticsearch/tree/v2.4.1#utc_index

容器云技术

Fluentd报错批处理索引队列满了，该怎么解决？

一、环境版本信息

Fluentd 1.1.0
Elasticsearch 6.3.0

二、报错信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
2018-12-20 03:50:41 +0000 [info]: [elasticsearch] Connection opened to Elasticsearch cluster => {:host=>"elasticsearch.logging", :port=>9200, :scheme=>"http"}
2018-12-20 20:28:23 +0000 [warn]: [elasticsearch] failed to flush the buffer. retry_time=0 next_retry_seconds=2018-12-20 20:28:24 +0000 chunk="57d79f767802e630777d6da9f12a1fd7" error_class=Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull error="Bulk index queue is full, retrying"
  2018-12-20 20:28:23 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-2.4.1/lib/fluent/plugin/elasticsearch_error_handler.rb:90:in `block in handle_error'
  2018-12-20 20:28:23 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-2.4.1/lib/fluent/plugin/elasticsearch_error_handler.rb:85:in `each_key'
  2018-12-20 20:28:23 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-2.4.1/lib/fluent/plugin/elasticsearch_error_handler.rb:85:in `handle_error'
  2018-12-20 20:28:23 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-2.4.1/lib/fluent/plugin/out_elasticsearch.rb:434:in `send_bulk'
  2018-12-20 20:28:23 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-2.4.1/lib/fluent/plugin/out_elasticsearch.rb:418:in `write'
  2018-12-20 20:28:23 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluentd-1.1.0/lib/fluent/plugin/output.rb:1094:in `try_flush'
  2018-12-20 20:28:23 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluentd-1.1.0/lib/fluent/plugin/output.rb:1319:in `flush_thread_run'
  2018-12-20 20:28:23 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluentd-1.1.0/lib/fluent/plugin/output.rb:439:in `block (2 levels) in start'
  2018-12-20 20:28:23 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluentd-1.1.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
2018-12-20 20:28:24 +0000 [warn]: [elasticsearch] failed to flush the buffer. retry_time=0 next_retry_seconds=2018-12-20 20:28:24 +0000 chunk="57d79f767802e630777d6da9f12a1fd7" error_class=Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull error="Bulk index queue is full, retrying"
  2018-12-20 20:28:24 +0000 [warn]: suppressed same stacktrace
2018-12-20 20:28:25 +0000 [warn]: [elasticsearch] failed to flush the buffer. retry_time=1 next_retry_seconds=2018-12-20 20:28:25 +0000 chunk="57d79f767802e630777d6da9f12a1fd7" error_class=Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull error="Bulk index queue is full, retrying"
  2018-12-20 20:28:25 +0000 [warn]: suppressed same stacktrace
2018-12-20 20:28:26 +0000 [warn]: [elasticsearch] failed to flush the buffer. retry_time=2 next_retry_seconds=2018-12-20 20:28:27 +0000 chunk="57d79f78b9fce084be8b1f544a046a09" error_class=Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull error="Bulk index queue is full, retrying"
  2018-12-20 20:28:26 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-2.4.1/lib/fluent/plugin/elasticsearch_error_handler.rb:90:in `block in handle_error'
  2018-12-20 20:28:26 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-2.4.1/lib/fluent/plugin/elasticsearch_error_handler.rb:85:in `each_key'
  2018-12-20 20:28:26 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-2.4.1/lib/fluent/plugin/elasticsearch_error_handler.rb:85:in `handle_error'
  2018-12-20 20:28:26 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-2.4.1/lib/fluent/plugin/out_elasticsearch.rb:434:in `send_bulk'
  2018-12-20 20:28:26 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-2.4.1/lib/fluent/plugin/out_elasticsearch.rb:418:in `write'
  2018-12-20 20:28:26 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluentd-1.1.0/lib/fluent/plugin/output.rb:1094:in `try_flush'
  2018-12-20 20:28:26 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluentd-1.1.0/lib/fluent/plugin/output.rb:1319:in `flush_thread_run'
  2018-12-20 20:28:26 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluentd-1.1.0/lib/fluent/plugin/output.rb:439:in `block (2 levels) in start'
  2018-12-20 20:28:26 +0000 [warn]: /var/lib/gems/2.3.0/gems/fluentd-1.1.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
2018-12-20 20:28:30 +0000 [warn]: [elasticsearch] failed to flush the buffer. retry_time=3 next_retry_seconds=2018-12-20 20:28:30 +0000 chunk="57d79f78b9fce084be8b1f544a046a09" error_class=Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull error="Bulk index queue is full, retrying"
  2018-12-20 20:28:30 +0000 [warn]: suppressed same stacktrace
2018-12-20 20:28:30 +0000 [warn]: [elasticsearch] failed to flush the buffer. retry_time=4 next_retry_seconds=2018-12-20 20:28:39 +0000 chunk="57d79f767802e630777d6da9f12a1fd7" error_class=Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull error="Bulk index queue is full, retrying"
  2018-12-20 20:28:30 +0000 [warn]: suppressed same stacktrace
2018-12-20 20:28:45 +0000 [warn]: [elasticsearch] failed to flush the buffer. retry_time=5 next_retry_seconds=2018-12-20 20:28:44 +0000 chunk="57d79f767802e630777d6da9f12a1fd7" error_class=Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull error="Bulk index queue is full, retrying"
  2018-12-20 20:28:45 +0000 [warn]: suppressed same stacktrace
2018-12-20 20:28:45 +0000 [warn]: [elasticsearch] failed to flush the buffer. retry_time=6 next_retry_seconds=2018-12-20 20:29:18 +0000 chunk="57d79f78b9fce084be8b1f544a046a09" error_class=Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull error="Bulk index queue is full, retrying"
  2018-12-20 20:28:45 +0000 [warn]: suppressed same stacktrace
2018-12-20 20:29:13 +0000 [warn]: [elasticsearch] retry succeeded. chunk_id="57d79f78b9fce084be8b1f544a046a09"

三、解决方法

在配置文件elasticsearch.yml中，增加如下配置：

1
2
3
4
5
6
7
...
thread_pool:
  index:
    queue_size: ${INDEX_QUEUE_SIZE:200}
  write:
    queue_size: ${WRITE_QUEUE_SIZE:200}
...

在pod、deployment或者statefulset等pod相关的yaml中，增加如下环境变量的配置：

1
2
3
4
5
6
7
...
        env:
        - name: INDEX_QUEUE_SIZE
          value: "1000"
        - name: WRITE_QUEUE_SIZE
          value: "1000"
...

如何查看配置的修改是否生效？即查看Elasticsearch的线程池配置。（注意：不通版本的Elasticsearch查看方式有所不同，详见参考资料中的链接文档）
提示：推荐使用 Kibana 的 Dev Tools进行查询，这种方式最为方便。

1
2
GET _cat/thread_pool/index?v&h=id,node_name,name,active,queue,rejected,completed,queue_size
GET _cat/thread_pool/write?v&h=id,node_name,name,active,queue,rejected,completed,queue_size

四、参考资料

https://blog.csdn.net/opensure/article/details/51491815
https://www.elastic.co/guide/en/elasticsearch/reference/2.4/modules-threadpool.html
https://www.elastic.co/guide/en/elasticsearch/reference/6.3/modules-threadpool.html
https://www.elastic.co/guide/en/elasticsearch/reference/6.3/cat-thread-pool.html

容器云技术

Elasticsearch 经典资料汇总

中文资料

https://es.xiaoleilu.com
https://github.com/elasticsearch-cn/elasticsearch-definitive-guide
https://www.elastic.co/guide/cn/elasticsearch/guide/current/index.html

Elasticsearch Client Golang SDK

https://github.com/olivere/elastic/tree/v6.2.11

深度分页问题

https://www.elastic.co/guide/en/elasticsearch/reference/6.3/search-request-search-after.html
https://blog.csdn.net/u011228889/article/details/79760167
https://blog.csdn.net/WangPing1223/article/details/79148244

Kubernetes上非常好用的Elasticsearch

https://github.com/pires/kubernetes-elasticsearch-cluster/tree/6.3.0

容器云技术

删除Docker容器或者重启Docker Daemon时，遇到device or resource busy，该怎么办？

前段时间，在内部的线上环境修复Kubernetes集群时，需要重启Docker Daemon，可以悲剧发生了，一重启就报错：device or resource busy。Google了好久，终于找到了类似的案例和解决方法。此时才知道，原来删除Docker容器，也可能会出现这个问题。这里简单记录一下，原创详见“参考资料”。

原文给出的脚本：

1
2
3
4
5
6
7
#!/bin/bash

for i in $(curl -s --unix /var/run/docker.sock http://localhost/info | jq -r .DockerRootDir) /var/lib/docker /run /var/run; do
    for m in $(tac /proc/mounts | awk '{print $2}' | grep ^${i}/); do
        umount $m || true
    done
done

在我们内部的环境中，DockerRootDir是固定的，就在/var/lib/docker，为了避免安装jq依赖，调整为下面的脚本：

1
2
3
4
5
6
7
#!/bin/bash

for i in /var/lib/docker /run /var/run; do
    for m in $(tac /proc/mounts | awk '{print $2}' | grep ^${i}/); do
        umount $m || true
    done
done

参考资料：
http://niusmallnan.com/2016/12/27/docker-device-resource-busy/

容器云技术