基于 Ceph RBD 实现 Docker 集群的分布式存储

# (1). 添加 ceph 源
cat >>/etc/yum.repos.d/ceph.repo <<EOF

 
[ceph-noarch]
name=Ceph noarch packages
baseurl=https://download.ceph.com/rpm-kraken/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc

[ceph]
name=Ceph packages for $basearch
baseurl=http://download.ceph.com/rpm-kraken/el7/$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc

 
[ceph-source]
name=Ceph source packages
baseurl=http://download.ceph.com/rpm-kraken/el7/SRPMS
enabled=0
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
EOF

#(2). 安装 ceph client
yum update & yum install -y ceph

#(3) 添加集群信息
cat >> /etc/hosts <<EOF
192.168.100 ceph-moni-0
192.168.1.101 ceph-moni-1
192.168.1.102 ceph-moni-2
192.168.1.103 ceph-osd-0
192.168.1.104 ceph-osd-1
192.168.1.105 ceph-osd-2
192.168.1.106 ceph-client
EOF

#(4). 拷贝集群配置信息和 admin 密钥
scp -r root@$ceph-moni-0:/etc/ceph/\{ceph.conf,ceph.client.admin.keyring\} /etc/ceph/

#(5) 创建 ceph rbd
sudo rbd create volume001 --size 1024 -m ceph-node1 -k
/etc/ceph/ceph.client.admin.keyring

#(6) 查看创建的 rbd 信息
rbd info volume001

# (1). 手动创建 ceph rbd
rbd create volume001 --size 1024 -m ceph-node1 -k /etc/ceph/ceph.client.admin.keyring
 
#(2) 禁用 rbd 的一些特性
rbd feature disable volume001 exclusive-lock, object-map, fast-diff, deep-flatten
 
#(3) 获取 ceph 集群 admin 的密钥的 base64
grep key /etc/ceph/ceph.client.admin.keyring |awk '{printf "%s", $NF}'|base64
#终端会输出 admin 密钥的 base64 形式${ceph-key-base64}
 
#(4) 创建 ceph 的 secret
# 在文件/home/ceph-secret.yaml 添加以下内容
apiVersion: v1
kind: Secret
metadata:
name: ceph-secret
type: "kubernetes.io/rbd"
data:
key: ${ceph-key-base64}
# 创建 ceph-secret
kubect create -f /home/ceph-secret.yaml
#(5) 创建使用 ceph rbd 的容器
#在/home/rbd.json 添加以下内容
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": "rbd"
},
"spec": {
"containers": [
{
"name": "rbd-rw",
"image": "kubernetes/pause",
"volumeMounts": [
{
"mountPath": "/mnt/rbd",
"name": "rbdpd"
}]}],
"volumes": [
{
"name": "rbdpd",
"rbd": {
"monitors": [
#ceph 集群的所有监视器的 IP
${ceph-monitors}
],
"pool": "rbd",
"image": "volume001",
"user": "admin",
"keyring": "/etc/ceph/ceph.client.admin.keyring",
"fsType": "ext4",
"readOnly": false
}}]}}
#创建使用 ceph rbd 的 pod
kubectl create -f /home/rbd.json

#(1) 安装依赖包
sudo apt-get install librados-dev librbd-dev -y
 
#(2) 安装 golang 环境
wget https://storage.googleapis.com/golang/go1.7.5.linux-amd64.tar.gz"
tar -C /usr/local -xzf go1.7.5.linux-amd64.tar.gz
 
#(3)配置 go
export PATH=$PATH:/usr/local/go/bin
 
#(4) 设置 go 项目工作目录
mkdir -p /root/ceph
export GOPATH=/root/ceph
export PATH=$GOPATH/bin:$PATH
 
#(5)安装 git 和 gcc
apt-get install -y git
apt-get install -y gcc
 
#(6) 下载 go-ceph
cd /root/ceph
go get github.com/ceph/go-ceph

# /root/ceph/src/github.com/ceph/go-ceph/rbd/rbd.go 中添加以下内容
#引入 ceph 获取 rbd 变化的函数 rbd_diff_iterate 同时实现该函数的回调函数 diff_callback
/*
int diff_callback(uint64_t offset,size_t len,int exists,void *arg) {
size_t *used_size = (size_t *)(arg);
if (exists){
(*used_size) +=len;
}
return 0;
}
int rbd_allocation(rbd_image_t image, const char *fromsnapname,uint64_t ofs, uint64_t len,
        void *arg){
return rbd_diff_iterate(image,fromsnapname,ofs,len,diff_callback,arg);
}
*/
//...
 
#添加获取 rbd 使用情况的 go 函数
//int rbd_diff_iterate(rbd_image_t image,const char *fromsnapname,uint64_t ofs, uint64_t
        len, int (*cb)(uint64_t, size_t, int, void *), void *arg);
func (image *Image)GetAllocation(imageName string,totalSize uint64)(allocation int64, err error){
if image.image == nil {
return 0, RbdErrorImageNotOpen
}
ret := int(C.rbd_allocation(
image.image,
nil,
(C.uint64_t)(image.offset),
(C.uint64_t)(totalSize),
(unsafe.Pointer)(&allocation)))
if ret < 0 {
return 0, RBDError(ret)
}
return allocation,nil
}
重新编译安装 go-ceph
go install github.com/ceph/go-ceph/rbd/

#编辑/root/ceph/src/rest-ceph/controllers/image.go
//Get Image Allocation
func getImageAllocation(poolName string, imageName string)(int64,string,error){
conn, err := newConn()
if err != nil {
return -1,"error",err
}
defer conn.Shutdown()
ioctx, err := conn.OpenIOContext(poolName)
if err != nil {
return -1,"error",err
}
defer ioctx.Destroy()
img := rbd.GetImage(ioctx, imageName)
img.Open()
totalSize,err:=img.GetSize()
if err!=nil {
fmt.Println("error when get size",err);
return -1,"error",err
}
allocation,err:=img.GetAllocation(imageName,totalSize);
if err!=nil {
fmt.Println("error when get allocation",err);
return -1,"error",err
}
img.Close()
return allocation,"success",nil
}
#其他操作 ceph 的接口...

#编辑 /root/ceph/src/rest-ceph/routers/router.go
func init() {
ns := beego.NewNamespace("/v1",
beego.NSNamespace("/image",
beego.NSInclude(
&controllers.ImageController{},
),
),
//其他的命名空间
)
beego.AddNamespace(ns)
}

#编辑 /root/ceph/src/rest-ceph/routers/commentsRouter_controllers.go
func init() {
beego.GlobalControllerRouter["rest-ceph/controllers:ImageController"] = append(beego.GlobalControllerRoute r["rest-ceph/controllers:ImageController"],
beego.ControllerComments{
Method: "GetImageAllocation",
Router: `/allocation`,
AllowHTTPMethods: []string{"get"},
 Params: nil})
//类似的添加其他 rest api 的具体路由
}

#mysql-deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: ${name}
spec:
replicas: 1
template:
metadata:
labels:
app: ${name}
spec:
containers:
- name: ${name}
image: ${mysql-repo}
imagePullPolicy: Always
env:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: mysql-pass
key: password.txt
ports:
- containerPort: 3306
name: mysql
imagePullSecrets:
- name: harbor.product.secret

#把清单 12 的${name}赋值为：mysql-deployment
#创建 mysql-deployment
kubectl create -f mysql-deployment.yaml
 
#查看 pod
kubectl get po|grep mysql-deployment
mysql-deployment-311776555-0zz28 1/1 Running 0 1m
 
#登录 mysql pod
kubectl exec -it mysql-deployment-311776555-0zz28 /bin/bash
root@mysql-deployment-311776555-0zz28:/#
 
#登录 mysql
root@mysql-deployment-311776555-0zz28:/# mysql -uroot
 
#创建数据库
mysql> create database mydb;
Query OK, 1 row affected (0.00 sec)
 
#查看数据库列表
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mydb               |
| mysql              |
| performance_schema |
| test               |
+--------------------+
5 rows in set (0.00 sec)
 
#删除 deployment：mysql-deployment
>kubectl delete deployment mysql-deployment
 
#把清单 12 的 mysql 数据库 deployment 的 yaml 文件的${name}赋值为 mysql-deployment-02
#得到 mysql-deployment-02.yaml 文件
#创建 mysql-deployment-02
kubectl create -f mysql-deployment-02.yaml
 
#获取 mysql-deployment-02 的 pod 的 name
kubectl get po|grep mysql-deployment
>mysql-deployment-02-423384506-d73q4 1/1 Running 0 52s
 
#登录 pod
kubectl exec -it mysql-deployment-02-423384506-d73q4 /bin/bash
 
#登录 mysql
root@mysql-deployment-02-423384506-d73q4:/# mysql -uroot
 
#列出所有的数据库
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test               |
+--------------------+
4 rows in set (0.00 sec)

#ceph-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: ceph-pv
spec:
capacity:
storage: 1Gi
accessModes:
- ReadWriteOnce
rbd:
monitors: [
"192.168.1.100:6789",
"192.168.1.101:6789",
"192.168.1.102:6789"
],
pool: rbd
image: ceph-rbd
user: admin
secretRef:
name: ceph-secret
fsType: ext4
readOnly: false
persistentVolumeReclaimPolicy: Recycle

#ceph-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ${name}
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi

apiVersion: v1
kind: Service
metadata:
name: mysql-service
labels:
app: mysql-statefulset
spec:
ports:
- port: 3306
selector:
app: mysql-statefulset
type: NodePort

apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: ${name}
spec:
serviceName: mysql-service
replicas: 1
template:
metadata:
labels:
app: mysql-statefulset
spec:
terminationGracePeriodSeconds: 10
containers:
- name: ${name}
image: ${image-repo}
ports:
- containerPort: 3306
name: mysql
volumeMounts:
- name: pvc
mountPath: /var/lib/mysql
volumeClaimTemplates:
- metadata:
name: pvc
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 1Gi

#创建 ceph-secret
kubectl create -f ceph-secret.yaml
 
#创建 ceph rbd
curl -X POST http://root:root@192.168.1.100/v1/image?
-d "poolName=rbd&imageName=ceph-rbd&size=1073741824"
 
#创建 pv
kubectl create -f ceph-pv.yaml
 
#把清单 16 的${name}赋值为 pvc-mysql-statefulset-0
#创建 pvc
Kubectl create -f ceph-pvc.yaml
 
#创建 service
Kubectl create -f mysql-statefulset-service.yaml
 
#把清单 18 的${name}赋值为 mysql-statefulset
#创建 mysql-statefulset
kubectl create -f mysql-statefulset.yaml
 
#获取 pod
kubectl get po|grep mysql-state
mysql-statefulset-0 1/1 Running 0 4m
 
#进入 pod
kubectl exec -it mysql-statefulset-0 /bin/bash
 
#登录 mysql
mysql -uroot
 
#创建 database
mysql> create database mydb;
Query OK, 1 row affected (0.00 sec)
 
#显示数据库列表
mysql>show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mydb               |
| mysql              |
| performance_schema |
| test               |
+--------------------+
5 rows in set (0.00 sec)
 
#退出 mysql 以及 pod
mysql> exit
Bye
root@mysql-statefulset-0:/# exit
exit
 
#删除 pv,pvc,statefulset
kubectl delete statefulset mysql-statefulse
 
#把清单 18 的${name}更改为"mysql-statefulset-02",得到 mysql-statefulset-02.yaml 文件
#把清单 16 的 name 更改为"pvc-mysql-statefulset-02-0"得到 pvc-mysql-statefulset-02.yaml 文件
#创建 pv,pvc 和 mysql-statefulset-02
kubectl creat -f ceph-pv.yaml
kubectl create -f ceph-pvc-02.yaml
kubectl create -f mysql-statefulset-02.yaml
 
#获取 pod name
kubectl get po|grep mysql-statefulset
mysql-statefulset-02-0 1/1 Running 0 10s
 
#登录 pod
kubectl exec -it mysql-statefulset-02-0 /bin/bash
 
#登录 mysql
mysql -uroot
 
#查看数据库列表
# (10). 列出 database
Mysql > show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mydb               |
| mysql              |
| performance_schema |
| test               |
+--------------------+
>5 rows in set (0.02 sec)