ceph中 RBD 使用

Last updated on 8 months ago

RBD

RBD是Ceph对外的三大存储服务组件之一，块存储接口Ceph ，企业环境中最常见的一种数据存储格式，支持的最大镜像为 16GB 这些镜像可以作为磁盘映射到物理裸机、虚拟机或者其他主机使用 ，此外Ceph RBD 驱动程序已经被集成到 Linux 内核( 2.6.39 或更高版本)中；还可以作为虚拟化平台的存储后端

创建 块设备rbd

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
# rbd create {poolname}/{rbdname} --size {rbdsize}
[root@node29 ~]# rbd create pool_A1/myrbd --size 102400
[root@node29 ~]# rbd -p pool_A1 ls
myrbd
[root@node29 ~]# 

# rbd扩容
[root@node29 ~]#  rbd resize pool_A1/myrbd --size 51200 --allow-shrink
Resizing image: 100% complete...done.


[root@node29 ~]# rbd info pool_A1/myrbd
rbd image 'myrbd':
	size 50 GiB in 12800 objects
	order 22 (4 MiB objects)
	snapshot_count: 0
	id: 42422a7512b3
	block_name_prefix: rbd_data.42422a7512b3
	format: 2
	features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
	op_features: 
	flags: 
	create_timestamp: Mon Aug  1 15:38:48 2022
	access_timestamp: Mon Aug  1 15:38:48 2022
	modify_timestamp: Mon Aug  1 15:38:48 2022
[root@node29 ~]# 

#查看pool里面的rbd
#这里不指定pool 默认的是 rbd
rbd ls {poolname}
[root@node29 ceph]# rbd ls pool_A1
myrbd

#查看rbd详细信息
[root@node29 ~]# rbd info  --pool pool_A1 --image myrbd
rbd image 'myrbd':
	size 50 GiB in 12800 objects
	order 22 (4 MiB objects)
	snapshot_count: 1             #快照数
	id: 42422a7512b3               
	block_name_prefix: rbd_data.42422a7512b3   #块唯一的前缀编号，
	format: 2
	features: layering            # 层级特性，这个和Linux有关  
	op_features: 
	flags: 
	create_timestamp: Mon Aug  1 15:38:48 2022
	access_timestamp: Mon Aug  1 15:38:48 2022
	modify_timestamp: Mon Aug  1 15:38:48 2022
	
	
	
#查看 rbd使用情况 
[root@node29 hrp]# rbd du pool_A1/myrbd
warning: fast-diff map is not enabled for myrbd. operation may be slow.
NAME        PROVISIONED USED    
myrbd@snap1      50 GiB 196 MiB 
myrbd@snap2      50 GiB 104 MiB 
myrbd@snap3      50 GiB  12 MiB 
myrbd@snap4      50 GiB  12 MiB 
myrbd            50 GiB 296 MiB 
<TOTAL>          50 GiB 620 MiB

删除& 移动回收站

1
2
3
4
5
6
7
8
9
10
11
12
13
# 移动到回收站
[root@node29 ~]# rbd trash move --pool pool_A1 --image myrbd
[root@node29 ~]# rbd status --pool pool_A1 --image myrbd
rbd: error opening image myrbd: (2) No such file or directory
#查看回收站列表
[root@node29 ~]# rbd trash list --pool pool_A1
42422a7512b3 myrbd
#恢复rbd --image-id 值得时候回收站的里 rbd对应的 id
[root@node29 ~]# rbd trash restore  --pool pool_A1 --image myrbd --image-id 42422a7512b3

[root@node29 ~]# rbd ls --pool pool_A1 -l
NAME         SIZE   PARENT FMT PROT LOCK 
myrbd        50 GiB          2

映射块设备

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
#rdb map {poolname}/{rbdname}
[root@node29 ~]# rbd map pool_A1/myrbd
rbd: sysfs write failed
RBD image feature set mismatch. You can disable features unsupported by the kernel with "rbd feature disable pool_A1/myrbd object-map fast-diff deep-flatten".
In some cases useful info is found in syslog - try "dmesg | tail".

#出现映射报错

[root@node29 ~]# dmesg | tail
[431601.152693] libceph: mon2 10.1.23.31:3300 socket closed (con state CONNECTING)
[431602.160874] libceph: mon3 10.1.23.31:6789 socket closed (con state CONNECTING)
[431603.152717] libceph: mon3 10.1.23.31:6789 socket closed (con state CONNECTING)
[431604.152924] libceph: mon3 10.1.23.31:6789 socket closed (con state CONNECTING)
[431605.152983] libceph: mon0 10.1.23.29:3300 socket closed (con state CONNECTING)
[431606.154832] libceph: mon0 10.1.23.29:3300 socket closed (con state CONNECTING)
[431607.154763] libceph: mon0 10.1.23.29:3300 socket closed (con state CONNECTING)
[431608.162428] libceph: mon1 10.1.23.29:6789 session established
[431608.168616] libceph: client16974 fsid 311a4057-0837-4af8-92ab-a8053e4a3a57
[431608.181694] rbd: image myrbd: [431608.181694] rbd: image myrbd: image uses unsupported features: 0x38 
 uses unsupported features: 0x38
 
#最后一行提示 imgage 不支持特性 ？？？？？？？
#用 rbd info 查看信息
[root@node29 ~]#  rbd info pool_A1/myrbd | grep feature
	features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
	
layering: 支持分层
striping: 支持条带化 v2
exclusive-lock: 支持独占锁
object-map: 支持对象映射（依赖 exclusive-lock ）
fast-diff: 快速计算差异（依赖 object-map ）
deep-flatten: 支持快照扁平化操作
journaling: 支持记录 IO 操作（依赖独占锁）
#CentOS的 内核仅支持其中的 layering feature，所以其他的得停用 

[root@node29 ~]# rbd feature disable  pool_A1/myrbd exclusive-lock, object-map, fast-diff, deep-flatten
#一劳永逸方法 ，在 ceph.conf 加入  rbd_default_features = 1
[root@node29 ~]# rbd map pool_A1/myrbd
/dev/rbd0

[root@node29 ~]# lsblk /dev/rbd0
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
rbd0 252:0    0  50G  0 disk 
You have new mail in /var/spool/mail/root
[root@node29 ~]# 

#取消映射

[root@node29 ~]#  rbd showmapped
id pool    namespace image snap device    
0  pool_A1           myrbd -    /dev/rbd0 

[root@node29 ~]# rbd --user lck -p pool_A1 unmap myrbd
[root@node29 ~]#  rbd showmapped

fio测试 设备块

随机写

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
[root@node29 home]# cat fio_test 
[global]
direct=1
ioengine=libaio
group_reporting=1
iodepth=32
size=1g
verify=md5
verify_fatal=1
verify_interval=512
verify_dump=1
runtime=30
rwmixread=70
rwmixwrite=30
time_based

[job1]
bssplit=4k/33:8k/33;16k/34
rw=randwrite
filename=/dev/rbd0

[root@node29 home]# fio fio_test 
job1: (g=0): rw=randwrite, bs=4K-8K/4K-8K, ioengine=libaio, iodepth=32
fio 2.0.7
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/5650K /s] [0 /861  iops] [eta 00m:00s]
job1: (groupid=0, jobs=1): err= 0: pid=52184
  write: io=158620KB, bw=5283.3KB/s, iops=809 , runt= 30023msec
    slat (usec): min=3 , max=461 , avg= 9.31, stdev= 9.32
    clat (msec): min=5 , max=448 , avg=39.51, stdev=35.73
     lat (msec): min=5 , max=448 , avg=39.52, stdev=35.73
    clat percentiles (msec):
     |  1.00th=[   12],  5.00th=[   15], 10.00th=[   18], 20.00th=[   21],
     | 30.00th=[   23], 40.00th=[   26], 50.00th=[   30], 60.00th=[   34],
     | 70.00th=[   42], 80.00th=[   52], 90.00th=[   70], 95.00th=[   87],
     | 99.00th=[  202], 99.50th=[  249], 99.90th=[  437], 99.95th=[  449],
     | 99.99th=[  449]
    bw (KB/s)  : min= 1551, max= 7474, per=100.00%, avg=5355.09, stdev=1441.25
    lat (msec) : 10=0.26%, 20=19.37%, 50=58.37%, 100=18.48%, 250=3.07%
    lat (msec) : 500=0.45%
  cpu          : usr=1.74%, sys=0.91%, ctx=5009, majf=0, minf=602
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=24296/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
  WRITE: io=158620KB, aggrb=5283KB/s, minb=5283KB/s, maxb=5283KB/s, mint=30023msec, maxt=30023msec

Disk stats (read/write):
  rbd0: ios=0/24216, merge=0/16, ticks=0/951410, in_queue=950541, util=99.97%


使用块设备

格式化、挂载后就可正常使用

1
2
3
4
5
6
7
8
9
10
11
[root@node29 ~]# mkfs.xfs -f /dev/rbd0
[root@node29 ~]# mount /dev/rbd0 /mnt/
[root@node29 mnt]# df -h | grep rbd
/dev/rbd0        50G   33M   50G   1% /mnt

 使用 dd命令写入 100M数据
[root@node29 rbd_test]# dd if=/dev/zero  of=/root/rbd_test/file1 count=100 bs=1M
100+0 records in
100+0 records out
104857600 bytes (105 MB) copied, 0.121638 s, 862 MB/s

创建快照

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
#此时文件
[root@node29 hrp]# touch 1 2 3 4 5
[root@node29 hrp]# ls
1  2  3  4  5
#打快照  #是写时复制 所以很快！ COW
[root@node29 hrp]# rbd snap create pool_A1/myrbd@snap8  

#查看创建的快照
[root@node29 hrp]# rbd snap ls pool_A1/myrbd
SNAPID NAME  SIZE   PROTECTED TIMESTAMP                
     6 snap1 50 GiB           Mon Aug  1 22:45:54 2022 
     7 snap2 50 GiB           Mon Aug  1 22:47:17 2022 
     8 snap3 50 GiB           Mon Aug  1 22:51:54 2022 
     9 snap4 50 GiB           Mon Aug  1 22:56:57 2022 
    10 snap5 50 GiB           Tue Aug  2 10:59:32 2022 
    11 snap6 50 GiB           Tue Aug  2 11:00:24 2022 
    12 new   50 GiB           Tue Aug  2 11:08:31 2022 
    13 snap8 50 GiB           Tue Aug  2 11:22:38 2022 
[root@node29 hrp]# 


#测试 删除后能否恢复
[root@node29 hrp]# rm -f  1 2 3 
[root@node29 hrp]# ls
4  5
[root@node29 hrp]# cd ..
#ummount 注意： umount 时不要在 挂载目录下，否则会报错
[root@node29 mnt]# umount /mnt/hrp/
#回滚
[root@node29 mnt]# rbd snap rollback pool_A1/myrbd@snap8
Rolling back to snapshot: 100% complete...done.
#重新挂载
[root@node29 mnt]# mount /dev/rbd0 /mnt/hrp/
#文件回来了
[root@node29 mnt]# cd hrp/ && ls
[root@node29 hrp]# ls
1  2  3  4  5

#删除快照
[root@node29 hrp]# rbd snap rm pool_A1/myrbd@snap1
Removing snap: 100% complete...done.
[root@node29 hrp]# rbd snap ls pool_A1/myrbd
SNAPID NAME  SIZE   PROTECTED TIMESTAMP                
     7 snap2 50 GiB           Mon Aug  1 22:47:17 2022 
     8 snap3 50 GiB           Mon Aug  1 22:51:54 2022 
     9 snap4 50 GiB           Mon Aug  1 22:56:57 2022 
    10 snap5 50 GiB           Tue Aug  2 10:59:32 2022 
    11 snap6 50 GiB           Tue Aug  2 11:00:24 2022 
    12 new   50 GiB           Tue Aug  2 11:08:31 2022 
    13 snap8 50 GiB           Tue Aug  2 11:22:38 2022 
    
#删除全部快照 purge
[root@node29 hrp]# rbd snap purge pool_A1/myrbd
Removing all snapshots: 100% complete...done.
[root@node29 hrp]#