ICode9
精准搜索请尝试： 精确搜索
首页
编程语言>
数据库
系统相关
互联网
其他分享
Python
Java
JavaScript
android
PHP
首页 > 其他分享> 文章详细

ceph 008 ceph多区域网关(ceph对象容灾) cephfs文件系统

2022-08-17 15:33:43  阅读：152  来源： 互联网

标签：网关 zone rgw -- ceph clienta cephfs root


clienta作为集群的管理人员。一部分。他是需要秘钥与配置文件的
但真正服务端只需要通过curl就好

ceph 多区域网关

对象存储容灾解决方案

zone与zone会做数据同步。
把会做同步的rgw放到一个组里
同步有方向的概念
有主zone  master
主zone会像其他zone同步
一个realm的数据可以同步（元数据）
我在realm创建一个bucket。那么realm下面的zonegroup也会看到
zone不可以跨集群
修改元数据得经过主zone，然后同步到其他zone


提交更改时

实践

官方文档
https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/5/html/object_gateway_guide/advanced-configuration#migrating-a-single-site-system-to-multi-site-rgw

[student@workstation ~]$ lab start object-radosgw

删除原有rgw

1.创建realm

[root@clienta ~]# radosgw-admin realm create --rgw-realm=myrealm --default
{
    "id": "084b1291-061d-4501-8ddd-bebccd4183bc",
    "name": "myrealm",
    "current_period": "b6d342c0-ae0b-4c59-9594-75e2cba281e0",
    "epoch": 1
}

[root@clienta ~]# radosgw-admin realm list
{
    "default_info": "084b1291-061d-4501-8ddd-bebccd4183bc",
    "realms": [
        "myrealm"
    ]
}
[root@clienta ~]#

默认有zone，zonegroup (default)。不用默认有的

2.创建zonegroup

[root@clienta ~]# radosgw-admin zonegroup create --rgw-realm=myrealm --rgw-zonegroup=myzonegroup --master  --default
{
    "id": "543c6500-a4d8-4653-9329-88d8e6b70fe3",
    "name": "myzonegroup",
    "api_name": "myzonegroup",
    "is_master": "true",
    "endpoints": [],
    "hostnames": [],
    "hostnames_s3website": [],
    "master_zone": "",
    "zones": [],
    "placement_targets": [],
    "default_placement": "",
    "realm_id": "084b1291-061d-4501-8ddd-bebccd4183bc",
    "sync_policy": {
        "groups": []
    }
}
[root@clienta ~]#

以后radosgw-admin 默认就是myzonegroup这个

3.创建zone

[root@clienta ~]# radosgw-admin zone create --rgw-realm=myrealm --rgw-zonegroup=myzonegroup --rgw-zone=myzone1 --master --default
{
    "id": "6d3b4396-1966-4b1d-a49a-3c48f908a286",
    "name": "myzone1",
    "domain_root": "myzone1.rgw.meta:root",
    "control_pool": "myzone1.rgw.control",
    "gc_pool": "myzone1.rgw.log:gc",
    "lc_pool": "myzone1.rgw.log:lc",
    "log_pool": "myzone1.rgw.log",
    "intent_log_pool": "myzone1.rgw.log:intent",
    "usage_log_pool": "myzone1.rgw.log:usage",
    "roles_pool": "myzone1.rgw.meta:roles",
    "reshard_pool": "myzone1.rgw.log:reshard",
    "user_keys_pool": "myzone1.rgw.meta:users.keys",
    "user_email_pool": "myzone1.rgw.meta:users.email",
    "user_swift_pool": "myzone1.rgw.meta:users.swift",
    "user_uid_pool": "myzone1.rgw.meta:users.uid",
    "otp_pool": "myzone1.rgw.otp",
    "system_key": {
        "access_key": "",
        "secret_key": ""
    },
    "placement_pools": [
        {
            "key": "default-placement",
            "val": {
                "index_pool": "myzone1.rgw.buckets.index",
                "storage_classes": {
                    "STANDARD": {
                        "data_pool": "myzone1.rgw.buckets.data"
                    }
                },
                "data_extra_pool": "myzone1.rgw.buckets.non-ec",
                "index_type": 0
            }
        }
    ],
    "realm_id": "084b1291-061d-4501-8ddd-bebccd4183bc",
    "notif_pool": "myzone1.rgw.log:notif"
}
[root@clienta ~]#

查验

[root@clienta ~]# radosgw-admin realm list
{
    "default_info": "084b1291-061d-4501-8ddd-bebccd4183bc",
    "realms": [
        "myrealm"
    ]
}
[root@clienta ~]# radosgw-admin zone list
{
    "default_info": "6d3b4396-1966-4b1d-a49a-3c48f908a286",
    "zones": [
        "myzone1",
        "default"
    ]
}
[root@clienta ~]# radosgw-admin zonegroup list
{
    "default_info": "543c6500-a4d8-4653-9329-88d8e6b70fe3",
    "zonegroups": [
        "myzonegroup",
        "default"
    ]
}
[root@clienta ~]# radosgw-admin zonegroup get --rgw-realm=myrealm
{
    "id": "543c6500-a4d8-4653-9329-88d8e6b70fe3",
    "name": "myzonegroup",
    "api_name": "myzonegroup",
    "is_master": "true",
    "endpoints": [],
    "hostnames": [],
    "hostnames_s3website": [],
    "master_zone": "6d3b4396-1966-4b1d-a49a-3c48f908a286",
    "zones": [
        {
            "id": "6d3b4396-1966-4b1d-a49a-3c48f908a286",
            "name": "myzone1",
            "endpoints": [],
            "log_meta": "false",
            "log_data": "false",
            "bucket_index_max_shards": 11,
            "read_only": "false",
            "tier_type": "",
            "sync_from_all": "true",
            "sync_from": [],
            "redirect_zone": ""
        }
    ],
    "placement_targets": [
        {
            "name": "default-placement",
            "tags": [],
            "storage_classes": [
                "STANDARD"
            ]
        }
    ],
    "default_placement": "default-placement",
    "realm_id": "084b1291-061d-4501-8ddd-bebccd4183bc",
    "sync_policy": {
        "groups": []
    }
}
[root@clienta ~]#

4.提交修改

[root@clienta ~]# radosgw-admin period update --rgw-realm=myrealm --commit

5.添加rgw

[root@clienta ~]# ceph orch apply rgw test  --realm=myrealm --zone=myzone1  --placement="2 serverc.lab.example.com serverd.lab.example.com" --port=80
Scheduled rgw.test update...

修改元数据

[root@clienta ~]# radosgw-admin zonegroup modify  --rgw-realm=myrealm --rgw-zonegroup=myzonegroup --endpoints http://serverc:80 --master --default
{
"id": "543c6500-a4d8-4653-9329-88d8e6b70fe3",
"name": "myzonegroup",
"api_name": "myzonegroup",
"is_master": "true",
"endpoints": [
    "http://serverc:80"
],

6.创建系统用户

同步，备集群拉取数据的时候，得通过用户

[root@clienta ~]# radosgw-admin user create --uid=user1 --display-name="user1" --access-key=123 --secret=456 --system

告诉zone系统账号

[root@clienta ~]# radosgw-admin zone modify  --rgw-realm=myrealm --rgw-zonegroup=myzonegroup --endpoints http://serverc:80 --access-key=123 --secret=456 --rgw-zone=myzone1 --master --default

7.更新ceph数据库配置并重启服务

[root@clienta ~]# ceph config set client.rgw.rgw.test.serverc.kpidkj rgw_realm myrealm 
[root@clienta ~]# ceph config set client.rgw.rgw.test.serverc.kpidkj rgw_zonegroup myzonegroup
[root@clienta ~]# ceph config set client.rgw.rgw.test.serverc.kpidkj rgw_zone myzone1
[root@clienta ~]# 
radosgw-admin period update --rgw-realm=myrealm --commit

在ceph集群拥有rgw节点那重启服务（在serverc上部署的rgw）

[root@serverc ~]# systemctl list-units | grep ceph | grep rgw
ceph-2ae6d05a-229a-11ec-925e-52540000fa0c@rgw.test.serverc.kpidkj.service                                        loaded    active running   Ceph rgw.test.serverc.kpidkj for 2ae6d05a-229a-11ec-925e-52540000fa0c                                           
[root@serverc ~]# systemctl restart ceph-2ae6d05a-229a-11ec-925e-52540000fa0c@rgw.test.serverc.kpidkj.service 
[root@serverc ~]# 
这是我加的rgw节点serverc

[root@clienta ~]# ceph orch restart rgw.test
Scheduled to restart rgw.test.serverc.kpidkj on host 'serverc.lab.example.com'
Scheduled to restart rgw.test.serverd.zhtcld on host 'serverd.lab.example.com'
[root@clienta ~]# 
这是整个rgw服务

Secondary zone

8.拉取realm （同步数据）

[root@serverf ~]# radosgw-admin realm pull --url=http://serverc:80 --access-key=123 --secret-key=456
2022-08-16T11:25:54.082-0400 7fe54eade380  1 error read_lastest_epoch .rgw.root:periods.7d9b13b1-e755-4b7b-908a-3deb39567dc9.latest_epoch
2022-08-16T11:25:54.183-0400 7fe54eade380  1 Set the period's master zonegroup 543c6500-a4d8-4653-9329-88d8e6b70fe3 as the default
{
    "id": "084b1291-061d-4501-8ddd-bebccd4183bc",
    "name": "myrealm",
    "current_period": "7d9b13b1-e755-4b7b-908a-3deb39567dc9",
    "epoch": 2
}
[root@serverf ~]#

查看

[root@serverf ~]# radosgw-admin realm list
{
    "default_info": "084b1291-061d-4501-8ddd-bebccd4183bc",
    "realms": [
        "myrealm"
    ]
}
[root@serverf ~]# 


[root@serverf ~]# radosgw-admin zonegroup list
{
    "default_info": "543c6500-a4d8-4653-9329-88d8e6b70fe3",
    "zonegroups": [
        "myzonegroup",
        "default"
    ]
}

zone不可以跨集群，所以zone看不到

[root@serverf ~]# radosgw-admin zone list
{
    "default_info": "",
    "zones": [
        "default"
    ]
}
[root@serverf ~]#

9.拉取配置信息

[root@serverf ~]# radosgw-admin period pull --url=http://serverc:80 --access-key=123 --secret-key=456

10.serverf创建zone

[root@serverf ~]# radosgw-admin zone create --rgw-zonegroup=myzonegroup --rgw-zone=myzone2  --endpoints=http://serverf:80  --access-key=123 --secret-key=456

查看
ceph orch ps #这里最好一条不能是error
rgw.realm.zone.serverf.qihrkc serverf.lab.example.com running (69m) 3m ago 10M *:80 16.2.0-117.el8cp 2142b60d7974 9820eb665dde
[root@serverf ~]# radosgw-admin zone list
{
"default_info": "bcb8d59c-66e6-423d-8a5b-8d5b3c2fa62b",
"zones": [
"myzone2",
"default"
]
}

11.配置数据库信息

[root@serverf ~]# ceph config set client.rgw.rgw.realm.zone.serverf.qihrkc  rgw_realm myrealm
[root@serverf ~]# ceph config set client.rgw.rgw.realm.zone.serverf.qihrkc  rgw_zonegroup myzonegroup
[root@serverf ~]# ceph config set client.rgw.rgw.realm.zone.serverf.qihrkc  rgw_zone myzone2
[root@serverf ~]# radosgw-admin period update  --commit

client.rgw这一部分是文档里的固定格式，这一部分rgw.realm.zone.serverf.qihrkc则要看ceph orch ps 的rgw信息

[root@serverf ~]# systemctl restart ceph-0bf7c358-25e1-11ec-ae02-52540000fa0f@rgw.realm.zone.serverf.qihrkc.service     
[root@serverf ~]# systemctl enable ceph-0bf7c358-25e1-11ec-ae02-52540000fa0f@rgw.realm.zone.serverf.qihrkc.service     
[root@serverf ~]# 
systemctl list-units | grep ceph

测试

配置完后，两边的用户同步了

[root@serverf ~]# radosgw-admin user list
[
    "user1"
]

yum -y install awscli
两个节点安装aws s3工具

[root@clienta ~]# radosgw-admin user create --uid=bob --display-name=bob --access-key=abc --secret=def
主创建用户
备端也会有
[root@serverf yum.repos.d]# radosgw-admin user list
[
    "user1",
    "bob"
]

[root@clienta ~]# aws configure
AWS Access Key ID [None]: abc
AWS Secret Access Key [None]: def
Default region name [None]: 
Default output format [None]: 
[root@clienta ~]# 

创建桶
[root@clienta ~]# aws --endpoint=http://serverc:80 s3 mb s3://test1
make_bucket: test1
[root@clienta ~]# aws --endpoint=http://serverc:80 s3 ls 
2022-08-17 09:37:29 test1

备集群查看自己有没有桶
[root@serverf yum.repos.d]# aws configure
AWS Access Key ID [None]: abc
AWS Secret Access Key [None]: def
Default region name [None]: 
Default output format [None]: 
[root@serverf yum.repos.d]# aws --endpoint=http://serverf:80 s3 ls
2022-08-17 09:37:29 test1
[root@serverf yum.repos.d]# 
桶被同步过来了

[root@clienta ~]# aws --endpoint=http://serverc:80 s3 cp /etc/profile s3://test1/
upload: ../etc/profile to s3://test1/profile  
上传对象

[root@serverf yum.repos.d]# aws --endpoint=http://serverf:80 s3 ls s3://test1
2022-08-17 09:42:04       2123 profile
[root@serverf yum.repos.d]# 
备节点也拥有该镜像

[root@serverf yum.repos.d]# aws --endpoint=http://serverf:80 s3 mb s3://test2
make_bucket: test2
备端创桶

[root@clienta ~]# aws --endpoint=http://serverc:80 s3 ls 
2022-08-17 09:37:29 test1
2022-08-17 09:45:34 test2
[root@clienta ~]# 
主端也会有

在备集群创建桶时，命令会被转发给主zone。主坏了，其他就可以成为主
zookeeper里只能一个写，三个都可以读，类似

[root@serverf yum.repos.d]# radosgw-admin user create --uid=boba --display-name=boba --access-key=abc --secret=def
Please run the command on master zone. Performing this operation on non-master zone leads to inconsistent metadata between zones
Are you sure you want to go ahead? (requires --yes-i-really-mean-it)
[root@serverf yum.repos.d]# 
备集群不能创建用户之类

故障切换

可以看到myzone1是主zone

[root@clienta ~]# radosgw-admin zonegroup get --rgw-realm=myrealm --rgw-zonegroup=myzonegroup
{
    "id": "543c6500-a4d8-4653-9329-88d8e6b70fe3",
    "name": "myzonegroup",
    "api_name": "myzonegroup",
    "is_master": "true",
    "endpoints": [
        "http://serverc:80"
    ],
    "hostnames": [],
    "hostnames_s3website": [],
    "master_zone": "6d3b4396-1966-4b1d-a49a-3c48f908a286",
    "zones": [
        {
            "id": "6d3b4396-1966-4b1d-a49a-3c48f908a286",
            "name": "myzone1",
            "endpoints": [
                "http://serverc:80"
            ],
            "log_meta": "false",
            "log_data": "true",
            "bucket_index_max_shards": 11,
            "read_only": "false",
            "tier_type": "",
            "sync_from_all": "true",
            "sync_from": [],
            "redirect_zone": ""
        },
        {
            "id": "bcb8d59c-66e6-423d-8a5b-8d5b3c2fa62b",
            "name": "myzone2",
            "endpoints": [
                "http://serverf:80"
            ],
            "log_meta": "false",
            "log_data": "true",
            "bucket_index_max_shards": 11,
            "read_only": "false",
            "tier_type": "",
            "sync_from_all": "true",
            "sync_from": [],
            "redirect_zone": ""
        }
    ],
    "placement_targets": [
        {
            "name": "default-placement",
            "tags": [],
            "storage_classes": [
                "STANDARD"
            ]
        }
    ],
    "default_placement": "default-placement",
    "realm_id": "084b1291-061d-4501-8ddd-bebccd4183bc",
    "sync_policy": {
        "groups": []
    }
}



[root@serverf yum.repos.d]# radosgw-admin zone modify --rgw-zone=myzone2 --master --default --read-only=false 
radosgw-admin period update --commit
更改主zone后，得提交元数据，让其他集群知道

period元数据
realm包含realm名字与zonegroup名字

cephfs 文件系统

mds管理文件系统元数据
mds服务拥有缓存提高效率
元数据存放在集群的osd中

实践

[root@clienta ~]# ceph osd pool create mycephfs_data
pool 'mycephfs_data' created
创建数据池
[root@clienta ~]# ceph osd pool create mycephfs_metadata
pool 'mycephfs_metadata' created
元数据池
[root@clienta ~]# ceph fs new mycephfs mycephfs_metadata mycephfs_data
文件系统
new fs with metadata pool 13 and data pool 12
[root@clienta ~]# ceph orch apply mds mycephfs --placement="2 serverc.lab.example.com serverd.lab.example.com"
Scheduled mds.mycephfs update...
[root@clienta ~]# ceph mds stat
mycephfs:1 {0=mycephfs.serverc.lkwfwl=up:active} 1 up:standby
[root@clienta ~]# 
一个活动的mds和一个备的

[root@clienta ~]# ceph fs status
mycephfs - 0 clients
========
RANK  STATE             MDS               ACTIVITY     DNS    INOS   DIRS   CAPS  
0    active  mycephfs.serverc.lkwfwl  Reqs:    0 /s    10     13     12      0   
    POOL          TYPE     USED  AVAIL  
mycephfs_metadata  metadata  96.0k  28.3G  
mycephfs_data      data       0   28.3G  
    STANDBY MDS        
mycephfs.serverd.epyssp  
MDS version: ceph version 16.2.0-117.el8cp (0e34bb74700060ebfaa22d99b7d2cdc037b28a57) pacific (stable)
[root@clienta ~]#

挂载两种方式

第一种linux内核

mount -t mount -t ceph serverc.lab.example.com:/  /mnt -o name=admin,fs=mycephfs
指定mon地址,文件系统都是根/

[root@clienta /]# cd /mnt/
[root@clienta mnt]# ls
[root@clienta mnt]# mkdir dir1
[root@clienta mnt]# mkdir dir33
[root@clienta mnt]# cd
[root@clienta ~]# umount /mnt
[root@clienta ~]# ceph fs authorize mycephfs client.mqy / r /dir33 rw > /etc/ceph/ceph.client.mqy.keyring 

[root@clienta ~]# mount -t ceph serverc.lab.example.com:/  /mnt -o name=mqy,fs=mycephfs
[root@clienta ~]# cd /mnt/
[root@clienta mnt]# ls
dir1  dir33
[root@clienta mnt]# touch flag
touch: cannot touch 'flag': Permission denied
[root@clienta mnt]# cd dir33
[root@clienta dir33]# ls
[root@clienta dir33]# touch flag

永久挂载

ceph auth get-key client.mqy > /root/secret
[root@clienta ~]# mount -a
[root@clienta ~]# cat /etc/fstab  | tail -n 1
serverc.lab.example.com:/  /mnt  ceph   name=mqy,fs=mycephfs,secretfile=/root/secret,_netdev 0 0
[root@clienta ~]# df -h  
172.25.250.12:/   29G     0   29G   0% /mnt

直接挂载其中的目录

[root@clienta ~]# mount -a
[root@clienta ~]# cd /mnt/
[root@clienta mnt]# ls
flag
[root@clienta mnt]# cat /etc/fstab  | tail -n 1
serverc.lab.example.com:/dir33  /mnt  ceph   name=mqy,fs=mycephfs,secretfile=/root/secret,_netdev 0 0
[root@clienta mnt]#

第二种ceph驱动 ceph-fuse

手动挂载

[root@clienta ~]# ceph-fuse -n client.mqy --client_fs mycephfs  /mnt
2022-08-17T10:53:23.822-0400 7f8f88d6d200 -1 init, newargv = 0x5596b7c3f9d0 newargc=15
ceph-fuse[19250]: starting ceph client
ceph-fuse[19250]: starting fuse
[root@clienta ~]# df -h /mnt/
Filesystem      Size  Used Avail Use% Mounted on
ceph-fuse        29G     0   29G   0% /mnt
[root@clienta ~]#

永久挂载

[root@clienta ~]# tail -n 1  /etc/fstab 
serverc.lab.example.com:/  /mnt/  fuse.ceph  ceph.id=mqy,_netdev 0 0

创建多个cephfs供于挂载

[root@clienta ~]# ceph osd pool create mycephfs2_data
pool 'mycephfs2_data' created
[root@clienta ~]# ceph osd pool create mycephfs2_metadata
pool 'mycephfs2_metadata' created
[root@clienta ~]# ceph fs new mycephfs2 mycephfs2_metadata mycephfs2_data
new fs with metadata pool 15 and data pool 14
[root@clienta ~]# ceph orch apply mds mycephfs2 --placement='1 servere.lab.example.com'
Scheduled mds.mycephfs2 update...
多个mds保证可靠性
[root@clienta ~]# ceph fs status
mycephfs - 1 clients
========
RANK  STATE             MDS               ACTIVITY     DNS    INOS   DIRS   CAPS  
0    active  mycephfs.serverc.lkwfwl  Reqs:    0 /s    13     16     14      1   
    POOL          TYPE     USED  AVAIL  
mycephfs_metadata  metadata   132k  28.3G  
mycephfs_data      data       0   28.3G  
mycephfs2 - 0 clients
=========
RANK  STATE             MDS                ACTIVITY     DNS    INOS   DIRS   CAPS  
0    active  mycephfs2.servere.cbglhs  Reqs:    0 /s    10     13     12      0   
    POOL           TYPE     USED  AVAIL  
mycephfs2_metadata  metadata  96.0k  28.3G  
mycephfs2_data      data       0   28.3G  
    STANDBY MDS        
mycephfs.serverd.epyssp  
MDS version: ceph version 16.2.0-117.el8cp (0e34bb74700060ebfaa22d99b7d2cdc037b28a57) pacific (stable)
[root@clienta ~]#

标签：网关,zone,rgw,--,ceph,clienta,cephfs,root
来源： https://www.cnblogs.com/supermao12/p/16595366.html

本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享；
2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关；
3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关；
4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除；
5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。

关于我们 | 联系我们 | 留言反馈

专注分享技术，共同学习，共同进步。侵权联系[81616952@qq.com]

Copyright (C)ICode9.com, All Rights Reserved.

ICode9版权所有