专用复制网络

摘要

Swift 的复制过程对于数据的一致性和可用性至关重要。默认情况下，复制活动将使用与其他集群操作相同的网络接口。但是，如果在环中为节点设置了复制接口，则该节点将在其指定的单独复制网络接口上发送复制流量。复制流量包括 REPLICATE 请求和 rsync 流量。

为了将集群内部的复制流量与客户端流量分离，可以使用单独的复制服务器。这些复制服务器基于标准的存储服务器，但它们监听复制 IP 并且仅响应 REPLICATE 请求。存储服务器可以服务 REPLICATE 请求，因此操作员可以在不中断集群的情况下过渡到使用单独的复制网络。

复制 IP 和端口信息存储在环中，按节点为单位。如果存在这些参数，将使用它们，但它们不是必需的。如果对于特定节点，此信息不存在或为空，则该节点的标准 IP 和端口将用于复制。

对于 SAIO 复制

  1. 在 ~/bin/ 中创建新脚本（例如：remakerings_new）

    #!/bin/bash
set -e
cd /etc/swift
rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz
swift-ring-builder object.builder create 10 3 1
swift-ring-builder object.builder add z1-127.0.0.1:6210R127.0.0.1:6250/sdb1 1
swift-ring-builder object.builder add z2-127.0.0.1:6220R127.0.0.1:6260/sdb2 1
swift-ring-builder object.builder add z3-127.0.0.1:6230R127.0.0.1:6270/sdb3 1
swift-ring-builder object.builder add z4-127.0.0.1:6240R127.0.0.1:6280/sdb4 1
swift-ring-builder object.builder rebalance
swift-ring-builder object-1.builder create 10 2 1
swift-ring-builder object-1.builder add z1-127.0.0.1:6210R127.0.0.1:6250/sdb1 1
swift-ring-builder object-1.builder add z2-127.0.0.1:6220R127.0.0.1:6260/sdb2 1
swift-ring-builder object-1.builder add z3-127.0.0.1:6230R127.0.0.1:6270/sdb3 1
swift-ring-builder object-1.builder add z4-127.0.0.1:6240R127.0.0.1:6280/sdb4 1
swift-ring-builder object-1.builder rebalance
swift-ring-builder object-2.builder create 10 6 1
swift-ring-builder object-2.builder add z1-127.0.0.1:6210R127.0.0.1:6250/sdb1 1
swift-ring-builder object-2.builder add z1-127.0.0.1:6210R127.0.0.1:6250/sdb5 1
swift-ring-builder object-2.builder add z2-127.0.0.1:6220R127.0.0.1:6260/sdb2 1
swift-ring-builder object-2.builder add z2-127.0.0.1:6220R127.0.0.1:6260/sdb6 1
swift-ring-builder object-2.builder add z3-127.0.0.1:6230R127.0.0.1:6270/sdb3 1
swift-ring-builder object-2.builder add z3-127.0.0.1:6230R127.0.0.1:6270/sdb7 1
swift-ring-builder object-2.builder add z4-127.0.0.1:6240R127.0.0.1:6280/sdb4 1
swift-ring-builder object-2.builder add z4-127.0.0.1:6240R127.0.0.1:6280/sdb8 1
swift-ring-builder object-2.builder rebalance
swift-ring-builder container.builder create 10 3 1
swift-ring-builder container.builder add z1-127.0.0.1:6211R127.0.0.1:6251/sdb1 1
swift-ring-builder container.builder add z2-127.0.0.1:6221R127.0.0.1:6261/sdb2 1
swift-ring-builder container.builder add z3-127.0.0.1:6231R127.0.0.1:6271/sdb3 1
swift-ring-builder container.builder add z4-127.0.0.1:6241R127.0.0.1:6281/sdb4 1
swift-ring-builder container.builder rebalance
swift-ring-builder account.builder create 10 3 1
swift-ring-builder account.builder add z1-127.0.0.1:6212R127.0.0.1:6252/sdb1 1
swift-ring-builder account.builder add z2-127.0.0.1:6222R127.0.0.1:6262/sdb2 1
swift-ring-builder account.builder add z3-127.0.0.1:6232R127.0.0.1:6272/sdb3 1
swift-ring-builder account.builder add z4-127.0.0.1:6242R127.0.0.1:6282/sdb4 1
swift-ring-builder account.builder rebalance

    注意

    添加设备的语法已更改：R<ip_replication>:<port_replication> 已添加到 z<zone>-<ip>:<port> 和 /<device_name>_<meta> <weight> 之间。添加的设备将使用 <ip_replication> 和 <port_replication> 进行复制活动。

  2. 在 /etc/rsyncd.conf 中添加下一行

    [account6252]
max connections = 25
path = /srv/1/node/
read only = false
lock file = /var/lock/account6252.lock

[account6262]
max connections = 25
path = /srv/2/node/
read only = false
lock file = /var/lock/account6262.lock

[account6272]
max connections = 25
path = /srv/3/node/
read only = false
lock file = /var/lock/account6272.lock

[account6282]
max connections = 25
path = /srv/4/node/
read only = false
lock file = /var/lock/account6282.lock


[container6251]
max connections = 25
path = /srv/1/node/
read only = false
lock file = /var/lock/container6251.lock

[container6261]
max connections = 25
path = /srv/2/node/
read only = false
lock file = /var/lock/container6261.lock

[container6271]
max connections = 25
path = /srv/3/node/
read only = false
lock file = /var/lock/container6271.lock

[container6281]
max connections = 25
path = /srv/4/node/
read only = false
lock file = /var/lock/container6281.lock


[object6250]
max connections = 25
path = /srv/1/node/
read only = false
lock file = /var/lock/object6250.lock

[object6260]
max connections = 25
path = /srv/2/node/
read only = false
lock file = /var/lock/object6260.lock

[object6270]
max connections = 25
path = /srv/3/node/
read only = false
lock file = /var/lock/object6270.lock

[object6280]
max connections = 25
path = /srv/4/node/
read only = false
lock file = /var/lock/object6280.lock

  3. 重启 rsync 守护进程

    service rsync restart

  4. 更新目录中的配置文件

    • /etc/swift/object-server(文件：1.conf, 2.conf, 3.conf, 4.conf)

    • /etc/swift/container-server(文件：1.conf, 2.conf, 3.conf, 4.conf)

    • /etc/swift/account-server(文件：1.conf, 2.conf, 3.conf, 4.conf)

    删除部分 [<*>-replicator] 中的所有配置选项

  5. 在 /etc/swift/object-server/ 中添加 object-server 的配置文件

    • 5.conf

      [DEFAULT]
devices = /srv/1/node
mount_check = false
disable_fallocate = true
bind_port = 6250
user = swift
log_facility = LOG_LOCAL2
recon_cache_path = /var/cache/swift

[pipeline:main]
pipeline = recon object-server

[app:object-server]
use = egg:swift#object
replication_server = True

[filter:recon]
use = egg:swift#recon

[object-replicator]
rsync_module = {replication_ip}::object{replication_port}

    • 6.conf

      [DEFAULT]
devices = /srv/2/node
mount_check = false
disable_fallocate = true
bind_port = 6260
user = swift
log_facility = LOG_LOCAL3
recon_cache_path = /var/cache/swift2

[pipeline:main]
pipeline = recon object-server

[app:object-server]
use = egg:swift#object
replication_server = True

[filter:recon]
use = egg:swift#recon

[object-replicator]
rsync_module = {replication_ip}::object{replication_port}

    • 7.conf

      [DEFAULT]
devices = /srv/3/node
mount_check = false
disable_fallocate = true
bind_port = 6270
user = swift
log_facility = LOG_LOCAL4
recon_cache_path = /var/cache/swift3

[pipeline:main]
pipeline = recon object-server

[app:object-server]
use = egg:swift#object
replication_server = True

[filter:recon]
use = egg:swift#recon

[object-replicator]
rsync_module = {replication_ip}::object{replication_port}

    • 8.conf

      [DEFAULT]
devices = /srv/4/node
mount_check = false
disable_fallocate = true
bind_port = 6280
user = swift
log_facility = LOG_LOCAL5
recon_cache_path = /var/cache/swift4

[pipeline:main]
pipeline = recon object-server

[app:object-server]
use = egg:swift#object
replication_server = True

[filter:recon]
use = egg:swift#recon

[object-replicator]
rsync_module = {replication_ip}::object{replication_port}

  6. 在 /etc/swift/container-server/ 中添加 container-server 的配置文件

    • 5.conf

      [DEFAULT]
devices = /srv/1/node
mount_check = false
disable_fallocate = true
bind_port = 6251
user = swift
log_facility = LOG_LOCAL2
recon_cache_path = /var/cache/swift

[pipeline:main]
pipeline = recon container-server

[app:container-server]
use = egg:swift#container
replication_server = True

[filter:recon]
use = egg:swift#recon

[container-replicator]
rsync_module = {replication_ip}::container{replication_port}

    • 6.conf

      [DEFAULT]
devices = /srv/2/node
mount_check = false
disable_fallocate = true
bind_port = 6261
user = swift
log_facility = LOG_LOCAL3
recon_cache_path = /var/cache/swift2

[pipeline:main]
pipeline = recon container-server

[app:container-server]
use = egg:swift#container
replication_server = True

[filter:recon]
use = egg:swift#recon

[container-replicator]
rsync_module = {replication_ip}::container{replication_port}

    • 7.conf

      [DEFAULT]
devices = /srv/3/node
mount_check = false
disable_fallocate = true
bind_port = 6271
user = swift
log_facility = LOG_LOCAL4
recon_cache_path = /var/cache/swift3

[pipeline:main]
pipeline = recon container-server

[app:container-server]
use = egg:swift#container
replication_server = True

[filter:recon]
use = egg:swift#recon

[container-replicator]
rsync_module = {replication_ip}::container{replication_port}

    • 8.conf

      [DEFAULT]
devices = /srv/4/node
mount_check = false
disable_fallocate = true
bind_port = 6281
user = swift
log_facility = LOG_LOCAL5
recon_cache_path = /var/cache/swift4

[pipeline:main]
pipeline = recon container-server

[app:container-server]
use = egg:swift#container
replication_server = True

[filter:recon]
use = egg:swift#recon

[container-replicator]
rsync_module = {replication_ip}::container{replication_port}

  7. 在 /etc/swift/account-server/ 中添加 account-server 的配置文件

    • 5.conf

      [DEFAULT]
devices = /srv/1/node
mount_check = false
disable_fallocate = true
bind_port = 6252
user = swift
log_facility = LOG_LOCAL2
recon_cache_path = /var/cache/swift

[pipeline:main]
pipeline = recon account-server

[app:account-server]
use = egg:swift#account
replication_server = True

[filter:recon]
use = egg:swift#recon

[account-replicator]
rsync_module = {replication_ip}::account{replication_port}

    • 6.conf

      [DEFAULT]
devices = /srv/2/node
mount_check = false
disable_fallocate = true
bind_port = 6262
user = swift
log_facility = LOG_LOCAL3
recon_cache_path = /var/cache/swift2

[pipeline:main]
pipeline = recon account-server

[app:account-server]
use = egg:swift#account
replication_server = True

[filter:recon]
use = egg:swift#recon

[account-replicator]
rsync_module = {replication_ip}::account{replication_port}

    • 7.conf

      [DEFAULT]
devices = /srv/3/node
mount_check = false
disable_fallocate = true
bind_port = 6272
user = swift
log_facility = LOG_LOCAL4
recon_cache_path = /var/cache/swift3

[pipeline:main]
pipeline = recon account-server

[app:account-server]
use = egg:swift#account
replication_server = True

[filter:recon]
use = egg:swift#recon

[account-replicator]
rsync_module = {replication_ip}::account{replication_port}

    • 8.conf

      [DEFAULT]
devices = /srv/4/node
mount_check = false
disable_fallocate = true
bind_port = 6282
user = swift
log_facility = LOG_LOCAL5
recon_cache_path = /var/cache/swift4

[pipeline:main]
pipeline = recon account-server

[app:account-server]
use = egg:swift#account
replication_server = True

[filter:recon]
use = egg:swift#recon

[account-replicator]
rsync_module = {replication_ip}::account{replication_port}

对于多服务器复制

  1. 移动配置文件。

    • object-server 的配置文件从 /etc/swift/object-server.conf 移动到 /etc/swift/object-server/1.conf

    • container-server 的配置文件从 /etc/swift/container-server.conf 移动到 /etc/swift/container-server/1.conf

    • account-server 的配置文件从 /etc/swift/account-server.conf 移动到 /etc/swift/account-server/1.conf

  2. 在目录中的配置文件中添加更改

    • /etc/swift/object-server(文件：1.conf)

    • /etc/swift/container-server(文件：1.conf)

    • /etc/swift/account-server(文件：1.conf)

    删除部分 [<*>-replicator] 中的所有配置选项

  3. 在 /etc/swift/object-server/2.conf 中添加 object-server 的配置文件

    [DEFAULT]
bind_ip = $STORAGE_LOCAL_NET_IP
workers = 2

[pipeline:main]
pipeline = object-server

[app:object-server]
use = egg:swift#object
replication_server = True

[object-replicator]

  4. 在 /etc/swift/container-server/2.conf 中添加 container-server 的配置文件

    [DEFAULT]
bind_ip = $STORAGE_LOCAL_NET_IP
workers = 2

[pipeline:main]
pipeline = container-server

[app:container-server]
use = egg:swift#container
replication_server = True

[container-replicator]

  5. 在 /etc/swift/account-server/2.conf 中添加 account-server 的配置文件

    [DEFAULT]
bind_ip = $STORAGE_LOCAL_NET_IP
workers = 2

[pipeline:main]
pipeline = account-server

[app:account-server]
use = egg:swift#account
replication_server = True

[account-replicator]