Difference between revisions of "DRBD"
(Created page with " = fedora = ln -sf /lib/systemd/system/runlevel3.target /etc/systemd/system/default.target yum -y groupinstall Clustering yum -y groupinstall “Cluster Storage” yum -y o...") |
|||
Line 15: | Line 15: | ||
# disable selinux (for testing) | # disable selinux (for testing) | ||
setenforce 0 (and also in /etc/selinux/config ) | setenforce 0 (and also in /etc/selinux/config ) | ||
= net = | = net = | ||
Line 22: | Line 23: | ||
192.168.55.101 virtualb02 | 192.168.55.101 virtualb02 | ||
192.168.55.102 virtualb01 | 192.168.55.102 virtualb01 | ||
yum remove NetworkManager | |||
systemctl stop NetworkManager.service | |||
systemctl disable NetworkManager.service | |||
systemctl enable network.service | |||
systemctl start network.service | |||
/etc/sysconfig/network-scripts/ifcfg-p2p1 | |||
/etc/sysconfig/network-scripts/ifcfg-p7p1 | |||
( host only adapter ) | |||
p2p1 Link encap:Ethernet HWaddr 08:00:27:E3:E1:EC | |||
inet addr:192.168.55.102 Bcast:192.168.55.255 Mask:255.255.255.0 | |||
inet6 addr: fe80::a00:27ff:fee3:e1ec/64 Scope:Link | |||
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 | |||
RX packets:68 errors:0 dropped:0 overruns:0 frame:0 | |||
TX packets:69 errors:0 dropped:0 overruns:0 carrier:0 | |||
collisions:0 txqueuelen:1000 | |||
RX bytes:15292 (14.9 KiB) TX bytes:12355 (12.0 KiB) | |||
(NAT adapter) | |||
p7p1 Link encap:Ethernet HWaddr 08:00:27:FA:C2:66 | |||
inet addr:10.0.3.15 Bcast:10.0.3.255 Mask:255.255.255.0 | |||
inet6 addr: fe80::a00:27ff:fefa:c266/64 Scope:Link | |||
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 | |||
RX packets:4 errors:0 dropped:0 overruns:0 frame:0 | |||
TX packets:26 errors:0 dropped:0 overruns:0 carrier:0 | |||
collisions:0 txqueuelen:1000 | |||
RX bytes:1300 (1.2 KiB) TX bytes:5634 (5.5 KiB) | |||
= DRBD = | = DRBD = | ||
Line 35: | Line 68: | ||
* disk | * disk | ||
/dev/sdb1 2048 2097151 1047552 83 Linux | /dev/sdb1 2048 2097151 1047552 83 Linux | ||
* our resource will be named "r0" | |||
== config == | == config == | ||
Line 147: | Line 180: | ||
New drbd meta data block successfully created. | New drbd meta data block successfully created. | ||
success | success | ||
* start the service at boot | |||
chkconfig drbd on | |||
[root@virtualb01 ~]# drbd-overview | |||
1:r0 Connected Primary/Primary UpToDate/UpToDate C r----- lvm-pv: clstorage 1.00g 0g | |||
[root@virtualb02 ~]# drbd-overview | |||
1:r0 Connected Primary/Primary UpToDate/UpToDate C r----- lvm-pv: clstorage 1.00g 0g | |||
Line 163: | Line 205: | ||
drbdadm resize resource | drbdadm resize resource | ||
drbdadm -- --size=new-size resize resource # shrink (online) | drbdadm -- --size=new-size resize resource # shrink (online) | ||
drbdadm dstate resource | |||
= GFS = | |||
http://www.drbd.org/users-guide-8.3/s-enable-dual-primary.html | |||
yum -y install gfs2-utils gfs2-cluster lvm2-cluster | |||
chkconfig gfs2 on | |||
chkconfig gfs2-cluster on | |||
chkconfig cman on | |||
== cman == | |||
/etc/cluster/cluster.conf | |||
== LVM to recognize the DRBD == | |||
*; using clustered locking: | |||
* in /etc/lvm/lvm.conf: | |||
locking_type = 3 | |||
* (default is) | |||
# Type 2 uses the external shared library locking_library. | |||
# Type 3 uses built-in clustered locking. | |||
# Type 4 uses read-only locking which forbids any operations that might | |||
# change metadata. | |||
locking_type = 1 | |||
*; allowing LVM scans for PV signatures. | |||
* in /etc/lvm/lvm.conf: | |||
filter = [ "a|drbd.*|", "r|.*|" ] | |||
*; disable the LVM cache | |||
* in /etc/lvm/lvm.conf: | |||
write_cache_state = 0 | |||
!!! run vgscan to reload lvm's cache etc. | |||
vgscan | |||
mkfs -t gfs2 -p lock_dlm -j 2 -t CLUSTER00:gfs2 /dev/drbd1 | |||
* in /etc/fstab | |||
/dev/drbd1 /mnt/cl_storage gfs2 defaults 0 0 | |||
mkdir /mnt/cl_storage | |||
* restarting the gfs2 deamon | |||
service gfs2 restart | |||
== cman "this also starts ccsd and fenced" == | |||
in /etc/cluster/cluster.conf (both nodes) | |||
<?xml version="1.0"?> | |||
<cluster alias="CLUSTER00" config_version="23" name="CLUSTER00"> | |||
<fence_daemon post_fail_delay="0" post_join_delay="3"/> | |||
<clusternodes> | |||
<clusternode name="virtualb01" nodeid="1" votes="1"> | |||
<fence> | |||
<method name="1"/> | |||
</fence> | |||
</clusternode> | |||
<clusternode name="virtualb02" nodeid="2" votes="1"> | |||
<fence> | |||
<method name="1"/> | |||
</fence> | |||
</clusternode> | |||
</clusternodes> | |||
<fencedevices/> | |||
</cluster> | |||
[root@virtualb01 ~]# cman_tool nodes | |||
Node Sts Inc Joined Name | |||
1 M 4 2012-05-08 16:31:14 virtualb01 | |||
2 M 8 2012-05-08 16:40:32 virtualb02 | |||
[root@virtualb02 ~]# cman_tool nodes | |||
Node Sts Inc Joined Name | |||
1 M 8 2012-05-08 16:40:31 virtualb01 | |||
2 M 4 2012-05-08 16:40:31 virtualb02 | |||
== clvmd == | |||
== remarks == | == remarks == | ||
*; "any DRBD resource to be used for storing a GFS filesystem must be configured in dual-primary mode." | |||
You’ll want these settings: | You’ll want these settings: | ||
Line 178: | Line 310: | ||
Line 265: | Line 396: | ||
total 0 | total 0 | ||
lrwxrwxrwx 1 root root 11 May 8 14:28 r0 -> ../../drbd1 | lrwxrwxrwx 1 root root 11 May 8 14:28 r0 -> ../../drbd1 | ||
[root@virtualb01 ~]# fdisk -l /dev/drbd1 | |||
Disk /dev/drbd1: 1072 MB, 1072623616 bytes | |||
255 heads, 63 sectors/track, 130 cylinders, total 2094968 sectors | |||
Units = sectors of 1 * 512 = 512 bytes | |||
Sector size (logical/physical): 512 bytes / 512 bytes | |||
I/O size (minimum/optimal): 512 bytes / 512 bytes | |||
Disk identifier: 0x00000000 |
Revision as of 14:59, 8 May 2012
fedora
ln -sf /lib/systemd/system/runlevel3.target /etc/systemd/system/default.target
yum -y groupinstall Clustering
yum -y groupinstall “Cluster Storage”
yum -y openssh-server
chkconfig sshd on
- disabled firewall (for testing)
chkconfig iptables off
- disable selinux (for testing)
setenforce 0 (and also in /etc/selinux/config )
net
/etc/hosts 192.168.55.1 hostsrv 192.168.55.101 virtualb02 192.168.55.102 virtualb01
yum remove NetworkManager
systemctl stop NetworkManager.service systemctl disable NetworkManager.service
systemctl enable network.service systemctl start network.service
/etc/sysconfig/network-scripts/ifcfg-p2p1 /etc/sysconfig/network-scripts/ifcfg-p7p1
( host only adapter ) p2p1 Link encap:Ethernet HWaddr 08:00:27:E3:E1:EC
inet addr:192.168.55.102 Bcast:192.168.55.255 Mask:255.255.255.0 inet6 addr: fe80::a00:27ff:fee3:e1ec/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:68 errors:0 dropped:0 overruns:0 frame:0 TX packets:69 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:15292 (14.9 KiB) TX bytes:12355 (12.0 KiB)
(NAT adapter) p7p1 Link encap:Ethernet HWaddr 08:00:27:FA:C2:66
inet addr:10.0.3.15 Bcast:10.0.3.255 Mask:255.255.255.0 inet6 addr: fe80::a00:27ff:fefa:c266/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:4 errors:0 dropped:0 overruns:0 frame:0 TX packets:26 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:1300 (1.2 KiB) TX bytes:5634 (5.5 KiB)
DRBD
- using official howto:
http://www.drbd.org/users-guide-8.3/ch-admin.html
install
- version 8.3.11
yum install drbd
- disk
/dev/sdb1 2048 2097151 1047552 83 Linux
- our resource will be named "r0"
config
/etc/drdb.d/
/etc/drdb.d/r0.res
global {
usage-count yes;
} common {
protocol C;
} resource r0 {
on virtualb01 { device /dev/drbd1; disk /dev/sdb1; address 192.168.55.102:7789; meta-disk internal; } on virtualb02 { device /dev/drbd1; disk /dev/sdb1; address 192.168.55.101:7789; meta-disk internal; }
}
resource r0 {
device /dev/drbd1; disk /dev/sdb1; meta-disk internal; on virtualb01 { address 192.168.55.102:7789; } on virtualb02 { address 192.168.55.101:7789; }
}
common (changes for dual-primary mode)
startup {
become-primary-on both;
net {
allow-two-primaries;
- split brain (and its policies):
handlers { split-brain "/usr/lib/drbd/notify-split-brain.sh root";
net {
after-sb-0pri discard-zero-changes; after-sb-1pri discard-secondary; after-sb-2pri disconnect;
-- meaning sequentially: [ If there is any host on which no changes occurred at all, simply apply all modifications made on the other and continue.] [ Whichever host is currently in the Secondary role, make that host the split brain victim.]
- limiting speed?
syncer { rate 16M;
- verification: (needed ?)
syncer { verify-alg sha1; }
- checksum ?
syncer { csums-alg sha1;
- error handling: (recommended)
disk { on-io-error detach;
- replication traffic integrity:
net { data-integrity-alg sha1
- disk flushes
- replicated data set
disk { no-disk-flushes;
- DRBD's meta data
disk { no-md-flushes;
+++++++++++++++++
drbdadm verify resource
- possible scheduling from crond
42 0 * * 0 root /sbin/drbdadm verify resource
- network:
ports 7788-7799
first start
[root@virtualb01 ~]# drbdadm create-md r0 Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success
- start the service at boot
chkconfig drbd on
[root@virtualb01 ~]# drbd-overview
1:r0 Connected Primary/Primary UpToDate/UpToDate C r----- lvm-pv: clstorage 1.00g 0g
[root@virtualb02 ~]# drbd-overview
1:r0 Connected Primary/Primary UpToDate/UpToDate C r----- lvm-pv: clstorage 1.00g 0g
quick commands
drbd-overview # Checking DRBD status watch cat /proc/drbd # same, but old school way.. drbdadm up resource drbdadm primary resource # promote node drbdadm disconnect resource drbdadm connect resource drbdadm verify resource drbdadm dump-md resource > /tmp/metadata # backing up metadata drbdsetup /dev/drbd1 syncer -r 110M # changing speed at runtime drbdadm adjust resource # revert to startup setting drbdadm resize resource drbdadm -- --size=new-size resize resource # shrink (online) drbdadm dstate resource
GFS
http://www.drbd.org/users-guide-8.3/s-enable-dual-primary.html
yum -y install gfs2-utils gfs2-cluster lvm2-cluster
chkconfig gfs2 on
chkconfig gfs2-cluster on
chkconfig cman on
cman
/etc/cluster/cluster.conf
LVM to recognize the DRBD
- using clustered locking
- in /etc/lvm/lvm.conf:
locking_type = 3
- (default is)
# Type 2 uses the external shared library locking_library. # Type 3 uses built-in clustered locking. # Type 4 uses read-only locking which forbids any operations that might # change metadata. locking_type = 1
- allowing LVM scans for PV signatures.
- in /etc/lvm/lvm.conf:
filter = [ "a|drbd.*|", "r|.*|" ]
- disable the LVM cache
- in /etc/lvm/lvm.conf:
write_cache_state = 0
!!! run vgscan to reload lvm's cache etc.
vgscan
mkfs -t gfs2 -p lock_dlm -j 2 -t CLUSTER00:gfs2 /dev/drbd1
- in /etc/fstab
/dev/drbd1 /mnt/cl_storage gfs2 defaults 0 0 mkdir /mnt/cl_storage
- restarting the gfs2 deamon
service gfs2 restart
cman "this also starts ccsd and fenced"
in /etc/cluster/cluster.conf (both nodes) <?xml version="1.0"?> <cluster alias="CLUSTER00" config_version="23" name="CLUSTER00"> <fence_daemon post_fail_delay="0" post_join_delay="3"/> <clusternodes> <clusternode name="virtualb01" nodeid="1" votes="1"> <fence> <method name="1"/> </fence> </clusternode> <clusternode name="virtualb02" nodeid="2" votes="1"> <fence> <method name="1"/> </fence> </clusternode> </clusternodes> <fencedevices/> </cluster>
[root@virtualb01 ~]# cman_tool nodes Node Sts Inc Joined Name
1 M 4 2012-05-08 16:31:14 virtualb01 2 M 8 2012-05-08 16:40:32 virtualb02
[root@virtualb02 ~]# cman_tool nodes Node Sts Inc Joined Name
1 M 8 2012-05-08 16:40:31 virtualb01 2 M 4 2012-05-08 16:40:31 virtualb02
clvmd
remarks
- "any DRBD resource to be used for storing a GFS filesystem must be configured in dual-primary mode."
You’ll want these settings:
disk {
fencing resource-and-stonith;
} handlers {
# Make sure the other node is confirmed # dead after this! outdate-peer "/sbin/kill-other-node.sh";
}
just notes
[root@virtualb01 ~]# drbdadm create-md r0 Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success [root@virtualb01 ~]# drbdadm attach r0 [root@virtualb01 ~]# drbdadm syncer r0 [root@virtualb01 ~]#
---
The server's response is:
node already registered Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success [root@virtualb02 ~]# drbdadm attach r0 [root@virtualb02 ~]# drbdadm syncer r0 [root@virtualb02 ~]# drbdadm connect r0 1: Failure: (125) Device has a net-config (use disconnect first) Command 'drbdsetup 1 net ipv4:192.168.55.101:7789 ipv4:192.168.55.102:7789 C --set-defaults --create-device --after-sb-2pri=disconnect --after-sb-1pri=discard-secondary --after-sb-0pri=discard-zero-changes --allow-two-primaries' terminated with exit code 10 [root@virtualb02 ~]#
[root@virtualb01 ~]# cat /proc/drbd
version: 8.3.11 (api:88/proto:86-96)
srcversion: 21CA73FE6D7D9C67B0C6AB2
1: cs:SyncTarget ro:Primary/Primary ds:Inconsistent/UpToDate C r----- ns:0 nr:24148 dw:23552 dr:76 al:0 bm:1 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:1023932
[>....................] sync'ed: 2.8% (1023932/1047484)K finish: 0:13:00 speed: 1,308 (1,308) want: 250 K/sec [root@virtualb01 ~]#
[root@virtualb02 ~]# cat /proc/drbd
version: 8.3.11 (api:88/proto:86-96)
srcversion: 21CA73FE6D7D9C67B0C6AB2
1: cs:SyncSource ro:Primary/Primary ds:UpToDate/Inconsistent C r----- ns:20308 nr:0 dw:0 dr:20980 al:0 bm:1 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:1027772
[>....................] sync'ed: 2.4% (1027772/1047484)K finish: 0:12:50 speed: 1,312 (1,312) K/sec
- resource devices #################
[root@virtualb02 ~]# ls -Rl /dev/drbd /dev/drbd: total 0 drwxr-xr-x 2 root root 60 May 8 14:28 by-disk drwxr-xr-x 2 root root 60 May 8 14:28 by-res
/dev/drbd/by-disk: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 sdb1 -> ../../drbd1
/dev/drbd/by-res: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 r0 -> ../../drbd1
++++++++++++++++++++++
[root@virtualb01 ~]# ls -Rl /dev/drbd /dev/drbd: total 0 drwxr-xr-x 2 root root 60 May 8 14:28 by-disk drwxr-xr-x 2 root root 60 May 8 14:28 by-res
/dev/drbd/by-disk: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 sdb1 -> ../../drbd1
/dev/drbd/by-res: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 r0 -> ../../drbd1
[root@virtualb01 ~]# fdisk -l /dev/drbd1
Disk /dev/drbd1: 1072 MB, 1072623616 bytes 255 heads, 63 sectors/track, 130 cylinders, total 2094968 sectors Units = sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disk identifier: 0x00000000