DRBD

From Alessandro's Wiki

fedora

ln -sf /lib/systemd/system/runlevel3.target /etc/systemd/system/default.target


yum -y groupinstall Clustering yum -y groupinstall “Cluster Storage” yum -y openssh-server chkconfig sshd on

  1. disabled firewall (for testing)

chkconfig iptables off

  1. disable selinux (for testing)

setenforce 0 (and also in /etc/selinux/config )


net

/etc/hosts 192.168.55.1 hostsrv 192.168.55.101 virtualb02 192.168.55.102 virtualb01


yum remove NetworkManager

systemctl stop NetworkManager.service systemctl disable NetworkManager.service

systemctl enable network.service systemctl start network.service

/etc/sysconfig/network-scripts/ifcfg-p2p1 /etc/sysconfig/network-scripts/ifcfg-p7p1

( host only adapter ) p2p1 Link encap:Ethernet HWaddr 08:00:27:E3:E1:EC

         inet addr:192.168.55.102  Bcast:192.168.55.255  Mask:255.255.255.0
         inet6 addr: fe80::a00:27ff:fee3:e1ec/64 Scope:Link
         UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
         RX packets:68 errors:0 dropped:0 overruns:0 frame:0
         TX packets:69 errors:0 dropped:0 overruns:0 carrier:0
         collisions:0 txqueuelen:1000 
         RX bytes:15292 (14.9 KiB)  TX bytes:12355 (12.0 KiB)

(NAT adapter) p7p1 Link encap:Ethernet HWaddr 08:00:27:FA:C2:66

         inet addr:10.0.3.15  Bcast:10.0.3.255  Mask:255.255.255.0
         inet6 addr: fe80::a00:27ff:fefa:c266/64 Scope:Link
         UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
         RX packets:4 errors:0 dropped:0 overruns:0 frame:0
         TX packets:26 errors:0 dropped:0 overruns:0 carrier:0
         collisions:0 txqueuelen:1000 
         RX bytes:1300 (1.2 KiB)  TX bytes:5634 (5.5 KiB)

DRBD

  • using official howto:

http://www.drbd.org/users-guide-8.3/ch-admin.html

install

  • version 8.3.11
yum install drbd
  • disk

/dev/sdb1 2048 2097151 1047552 83 Linux

  • our resource will be named "r0"

config

/etc/drdb.d/
/etc/drdb.d/r0.res
global {
  usage-count yes;
}
common {
  protocol C;
}
resource r0 {
  on virtualb01 {
    device    /dev/drbd1;
    disk      /dev/sdb1;
    address   192.168.55.102:7789;
    meta-disk internal;
  }
  on virtualb02 {
    device    /dev/drbd1;
    disk      /dev/sdb1;
    address   192.168.55.101:7789;
    meta-disk internal;
  }
}
resource r0 {
  device    /dev/drbd1;
  disk      /dev/sdb1;
  meta-disk internal;
  on virtualb01 {
    address   192.168.55.102:7789;
  }
  on virtualb02 {
    address   192.168.55.101:7789;
  }
}
  • common (changes for dual-primary mode)
startup {
               become-primary-on both;
net {
                allow-two-primaries;
  • split brain (and its policies):
 handlers {
   split-brain "/usr/lib/drbd/notify-split-brain.sh root";

net {

               after-sb-0pri discard-zero-changes;
               after-sb-1pri discard-secondary;
               after-sb-2pri disconnect;

-- meaning sequentially: [ If there is any host on which no changes occurred at all, simply apply all modifications made on the other and continue.] [ Whichever host is currently in the Secondary role, make that host the split brain victim.]

  • limiting speed?
 syncer {
   rate 16M;
  • verification:
syncer {
 verify-alg sha1;
}
  • checksum ?
syncer {
 csums-alg sha1;
  • error handling: (recommended)
disk {
 on-io-error detach;
  • replication traffic integrity:
net {
 data-integrity-alg sha1
  • disk flushes
    • replicated data set
 disk {
   no-disk-flushes;
    • DRBD's meta data
 disk {
   no-md-flushes;
drbdadm verify resource
  • possible scheduling from crond
42 0 * * 0    root    /sbin/drbdadm verify resource


  • network ports: 7788-7799

first start

[root@virtualb01 ~]# drbdadm create-md r0 Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success

  • start the service at boot

chkconfig drbd on

[root@virtualb01 ~]# drbd-overview

 1:r0  Connected Primary/Primary UpToDate/UpToDate C r----- lvm-pv: clstorage 1.00g 0g 

[root@virtualb02 ~]# drbd-overview

 1:r0  Connected Primary/Primary UpToDate/UpToDate C r----- lvm-pv: clstorage 1.00g 0g 


quick commands

drbd-overview # Checking DRBD status watch cat /proc/drbd # same, but old school way.. drbdadm up resource drbdadm primary resource # promote node drbdadm disconnect resource drbdadm connect resource drbdadm verify resource drbdadm dump-md resource > /tmp/metadata # backing up metadata drbdsetup /dev/drbd1 syncer -r 110M # changing speed at runtime drbdadm adjust resource # revert to startup setting drbdadm resize resource drbdadm -- --size=new-size resize resource # shrink (online) drbdadm dstate resource


GFS

http://www.drbd.org/users-guide-8.3/s-enable-dual-primary.html

yum -y install gfs2-utils gfs2-cluster lvm2-cluster


chkconfig gfs2 on chkconfig gfs2-cluster on chkconfig cman on

cman

/etc/cluster/cluster.conf


LVM to recognize the DRBD

  • using clustered locking
  • in /etc/lvm/lvm.conf:
locking_type = 3
  • (default is)
   # Type 2 uses the external shared library locking_library.
   # Type 3 uses built-in clustered locking.
   # Type 4 uses read-only locking which forbids any operations that might
   # change metadata.
   locking_type = 1
  • allowing LVM scans for PV signatures.
  • in /etc/lvm/lvm.conf:
filter = [ "a|drbd.*|", "r|.*|" ]
  • disable the LVM cache
  • in /etc/lvm/lvm.conf:
write_cache_state = 0


!!! run vgscan to reload lvm's cache etc.

vgscan 


mkfs -t gfs2 -p lock_dlm -j 2 -t CLUSTER00:gfs2 /dev/drbd1

  • in /etc/fstab

/dev/drbd1 /mnt/cl_storage gfs2 defaults 0 0 mkdir /mnt/cl_storage

  • restarting the gfs2 deamon

service gfs2 restart

cman "this also starts ccsd and fenced"

in /etc/cluster/cluster.conf (both nodes) <?xml version="1.0"?> <cluster alias="CLUSTER00" config_version="23" name="CLUSTER00"> <fence_daemon post_fail_delay="0" post_join_delay="3"/> <clusternodes> <clusternode name="virtualb01" nodeid="1" votes="1"> <fence> <method name="1"/> </fence> </clusternode> <clusternode name="virtualb02" nodeid="2" votes="1"> <fence> <method name="1"/> </fence> </clusternode> </clusternodes> <fencedevices/> </cluster>

[root@virtualb01 ~]# cman_tool nodes Node Sts Inc Joined Name

  1   M      4   2012-05-08 16:31:14  virtualb01
  2   M      8   2012-05-08 16:40:32  virtualb02

[root@virtualb02 ~]# cman_tool nodes Node Sts Inc Joined Name

  1   M      8   2012-05-08 16:40:31  virtualb01
  2   M      4   2012-05-08 16:40:31  virtualb02


clvmd

remarks

  • "any DRBD resource to be used for storing a GFS filesystem must be configured in dual-primary mode."

You’ll want these settings:

disk {

       fencing resource-and-stonith;

} handlers {

       # Make sure the other node is confirmed
       # dead after this!
       outdate-peer "/sbin/kill-other-node.sh";

}



just notes

[root@virtualb01 ~]# drbdadm create-md r0 Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success [root@virtualb01 ~]# drbdadm attach r0 [root@virtualb01 ~]# drbdadm syncer r0 [root@virtualb01 ~]#


--- The server's response is:

node already registered Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success [root@virtualb02 ~]# drbdadm attach r0 [root@virtualb02 ~]# drbdadm syncer r0 [root@virtualb02 ~]# drbdadm connect r0 1: Failure: (125) Device has a net-config (use disconnect first) Command 'drbdsetup 1 net ipv4:192.168.55.101:7789 ipv4:192.168.55.102:7789 C --set-defaults --create-device --after-sb-2pri=disconnect --after-sb-1pri=discard-secondary --after-sb-0pri=discard-zero-changes --allow-two-primaries' terminated with exit code 10 [root@virtualb02 ~]#



[root@virtualb01 ~]# cat /proc/drbd version: 8.3.11 (api:88/proto:86-96) srcversion: 21CA73FE6D7D9C67B0C6AB2

1: cs:SyncTarget ro:Primary/Primary ds:Inconsistent/UpToDate C r-----
   ns:0 nr:24148 dw:23552 dr:76 al:0 bm:1 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:1023932

[>....................] sync'ed: 2.8% (1023932/1047484)K finish: 0:13:00 speed: 1,308 (1,308) want: 250 K/sec [root@virtualb01 ~]#


[root@virtualb02 ~]# cat /proc/drbd version: 8.3.11 (api:88/proto:86-96) srcversion: 21CA73FE6D7D9C67B0C6AB2

1: cs:SyncSource ro:Primary/Primary ds:UpToDate/Inconsistent C r-----
   ns:20308 nr:0 dw:0 dr:20980 al:0 bm:1 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:1027772

[>....................] sync'ed: 2.4% (1027772/1047484)K finish: 0:12:50 speed: 1,312 (1,312) K/sec

    1. resource devices #################

[root@virtualb02 ~]# ls -Rl /dev/drbd /dev/drbd: total 0 drwxr-xr-x 2 root root 60 May 8 14:28 by-disk drwxr-xr-x 2 root root 60 May 8 14:28 by-res

/dev/drbd/by-disk: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 sdb1 -> ../../drbd1

/dev/drbd/by-res: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 r0 -> ../../drbd1

++++++++++++++++++++++

[root@virtualb01 ~]# ls -Rl /dev/drbd /dev/drbd: total 0 drwxr-xr-x 2 root root 60 May 8 14:28 by-disk drwxr-xr-x 2 root root 60 May 8 14:28 by-res

/dev/drbd/by-disk: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 sdb1 -> ../../drbd1

/dev/drbd/by-res: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 r0 -> ../../drbd1



[root@virtualb01 ~]# fdisk -l /dev/drbd1

Disk /dev/drbd1: 1072 MB, 1072623616 bytes 255 heads, 63 sectors/track, 130 cylinders, total 2094968 sectors Units = sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disk identifier: 0x00000000