DRBD

From Alessandro's Wiki
The printable version is no longer supported and may have rendering errors. Please update your browser bookmarks and please use the default browser print function instead.

fedora

ln -sf /lib/systemd/system/runlevel3.target /etc/systemd/system/default.target


yum -y groupinstall Clustering yum -y groupinstall “Cluster Storage” yum -y openssh-server chkconfig sshd on

  1. disabled firewall (for testing)

chkconfig iptables off

  1. disable selinux (for testing)

setenforce 0 (and also in /etc/selinux/config )


net

/etc/hosts 192.168.55.1 hostsrv 192.168.55.101 virtualb02 192.168.55.102 virtualb01


yum remove NetworkManager

systemctl stop NetworkManager.service systemctl disable NetworkManager.service

systemctl enable network.service systemctl start network.service

/etc/sysconfig/network-scripts/ifcfg-p2p1 /etc/sysconfig/network-scripts/ifcfg-p7p1

( host only adapter ) p2p1 Link encap:Ethernet HWaddr 08:00:27:E3:E1:EC 

         inet addr:192.168.55.102  Bcast:192.168.55.255  Mask:255.255.255.0
         inet6 addr: fe80::a00:27ff:fee3:e1ec/64 Scope:Link
         UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
         RX packets:68 errors:0 dropped:0 overruns:0 frame:0
         TX packets:69 errors:0 dropped:0 overruns:0 carrier:0
         collisions:0 txqueuelen:1000 
         RX bytes:15292 (14.9 KiB)  TX bytes:12355 (12.0 KiB)

(NAT adapter) p7p1 Link encap:Ethernet HWaddr 08:00:27:FA:C2:66 

         inet addr:10.0.3.15  Bcast:10.0.3.255  Mask:255.255.255.0
         inet6 addr: fe80::a00:27ff:fefa:c266/64 Scope:Link
         UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
         RX packets:4 errors:0 dropped:0 overruns:0 frame:0
         TX packets:26 errors:0 dropped:0 overruns:0 carrier:0
         collisions:0 txqueuelen:1000 
         RX bytes:1300 (1.2 KiB)  TX bytes:5634 (5.5 KiB)

DRBD

  • using official howto:

http://www.drbd.org/users-guide-8.3/ch-admin.html

install

  • version 8.3.11
yum install drbd
  • disk

/dev/sdb1 2048 2097151 1047552 83 Linux

  • our resource will be named "r0"

config

/etc/drdb.d/

/etc/drdb.d/r0.res

global {
  usage-count yes;
}
common {
  protocol C;
}
resource r0 {
  on virtualb01 {
    device    /dev/drbd1;
    disk      /dev/sdb1;
    address   192.168.55.102:7789;
    meta-disk internal;
  }
  on virtualb02 {
    device    /dev/drbd1;
    disk      /dev/sdb1;
    address   192.168.55.101:7789;
    meta-disk internal;
  }
}

resource r0 {
  device    /dev/drbd1;
  disk      /dev/sdb1;
  meta-disk internal;
  on virtualb01 {
    address   192.168.55.102:7789;
  }
  on virtualb02 {
    address   192.168.55.101:7789;
  }
}
  • common (changes for dual-primary mode)
startup {
               become-primary-on both;
net {
                allow-two-primaries;
  • split brain (and its policies):
 handlers {
   split-brain "/usr/lib/drbd/notify-split-brain.sh root";

net { 

               after-sb-0pri discard-zero-changes;
               after-sb-1pri discard-secondary;
               after-sb-2pri disconnect;

-- meaning sequentially: [ If there is any host on which no changes occurred at all, simply apply all modifications made on the other and continue.] [ Whichever host is currently in the Secondary role, make that host the split brain victim.]

  • limiting speed?
 syncer {
   rate 16M;
  • verification:
syncer {
 verify-alg sha1;
}
  • checksum ?
syncer {
 csums-alg sha1;
  • error handling: (recommended)
disk {
 on-io-error detach;
  • replication traffic integrity:
net {
 data-integrity-alg sha1
  • disk flushes
    • replicated data set
 disk {
   no-disk-flushes;
    • DRBD's meta data
 disk {
   no-md-flushes;

drbdadm verify resource
  • possible scheduling from crond
42 0 * * 0    root    /sbin/drbdadm verify resource


  • network ports: 7788-7799

first start

[root@virtualb01 ~]# drbdadm create-md r0 Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success

  • start the service at boot

chkconfig drbd on

[root@virtualb01 ~]# drbd-overview 

 1:r0  Connected Primary/Primary UpToDate/UpToDate C r----- lvm-pv: clstorage 1.00g 0g

[root@virtualb02 ~]# drbd-overview 

 1:r0  Connected Primary/Primary UpToDate/UpToDate C r----- lvm-pv: clstorage 1.00g 0g


quick commands

drbd-overview # Checking DRBD status watch cat /proc/drbd # same, but old school way.. drbdadm up resource drbdadm primary resource # promote node drbdadm disconnect resource drbdadm connect resource drbdadm verify resource drbdadm dump-md resource > /tmp/metadata # backing up metadata drbdsetup /dev/drbd1 syncer -r 110M # changing speed at runtime drbdadm adjust resource # revert to startup setting drbdadm resize resource drbdadm -- --size=new-size resize resource # shrink (online) drbdadm dstate resource


GFS

http://www.drbd.org/users-guide-8.3/s-enable-dual-primary.html

yum -y install gfs2-utils gfs2-cluster lvm2-cluster


chkconfig gfs2 on chkconfig gfs2-cluster on chkconfig cman on

cman

/etc/cluster/cluster.conf


LVM to recognize the DRBD

  • using clustered locking
  • in /etc/lvm/lvm.conf:
locking_type = 3
  • (default is)
   # Type 2 uses the external shared library locking_library.
   # Type 3 uses built-in clustered locking.
   # Type 4 uses read-only locking which forbids any operations that might
   # change metadata.
   locking_type = 1
  • allowing LVM scans for PV signatures.
  • in /etc/lvm/lvm.conf:
filter = [ "a|drbd.*|", "r|.*|" ]
  • disable the LVM cache
  • in /etc/lvm/lvm.conf:
write_cache_state = 0


!!! run vgscan to reload lvm's cache etc. 

vgscan


mkfs -t gfs2 -p lock_dlm -j 2 -t CLUSTER00:gfs2 /dev/drbd1

  • in /etc/fstab

/dev/drbd1 /mnt/cl_storage gfs2 defaults 0 0 mkdir /mnt/cl_storage

  • restarting the gfs2 deamon

service gfs2 restart

cman "this also starts ccsd and fenced"

in /etc/cluster/cluster.conf (both nodes) <?xml version="1.0"?> <cluster alias="CLUSTER00" config_version="23" name="CLUSTER00"> <fence_daemon post_fail_delay="0" post_join_delay="3"/> <clusternodes> <clusternode name="virtualb01" nodeid="1" votes="1"> <fence> <method name="1"/> </fence> </clusternode> <clusternode name="virtualb02" nodeid="2" votes="1"> <fence> <method name="1"/> </fence> </clusternode> </clusternodes> <fencedevices/> </cluster>

[root@virtualb01 ~]# cman_tool nodes Node Sts Inc Joined Name 

  1   M      4   2012-05-08 16:31:14  virtualb01
  2   M      8   2012-05-08 16:40:32  virtualb02

[root@virtualb02 ~]# cman_tool nodes Node Sts Inc Joined Name 

  1   M      8   2012-05-08 16:40:31  virtualb01
  2   M      4   2012-05-08 16:40:31  virtualb02


clvmd

remarks

  • "any DRBD resource to be used for storing a GFS filesystem must be configured in dual-primary mode."

You’ll want these settings:

disk { 

       fencing resource-and-stonith;

} handlers { 

       # Make sure the other node is confirmed
       # dead after this!
       outdate-peer "/sbin/kill-other-node.sh";

}



just notes

[root@virtualb01 ~]# drbdadm create-md r0 Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success [root@virtualb01 ~]# drbdadm attach r0 [root@virtualb01 ~]# drbdadm syncer r0 [root@virtualb01 ~]#


--- The server's response is:

node already registered Writing meta data... initializing activity log NOT initialized bitmap New drbd meta data block successfully created. success [root@virtualb02 ~]# drbdadm attach r0 [root@virtualb02 ~]# drbdadm syncer r0 [root@virtualb02 ~]# drbdadm connect r0 1: Failure: (125) Device has a net-config (use disconnect first) Command 'drbdsetup 1 net ipv4:192.168.55.101:7789 ipv4:192.168.55.102:7789 C --set-defaults --create-device --after-sb-2pri=disconnect --after-sb-1pri=discard-secondary --after-sb-0pri=discard-zero-changes --allow-two-primaries' terminated with exit code 10 [root@virtualb02 ~]#



[root@virtualb01 ~]# cat /proc/drbd version: 8.3.11 (api:88/proto:86-96) srcversion: 21CA73FE6D7D9C67B0C6AB2 

1: cs:SyncTarget ro:Primary/Primary ds:Inconsistent/UpToDate C r-----
   ns:0 nr:24148 dw:23552 dr:76 al:0 bm:1 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:1023932

[>....................] sync'ed: 2.8% (1023932/1047484)K finish: 0:13:00 speed: 1,308 (1,308) want: 250 K/sec [root@virtualb01 ~]#


[root@virtualb02 ~]# cat /proc/drbd version: 8.3.11 (api:88/proto:86-96) srcversion: 21CA73FE6D7D9C67B0C6AB2 

1: cs:SyncSource ro:Primary/Primary ds:UpToDate/Inconsistent C r-----
   ns:20308 nr:0 dw:0 dr:20980 al:0 bm:1 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:1027772

[>....................] sync'ed: 2.4% (1027772/1047484)K finish: 0:12:50 speed: 1,312 (1,312) K/sec

    1. resource devices #################

[root@virtualb02 ~]# ls -Rl /dev/drbd /dev/drbd: total 0 drwxr-xr-x 2 root root 60 May 8 14:28 by-disk drwxr-xr-x 2 root root 60 May 8 14:28 by-res

/dev/drbd/by-disk: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 sdb1 -> ../../drbd1

/dev/drbd/by-res: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 r0 -> ../../drbd1

++++++++++++++++++++++

[root@virtualb01 ~]# ls -Rl /dev/drbd /dev/drbd: total 0 drwxr-xr-x 2 root root 60 May 8 14:28 by-disk drwxr-xr-x 2 root root 60 May 8 14:28 by-res

/dev/drbd/by-disk: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 sdb1 -> ../../drbd1

/dev/drbd/by-res: total 0 lrwxrwxrwx 1 root root 11 May 8 14:28 r0 -> ../../drbd1



[root@virtualb01 ~]# fdisk -l /dev/drbd1

Disk /dev/drbd1: 1072 MB, 1072623616 bytes 255 heads, 63 sectors/track, 130 cylinders, total 2094968 sectors Units = sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disk identifier: 0x00000000

Retrieved from "https://wiki.alessandro.delgallo.net//index.php?title=DRBD&oldid=5991"