1#!/bin/bash
2##################################################################################
3# This script is used for synchronizing the RBDs (Rados Block Devices) between a
4# production Ceph cluster and a backup Ceph cluster. It checks whether the images exist in
5# the remote pool and creates them if not. It then checks for the last synced ID and
6# performs an initial or incremental sync. It also deletes old snapshots from both the
7# source and destination clusters.
8##################################################################################
9echo "Starting"
10rm /tmp/rbd-sync-snaplistlocal
11rm /tmp/rbd-sync-snaplistremote
12
13mirrorPool() {
14 #list all images in the pool
15 IMAGES=`$SOURCERBDCMD ls $SOURCEPOOL`
16
17 for LOCAL_IMAGE in $IMAGES; do
18 #check whether remote host/pool has image
19 if [[ -z $($DESTRBDCMD ls $DESTPOOL | grep $LOCAL_IMAGE) ]];then
20 echo "info: image does not exist in remote pool. creating new image"
21 #todo: check succesful creation
22 echo $DESTRBDCMD create $DESTPOOL/$LOCAL_IMAGE -s 1
23 $DESTRBDCMD create $DESTPOOL/$LOCAL_IMAGE -s 1
24 fi
25
26 # Retreive last synced id
27 expr=" $SNAPNAME\([[:digit:]]\+\)"
28 echo '$SOURCERBDCMD snap ls $SOURCEPOOL/$LOCAL_IMAGE | grep "$expr" | sed "s/.*$expr.*/\1/g" | sort -n > /tmp/rbd-sync-snaplistlocal'
29 $SOURCERBDCMD snap ls $SOURCEPOOL/$LOCAL_IMAGE | grep "$expr" | sed "s/.*$expr.*/\1/g" | sort -n > /tmp/rbd-sync-snaplistlocal
30
31 if $DESTRBDCMD info $DESTPOOL/$LOCAL_IMAGE >/dev/null 2>&1; then
32 echo "Image in destination pool exists $DESTPOOL/$LOCAL_IMAGE"
33 echo "Listing snaps on destination image"
34 echo $DESTRBDCMD snap ls $DESTPOOL/$LOCAL_IMAGE | grep "$expr" | sed "s/.*$expr.*/\1/g" | sort -n > /tmp/rbd-sync-snaplistremote
35 $DESTRBDCMD snap ls $DESTPOOL/$LOCAL_IMAGE | grep "$expr" | sed "s/.*$expr.*/\1/g" | sort -n > /tmp/rbd-sync-snaplistremote
36 else
37 echo "Image does not exist in destination"
38 echo "" > /tmp/rbd-sync-snaplistremote
39 fi
40 syncid=$(comm -12 /tmp/rbd-sync-snaplistlocal /tmp/rbd-sync-snaplistremote | tail -n1)
41 lastid=$(cat /tmp/rbd-sync-snaplistlocal /tmp/rbd-sync-snaplistremote | sort -n | tail -n1)
42 nextid=$(($lastid + 1))
43
44
45
46 # Initial sync
47 if [ "$syncid" = "" ]; then
48 echo "Initial sync with id $nextid"
49 echo $SOURCERBDCMD snap create $SOURCEPOOL/$LOCAL_IMAGE@$SNAPNAME$nextid
50 $SOURCERBDCMD snap create $SOURCEPOOL/$LOCAL_IMAGE@$SNAPNAME$nextid
51
52 echo '$SOURCERBDCMD export-diff --no-progress $SOURCEPOOL/$LOCAL_IMAGE@$SNAPNAME$nextid - | $DESTRBDCMD import-diff --no-progress - $DESTPOOL/$LOCAL_IMAGE'
53 $SOURCERBDCMD export-diff --no-progress $SOURCEPOOL/$LOCAL_IMAGE@$SNAPNAME$nextid - | $DESTRBDCMD import-diff --no-progress - $DESTPOOL/$LOCAL_IMAGE
54
55
56 # Incremental sync
57 else
58 echo "Found synced id : $syncid"
59 echo $SOURCERBDCMD snap create $SOURCEPOOL/$LOCAL_IMAGE@$SNAPNAME$nextid
60 $SOURCERBDCMD snap create $SOURCEPOOL/$LOCAL_IMAGE@$SNAPNAME$nextid
61
62 echo "Sync $syncid -> $nextid"
63
64 $SOURCERBDCMD export-diff --no-progress --from-snap $SNAPNAME$syncid $SOURCEPOOL/$LOCAL_IMAGE@$SNAPNAME$nextid - |$DESTRBDCMD import-diff --no-progress - $DESTPOOL/$LOCAL_IMAGE
65
66
67 echo "Checking for old snaps to delete on source"
68 deleteOldSnapsOnSource
69 echo "Checking for old snaps to delete on destination"
70 deleteOldSnapsOnDest
71 fi
72
73 done
74}
75
76#Delete snapshots from source cluster
77deleteOldSnapsOnSource() {
78 #get all the snapshots of the image
79 while IFS= read -r line; do
80 # echo $line
81 timestamp=$(echo "$line" | jq -r '.timestamp')
82 a=$(date +%s)
83 b=$(date -d "$timestamp" +%s)
84 date_diff=$(((a-b)/86400))
85
86 if [ "$date_diff" -gt "$THRESHOLD_DAYS_SOURCE" ]; then
87 echo "Alert: Item $(echo "$line" | jq -r '.name') with ID $(echo "$line" | jq -r '.id') is older than $THRESHOLD_DAYS_SOURCE days($date_diff days)!"
88 echo $SOURCERBDCMD snap rm $SOURCEPOOL/$LOCAL_IMAGE@$(echo "$line" | jq -r '.name')
89 $SOURCERBDCMD snap rm $SOURCEPOOL/$LOCAL_IMAGE@$(echo "$line" | jq -r '.name')
90 else
91 echo "Item $(echo "$line" | jq -r '.name') with ID $(echo "$line" | jq -r '.id') is only $date_diff days"
92 fi
93 done < <($SOURCERBDCMD snap ls $SOURCEPOOL/$LOCAL_IMAGE --format json | jq -c '.[] | select(.name|startswith("rbd-sync"))')
94}
95
96
97#Delete snapshots from destination cluster
98deleteOldSnapsOnDest() {
99 #get all the snapshots of the image
100 while IFS= read -r line; do
101 # echo $line
102 timestamp=$(echo "$line" | jq -r '.timestamp')
103 a=$(date +%s)
104 b=$(date -d "$timestamp" +%s)
105 date_diff=$(((a-b)/86400))
106
107 if [ "$date_diff" -gt "$THRESHOLD_DAYS_DEST" ]; then
108 echo "Alert: Item $(echo "$line" | jq -r '.name') with ID $(echo "$line" | jq -r '.id') is older than $THRESHOLD_DAYS_DEST days($date_diff days)!"
109 echo $DESTRBDCMD snap rm $DESTPOOL/$LOCAL_IMAGE@$(echo "$line" | jq -r '.name')
110 $DESTRBDCMD snap rm $DESTPOOL/$LOCAL_IMAGE@$(echo "$line" | jq -r '.name')
111 else
112 echo "Item $(echo "$line" | jq -r '.name') with ID $(echo "$line" | jq -r '.id') is only $date_diff days"
113 fi
114 done < <($DESTRBDCMD snap ls $DESTPOOL/$LOCAL_IMAGE --format json | jq -c '.[] | select(.name|startswith("rbd-sync"))')
115}
116
117THRESHOLD_DAYS_DEST=7
118THRESHOLD_DAYS_SOURCE=1=
119SOURCECLUSTER="ceph"
120DESTCLUSTER="backup"
121SOURCERBDCMD="rbd --cluster $SOURCECLUSTER"
122DESTRBDCMD="rbd --cluster $DESTCLUSTER"
123SNAPNAME='rbd-sync-'
124SOURCEPOOL="volumes"
125DESTPOOL="backup"
126
127exec > >(tee ceph_sync.log) 2>&1
128date
129mirrorPool
130
131# if [[ -z $(grep error /var/tmp/ceph_sync.log) ]]; then
132# cat /var/tmp/ceph_sync.log|mailx -s "Ceph sync- Successfull" XX@YY.com.au
133# else
134# cat /var/tmp/ceph_sync.log|mailx -s "Ceph sync- FAILED" XX@YY.com.au
135# fi
136date
137rm /tmp/rbd-sync-snaplistlocal
138rm /tmp/rbd-sync-snaplistremote