Install Ceph by hand Ubuntu 22.04

Jun 28, 2023 · 3 min read · ceph ·

It's not pretty but sometimes it has to be done Here is the set of commands I use to install Ceph MONs, MGRs or OSDs manually

This came about becuase I was trying to use cephadm but the tool was broken for Ubuntu 22.04 at the time(Wouldn't bootstrap) so I did a package install using apt install ceph which pulled Quincy from the Ubuntu repos then I could do a simple(But yukky) manual install

I used this excellent site as a reference https://www.server-world.info/en/note?os=Ubuntu_20.04&p=ceph15&f=1

Single script spin up a cluster

 1mkdir -p /var/lib/ceph/bootstrap-osd
 2mkdir -p /var/lib/ceph/mon/
 3mkdir -p /var/lib/ceph/mgr/
 4mkdir /var/lib/ceph/osd
 5chown -R ceph. /var/lib/ceph/
 6
 7ceph-authtool --create-keyring /etc/ceph/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
 8# generate secret key for Cluster admin
 9ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
10# generate key for bootstrap
11ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
12# import generated key
13ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
14ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
15# generate monitor map
16FSID=$(grep "^fsid" /etc/ceph/ceph.conf | awk {'print $NF'})
17NODENAME=$(grep -e "^mon initial" -e "^mon_initial" /etc/ceph/ceph.conf | awk {'print $NF'})
18NODEIP=$(grep -e "^mon host" -e "^mon_host" /etc/ceph/ceph.conf | awk {'print $NF'})
19monmaptool --create --add $NODENAME $NODEIP --fsid $FSID /etc/ceph/monmap
20
21
22mkdir /var/lib/ceph/mon/ceph-$NODENAME
23ceph-mon --cluster ceph --mkfs -i $NODENAME --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring
24
25chown ceph. /etc/ceph/ceph.*
26chown -R ceph. /var/lib/ceph/mon/ceph-$NODENAME /var/lib/ceph/bootstrap-osd
27systemctl enable --now ceph-mon@$NODENAME
28
29ceph mon enable-msgr2
30
31ceph mgr module enable pg_autoscaler
32mkdir /var/lib/ceph/mgr/ceph-$NODENAME
33ceph auth get-or-create mgr.$NODENAME mon 'allow profile mgr' osd 'allow *' mds 'allow *'
34ceph auth get-or-create mgr.$NODENAME | tee /etc/ceph/ceph.mgr.admin.keyring
35cp /etc/ceph/ceph.mgr.admin.keyring /var/lib/ceph/mgr/ceph-$NODENAME/keyring
36chown ceph. /etc/ceph/ceph.mgr.admin.keyring
37chown -R ceph. /var/lib/ceph/mgr/ceph-$NODENAME
38systemctl enable --now ceph-mgr@$NODENAME

Made a mistake and need to start again?

systemctl stop ceph.target rm -rf /var/lib/ceph/* rm /etc/ceph/.keyring rm /etc/ceph/monmap rm /var/log/ceph/

Bootstrap

 1for NODE in mon01 mon02 mon03 osd01 osd02 osd03
 2do
 3    if [ ! ${NODE} = "mon01" ]
 4    then
 5        ssh $NODE -p 1812 "mkdir -p /var/lib/ceph/bootstrap-osd"
 6        scp -P 1812 /etc/ceph/ceph.conf ${NODE}:/etc/ceph/ceph.conf
 7        scp -P 1812 /etc/ceph/ceph.client.admin.keyring ${NODE}:/etc/ceph
 8        scp -P 1812 /var/lib/ceph/bootstrap-osd/ceph.keyring ${NODE}:/var/lib/ceph/bootstrap-osd
 9        ssh $NODE -p 1812 "apt update && apt install ceph -y"
10        ssh $NODE -p 1812 "chown ceph /etc/ceph/ceph.* /var/lib/ceph/bootstrap-osd/*;"
11    fi
12
13done

MONs

 1sudo mkdir /var/lib/ceph/mon/ceph-$(hostname)
 2ceph auth get mon. -o /tmp/{key-filename}
 3
 4TEMPDIR=/tmp
 5mkdir $TEMPDIR
 6ceph auth get mon. -o $TEMPDIR/mon-key
 7ceph mon getmap -o $TEMPDIR/mon-map
 8ceph-mon -i $(hostname) --mkfs --monmap $TEMPDIR/mon-map --keyring $TEMPDIR/mon-key
 9chown ceph:ceph /var/lib/ceph/mon/ -R
10systemctl enable ceph-mon@$(hostname)
11systemctl start ceph-mon@$(hostname)

MGR

1mkdir /var/lib/ceph/mgr/ceph-$(hostname)/
2ceph auth get-or-create mgr.$(hostname) mon 'allow profile mgr' osd 'allow *' mds 'allow *' -o /var/lib/ceph/mgr/ceph-$(hostname)/keyring
3systemctl enable ceph-mgr@$(hostname)
4systemctl start ceph-mgr@$(hostname)

OSD

 1echo "Gathering available disks for OSD setup..."
 2DISKS=$(lsblk -dno NAME | grep -Ev '^(sda|loop|sr|nvme0n1)$') # Exclude boot drive and system disks
 3
 4if [[ -z "$DISKS" ]]; then
 5    error "No additional disks found for OSDs. Please attach disks and re-run the script."
 6fi
 7
 8for DISK in $DISKS; do
 9    echo "Preparing disk /dev/$DISK for OSD..."
10    ceph-volume lvm prepare --data /dev/$DISK
11done
12
13echo "Activating OSDs..."
14ceph-volume lvm activate --all
15
16ceph osd pool set device_health_metrics  size 1
17
18ceph -s