It's not pretty but sometimes it has to be done Here is the set of commands I use to install Ceph MONs, MGRs or OSDs manually

This came about becuase I was trying to use cephadm but the tool was broken for Ubuntu 22.04 at the time(Wouldn't bootstrap) so I did a package install using apt install ceph which pulled Quincy from the Ubuntu repos then I could do a simple(But yukky) manual install

I used this excellent site as a reference https://www.server-world.info/en/note?os=Ubuntu_20.04&p=ceph15&f=1

#Single script spin up a cluster

mkdir -p /var/lib/ceph/bootstrap-osd mkdir -p /var/lib/ceph/mon/ mkdir -p /var/lib/ceph/mgr/ mkdir /var/lib/ceph/osd chown -R ceph. /var/lib/ceph/

ceph-authtool --create-keyring /etc/ceph/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'

generate secret key for Cluster admin

ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'

generate key for bootstrap

ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'

import generated key

ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring

generate monitor map

FSID=$(grep "^fsid" /etc/ceph/ceph.conf | awk {'print $NF'}) NODENAME=$(grep "^mon initial" /etc/ceph/ceph.conf | awk {'print $NF'}) NODEIP=$(grep "^mon host" /etc/ceph/ceph.conf | awk {'print $NF'}) monmaptool --create --add $NODENAME $NODEIP --fsid $FSID /etc/ceph/monmap

mkdir /var/lib/ceph/mon/ceph-jumpbox-backup ceph-mon --cluster ceph --mkfs -i $NODENAME --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring

chown ceph. /etc/ceph/ceph.* chown -R ceph. /var/lib/ceph/mon/ceph-jumpbox-backup /var/lib/ceph/bootstrap-osd systemctl enable --now ceph-mon@$NODENAME

ceph mon enable-msgr2

ceph mgr module enable pg_autoscaler mkdir /var/lib/ceph/mgr/ceph-jumpbox-backup ceph auth get-or-create mgr.$NODENAME mon 'allow profile mgr' osd 'allow *' mds 'allow *' ceph auth get-or-create mgr.jumpbox-backup | tee /etc/ceph/ceph.mgr.admin.keyring cp /etc/ceph/ceph.mgr.admin.keyring /var/lib/ceph/mgr/ceph-jumpbox-backup/keyring chown ceph. /etc/ceph/ceph.mgr.admin.keyring chown -R ceph. /var/lib/ceph/mgr/ceph-jumpbox-backup systemctl enable --now ceph-mgr@$NODENAME

Made a mistake and need to start again?

systemctl stop ceph.target rm -rf /var/lib/ceph/* rm /etc/ceph/.keyring rm /etc/ceph/monmap rm /var/log/ceph/

Bootstrap

 1for NODE in mon01 mon02 mon03 osd01 osd02 osd03
 2do
 3    if [ ! ${NODE} = "mon01" ]
 4    then
 5        ssh $NODE -p 1812 "mkdir -p /var/lib/ceph/bootstrap-osd"
 6        scp -P 1812 /etc/ceph/ceph.conf ${NODE}:/etc/ceph/ceph.conf
 7        scp -P 1812 /etc/ceph/ceph.client.admin.keyring ${NODE}:/etc/ceph
 8        scp -P 1812 /var/lib/ceph/bootstrap-osd/ceph.keyring ${NODE}:/var/lib/ceph/bootstrap-osd
 9        ssh $NODE -p 1812 "apt update && apt install ceph -y"
10        ssh $NODE -p 1812 "chown ceph /etc/ceph/ceph.* /var/lib/ceph/bootstrap-osd/*;"
11    fi
12
13done 

MONs

 1sudo mkdir /var/lib/ceph/mon/ceph-$(hostname)
 2ceph auth get mon. -o /tmp/{key-filename}
 3
 4TEMPDIR=/tmp
 5mkdir $TEMPDIR
 6ceph auth get mon. -o $TEMPDIR/mon-key
 7ceph mon getmap -o $TEMPDIR/mon-map
 8ceph-mon -i $(hostname) --mkfs --monmap $TEMPDIR/mon-map --keyring $TEMPDIR/mon-key
 9chown ceph:ceph /var/lib/ceph/mon/ -R
10systemctl enable ceph-mon@$(hostname)
11systemctl start ceph-mon@$(hostname)

MGR

 1mkdir /var/lib/ceph/mgr/ceph-$(hostname)/
 2ceph auth get-or-create mgr.$(hostname) mon 'allow profile mgr' osd 'allow *' mds 'allow *' -o /var/lib/ceph/mgr/ceph-$(hostname)/keyring
 3systemctl enable ceph-mgr@$(hostname)
 4systemctl start ceph-mgr@$(hostname)
 5
 6# OSD
 7
 8<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"><span class="ln"> 1</span>ceph-volume lvm prepare --data /dev/nvme11n1 
 9<span class="ln"> 2</span>ceph-volume lvm prepare --data /dev/nvme9n1  
10<span class="ln"> 3</span>ceph-volume lvm prepare --data /dev/nvme12n1 
11<span class="ln"> 4</span>ceph-volume lvm prepare --data /dev/nvme2n1  
12<span class="ln"> 5</span>ceph-volume lvm prepare --data /dev/nvme7n1  
13<span class="ln"> 6</span>ceph-volume lvm prepare --data /dev/nvme6n1  
14<span class="ln"> 7</span>ceph-volume lvm prepare --data /dev/nvme1n1  
15<span class="ln"> 8</span>ceph-volume lvm prepare --data /dev/nvme3n1  
16<span class="ln"> 9</span>ceph-volume lvm prepare --data /dev/nvme13n1 
17<span class="ln">10</span>ceph-volume lvm prepare --data /dev/nvme0n1  
18<span class="ln">11</span>ceph-volume lvm prepare --data /dev/nvme10n1 
19<span class="ln">12</span>ceph-volume lvm prepare --data /dev/nvme8n1 
20<span class="ln">13</span>
21<span class="ln">14</span>ceph-volume lvm activate --all</code></pre></div>