Ceph Storage on Kubernetes Cluster

sudo qemu-img create -f raw ceph-osd-h1-n1.img 30G

sudo virsh attach-disk ds-h1-n1 \
  /var/lib/libvirt/images/ubuntu24/ceph-osd-h1-n1.img \
  vdb \
  --persistent \
  --cache none \
  --io native

# Adjust also all nodes memory size

sudo qemu-img resize node-h1-n1.qcow2 +10G

sudo apt install -y chrony

sudo systemctl restart chrony.service

helm repo add rook-release https://charts.rook.io/release
helm repo update

helm install rook-ceph rook-release/rook-ceph --create-namespace --namespace rook-ceph --set crds.enabled=true

kubectl -n rook-ceph get pods

# ceph-cluster-values.yaml

operatorNamespace: rook-ceph

cephClusterSpec:
  cephVersion:
    image: quay.io/ceph/ceph:v19.2.0  # Squid
    allowUnsupported: true

  dataDirHostPath: /var/lib/rook

  mon:
    count: 3
    allowMultiplePerNode: false

  mgr:
    count: 2

  dashboard:
    enabled: true
    ssl: false

  storage:
    useAllNodes: false
    useAllDevices: false

    config:
      osdsPerDevice: "1"

    nodes:
      - name: "ds-h1-n1"
        devices:
          - name: "vdb"
      - name: "ds-h1-n2"
        devices:
          - name: "vdb"
      - name: "ds-h1-n3"
        devices:
          - name: "vdb"

  resources:
    osd:
      limits:
        memory: "4Gi"
      requests:
        memory: "2Gi"

  healthCheck:
    daemonHealth:
      mon:
        disabled: false
      osd:
        disabled: false

helm upgrade --install rook-ceph-cluster rook-release/rook-ceph-cluster -n rook-ceph -f ceph-cluster-values.yaml

# certificate-rook-ceph.yaml

apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: syncroze-tls
  namespace: rook-ceph
spec:
  dnsNames:
  - ceph.syncroze.com
  privateKey:
    rotationPolicy: Always
  issuerRef:
    kind: ClusterIssuer
    name: letsencrypt-prod
  secretName: syncroze-tls

kubectl apply -f certificate-rook-ceph.yaml

# ingress-rook-ceph.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-rook-ceph
  namespace: rook-ceph
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
    nginx.org/ssl-redirect: "true"
    nginx.org/proxy-read-timeout: "3600s"
    nginx.org/proxy-send-timeout: "3600s"
    nginx.org/client-max-body-size: "0"
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - ceph.syncroze.com
    secretName: syncroze-tls
  rules:
  - host: ceph.syncroze.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: rook-ceph-mgr-dashboard
            port:
              number: 7000

kubectl apply -f ingress-rook-ceph.yaml

# username "admin" and password from below
kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath='{.data.password}' | base64 -d && echo

git clone --single-branch --branch v1.18.8 https://github.com/rook/rook.git
kubectl create -f rook/deploy/examples/toolbox.yaml

# Wait for toolbox to be Running
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- bash
ceph status
ceph osd status
ceph health detail
ceph df
rados df

# Below fixed the Too Many PGs warning
ceph osd df
ceph osd pool ls detail

ceph osd pool set ceph-objectstore.rgw.buckets.data pg_num 64
ceph osd pool set ceph-objectstore.rgw.buckets.data pgp_num 64

kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph mgr module disable prometheus

kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph crash archive-all

kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph dashboard set-prometheus-api-host \
http://kube-prometheus-stack-prometheus.monitoring.svc:9090