Puppet Class: profile::ceph::osd

Defined in:
modules/profile/manifests/ceph/osd.pp

Overview

Class: profile::ceph::osd

This profile configures Ceph object storage hosts with the osd daemon

Parameters:

  • mon_hosts (Hash[String,Hash]) (defaults to: lookup('profile::ceph::mon::hosts'))
  • osd_hosts (Hash[String,Hash]) (defaults to: lookup('profile::ceph::osd::hosts'))
  • admin_keyring (Stdlib::AbsolutePath) (defaults to: lookup('profile::ceph::admin_keyring'))
  • cluster_network (Stdlib::IP::Address) (defaults to: lookup('profile::ceph::cluster_network'))
  • public_network (Stdlib::IP::Address) (defaults to: lookup('profile::ceph::public_network'))
  • data_dir (Stdlib::Unixpath) (defaults to: lookup('profile::ceph::data_dir'))
  • admin_keydata (String) (defaults to: lookup('profile::ceph::admin_keydata'))
  • fsid (String) (defaults to: lookup('profile::ceph::fsid'))
  • bootstrap_keyring (Stdlib::AbsolutePath) (defaults to: lookup('profile::ceph::osd::bootstrap_keyring'))
  • bootstrap_keydata (String) (defaults to: lookup('profile::ceph::osd::bootstrap_keydata'))


4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
# File 'modules/profile/manifests/ceph/osd.pp', line 4

class profile::ceph::osd(
    Hash[String,Hash]    $mon_hosts         = lookup('profile::ceph::mon::hosts'),
    Hash[String,Hash]    $osd_hosts         = lookup('profile::ceph::osd::hosts'),
    Stdlib::AbsolutePath $admin_keyring     = lookup('profile::ceph::admin_keyring'),
    Stdlib::IP::Address  $cluster_network   = lookup('profile::ceph::cluster_network'),
    Stdlib::IP::Address  $public_network    = lookup('profile::ceph::public_network'),
    Stdlib::Unixpath     $data_dir          = lookup('profile::ceph::data_dir'),
    String               $admin_keydata     = lookup('profile::ceph::admin_keydata'),
    String               $fsid              = lookup('profile::ceph::fsid'),
    Stdlib::AbsolutePath $bootstrap_keyring = lookup('profile::ceph::osd::bootstrap_keyring'),
    String               $bootstrap_keydata = lookup('profile::ceph::osd::bootstrap_keydata'),
) {
    include ::network::constants
    # Limit the client connections to the hypervisors in eqiad and codfw
    $client_networks = [
        $network::constants::all_network_subnets['production']['eqiad']['private']['labs-hosts1-b-eqiad']['ipv4'],
        $network::constants::all_network_subnets['production']['codfw']['private']['labs-hosts1-b-codfw']['ipv4'],
    ]

    # Each ceph osd server runs multiple daemons, each daemon listens on 6 ports
    # The ports can range anywhere between 6800 and 7100. This can be controlled
    # with the `ms bind port min` and `ms bind port max` ceph config parameters.

    # The cluster interface is used for OSD data replication and heartbeat network traffic
    interface::manual{ 'osd-cluster':
        interface => $osd_hosts["$::fqdn"]['cluster']['iface'],
    }
    interface::ip { 'osd-cluster-ip':
        interface => $osd_hosts["$::fqdn"]['cluster']['iface'],
        address   => $osd_hosts["$::fqdn"]['cluster']['addr'],
        prefixlen => $osd_hosts["$::fqdn"]['cluster']['prefix'],
        require   => Interface::Manual['osd-cluster'],
        before    => Class['ceph::common'],
    }

    # Tune the MTU on both the cluster and public network
    interface::setting { 'osd-cluster-mtu':
        interface => $osd_hosts["$::fqdn"]['cluster']['iface'],
        setting   => 'mtu',
        value     => '9000',
        before    => Class['ceph::common'],
        notify    => Exec['set-osd-cluster-mtu'],
    }
    interface::setting { 'osd-public-mtu':
        interface => $osd_hosts["$::fqdn"]['public']['iface'],
        setting   => 'mtu',
        value     => '9000',
        before    => Class['ceph::common'],
        notify    => Exec['set-osd-public-mtu'],
    }
    # Make sure the interface is in sync with configuration changes
    exec { 'set-osd-cluster-mtu':
        command     => "/usr/sbin/ip link set mtu 9000 ${osd_hosts["$::fqdn"]['cluster']['iface']}",
        refreshonly => true,
    }
    exec { 'set-osd-public-mtu':
        command     => "/usr/sbin/ip link set mtu 9000 ${osd_hosts["$::fqdn"]['public']['iface']}",
        refreshonly => true,
    }

    $ferm_cluster_srange = join($osd_hosts.map | $key, $value | { $value['cluster']['addr'] }, ' ')
    ferm::service { 'ceph_osd_cluster_range':
        proto  => 'tcp',
        port   => '6800:7100',
        srange => "(${ferm_cluster_srange})",
        drange => $osd_hosts["$::fqdn"]['cluster']['addr'],
        before => Class['ceph::common'],
    }

    # The public network is used for communication between Ceph serivces and client traffic
    $mon_addrs = $mon_hosts.map | $key, $value | { $value['public']['addr'] }
    $osd_addrs = $osd_hosts.map | $key, $value | { $value['public']['addr'] }
    $ferm_public_srange = join(concat($mon_addrs, $osd_addrs, $client_networks), ' ')
    ferm::service { 'ceph_osd_range':
        proto  => 'tcp',
        port   => '6800:7100',
        srange => "(${ferm_public_srange})",
        drange => $osd_hosts["$::fqdn"]['public']['addr'],
        before => Class['ceph::common'],
    }

    if os_version('debian == buster') {
        apt::repository { 'thirdparty-ceph-nautilus-buster':
            uri        => 'http://apt.wikimedia.org/wikimedia',
            dist       => 'buster-wikimedia',
            components => 'thirdparty/ceph-nautilus-buster',
            source     => false,
            before     => Class['ceph::common'],
        }
    }

    class { 'ceph::common':
        home_dir => $data_dir,
    }

    class { 'ceph::config':
        cluster_network     => $cluster_network,
        enable_libvirt_rbd  => false,
        enable_v2_messenger => true,
        fsid                => $fsid,
        mon_hosts           => $mon_hosts,
        osd_hosts           => $osd_hosts,
        public_network      => $public_network,
    }

    class { 'ceph::admin':
        admin_keyring => $admin_keyring,
        admin_keydata => $admin_keydata,
        data_dir      => $data_dir,
    }

    # We need this to finish initial osd setup
    ceph::keyring { 'client.bootstrap-osd':
        keyring => $bootstrap_keyring,
        keydata => $bootstrap_keydata,
    }
}