Puppet Class: profile::ceph::osd

Defined in:
modules/profile/manifests/ceph/osd.pp

Overview

Class: profile::ceph::osd

This profile configures Ceph object storage hosts with the osd daemon

Parameters:

  • openstack_controllers (Array[Stdlib::Fqdn]) (defaults to: lookup('profile::ceph::openstack_controllers'))
  • mon_hosts (Hash[String,Hash]) (defaults to: lookup('profile::ceph::mon::hosts'))
  • osd_hosts (Hash[String,Hash]) (defaults to: lookup('profile::ceph::osd::hosts'))
  • cluster_network (Stdlib::IP::Address) (defaults to: lookup('profile::ceph::cluster_network'))
  • public_network (Stdlib::IP::Address) (defaults to: lookup('profile::ceph::public_network'))
  • data_dir (Stdlib::Unixpath) (defaults to: lookup('profile::ceph::data_dir'))
  • fsid (String) (defaults to: lookup('profile::ceph::fsid'))
  • disk_models_without_write_cache (Array[String]) (defaults to: lookup('profile::ceph::osd::disk_models_without_write_cache'))
  • os_disks (Array[String]) (defaults to: lookup('profile::ceph::osd::os_disks'))
  • disks_io_scheduler (String) (defaults to: lookup('profile::ceph::osd::disks_io_scheduler', { default_value => 'mq-deadline'}))
  • ceph_repository_component (String) (defaults to: lookup('profile::ceph::ceph_repository_component'))
  • cinder_backup_nodes (Array[Stdlib::Fqdn]) (defaults to: lookup('profile::ceph::cinder_backup_nodes'))


4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
# File 'modules/profile/manifests/ceph/osd.pp', line 4

class profile::ceph::osd(
    Array[Stdlib::Fqdn]  $openstack_controllers = lookup('profile::ceph::openstack_controllers'),
    Hash[String,Hash]    $mon_hosts         = lookup('profile::ceph::mon::hosts'),
    Hash[String,Hash]    $osd_hosts         = lookup('profile::ceph::osd::hosts'),
    Stdlib::IP::Address  $cluster_network   = lookup('profile::ceph::cluster_network'),
    Stdlib::IP::Address  $public_network    = lookup('profile::ceph::public_network'),
    Stdlib::Unixpath     $data_dir          = lookup('profile::ceph::data_dir'),
    String               $fsid              = lookup('profile::ceph::fsid'),
    Array[String]        $disk_models_without_write_cache = lookup('profile::ceph::osd::disk_models_without_write_cache'),
    Array[String]        $os_disks = lookup('profile::ceph::osd::os_disks'),
    String               $disks_io_scheduler = lookup('profile::ceph::osd::disks_io_scheduler', { default_value => 'mq-deadline'}),
    String               $ceph_repository_component  = lookup('profile::ceph::ceph_repository_component'),
    Array[Stdlib::Fqdn]  $cinder_backup_nodes        = lookup('profile::ceph::cinder_backup_nodes'),
) {
    require profile::ceph::auth::deploy
    if ! defined(Ceph::Auth::Keyring['admin']) {
        notify{'profile::ceph::osd: Admin keyring not defined, things might not work as expected.': }
    }
    if ! defined(Ceph::Auth::Keyring['bootstrap-osd']) {
        notify{'profile::ceph::osd: bootstrap-osd keyring not defined, things might not work as expected.': }
    }

    # Ceph OSDs should use the performance governor, not the default 'powersave'
    # governor
    class { 'cpufrequtils': }

    # Each ceph osd server runs multiple daemons, each daemon listens on 6 ports
    # The ports can range anywhere between 6800 and 7100. This can be controlled
    # with the `ms bind port min` and `ms bind port max` ceph config parameters.

    # The cluster interface is used for OSD data replication and heartbeat network traffic
    interface::manual{ 'osd-cluster':
        interface => $osd_hosts["$::fqdn"]['cluster']['iface'],
    }
    interface::ip { 'osd-cluster-ip':
        interface => $osd_hosts["$::fqdn"]['cluster']['iface'],
        address   => $osd_hosts["$::fqdn"]['cluster']['addr'],
        prefixlen => $osd_hosts["$::fqdn"]['cluster']['prefix'],
        require   => Interface::Manual['osd-cluster'],
        before    => Class['ceph::common'],
    }

    # Tune the MTU on both the cluster and public network
    interface::setting { 'osd-cluster-mtu':
        interface => $osd_hosts["$::fqdn"]['cluster']['iface'],
        setting   => 'mtu',
        value     => '9000',
        before    => Class['ceph::common'],
        notify    => Exec['set-osd-cluster-mtu'],
    }
    interface::setting { 'osd-public-mtu':
        interface => $osd_hosts["$::fqdn"]['public']['iface'],
        setting   => 'mtu',
        value     => '9000',
        before    => Class['ceph::common'],
        notify    => Exec['set-osd-public-mtu'],
    }
    # Make sure the interface is in sync with configuration changes
    exec { 'set-osd-cluster-mtu':
        command     => "/usr/sbin/ip link set mtu 9000 ${osd_hosts[$facts['fqdn']]['cluster']['iface']}",
        refreshonly => true,
    }
    exec { 'set-osd-public-mtu':
        command     => "/usr/sbin/ip link set mtu 9000 ${osd_hosts[$facts['fqdn']]['public']['iface']}",
        refreshonly => true,
    }

    $ferm_cluster_srange = join($osd_hosts.map | $key, $value | { $value['cluster']['addr'] }, ' ')
    ferm::service { 'ceph_osd_cluster_range':
        proto  => 'tcp',
        port   => '6800:7100',
        srange => "(${ferm_cluster_srange})",
        drange => $osd_hosts["$::fqdn"]['cluster']['addr'],
        before => Class['ceph::common'],
    }

    include network::constants

    $client_networks = [
        $network::constants::all_network_subnets['production']['eqiad']['private']['labs-hosts1-b-eqiad']['ipv4'],
        $network::constants::all_network_subnets['production']['codfw']['private']['labs-hosts1-b-codfw']['ipv4'],
    ]

    $mon_addrs = $mon_hosts.map | $key, $value | { $value['public']['addr'] }
    $osd_addrs = $osd_hosts.map | $key, $value | { $value['public']['addr'] }
    $openstack_controller_ips = $openstack_controllers.map |$host| { ipresolve($host, 4) }
    $cinder_backup_nodes_ips  = $cinder_backup_nodes.map |$host| { ipresolve($host, 4) }
    $ferm_public_srange = join(concat($mon_addrs, $osd_addrs, $client_networks, $openstack_controller_ips, $cinder_backup_nodes_ips), ' ')
    ferm::service { 'ceph_osd_range':
        proto  => 'tcp',
        port   => '6800:7100',
        srange => "(${ferm_public_srange})",
        drange => $osd_hosts["$::fqdn"]['public']['addr'],
        before => Class['ceph::common'],
    }

    class { 'ceph::common':
        home_dir                  => $data_dir,
        ceph_repository_component => $ceph_repository_component,
    }

    class { 'ceph::config':
        cluster_network     => $cluster_network,
        enable_libvirt_rbd  => false,
        enable_v2_messenger => true,
        fsid                => $fsid,
        mon_hosts           => $mon_hosts,
        osd_hosts           => $osd_hosts,
        public_network      => $public_network,
    }

    # This adds latency stats between from this osd to the rest of the ceph fleet
    class { 'prometheus::node_pinger':
        nodes_to_ping => $osd_hosts.keys() + $mon_hosts.keys(),
    }

    $facts['disks'].each |String $device, Hash $device_info| {
        if ! ( $device in $os_disks) {
            if ('model' in $device_info and $device_info['model'] in $disk_models_without_write_cache) {
                exec { "Disable write cache on device /dev/${device}":
                    # 0->disable, 1->enable
                    command => "hdparm -W 0 /dev/${device}",
                    user    => 'root',
                    unless  => "hdparm -W /dev/${device} | grep write-caching | grep -q off",
                    path    => ['/usr/sbin', '/usr/bin'],
                }
            }

            exec { "Set IO scheduler on device /dev/${device} to ${disks_io_scheduler}":
                command => "echo ${disks_io_scheduler} > /sys/block/${device}/queue/scheduler",
                user    => 'root',
                unless  => "grep -q '\\[${disks_io_scheduler}\\]' /sys/block/${device}/queue/scheduler",
                path    => ['/usr/sbin', '/usr/bin'],
            }
        }
    }
}