Puppet Class: profile::opensearch::cirrus::server

Defined in:
modules/profile/manifests/opensearch/cirrus/server.pp

Overview

SPDX-License-Identifier: Apache-2.0

This class configures OpenSearch for serving CirrusSearch

Parameters:

For documentation of parameters, see the opensearch profile.

Parameters:

  • cluster (String) (defaults to: lookup('cluster'))
  • ferm_srange (String) (defaults to: lookup('profile::opensearch::cirrus::ferm_srange'))
  • ferm_ro_srange (String) (defaults to: lookup('profile::opensearch::cirrus::ferm_ro_srange', { default_value => '' }))
  • expose_http (Boolean) (defaults to: lookup('profile::opensearch::cirrus::expose_http'))
  • storage_device (String) (defaults to: lookup('profile::opensearch::cirrus::storage_device'))
  • enable_remote_search (Boolean) (defaults to: lookup('profile::opensearch::cirrus::enable_remote_search'))
  • ssl_provider (Profile::Pki::Provider) (defaults to: lookup('profile::opensearch::cirrus::ssl_provider'))
  • base_data_dir (Stdlib::AbsolutePath) (defaults to: lookup('profile::opensearch::base_data_dir'))
  • certificate_domains (Array) (defaults to: lookup('profile::opensearch::cirrus::certificate_domains'))
  • enable_performance_cpu_governor (Boolean) (defaults to: lookup('profile::opensearch::cirrus::enable_performance_cpu_governor', { 'default_value' => false })) 


9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'modules/profile/manifests/opensearch/cirrus/server.pp', line 9

class profile::opensearch::cirrus::server(
    String $cluster = lookup('cluster'),
    String $ferm_srange = lookup('profile::opensearch::cirrus::ferm_srange'),
    String $ferm_ro_srange = lookup('profile::opensearch::cirrus::ferm_ro_srange', { default_value => '' }),
    Boolean $expose_http = lookup('profile::opensearch::cirrus::expose_http'),
    String $storage_device = lookup('profile::opensearch::cirrus::storage_device'),
    Boolean $enable_remote_search = lookup('profile::opensearch::cirrus::enable_remote_search'),
    Profile::Pki::Provider $ssl_provider = lookup('profile::opensearch::cirrus::ssl_provider'),
    Stdlib::AbsolutePath $base_data_dir = lookup('profile::opensearch::base_data_dir'),
    Array $certificate_domains = lookup('profile::opensearch::cirrus::certificate_domains'),
    Boolean $enable_performance_cpu_governor = lookup('profile::opensearch::cirrus::enable_performance_cpu_governor', { 'default_value' => false }),
) {

    if $enable_performance_cpu_governor {
        # enable CPU performance governor; see T386860
        class { 'cpufrequtils': }
    }

    # Also brings in ::profile::opensearch::server
    include ::profile::opensearch::monitoring::base_checks

    # syslog logstash transport type depends on this. See T225125.
    # TODO: Check if still necessary w/opensearch
    include ::profile::rsyslog::udp_json_logback_compat

    # nginx, which terminates tls for elasticsearch, needs `/etc/ssl/dhparam.pem` to be in place in order to function.
    class { '::sslcert::dhparam': }

    # Install curator for opensearch
    $apt_component = 'opensearch13'
    apt::repository { 'wikimedia-opensearch-plugins':
        uri        => 'http://apt.wikimedia.org/wikimedia',
        dist       => "${::lsbdistcodename}-wikimedia",
        components => "component/${apt_component}",
    }

    package { 'wmf-opensearch-search-plugins':
        ensure  => present,
        require => [Class['Java'], Package['opensearch']],
    }

    # Since the opensearch service is dynamically named after the cluster
    # name, and because there can be multiple opensearch services on the
    # same node we need to use collectors.
    Package['wmf-opensearch-search-plugins'] -> Service <| tag == 'opensearch_services' |>

    $::profile::opensearch::server::filtered_instances.each |$instance_title, $instance_params| {
        $cluster_name = $instance_params['cluster_name']
        $http_port = $instance_params['http_port']
        $tls_port = $instance_params['tls_port']
        opensearch::log::hot_threads_cluster { $cluster_name:
            http_port => $http_port,
        }

        # Also limit these checks to only the master nodes to reduce duplication
        # of these checks on all nodes until we find a better way to run these checks
        # only on icinga nodes
        if $facts['fqdn'] in $instance_params['unicast_hosts'] {
            opensearch::cross_cluster_settings { $instance_title:
                settings             => $::profile::opensearch::server::configured_instances,
                enable_remote_search => $enable_remote_search,
            }
        }
    }

    $read_ahead_kb = 16
    udev::rule { 'opensearch-readahead':
        content => "SUBSYSTEM==\"block\", KERNEL==\"${storage_device}\", ACTION==\"add|change\", ATTR{bdi/read_ahead_kb}=\"${read_ahead_kb}\"",
    }

    # BEGIN Temporary mitigation put in place for T264053
    # Source code lives here: https://gitlab.wikimedia.org/repos/search-platform/opensearch-madvise
    package {'opensearch-madvise':
        ensure => present,
    }

    # Add opensearch bin to root's PATH
    file_line { 'opensearch_bin_bashrc':
      ensure => present,
      path   => '/root/.bashrc',
      line   => "PATH=\${PATH}:/usr/share/opensearch/bin  # Managed by puppet",
    }

    # Wrapper script to run opensearch-madvise-random once per opensearch process, passing PID
    file { '/usr/local/bin/opensearch-disable-readahead.sh':
        ensure => file,
        owner  => 'root',
        group  => 'root',
        mode   => '0555',
        source => 'puppet:///modules/profile/opensearch/cirrus/opensearch-disable-readahead.sh',
    }

    systemd::timer::job { 'opensearch-disable-readahead':
        ensure      => absent,
        description => 'Disables readahead on all open files every 30 minutes to alleviate Cirrussearch / opensearch IO load spikes',
        command     => '/usr/local/bin/opensearch-disable-readahead.sh',
        user        => 'root',
        interval    => [{'start' => 'OnUnitActiveSec', 'interval' => '30min'}, {'start' => 'OnBootSec', 'interval' => '1min'}],
    }

    # Run the wrapper every 30 mins for each installed cluster
    $::profile::opensearch::server::filtered_instances.each |$instance_title, $instance_params| {
        $cluster_name = $instance_params['cluster_name']

        systemd::timer::job { "opensearch-disable-readahead-${cluster_name}":
            description => 'Disables readahead on all open files every 30 minutes to alleviate Cirrussearch / opensearch IO load spikes',
            command     => "/usr/local/bin/opensearch-disable-readahead.sh ${cluster_name} ${base_data_dir}",
            user        => 'root',
            interval    => [{'start' => 'OnUnitActiveSec', 'interval' => '30min'}, {'start' => 'OnBootSec', 'interval' => '1min'}],
        }
    }

    # END   Temporary mitigation put in place for T264053

    # Install custom prometheus data collection. Standard data collection is
    # configured from profile::opensearch::server.
    $::profile::opensearch::server::filtered_instances.reduce(9120) |$prometheus_port, $kv_pair| {
        $instance_params = $kv_pair[1]
        $http_port = $instance_params['http_port']
        $indices_to_monitor = $instance_params['indices_to_monitor'] ? {
            undef   => [],
            default => $instance_params['indices_to_monitor']
        }

        profile::prometheus::wmf_elasticsearch_exporter { "${::hostname}:${http_port}":
            prometheus_port    => $prometheus_port,
            elasticsearch_port => $http_port,
            indices_to_monitor => $indices_to_monitor,
        }
        $prometheus_port + 1
    }

    motd::script { 'cluster_memberships':
      ensure   => present,
      priority => 96,
      source   => 'puppet:///modules/opensearch/opensearch.motd',
    }

    # symlink elasticsearch to opensearch, so we can run our rolling-operation
    # cookbook without patching Spicerack
    # (ref https://gerrit.wikimedia.org/r/plugins/gitiles/operations/software/spicerack/+/refs/heads
    # /master/spicerack/elasticsearch_cluster.py#111
    file { '/etc/elasticsearch':
        ensure  => link,
        target  => '/etc/opensearch',
        require => File['/etc/opensearch/instances'],
    }

    # TLS configuration
    # For legacy reasons this reuses elasticsearch::tlsproxy until we can
    # enable the opensearch security plugin for native tls.
    $::profile::opensearch::server::filtered_instances.each |$instance_title, $instance_params| {
        $cluster_name = $instance_params['cluster_name']
        $http_port = $instance_params['http_port']
        $tls_port = $instance_params['tls_port']
        $tls_ro_port = $instance_params['tls_ro_port']

        if $expose_http {
            ferm::service { "opensearch-http-${http_port}":
                proto   => 'tcp',
                port    => $http_port,
                notrack => true,
                srange  => $ferm_srange,
            }
        }

        ferm::service { "opensearch-https-${tls_port}":
            proto  => 'tcp',
            port   => $tls_port,
            srange => $ferm_srange,
        }
        if $ssl_provider == 'acme_chief' {
            $proxy_cert_params = {
                acme_chief        => true,
                acme_certname     => $cluster,
                server_name       => $instance_params['certificate_name'],
            }
        }

        if $ssl_provider == 'cfssl' {
            $cfssl_paths = profile::pki::get_cert('discovery', $facts['networking']['fqdn'], {
                hosts => $certificate_domains,
            })

            $proxy_cert_params = {
                'cfssl_paths'  => $cfssl_paths,
                server_aliases => $certificate_domains,
            }
        }

        $proxy_params = merge($proxy_cert_params, {
            upstream_port => $http_port,
            tls_port      => $tls_port,
            enable_http2  => false,
        })

        elasticsearch::tlsproxy { $cluster_name:
            * => $proxy_params,
        }
        if $tls_ro_port {
            if empty($ferm_ro_srange) {
                fail('Read only port specified without a read only srange')
            }

            ferm::service { "opensearch-ro-https-${tls_ro_port}":
                proto  => 'tcp',
                port   => $tls_ro_port,
                srange => $ferm_ro_srange,
            }

            elasticsearch::tlsproxy { "${cluster_name}-ro":
                * => merge($proxy_params, {
                    tls_port  => $tls_ro_port,
                    read_only => true,
                })
            }
        }
    }
}