Puppet Class: profile::elasticsearch

Defined in:
modules/profile/manifests/elasticsearch.pp

Overview

This class configures elasticsearch

Parameters:

  • $dc_settings: data center specific overrides for ::elasticsearch::instance

  • $common_settings: global overrides for ::elasticsearch::instance

  • $logstash_host: Host to send logs to

  • $logstash_gelf_port: Tcp port on $logstash_host to send gelf formatted logs to.

  • $logstash_logback_port: Tcp port on localhost to send structured logs to.

  • $logstash_transport: Transport mechanism for logs.

  • $rack: Rack server is in. Used for allocation awareness.

  • $row: Row server is in. Used for allocation awareness.

  • $version: version of the package to install

  • $config_version: configure for this versionof elastic. This is independant from $version as during the transition

    ES5 -> ES6, we need to deploy the new package before applying the configuration.
    TODO: remove this configuration option once all instances have been migrated to ES6.
  • $java_home: optionally specify the JAVA_HOME path in the elasticsearch systemd unit.

    note: used only with ES7.

Parameters:

  • instances (Hash[String, Elasticsearch::InstanceParams]) (defaults to: hiera('profile::elasticsearch::instances'))
  • dc_settings (Elasticsearch::InstanceParams) (defaults to: hiera('profile::elasticsearch::dc_settings'))
  • common_settings (Elasticsearch::InstanceParams) (defaults to: hiera('profile::elasticsearch::common_settings'))
  • base_data_dir (Stdlib::AbsolutePath) (defaults to: hiera('profile::elasticsearch::base_data_dir'))
  • logstash_host (String) (defaults to: hiera('logstash_host'))
  • logstash_gelf_port (Stdlib::Port) (defaults to: hiera('logstash_gelf_port'))
  • logstash_logback_port (Stdlib::Port) (defaults to: hiera('logstash_logback_port'))
  • logstash_transport (Enum['Gelf', 'syslog']) (defaults to: hiera('profile::elasticsearch::logstash_transport', 'Gelf'))
  • rack (String) (defaults to: hiera('profile::elasticsearch::rack'))
  • row (String) (defaults to: hiera('profile::elasticsearch::row'))
  • version (Enum['5.5', '5.6', '6.5', '7.4', '7.8']) (defaults to: hiera('profile::elasticsearch::version', '5.5'))
  • config_version (Enum['5', '6', '7']) (defaults to: hiera('profile::elasticsearch::config_version', '5'))
  • java_home (Optional[String]) (defaults to: lookup('profile::elasticsearch::java_home', { 'default_value' => undef }))


21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
# File 'modules/profile/manifests/elasticsearch.pp', line 21

class profile::elasticsearch(
    Hash[String, Elasticsearch::InstanceParams] $instances = hiera('profile::elasticsearch::instances'),
    Elasticsearch::InstanceParams $dc_settings = hiera('profile::elasticsearch::dc_settings'),
    Elasticsearch::InstanceParams $common_settings = hiera('profile::elasticsearch::common_settings'),
    Stdlib::AbsolutePath $base_data_dir = hiera('profile::elasticsearch::base_data_dir'),

    String $logstash_host = hiera('logstash_host'),
    Stdlib::Port $logstash_gelf_port = hiera('logstash_gelf_port'),
    Stdlib::Port $logstash_logback_port = hiera('logstash_logback_port'),
    Enum['Gelf', 'syslog'] $logstash_transport = hiera('profile::elasticsearch::logstash_transport', 'Gelf'),
    String $rack = hiera('profile::elasticsearch::rack'),
    String $row = hiera('profile::elasticsearch::row'),
    Enum['5.5', '5.6', '6.5', '7.4', '7.8'] $version = hiera('profile::elasticsearch::version', '5.5'),
    Enum['5', '6', '7'] $config_version = hiera('profile::elasticsearch::config_version', '5'),
    Optional[String] $java_home = lookup('profile::elasticsearch::java_home', { 'default_value' => undef }),
) {

    require ::profile::java

    # Rather than asking hiera to magically merge these settings for us, we
    # explicitly take two sets of defaults for global defaults and per-dc
    # defaults. Per cluster overrides are then provided in $instances.
    $settings = $common_settings + $dc_settings

    # Sane defaults to simplify single instance configuration
    $defaults_for_single_instance = {
        http_port          => 9200,
        transport_tcp_port => 9300,
    }

    # Resolve instance configuration here, rather than in the elasticsearch
    # define, so we have access to final configuration, such as http ports,
    # for configuring firewalls and such.
    # Also accessed from profile::elasticsearch::* for firewalls, proxies, etc.
    $configured_instances = empty($instances) ? {
        true    => {
            'default' => $defaults_for_single_instance + $settings,
        },
        default => $instances.reduce({}) |$agg, $kv_pair| {
            $instance_title = $kv_pair[0]
            $instance_params = $kv_pair[1]
            $final_params = $settings + $instance_params
            $agg + [$instance_title, $final_params]
        }
    }

    # Get all unique elastic nodes across all instances.
    # This is needed to set ferm rules for cross cluster communication
    $all_elastic_nodes = unique($configured_instances.reduce([]) |$result, $instance_params| {
        $result + $instance_params[1]['cluster_hosts']
    })

    # filter out instances that should not be deployed on this node
    # this is used for the cirrus clusters, where multiple sub clusters are defined
    # on a subset of all nodes.
    #
    # note in filter |$instance| below, $instance is an array [ key, value ]
    # see https://puppet.com/docs/puppet/5.5/function.html#filter for details
    $filtered_instances = $configured_instances.filter |$instance| { $facts['fqdn'] in $instance[1]['cluster_hosts'] }

    # Accessed from profile::elasticsearch::* for firewalls, proxies, etc.
    $filtered_instances.each |$instance_title, $instance_params| {
        $transport_tcp_port = pick_default($instance_params['transport_tcp_port'], 9300)
        $elastic_nodes_ferm = join(pick_default($all_elastic_nodes, [$::fqdn]), ' ')

        ferm::service { "elastic-inter-node-${transport_tcp_port}":
            proto   => 'tcp',
            port    => $transport_tcp_port,
            notrack => true,
            srange  => "@resolve((${elastic_nodes_ferm}))",
        }

        # Let deploy this check per node/ per cluster. see T231516
        icinga::monitor::elasticsearch::old_jvm_gc_checks { $instance_params['cluster_name']: }
    }

    $apt_component = $version ? {
        '5.5' => 'elastic55',
        '5.6' => 'elastic56',
        '6.5' => 'elastic65',
        '7.4' => 'elastic74',
        '7.8' => 'elastic78',
    }

    apt::repository { 'wikimedia-elastic':
        uri        => 'http://apt.wikimedia.org/wikimedia',
        dist       => "${::lsbdistcodename}-wikimedia",
        components => "component/${apt_component} thirdparty/${apt_component}",
        before     => Class['::elasticsearch'],
    }

    apt::repository { 'wikimedia-curator':
        uri        => 'http://apt.wikimedia.org/wikimedia',
        dist       => "${::lsbdistcodename}-wikimedia",
        components => 'thirdparty/elasticsearch-curator5',
        before     => Class['::elasticsearch::curator'],
    }

    # ensure that apt is refreshed before installing elasticsearch
    Exec['apt-get update'] -> Class['::elasticsearch']

    # Install
    class { '::elasticsearch':
        version               => $config_version,
        instances             => $filtered_instances,
        base_data_dir         => $base_data_dir,
        logstash_host         => $logstash_host,
        logstash_gelf_port    => $logstash_gelf_port,
        logstash_logback_port => $logstash_logback_port,
        logstash_transport    => $logstash_transport,
        rack                  => $rack,
        row                   => $row,
        java_home             => $java_home,
    }
}