Puppet Class: profile::apifeatureusage::logstash

Defined in:
modules/profile/manifests/apifeatureusage/logstash.pp

Overview

Class: profile::apifeatureusage::logstash

Loads api-feature-usage logs into ElasticSearch for MediaWiki Extension:ApiFeatureUsage

Parameters:

  • targets (Array[Stdlib::Host]) (defaults to: lookup('profile::apifeatureusage::logstash::targets'))
  • curator_actions (Hash) (defaults to: lookup('profile::apifeatureusage::logstash::curator_actions'))
  • input_kafka_consumer_group_id (Optional[String]) (defaults to: lookup('profile::apifeatureusage::logstash::input_kafka_consumer_group_id', { default_value => undef }))
  • jobs_host (Optional[Stdlib::Fqdn]) (defaults to: lookup('profile::apifeatureusage::logstash::jobs_host', { default_value => undef }))
  • jmx_exporter_port (Stdlib::Port) (defaults to: lookup('profile::apifeatureusage::logstash::jmx_exporter_port', { default_value => 7800 }))
  • java_home (Optional[Stdlib::Unixpath]) (defaults to: lookup('profile::apifeatureusage::logstash::java_home', { default_value => undef })) 


6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'modules/profile/manifests/apifeatureusage/logstash.pp', line 6

class profile::apifeatureusage::logstash (
  Array[Stdlib::Host]        $targets                              = lookup('profile::apifeatureusage::logstash::targets'),
  Hash                       $curator_actions                      = lookup('profile::apifeatureusage::logstash::curator_actions'),
  Optional[String]           $input_kafka_consumer_group_id        = lookup('profile::apifeatureusage::logstash::input_kafka_consumer_group_id', { default_value => undef }),
  Optional[Stdlib::Fqdn]     $jobs_host                            = lookup('profile::apifeatureusage::logstash::jobs_host',                     { default_value => undef }),
  Stdlib::Port               $jmx_exporter_port                    = lookup('profile::apifeatureusage::logstash::jmx_exporter_port',             { default_value => 7800  }),
  Optional[Stdlib::Unixpath] $java_home                            = lookup('profile::apifeatureusage::logstash::java_home',                     { default_value => undef }),
) {
  require ::profile::java
  include profile::base::certificates
  $ssl_truststore_location = profile::base::certificates::get_trusted_ca_jks_path()
  $ssl_truststore_password = profile::base::certificates::get_trusted_ca_jks_password()
  $manage_truststore = false

  class { 'elasticsearch::curator': }

  $config_dir = '/etc/prometheus'
  $jmx_exporter_config_file = "${config_dir}/logstash_jmx_exporter.yaml"

  # Prometheus JVM metrics
  profile::prometheus::jmx_exporter { "logstash_collector_${::hostname}":
    hostname    => $::hostname,
    port        => $jmx_exporter_port,
    config_file => $jmx_exporter_config_file,
    config_dir  => $config_dir,
    source      => 'puppet:///modules/profile/logstash/jmx_exporter.yaml',
  }

  sysctl::parameters { 'logstash_receive_skbuf':
    values => {
      'net.core.rmem_default' => 8388608,
    },
  }

  $apt_component = 'elastic710'
  apt::repository { 'wikimedia-elastic':
    uri        => 'http://apt.wikimedia.org/wikimedia',
    dist       => "${::lsbdistcodename}-wikimedia",
    components => "thirdparty/${apt_component}",
    before     => Class['::logstash'],
  }

  class { '::logstash':
    jmx_exporter_port   => $jmx_exporter_port,
    jmx_exporter_config => $jmx_exporter_config_file,
    pipeline_workers    => $::processorcount * 2,
    java_package        => 'openjdk-11-jdk',
    logstash_package    => 'logstash-oss',
    logstash_version    => 7,
    log_format          => 'json',
    gc_log              => false,
    java_home           => pick($java_home, $profile::java::default_java_home),
    manage_service      => false,
  }

  systemd::service { 'logstash':
      ensure   => present,
      content  => init_template('logstash', 'systemd_override'),
      override => true,
      restart  => true,
  }

  # Ship logstash service logs to ELK
  rsyslog::input::file { 'logstash-json':
    path => '/var/log/logstash/logstash-json.log'
  }

  # Inputs
  logstash::input::kafka { 'rsyslog-udp-localhost-eqiad':
    kafka_cluster_name                    => 'logging-eqiad',
    topics_pattern                        => 'udp_localhost-.*',
    group_id                              => $input_kafka_consumer_group_id,
    type                                  => 'syslog',
    tags                                  => ['input-kafka-rsyslog-udp-localhost', 'rsyslog-udp-localhost', 'kafka'],
    codec                                 => 'json',
    security_protocol                     => 'SSL',
    ssl_truststore_location               => $ssl_truststore_location,
    ssl_truststore_password               => $ssl_truststore_password,
    manage_truststore                     => $manage_truststore,
    ssl_endpoint_identification_algorithm => '',
    consumer_threads                      => 3,
  }

  logstash::input::kafka { 'rsyslog-udp-localhost-codfw':
    kafka_cluster_name                    => 'logging-codfw',
    topics_pattern                        => 'udp_localhost-.*',
    group_id                              => $input_kafka_consumer_group_id,
    type                                  => 'syslog',
    tags                                  => ['input-kafka-rsyslog-udp-localhost', 'rsyslog-udp-localhost', 'kafka'],
    codec                                 => 'json',
    security_protocol                     => 'SSL',
    ssl_truststore_location               => $ssl_truststore_location,
    ssl_truststore_password               => $ssl_truststore_password,
    manage_truststore                     => $manage_truststore,
    ssl_endpoint_identification_algorithm => '',
    consumer_threads                      => 3,
  }

  # Filters
  file { '/etc/logstash/conf.d':
    ensure  => directory,
    source  => 'puppet:///modules/profile/apifeatureusage/filters',
    owner   => 'logstash',
    group   => 'logstash',
    mode    => '0440',
    recurse => true,
    purge   => true,
    force   => true,
    notify  => Service['logstash'],
  }

  # Outputs
  $targets.each |Stdlib::Host $cluster| {
    logstash::output::elasticsearch { "apifeatureusage-${cluster}":
      host            => $cluster,
      index           => 'apifeatureusage-%{+YYYY.MM.dd}',
      guard_condition => '[type] == "api-feature-usage-sanitized"',
      priority        => 95,
      template        => '/etc/logstash/templates/apifeatureusage_7.0-1.json',
      document_type   => '_doc',
      require         => File['/etc/logstash/templates'],
    }

    # Curator
    $dc = $cluster.split('[.]')[-2]
    $cluster_name = "production-search-${dc}"
    $curator_hosts = [$cluster]
    $http_port = 9200
    if $jobs_host == $::fqdn {
      elasticsearch::curator::config { $cluster_name:
        content => template('elasticsearch/curator_cluster.yaml.erb'),
      }

      elasticsearch::curator::job { "apifeatureusage_${dc}":
        cluster_name => $cluster_name,
        actions      => $curator_actions,
      }
    } else {
      elasticsearch::curator::job { "apifeatureusage_${dc}":
        ensure       => 'absent',
        cluster_name => $cluster_name,
      }
    }
  }

  # Templates
  file { '/etc/logstash/templates':
    ensure  => directory,
    source  => 'puppet:///modules/profile/apifeatureusage/templates',
    owner   => 'logstash',
    group   => 'logstash',
    mode    => '0444',
    recurse => true,
    purge   => true,
    force   => true,
    notify  => Service['logstash'],
  }
}