Puppet Class: profile::druid::historical

Defined in:
modules/profile/manifests/druid/historical.pp

Overview

SPDX-License-Identifier: Apache-2.0 Class: profile::druid::historical

Parameters:

  • properties (Hash[String, Any]) (defaults to: lookup('profile::druid::historical::properties', {'default_value' => {}}))
  • env (Hash[String, String]) (defaults to: lookup('profile::druid::historical::env', {'default_value' => {}}))
  • daemon_autoreload (Boolean) (defaults to: lookup('profile::druid::daemons_autoreload', {'default_value' => true}))
  • firewall_access (Optional[Array[String]]) (defaults to: lookup('profile::druid::firewall_access'))
  • monitoring_enabled (Boolean) (defaults to: lookup('profile::druid::historical::monitoring_enabled', {'default_value' => false})) 


4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'modules/profile/manifests/druid/historical.pp', line 4

class profile::druid::historical(
    Hash[String, Any] $properties            = lookup('profile::druid::historical::properties', {'default_value' => {}}),
    Hash[String, String] $env                = lookup('profile::druid::historical::env', {'default_value' => {}}),
    Boolean $daemon_autoreload               = lookup('profile::druid::daemons_autoreload', {'default_value' => true}),
    Optional[Array[String]] $firewall_access = lookup('profile::druid::firewall_access'),
    Boolean $monitoring_enabled              = lookup('profile::druid::historical::monitoring_enabled', {'default_value' => false}),
) {

    require ::profile::druid::common

    # If monitoring is enabled, then include the monitoring profile and set $java_opts
    # for exposing the Prometheus JMX Exporter in the Druid Broker process.
    if $monitoring_enabled {
        require ::profile::druid::monitoring::historical
        $monitoring_java_opts = $::profile::druid::monitoring::historical::java_opts
    } else {
        $monitoring_java_opts = undef
    }

    # The suggestion for the Historical daemon is to set the
    # druid.processing.numThreads to ncores.
    # https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#process-specific-guidelines
    # This is useful since we have historicals running on different hw.
    # As consequence, the MaxDirectMemorySize java opt needs to be set according to a formula, otherwise
    # the Historical will refuse to start.
    if !has_key($properties, 'druid.processing.numThreads') {
        $extra_properties = {
            'druid.processing.numThreads' => $facts['processors']['count']
        }
        # max direct memory = druid.processing.buffer.sizeBytes[268,435,456] * (druid.processing.numMergeBuffers[10] + druid.processing.numThreads[64] + 1)
        $max_direct_memory = Integer($properties['druid.processing.buffer.sizeBytes']) * (Integer($properties['druid.processing.numMergeBuffers']) + Integer($facts['processors']['count']) + 1)
        $max_direct_memory_java_opt = "-XX:MaxDirectMemorySize=${max_direct_memory}"
    } else {
        $extra_properties = {}
        $max_direct_memory = undef
        $max_direct_memory_java_opt = undef
    }

    if $monitoring_java_opts or $max_direct_memory_java_opt {
        if $env['DRUID_EXTRA_JVM_OPTS'] {
            $druid_extra_env_var = {
                'DRUID_EXTRA_JVM_OPTS' => "${env['DRUID_EXTRA_JVM_OPTS']} ${max_direct_memory_java_opt} ${monitoring_java_opts}"
            }
        } else {
            $druid_extra_env_var = {
                'DRUID_EXTRA_JVM_OPTS' => "${max_direct_memory} ${monitoring_java_opts}"
            }
        }
    }

    $class_prefix = 'org.apache.druid'

    # Druid historical Service
    class { '::druid::historical':
        properties       => merge($properties, $extra_properties),
        env              => merge($env, $druid_extra_env_var),
        should_subscribe => $daemon_autoreload,
        logger_prefix    => $class_prefix,
    }

    firewall::service { 'druid-historical':
        proto    => 'tcp',
        port     => $::druid::historical::runtime_properties['druid.port'],
        src_sets => $firewall_access,
    }

    if $monitoring_enabled {
        nrpe::monitor_service { 'druid-historical':
            description  => 'Druid historical',
            nrpe_command => "/usr/lib/nagios/plugins/check_procs -c 1:1 -C java -a \'${class_prefix}.cli.Main server historical\'",
            critical     => false,
            notes_url    => 'https://wikitech.wikimedia.org/wiki/Analytics/Systems/Druid',
        }
    }
}