Puppet Class: cdh::hadoop::jmxtrans::datanode

Defined in:
modules/cdh/manifests/hadoop/jmxtrans/datanode.pp

Overview

Class cdh::hadoop::jmxtrans::datanode

Sets up a jmxtrans instance for a Hadoop ResourceManager running on the current host. Note: This requires the jmxtrans puppet module found at github.com/wikimedia/puppet-jmxtrans.

Parameters

$jmx_port - DataNode JMX port. Default: 9981 $ganglia - Ganglia host:port $graphite - Graphite host:port $statsd - statsd host:port $outfile - outfile to which Kafka stats will be written. $objects - objects parameter to pass to jmxtrans::metrics. Only use

this if you need to override the default ones that this
class provides.

Usage

class { 'cdh::hadoop::jmxtrans::datanode':

ganglia => 'ganglia.example.org:8649'

}

Parameters:

  • jmx_port (Any) (defaults to: 9981)
  • ganglia (Any) (defaults to: undef)
  • graphite (Any) (defaults to: undef)
  • statsd (Any) (defaults to: undef)
  • outfile (Any) (defaults to: undef)
  • objects (Any) (defaults to: undef)


22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
# File 'modules/cdh/manifests/hadoop/jmxtrans/datanode.pp', line 22

class cdh::hadoop::jmxtrans::datanode(
    $jmx_port       = 9981,
    $ganglia        = undef,
    $graphite       = undef,
    $statsd         = undef,
    $outfile        = undef,
    $objects        = undef,
) {
    $jmx = "${::fqdn}:${jmx_port}"
    $group_name = 'Hadoop.DataNode'

    # query for metrics from Hadoop DataNode's JVM
    jmxtrans::metrics::jvm { 'hadoop-yarn-datanode':
        jmx          => $jmx,
        group_prefix => "${group_name}.",
        outfile      => $outfile,
        ganglia      => $ganglia,
        graphite     => $graphite,
        statsd       => $statsd,
    }


    $datanode_objects = $objects ? {
        # if $objects was not set, then use this as the
        # default set of JMX MBean objects to query.
        undef   => [
            {
                'name'          =>          'Hadoop:name=DataNodeActivity-analytics1011.eqiad.wmnet-50010,service=DataNode',
                'resultAlias'   => "${group_name}.Activity",
                'attrs'         => {
                    'BlockChecksumOpAvgTime'                     => { 'slope' => 'both' },
                    'BlockChecksumOpNumOps'                      => { 'slope' => 'positive' },
                    'BlockReportsAvgTime'                        => { 'slope' => 'both' },
                    'BlockReportsNumOps'                         => { 'slope' => 'positive' },
                    'BlockVerificationFailures'                  => { 'slope' => 'positive' },
                    'BlocksGetLocalPathInfo'                     => { 'slope' => 'both' },
                    'BlocksRead'                                 => { 'slope' => 'positive' },
                    'BlocksRemoved'                              => { 'slope' => 'positive' },
                    'BlocksReplicated'                           => { 'slope' => 'positive' },
                    'BlocksVerified'                             => { 'slope' => 'positive' },
                    'BlocksWritten'                              => { 'slope' => 'positive' },
                    'BytesRead'                                  => { 'slope' => 'positive' },
                    'BytesWritten'                               => { 'slope' => 'positive' },
                    'CopyBlockOpAvgTime'                         => { 'slope' => 'both' },
                    'CopyBlockOpNumOps'                          => { 'slope' => 'positive' },
                    'FlushNanosAvgTime'                          => { 'slope' => 'both' },
                    'FlushNanosNumOps'                           => { 'slope' => 'positive' },
                    'FsyncCount'                                 => { 'slope' => 'positive' },
                    'FsyncNanosAvgTime'                          => { 'slope' => 'both' },
                    'FsyncNanosNumOps'                           => { 'slope' => 'positive' },
                    'HeartbeatsAvgTime'                          => { 'slope' => 'both' },
                    'HeartbeatsNumOps'                           => { 'slope' => 'positive' },
                    'PacketAckRoundTripTimeNanosAvgTime'         => { 'slope' => 'both' },
                    'PacketAckRoundTripTimeNanosNumOps'          => { 'slope' => 'positive' },
                    'ReadBlockOpAvgTime'                         => { 'slope' => 'both' },
                    'ReadBlockOpNumOps'                          => { 'slope' => 'positive' },
                    'ReadsFromLocalClient'                       => { 'slope' => 'positive' },
                    'ReadsFromRemoteClient'                      => { 'slope' => 'positive' },
                    'ReplaceBlockOpAvgTime'                      => { 'slope' => 'both' },
                    'ReplaceBlockOpNumOps'                       => { 'slope' => 'positive' },
                    'SendDataPacketBlockedOnNetworkNanosAvgTime' => { 'slope' => 'both' },
                    'SendDataPacketBlockedOnNetworkNanosNumOps'  => { 'slope' => 'positive' },
                    'SendDataPacketTransferNanosAvgTime'         => { 'slope' => 'both' },
                    'SendDataPacketTransferNanosNumOps'          => { 'slope' => 'positive' },
                    'VolumeFailures'                             => { 'slope' => 'positive' },
                    'WriteBlockOpAvgTime'                        => { 'slope' => 'both' },
                    'WriteBlockOpNumOps'                         => { 'slope' => 'positive' },
                    'WritesFromLocalClient'                      => { 'slope' => 'both' },
                    'WritesFromRemoteClient'                     => { 'slope' => 'both' },
                },
            },

            {
                'name'          =>          'Hadoop:name=FSDatasetState-DS-443972865-10.64.36.111-50010-1377634552935,service=DataNode',
                'resultAlias'   => "${group_name}.FSDatasetState",
                'attrs'         => {
                    'Capacity'                                  => { 'slope' => 'both' },
                    'DfsUsed'                                   => { 'slope' => 'both' },
                    'NumFailedVolumes'                          => { 'slope' => 'both' },
                    'Remaining'                                 => { 'slope' => 'both' },
                    'StorageInfo'                               => { 'slope' => 'both' },
                },
            },

            {
                'name' =>          'Hadoop:name=JvmMetrics,service=DataNode',
                'resultAlias'   => "${group_name}.JvmMetrics",
                'attrs'         => {
                    'GcCount'                                   => { 'slope' => 'positive' },
                    'GcCountPS MarkSweep'                       => { 'slope' => 'positive' },
                    'GcCountPS Scavenge'                        => { 'slope' => 'positive' },
                    'GcTimeMillis'                              => { 'slope' => 'both' },
                    'GcTimeMillisPS MarkSweep'                  => { 'slope' => 'both' },
                    'GcTimeMillisPS Scavenge'                   => { 'slope' => 'both' },
                    'LogError'                                  => { 'slope' => 'positive' },
                    'LogFatal'                                  => { 'slope' => 'positive' },
                    'LogInfo'                                   => { 'slope' => 'both' },
                    'LogWarn'                                   => { 'slope' => 'positive' },
                    'MemHeapCommittedM'                         => { 'slope' => 'both' },
                    'MemHeapUsedM'                              => { 'slope' => 'both' },
                    'MemNonHeapCommittedM'                      => { 'slope' => 'both' },
                    'MemNonHeapUsedM'                           => { 'slope' => 'both' },
                    'ThreadsBlocked'                            => { 'slope' => 'both' },
                    'ThreadsNew'                                => { 'slope' => 'both' },
                    'ThreadsRunnable'                           => { 'slope' => 'both' },
                    'ThreadsTerminated'                         => { 'slope' => 'both' },
                    'ThreadsTimedWaiting'                       => { 'slope' => 'both' },
                    'ThreadsWaiting'                            => { 'slope' => 'both' },
                },
            },

            {
                'name' =>          'Hadoop:name=RpcActivityForPort50020,service=DataNode',
                'resultAlias'   => "${group_name}.RpcActivityForPort50020",
                'attrs'         => {
                    'CallQueueLength'                           => { 'slope' => 'both' },
                    'NumOpenConnections'                        => { 'slope' => 'both' },
                    'ReceivedBytes'                             => { 'slope' => 'positive' },
                    'RpcAuthenticationFailures'                 => { 'slope' => 'positive' },
                    'RpcAuthenticationSuccesses'                => { 'slope' => 'positive' },
                    'RpcAuthorizationFailures'                  => { 'slope' => 'positive' },
                    'RpcAuthorizationSuccesses'                 => { 'slope' => 'positive' },
                    'RpcProcessingTimeAvgTime'                  => { 'slope' => 'both' },
                    'RpcProcessingTimeNumOps'                   => { 'slope' => 'positive' },
                    'RpcQueueTimeAvgTime'                       => { 'slope' => 'both' },
                    'RpcQueueTimeNumOps'                        => { 'slope' => 'positive' },
                    'SentBytes'                                 => { 'slope' => 'positive' },

                },
            },
            {
                'name' =>          'Hadoop:name=RpcDetailedActivityForPort50020,service=DataNode',
                'resultAlias'   => "${group_name}.RpcDetailedActivityForPort50020",
                'attrs'         => {
                    'InitReplicaRecoveryAvgTime'                => { 'slope' => 'both' },
                    'InitReplicaRecoveryNumOps'                 => { 'slope' => 'positive' },
                    'UpdateReplicaUnderRecoveryAvgTime'         => { 'slope' => 'both' },
                    'UpdateReplicaUnderRecoveryNumOps'          => { 'slope' => 'positive' },
                },
            },
        ],
        # else use $objects
        default => $objects,
    }

    # query for jmx metrics
    jmxtrans::metrics { "hadoop-hdfs-datanode-${::hostname}-${jmx_port}":
        jmx                  => $jmx,
        outfile              => $outfile,
        ganglia              => $ganglia,
        ganglia_group_name   => $group_name,
        graphite             => $graphite,
        graphite_root_prefix => $group_name,
        statsd               => $statsd,
        statsd_root_prefix   => $group_name,
        objects              => $datanode_objects,
    }
}