Puppet Class: eventgate

Defined in:
puppet/modules/eventgate/manifests/init.pp

Overview

Class eventgate

Installs an EventGate service listening on internally port 8192 and producing to Kafka.

EventGate is made available to POST events publicly at

http://eventgate.local.wmftest.net:8080/v1/events

You can reference this URL in other puppet configs via the $::eventgate::url variable.

This class does not set up any stream configs to enforce schemas in topics.

NOTE: to use this class, you must set the npm::node_version hiera key to 10. Doing so will install NodeJS 10 instead of 6 and may break other NodeJS services if they are not compatible NodeJS 10.

Parameters:

  • port (Any) (defaults to: 8192)
  • log_level (Any) (defaults to: undef)


20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
# File 'puppet/modules/eventgate/manifests/init.pp', line 20

class eventgate(
    $port      = 8192,
    $log_level = undef,
) {
    $node_version = hiera('npm::node_version', '6')
    if $node_version != 10 {
        warning("eventgate requires NodeJS version 10 but was ${node_version}. To use it, run `vagrant hiera npm::node_version 10 && vagrant provision`. (Might break other services.)")
    }

    require ::eventschemas
    include ::kafka
    include ::mwv # to get $::mwv::tld

    require_package('librdkafka1', 'librdkafka++1', 'librdkafka-dev')

    # A simple config suitable for producing valid events to to Kafka.
    $config = {
        'user_agent' => 'eventgate',
        'eventgate_factory_module' => 'eventgate-wikimedia.js',

        # This field in each event will be used to extract a
        # (possibly relative) schema uri.  The default is $schema.
        # An array of field names will cause EventGate to search for
        # fields by these names in each event, using the first match.
        'schema_uri_field' => '$schema',

        # If set, this URI will be prepended to any relative schema URI
        # extracted from each event's schema_field./
        'schema_base_uris' =>  [
            "${::eventschemas::path}/primary/jsonschema/",
            "${::eventschemas::path}/secondary/jsonschema/",
        ],

        # This field in each event will be used to extract a destination 'stream' name.
        # This will equal the destination Kafka topic, unless a topic prefix
        # is also configured.
        'stream_field' => 'meta.stream',
        'topic_prefix' => 'datacenter1.',

        # This field will be used in log messages to uniquely ID each event.
        'id_field' => 'meta.id',

        # If a validation error is encountered, a validation error event
        # will be produced to this stream.
        'error_stream' =>  'eventgate.error.validation',

        # kafka configs go here.
        'kafka' => {
              'conf' => {
                    'metadata.broker.list' => 'localhost:9092',
            },
        },
    }

    # eventgate-wikimedia has the WMF specific implementation of EventGate.
    # It specifies the eventgate (service-runner) package as a dependency
    # and runs the eventgate from it.
    service::node { 'eventgate-wikimedia':
        git_remote      => 'https://gerrit.wikimedia.org/r/eventgate-wikimedia',
        port            => $port,
        log_level       => $log_level,
        module          => 'eventgate',
        entrypoint      => 'app',
        script          => 'node_modules/eventgate/server.js',
        config          => $config,
        # Use debian librdkafka package
        npm_environment => ['BUILD_LIBRDKAFKA=0'],
        require         => Systemd::Service['kafka'],
    }

    # Add a reverse proxy from eventgate.local.wmftest.net to the
    # eventgate-wikimedia service.
    apache::reverse_proxy { 'eventgate':
        port => $port,
    }
    # The reverse_proxy will make this URL publicly addressable.
    # This is used by event clients (EventBus, EventLogging, etc.) to POST event data.
    $url = "http://eventgate${::mwv::tld}${::port_fragment}/v1/events"

    # make a symlink from srv/eventgate -> eventgate-wikimedia/node_modules/eventgate for
    # easier access to the EventGate package code for development purposes.
    file { '/vagrant/srv/eventgate':
        ensure  => 'link',
        target  => 'eventgate-wikimedia/node_modules/eventgate',
        require => Service::Node['eventgate-wikimedia'],
    }
}