Puppet Class: profile::wmcs::services::toolsdb_replica_cnf

Defined in:
modules/profile/manifests/wmcs/services/toolsdb_replica_cnf.pp

Overview

SPDX-License-Identifier: Apache-2.0

Parameters:

  • tool_replica_cnf_path (String) (defaults to: lookup('profile::wmcs::services::toolsdb_replica_cnf::tool_replica_cnf_path'))
  • paws_replica_cnf_path (String) (defaults to: lookup('profile::wmcs::services::toolsdb_replica_cnf::paws_replica_cnf_path'))
  • user_replica_cnf_path (String) (defaults to: lookup('profile::wmcs::services::toolsdb_replica_cnf::user_replica_cnf_path'))
  • htuser (String) (defaults to: lookup('profile::wmcs::services::toolsdb_replica_cnf::htuser'))
  • htpassword (String) (defaults to: lookup('profile::wmcs::services::toolsdb_replica_cnf::htpassword'))
  • htpassword_salt (String) (defaults to: lookup('profile::wmcs::services::toolsdb_replica_cnf::htpassword_salt'))
  • tools_project_prefix (String) (defaults to: lookup('profile::wmcs::services::toolsdb_replica_cnf::tools_project_prefix'))
  • kubeconfig_path_template (String) (defaults to: lookup('profile::wmcs::services::toolsdb_replica_cnf::kubeconfig_path_template'))
  • cloudcontrol (Array[Stdlib::Fqdn]) (defaults to: lookup('profile::openstack::eqiad1::openstack_controllers'))
  • redirect_to_https (Boolean) (defaults to: lookup('profile::wmcs::services::toolsdb_replica_cnf::redirect_to_https'))
  • include_tools_repo (Boolean) (defaults to: lookup('profile::wmcs::services::toolsdb_replica_cnf::include_tools_repo'))


2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
# File 'modules/profile/manifests/wmcs/services/toolsdb_replica_cnf.pp', line 2

class profile::wmcs::services::toolsdb_replica_cnf(
    String $tool_replica_cnf_path    = lookup('profile::wmcs::services::toolsdb_replica_cnf::tool_replica_cnf_path'),
    String $paws_replica_cnf_path    = lookup('profile::wmcs::services::toolsdb_replica_cnf::paws_replica_cnf_path'),
    String $user_replica_cnf_path    = lookup('profile::wmcs::services::toolsdb_replica_cnf::user_replica_cnf_path'),
    String $htuser                   = lookup('profile::wmcs::services::toolsdb_replica_cnf::htuser'),
    String $htpassword               = lookup('profile::wmcs::services::toolsdb_replica_cnf::htpassword'),
    String $htpassword_salt          = lookup('profile::wmcs::services::toolsdb_replica_cnf::htpassword_salt'),
    String $tools_project_prefix     = lookup('profile::wmcs::services::toolsdb_replica_cnf::tools_project_prefix'),
    String $kubeconfig_path_template = lookup('profile::wmcs::services::toolsdb_replica_cnf::kubeconfig_path_template'),
    Array[Stdlib::Fqdn]$cloudcontrol = lookup('profile::openstack::eqiad1::openstack_controllers'),
    Boolean $redirect_to_https       = lookup('profile::wmcs::services::toolsdb_replica_cnf::redirect_to_https'),
    # might be needed to get toolforge weld
    Boolean $include_tools_repo      = lookup('profile::wmcs::services::toolsdb_replica_cnf::include_tools_repo'),
) {
    $user                           = 'www-data'
    $group                          = 'www-data'
    $modules_uri                    = 'puppet:///modules/'
    $base_path                      = "/home/${user}"
    $api_service_base_path          = "${base_path}/replica_cnf_api_service"
    $api_service_app_path           = "${api_service_base_path}/replica_cnf_api_service"
    $api_service_base_path_in_repo  = "${modules_uri}profile/wmcs/nfs/replica_cnf_api_service"
    $api_service_app_path_in_repo   = "${api_service_base_path_in_repo}/replica_cnf_api_service"
    $replica_cnf_config_file_path   = '/etc/replica_cnf_config.yaml'
    $scripts_path                   = '/usr/local/bin'
    $write_replica_cnf_script_path  = "${scripts_path}/write_replica_cnf.sh"
    $read_replica_cnf_script_path   = "${scripts_path}/read_replica_cnf.sh"
    $delete_replica_cnf_script_path = "${scripts_path}/delete_replica_cnf.sh"
    $load_kubeconfig_script_path = "${scripts_path}/load_user_kubeconfig.py"
    $metrics_dir                    = '/run/toolsdb-replica-cnf-metrics'
    $htpassword_file                = '/etc/nginx/toolsdb-replica-cnf.htpasswd';
    $htpassword_hash                = htpasswd($htpassword, $htpassword_salt);


    if $include_tools_repo {
        apt::repository { 'toolforge':
            uri        => 'https://deb-tools.wmcloud.org/repo',
            dist       => "${::lsbdistcodename}-tools",
            components => 'main',
            trust_repo => true,
            source     => false,
        }

    }

    ensure_packages(['python3-flask', 'python3-toolforge-weld'])

    file { $replica_cnf_config_file_path:
        ensure  => 'file',
        owner   => 'root',
        group   => 'root',
        mode    => '0444',
        content => to_yaml({
          'TOOL_REPLICA_CNF_PATH' => $tool_replica_cnf_path,
          'PAWS_REPLICA_CNF_PATH' => $paws_replica_cnf_path,
          'USER_REPLICA_CNF_PATH' => $user_replica_cnf_path,
          'BACKENDS'              => {
            'ToolforgeToolFileBackend'    => {
              'ToolforgeToolBackendConfig' => {
                'replica_cnf_path'     => $tool_replica_cnf_path,
                'scripts_path'         => $scripts_path,
                'tools_project_prefix' => $tools_project_prefix,
                'use_sudo'             => true,
              },
            },
            'ToolforgeUserFileBackend'    => {
              'FileConfig' => {
                'replica_cnf_path' => $user_replica_cnf_path,
                'scripts_path'     => $scripts_path,
                'use_sudo'         => true,
              },
            },
            'PawsUserFileBackend'         => {
              'FileConfig' => {
                'replica_cnf_path' => $paws_replica_cnf_path,
                'scripts_path'     => $scripts_path,
                'use_sudo'         => true,
              },
            },
            'ToolforgeToolEnvvarsBackend' => {
              'EnvvarsConfig' => {
                'kubeconfig_path_template' => $kubeconfig_path_template,
                'toolforge_api_endpoint'   => "https://api.svc.${::wmcs_project}.${::wmcs_deployment}.wikimedia.cloud:30003",
                'scripts_path'             => $scripts_path,
                'use_sudo'                 => true,
              },
            },
          }
        })
    }

    file { $write_replica_cnf_script_path:
        ensure => 'file',
        owner  => 'root',
        group  => 'root',
        mode   => '0500',
        source => "${api_service_base_path_in_repo}/write_replica_cnf.sh"
    }

    file { $read_replica_cnf_script_path:
        ensure => 'file',
        owner  => 'root',
        group  => 'root',
        mode   => '0500',
        source => "${api_service_base_path_in_repo}/read_replica_cnf.sh"
    }

    file { $delete_replica_cnf_script_path:
        ensure => 'file',
        owner  => 'root',
        group  => 'root',
        mode   => '0500',
        source => "${api_service_base_path_in_repo}/delete_replica_cnf.sh"
    }
    file { $load_kubeconfig_script_path:
        ensure => 'file',
        owner  => 'root',
        group  => 'root',
        mode   => '0500',
        source => "${api_service_base_path_in_repo}/load_user_kubeconfig.py"
    }

    sudo::user { $user:
        ensure     => present,
        privileges => [
            "ALL = (ALL) NOPASSWD: ${write_replica_cnf_script_path}",
            "ALL = (ALL) NOPASSWD: ${read_replica_cnf_script_path}",
            "ALL = (ALL) NOPASSWD: ${delete_replica_cnf_script_path}",
            "ALL = (ALL) NOPASSWD: ${load_kubeconfig_script_path}",
        ]
    }


    file { [$base_path, $api_service_base_path]:
        ensure  => 'directory',
        owner   => $user,
        group   => $group,
        require => Sudo::User[ $user ],
        recurse => true,
        purge   => true
    }

    file { $api_service_app_path:
        ensure  => 'directory',
        owner   => $user,
        group   => $group,
        require => File[ $base_path, $api_service_base_path ],
        recurse => true,
        source  => $api_service_app_path_in_repo,
        }

    # Needed for prometheus exporter to share metrics between uwsgi processes
    file { $metrics_dir:
        ensure => 'directory',
        owner  => $user,
        group  => $group,
    }

    systemd::tmpfile { 'toolsdb-replica-cnf-shared-metrics':
        content => "d ${metrics_dir} 0755 ${user} ${group}",
    }

    uwsgi::app { 'toolsdb-replica-cnf-web':
        ensure             => 'present',
        subscribe          => [
            Package['python3-flask'],
            File[ $replica_cnf_config_file_path ],
            File[ $api_service_app_path ]
            ],
        settings           => {
            uwsgi              => {
                'plugins'      => 'python3',
                'socket'       => '/run/uwsgi/toolsdb-replica-cnf-web.sock',
                'module'       => 'replica_cnf_api_service.views:create_app()',
                'chmod-socket' => 664,
                'die-on-term'  => true,
                'vacuum'       => true,
                'master'       => true,
                'processes'    => 8,
                'chdir'        => $api_service_base_path,
                'env'          => [
                    # fix prometheus exporter for multiple uwsgi processes/workers
                    "PROMETHEUS_MULTIPROC_DIR=${metrics_dir}",
                ],
                'pythonpath'   => $api_service_base_path,
            },
        },
        extra_systemd_opts => {
            'ExecStartPre' => [
                # Clear out metrics caches for previous runs
                "/bin/bash -c \"rm -rf ${metrics_dir}/*\"",
            ],
        },
    }

    file { $htpassword_file:
            content => "${htuser}:${htpassword_hash}",
            owner   => $user,
            group   => $group,
            mode    => '0440',
            before  => Service['nginx'],
            require => Package['nginx-common'],
    }

    nginx::site { 'toolsdb-replica-cnf-web-nginx':
        require => Uwsgi::App['toolsdb-replica-cnf-web'],
        content => epp(
          'profile/wmcs/nfs/toolsdb-replica-cnf-web.nginx.epp',
          {
            'redirect_to_https' => $redirect_to_https,
          }
        ),
    }


    ensure_packages(['bats'])
    file { '/srv/ops':
      ensure => 'directory',
      mode   => '0500',
    }
    file { '/srv/ops/replica_cnf_web':
      ensure => 'directory',
      mode   => '0500',
    }
    $func_tests_dir='/srv/ops/replica_cnf_web/functional_tests'
    $puppet_path='profile/wmcs/nfs/replica_cnf_web_fullstack_tests'
    file { $func_tests_dir:
      ensure => 'directory',
      mode   => '0500',
    }
    file { "${func_tests_dir}/run.sh":
      content => epp(
        "${puppet_path}/run.sh.epp",
        {
          'http_user'     => $htuser,
          'http_password' => $htpassword,
        }
      ),
      mode    => '0500',
    }
    file { "${func_tests_dir}/paws_account.bats":
      source => "puppet:///modules/${puppet_path}/paws_accounts.bats",
      mode   => '0400',
    }
    file { "${func_tests_dir}/user_account.bats":
      source => "puppet:///modules/${puppet_path}/user_accounts.bats",
      mode   => '0400',
    }
    file { "${func_tests_dir}/tool_account.bats":
      source => "puppet:///modules/${puppet_path}/tool_accounts.bats",
      mode   => '0400',
    }
    file { "${func_tests_dir}/helpers.bash":
      source => "puppet:///modules/${puppet_path}/helpers.bash",
      mode   => '0400',
    }

    # nginx runs a REST API
    ferm::service{ 'rest_api_from_cloudcontrol':
        proto  => 'tcp',
        port   => '80',
        srange => "(@resolve((${join($cloudcontrol,' ')})))",
    }
}