Top Level Namespace

Includes:
RspecPuppetFacts

Defined Under Namespace

Modules: DomainRedirects, Helpers, Matchers, Puppet, PuppetDB, PuppetSyntax, Puppet_X, SharedData, URI Classes: AlsoString, Apr1Md5, BasicTTLCache, Class, DNSCacheEntry, DNSCached, File, GitOps, Hiera, Object, PuppetECDSAGen, PuppetECDSAGenError, PybalError, Service, SpecDependencies, String, TaskGen, UnknownExtensionError, WMFConfig

Constant Summary collapse

DIR =
ENV['PUPPETDIR'] || File.expand_path('../..', __FILE__)
HOSTNAME =
ENV.fetch('PUPPET_HOSTNAME', "#{ROLE}1001.eqiad.wmnet")
SPDX_GLOB =
"{modules,manifests,rake_module,utils}/{**/*,*}"
SPDX_TAG =
"SPDX-License-Identifier: Apache-2.0"
IGNORE_EXT_PATTERN =
/\A\.(?:json|pem|key|csr)\z/i
IGNORE_FILE =
['README', 'CONTRIBUTOR']
REALM =
ENV.fetch('PUPPET_REALM', 'production')
ROLE =
ENV.fetch('PUPPET_ROLE', 'insetup')
SITE =
ENV.fetch('PUPPET_SITE', 'eqiad')
OTHER_SITE =
SITE == 'eqiad' ? 'codfw' : 'eqiad'
PP_HEADER =
<<-PP
$realm = '#{REALM}'
$site = '#{SITE}'
$other_site = '#{OTHER_SITE}'
$numa_networking = 'off'
$wikimail_smarthost = lookup('wikimail_smarthost')
$mail_smarthost = lookup('mail_smarthost')
$acmechief_host = lookup('acmechief_host')
$ntp_peers = lookup('ntp_peers')
role('#{ROLE}')
PP
REPO_ROOT =
File.expand_path(File.join(__dir__, '..'))
PATH_ROOT =
'/usr/local/bin/'
Log =
Logger.new(STDOUT)
3
STATUS =
'WARNING'
EXIT =
1

Instance Method Summary collapse

Instance Method Details

#add_spdx_tags(files) ⇒ Object



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
# File 'rake_modules/tasks/spdx.rb', line 78

def add_spdx_tags(files)
  # Add the SPDX_TAGS near the top of a each file passed
  unknown_files = []
  files.each do |filename|
    begin
      tag = comment_line(filename, SPDX_TAG)
    rescue UnknownExtensionError => error
      unknown_files << error.filename
      next
    end
    puts "#{filename}: adding spdx licence"
    File.open(filename, 'r+') do |fh|
      while line = fh.readline  # rubocop:disable Lint/AssignmentInCondition
        break unless line[0..1] == '#!'
      end
      rewind_pos = fh.pos - line.size
      file_end = line + fh.read
      fh.seek(rewind_pos)
      fh.write(tag)
      fh.write(file_end)
    end
  end
  unless unknown_files.empty?
    puts(("Unable to add tag to the following files:\n" + unknown_files.join("\n")).yellow)
  end
end

#bootstrap(host) ⇒ Object



28
29
30
31
# File 'spec/spec_helper_acceptance.rb', line 28

def bootstrap(host)
  on(host, "/production/utils/beaker_bootstrap.rb #{HOSTNAME}")
  on(host, 'apt-get install -y vim')
end

#check_path_contributors(path) ⇒ Object



20
21
22
23
24
25
# File 'rake_modules/tasks/spdx.rb', line 20

def check_path_contributors(path)
  module_contributors = extract_email(`git shortlog -se -- #{path}`)
                          .reject {|email| email.end_with?('@wikimedia.org') }
  allowed_contributors = extract_email(File.read('CONTRIBUTORS'))
  module_contributors - allowed_contributors
end

#check_pooled_state(ip, port, pool, host, want_pooled) ⇒ Object



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
# File 'modules/conftool/files/pooler_loop.rb', line 70

def check_pooled_state(ip, port, pool, host, want_pooled)
  # Manage down or unresponsive pybals
  http = Net::HTTP.new(ip, port)
  http.open_timeout = 1
  http.read_timeout = 2

  resp = http.start do |http|
    http.get "/pools/#{pool}/#{host}"
  end

  # ignore 404s
  # rubocop:disable Style/CaseEquality
  return true unless resp === Net::HTTPSuccess
  # rubocop:enable Style/CaseEquality
  enabled, active, pooled = resp.body.strip.split '/'
  if want_pooled
    # A pooled server must be enabled/up/pooled
    # anything else is not ok
    if enabled != 'enabled'
      raise PybalError, "#{host} not logically pooled, lb #{ip}"
    elsif active != 'up'
      raise PybalError, "The service is not up on #{host}, lb #{ip}"
    elsif pooled != 'pooled'
      raise PybalError, "Service not pooled"
    end
  else
    # A disabled server should be disabled/{up,down}/not pooled
    if enabled == 'enabled'
      raise PybalError, "#{host} is still logically pooled, lb #{ip}"
    elsif pooled == 'pooled'
      raise PybalError, "#{host} is still pooled, maybe too many depooled? lb #{ip}"
    end
  end
end

#check_spdx_licence(file_list) ⇒ Object



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# File 'rake_modules/tasks/spdx.rb', line 34

def check_spdx_licence(file_list)
  # Check a list of files for an spdx licence header
  missing_licence = []
  file_list.each do |filename|
    next unless File.file?(filename)
    next if File.empty?(filename)
    next if filename.end_with?('.original.py')
    next if filename.start_with?('modules/admin/files/home')
    next unless File.text?(filename)
    next if IGNORE_FILE.include?(filename)
    next if File.extname(filename).match?(IGNORE_EXT_PATTERN)
    begin
      missing_licence << filename unless File.foreach(filename).grep(/SPDX-License-Identifier:/).any?
    rescue ArgumentError => error
      STDERR.puts "Error Could not read #{filename}: #{error}".red
    end
  end
  missing_licence
end

#comment_line(filename, line) ⇒ Object



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
# File 'rake_modules/tasks/spdx.rb', line 54

def comment_line(filename, line)
  # format a line as a comment using the file type specific comment
  # filetype is calculated based on the file extension
  ext = File.extname(filename)[1..-1]
  case ext
  when /\A(?:erb|epp)\z/
    "<%#- #{line} -%>\n"
  when /\A(?:jinja)\z/
    "{# #{line} #}\n"
  when /\A(?:html|md|markdown|xml)\z/
    "<!-- #{line} -->\n"
  when /\A(?:css)\z/
    "/* #{line} */\n"
  when /\A(?:vcl|php|groovy|js)\z/
    "// #{line}\n"
  when /\A(?:conf|cf|cfg|csh|ini|pl|pp|properties|py|R|Rakefile|rb|rc|service|sh|stp|vtc|yaml|yml)\z/
    "# #{line}\n"
  when /\A(?:lua|sql)\z/
    "-- #{line}\n"
  else
    raise UnknownExtensionError, filename
  end
end

#config_to_hash(conf) ⇒ Object

Function: merge_config(string|hash main_conf, string|hash service_conf)

Merges the service-specific service_conf into main_conf. Both arguments can be either hashes or YAML-formatted strings. It returns the merged configuration hash.



8
9
10
11
# File 'modules/service/lib/puppet/parser/functions/merge_config.rb', line 8

def config_to_hash(conf)
  return YAML.load(conf) unless conf.is_a?(Hash)
  conf
end

#configparser_format(config) ⇒ Object



17
18
19
20
21
22
23
24
25
26
27
28
29
30
# File 'modules/graphite/lib/puppet/parser/functions/configparser_format.rb', line 17

def configparser_format(config)
  # Serialize a hash to Python ConfigParser format.
  config.sort.map { |section, items|
    ["[#{section}]"].concat items.sort.map { |k, v|
      if v.is_a?(Array)
        v = v.join(',')
      else
        v = v == :undef ? '' : v
      end

      "#{k} = #{v}"
    }.push []
  }.join("\n")
end

#contributors_missing_permissionObject



27
28
29
30
31
32
# File 'rake_modules/tasks/spdx.rb', line 27

def contributors_missing_permission
  module_contributors = extract_email(`git shortlog -se`)
                          .reject {|email| email.end_with?('@wikimedia.org') }
  allowed_contributors = extract_email(File.read('CONTRIBUTORS'))
  module_contributors - allowed_contributors
end

#deep_merge(source, dest) ⇒ Object

rubocop:disable Metrics/BlockNesting



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# File 'modules/profile/files/logstash/filter_scripts/dot_expander.rb', line 18

def deep_merge(source, dest)
# Merges source hash into dest hash and returns dest
#
# adapted from https://github.com/danielsdeleo/deep_merge
# The MIT License (MIT)
#
# Copyright (c) 2008-2016 Steve Midgley, Daniel DeLeo
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
  if source.kind_of?(Hash)
    source.each do |src_key, src_value|
      if dest.kind_of?(Hash)
        if dest[src_key]
          dest[src_key] = deep_merge(src_value, dest[src_key])
        else # dest[src_key] doesn't exist so we want to create and overwrite it (but we do this via deep_merge)
          # note: we rescue here b/c some classes respond to "dup" but don't implement it (Numeric, TrueClass, FalseClass, NilClass among maybe others)
          begin
            src_dup = src_value.dup # we dup src_value if possible because we're going to merge into it (since dest is empty)
          rescue TypeError
            src_dup = src_value
          end
          dest[src_key] = deep_merge(src_value, src_dup)
        end
      end
    end
  else
    dest = source
  end
  dest
end

#determine_scopeObject

Determine the scope from the command line arguments



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
# File 'utils/hiera_lookup.rb', line 40

def determine_scope
  # Transform the CL arguments in a dictionary in the form "::option" => argument
  scope = Hash[ARGV.map { |kv| "::#{kv.sub('--', '')}".split('=') }]
  abort(usage "Error: --fqdn=FQDN is required.") unless scope['::fqdn']
  scope['::hostname'] = scope['::fqdn'][/^[^.]+/]
  if scope['::fqdn'].end_with?('.wmflabs') || scope['::fqdn'].end_with?('.labtest') || scope['::fqdn'].end_with?('.cloud')
    scope['::realm'] = 'labs'
  else
    scope['::realm'] = 'production'
  end

  # Detect labs project/site
  if scope['::realm'] == 'labs'
    bits = scope['::fqdn'].split('.')
    unless bits.length == 4
      Kernel.abort('labs FQDN must be <hostname>.<project>.<site>.(wmflabs|labtest|cloud)')
    end
    scope['::labsproject'] = bits[1]
    scope['::site'] = bits[2]
  end

  # Attempt at determining the site
  scope['::site'] ||= scope['::fqdn'].split('.')[-2]
  if scope['::site'] == 'wikimedia' || scope['::site'].nil?
    abort(usage "\nError: --site=SITE is required if the FQDN doesn't contain it")
  end

  # Transform roles in the form puppet expects
  if scope['::roles']
    scope['_roles'] = Hash[scope['::roles'].split(',').map { |role| [role.gsub('role::', ''), true] }]
    # Mimic the changes in the role() function
    scope['::_role'] = scope['_roles'].keys[0].gsub(/::/, '/')
    scope.delete('::roles')
  end
  scope
end

#ensure_module_defined(module_name) ⇒ Object

Ensures that a module is defined

Parameters:

  • module_name

    Name of the module



67
68
69
70
71
72
# File 'vendor_modules/stdlib/spec/spec_helper.rb', line 67

def ensure_module_defined(module_name)
  module_name.split('::').reduce(Object) do |last_module, next_module|
    last_module.const_set(next_module, Module.new) unless last_module.const_defined?(next_module, false)
    last_module.const_get(next_module, false)
  end
end

#event_with_tag(event, value) ⇒ Object

get the event with an additional tag



44
45
46
47
48
49
# File 'modules/profile/files/logstash/filter_scripts/nest_root_fields.rb', line 44

def event_with_tag(event, value)
  tags = event.get("tags")
  tags.push(value)
  event.set("tags", tags)
  event
end

#expand_dots(key, value) ⇒ Object



7
8
9
10
11
12
13
14
15
# File 'modules/profile/files/logstash/filter_scripts/dot_expander.rb', line 7

def expand_dots(key, value)
  # Transforms a dot-delimited string "key" into nested objects with the final key set to value
  # ex. |'a.b.c.d', 'value'| to {'a' => {'b' => {'c' => {'d' => 'value'}}}}
  o = {}
  keys = key.split('.')
  o.default_proc = -> (h, k) { h[k] = Hash.new(&h.default_proc) }
  o.dig(*keys[0..-2])[keys.fetch(-1)] = value
  o
end

#extract_email(string) ⇒ Object



16
17
18
# File 'rake_modules/tasks/spdx.rb', line 16

def extract_email(string)
  string.scan(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/).sort.uniq
end

#filter(event) ⇒ Object

rubocop:enable Metrics/BlockNesting



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
# File 'modules/profile/files/logstash/filter_scripts/dot_expander.rb', line 66

def filter(event)
  # event timestamp to nanoseconds
  ts = (event.get('@timestamp').to_i * 1e9).to_s.split('.')[0]

  case @event_type
  when 'alert'
    stream = { :type => 'alert', :host => event.get('host')}
    values = [[ts, "#{event.get('icinga_state')} -- #{event.get('icinga_check_descr')}: #{event.get('icinga_message')}"]]
  when 'sal'
    stream = { :type => 'sal', :project => event['project'] }
    values = [[ts, "#{event.get('nick')}: #{event.get('message')}"]]
  when 'deploy'
    stream = { :type => 'deploy' }
    values = [[ts, "#{event.get('user')}: #{event.get('message')}"]]
  else
    # do nothing
    return [event]
  end

  event.set('streams', [{ :stream => stream, :values => values }])
  [event]
end

#fix_sorces_listObject



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# File 'utils/beaker_bootstrap.rb', line 33

def fix_sorces_list
  # Add contrib and none-free to the sources list
  orig_content = File.open('/etc/apt/sources.list').read
  File.open('/etc/apt/sources.list', 'w') do |sources_list|
    orig_content.each_line do |line|
      if line.start_with?("deb\s") && !line.include?('apt.wikimedia.org')
        ['contrib', 'non-free'].each do |component|
          line = "#{line.chomp} #{component}\n" unless line.include?(component)
        end
      end
      sources_list.write(line)
    end
  end
  `apt-get update -y`
end

#generate_ssl_certs(hostname) ⇒ Object



76
77
78
79
80
# File 'utils/beaker_bootstrap.rb', line 76

def generate_ssl_certs(hostname)
  # Generate some puppet certs for the hostname being tested
  FileUtils.rm_rf('/var/lib/puppet/ssl')
  `/usr/bin/puppet cert generate '#{hostname}'`
end

#get_facility(field) ⇒ Object



82
83
84
85
86
87
88
89
90
# File 'modules/profile/files/logstash/filter_scripts/normalize_level.rb', line 82

def get_facility(field)
  # Returns normalized facility field
  unless field.nil?
    unless @facilities[field].nil?
      return [field, @facilities[field]]
    end
  end
  ["local7", 23]
end

#get_level(event) ⇒ Object



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
# File 'modules/profile/files/logstash/filter_scripts/normalize_level.rb', line 108

def get_level(event)
  # Returns normalized level field.
  # dependent on the availability of either `event.level` or `event.severity`

  level = [
    event.get('[log][level]'),
    event.get('level'),
    event.get('severity'),
  ].find(-> { return 'NOTSET' }) { |v| !v.nil? }

  # bunyan sends levels as integers
  if level.is_a? Numeric
    idx = level / 10 - 1  # transform to array index
    level = @bunyan_levels[idx]
  end

  level.upcase
end

#get_severity(field) ⇒ Object



92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
# File 'modules/profile/files/logstash/filter_scripts/normalize_level.rb', line 92

def get_severity(field)
  # Returns normalized severity field
  # Defaults to alert because this severity level is seldom hit.
  # Mismatches between `log.syslog.severity` and `log.level` should be addressed
  # Default ["alert", 1]
  unless field.nil?
    field = field.downcase
    @severity_map.each do |k, v|
      if (v[:aliases] + [k]).include?(field)
        return k, v[:code]
      end
    end
  end
  ["alert", 1]
end

#git_files(path = '.') ⇒ Object



158
159
160
# File 'rake_modules/tasks/spdx.rb', line 158

def git_files(path = '.')
  `git ls-files -- #{path}`.split("\n")
end

#install_modules(host, modules) ⇒ Object



16
17
18
19
20
21
22
23
24
# File 'modules/cfssl/spec/spec_helper_acceptance.rb', line 16

def install_modules(host, modules)
  module_root = File.expand_path(File.join(__dir__, '..'))
  install_dev_puppet_module_on(
    host, source: module_root, module_name: File.basename(module_root))
  modules.each do |m|
    source = File.expand_path(File.join(module_root, '..', m))
    install_dev_puppet_module_on(host, source: source, module_name: m)
  end
end

#is_correct_format?(agent, volume_group, logical_volume, format_type) ⇒ Boolean

Verify if a filesystem resource type is successfully created

Attributes

  • volume_group - resorce type name, i.e 'VolumeGroup_1234'

  • logical_volume - resorce type name, i.e 'LogicalVolume_a2b3'

  • fromat_type - type of the format of the logical volume, i.e 'ext3'

Returns

nil

Raises

assert_match failure message

Examples

is_correct_format?(agent, VolumeGroup_1234, LogicalVolume_a2b3, ext3)

Returns:

  • (Boolean)


73
74
75
76
77
# File 'vendor_modules/lvm/tests/beaker/lib/lvm_helper.rb', line 73

def is_correct_format?(agent, volume_group, logical_volume, format_type)
  on(agent, "file -sL /dev/#{volume_group}/#{logical_volume}") do |result|
    assert_match(/#{format_type}/, result.stdout, "Unexpected error was detected")
  end
end

#nodes_listObject



53
54
55
56
# File 'modules/base/lib/facter/numa.rb', line 53

def nodes_list
  Pathname.glob('/sys/devices/system/node/node[0-9]*')
          .map { |n| /([0-9]+)$/.match(n.to_s)[0].to_i }.sort
end

#nodes_to_htsets(nodes) ⇒ Object



58
59
60
61
62
63
64
65
66
67
68
69
# File 'modules/base/lib/facter/numa.rb', line 58

def nodes_to_htsets(nodes)
  tsl_glob = 'cpu[0-9]*/topology/thread_siblings_list'
  node_to_htset = {}
  nodes.each do |n|
    node_to_htset[n] =
      Pathname.glob("/sys/devices/system/node/node#{n}/#{tsl_glob}")
              .map do |c|
                File.open(c).read.split(',').map(&:to_i).sort
              end.sort.uniq
  end
  node_to_htset
end

#register(params) ⇒ Object

filter_on_templates.rb Logstash Ruby script to strip incompatible top-level fields based on type from a set of index templates

Version:

  • 1.0.0



5
6
7
# File 'modules/profile/files/logstash/filter_scripts/dot_expander.rb', line 5

def register(params)
  @event_type = params["type"]
end

#remove_all(agent, pv = nil, vg = nil, lv = nil, aix = false) ⇒ Object

Clean the box after each test, make sure the newly created logical volumes, volume groups, and physical volumes are removed at the end of each test to make the server ready for the next test case.

Attributes

  • pv - physical volume, can be one volume or an array of multiple volumes

  • vg - volume group, can be one group or an array of multiple volume groups

  • lv - logical volume, can be one volume or an array of multiple volumes

  • aix - if the agent is an AIX server.

Returns

nil

Raises

nil

Examples

remove_all(agent, '/dev/sdb', 'VolumeGroup_1234', 'LogicalVolume_fa13')



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
# File 'vendor_modules/lvm/tests/beaker/lib/lvm_helper.rb', line 99

def remove_all(agent, pv=nil, vg=nil, lv=nil, aix=false)
  if aix
    step 'remove aix volume group, physical/logical volume '
    on(agent, "reducevg -d -f #{vg} #{pv}")
    on(agent, "rm -rf /dev/#{vg} /dev/#{lv}")
  else
    step 'remove logical volume if any:'
    if lv
      if lv.kind_of?(Array)
        lv.each do |logical_volume|
          on(agent, "umount /dev/#{vg}/#{logical_volume}", :acceptable_exit_codes => [0,1])
          on(agent, "lvremove /dev/#{vg}/#{logical_volume} --force")
        end
      else
        #note: in some test cases, for example, the test case 'create_vg_property_logical_volume'
        # the logical volume must be unmount before being able to delete it
        on(agent, "umount /dev/#{vg}/#{lv}", :acceptable_exit_codes => [0,1])
        on(agent, "lvremove /dev/#{vg}/#{lv} --force")
      end
    end

    step 'remove volume group if any:'
    if vg
      if vg.kind_of?(Array)
        vg.each do |volume_group|
          on(agent, "vgremove /dev/#{volume_group}")
        end
      else
        on(agent, "vgremove /dev/#{vg}")
      end
    end

    step 'remove logical volume if any:'
    if pv
      if pv.kind_of?(Array)
        pv.each do |physical_volume|
          on(agent, "pvremove #{physical_volume}")
        end
      else
        on(agent, "pvremove #{pv}")
      end
    end
  end
end

#rmerge(*args) ⇒ Object

Function: configparser_format

Serialize a hash to Python ConfigParser format. See <docs.python.org/2/library/configparser.html>



6
7
8
9
10
11
12
13
14
15
# File 'modules/graphite/lib/puppet/parser/functions/configparser_format.rb', line 6

def rmerge(*args)
  # Recursively merge hashes.
  merged = args.shift.clone
  args.each do |hash|
    merged.merge!(hash) do |k, old, new|
      merged[k] = old.is_a?(Hash) && new.is_a?(Hash) ? rmerge(old, new) : new
    end
  end
  merged
end

#setup_spdx(git) ⇒ Object



105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
# File 'rake_modules/tasks/spdx.rb', line 105

def setup_spdx(git)
  changed_files = git.changes_in_head.select{ |f| File.fnmatch(SPDX_GLOB, f, File::FNM_EXTGLOB) }
  new_files = git.new_files_in_head.select{ |f| File.fnmatch(SPDX_GLOB, f, File::FNM_EXTGLOB) }
  tasks = []
  unless changed_files.empty?
    namespace :'spdx:check' do
      desc "Check changed files"
      task :changed do
        missing_licence = check_spdx_licence(changed_files)
        if missing_licence.empty?
          abort("The following are missing a SPDX licence header:\n#{missing_licence.join("\n")}".red)
        end
        puts 'SPDX licence: OK'.green
      end
    end
  end
  unless new_files.empty?
    missing_licence = check_spdx_licence(new_files)
    namespace :'spdx:check' do
      desc "Check changed files"
      task :new_files do
        unless missing_licence.empty?
          msg = <<~ERROR
          The following are missing a SPDX licence header:

          #{missing_licence.join("\n")}

          Use the following command to automatically add tags

            `bundle exec rake spdx:convert:new_files`

          ERROR
          abort(msg.red)
        end
        puts 'SPDX licence: OK'.green
      end
    end
    tasks << 'spdx:check:new_files'
    namespace :'spdx:convert' do
      desc "Convert a module to SPDX"
      task :new_files do
        if missing_licence.empty?
          puts 'OK: all new files an spdx header'
          exit
        end
        add_spdx_tags(missing_licence)
      end
    end
    tasks << 'spdx:check:new_files'
  end
  tasks
end

#sync_puppet_dirs(prod_source, private_source = '/etc/puppet/private') ⇒ Object



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
# File 'utils/beaker_bootstrap.rb', line 59

def sync_puppet_dirs(prod_source, private_source = '/etc/puppet/private')
  # link all directories into the correct folder.
  modules_dir = '/etc/puppet/code/modules'
  hieradata_dir = '/etc/puppet/hieradata'
  FileUtils.rm_rf(Dir["#{modules_dir}/*"])
  FileUtils.mkdir_p(modules_dir)
  Dir[
    "#{prod_source}/modules/*",
    "#{prod_source}/vendor_modules/*",
    "#{private_source}/modules/*"
  ].each do |mod|
    File.symlink(mod, File.join(modules_dir, File.basename(mod)))
  end
  FileUtils.rm_rf(hieradata_dir)
  File.symlink(File.join(prod_source, 'hieradata'), hieradata_dir)
end

#time_ago(s) ⇒ Object



132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
# File 'modules/profile/files/monitoring/check_puppetrun.rb', line 132

def time_ago(s)
  units = {
    24 * 60 * 60 => 'day',
    60 * 60      => 'hour',
    60           => 'minute',
    1            => 'second',
  }
  if s.zero?
    return "0 seconds"
  end
  units.sort.reverse.each do |len, unit|
    return "#{s / len} #{unit}#{'s' if s / len > 1}" if s >= len
  end
  "Indeterminate amount of time (see time_ago)"
end

#update_repo(repo_dir, repo_url, sha1_url) ⇒ Object



49
50
51
52
53
54
55
56
57
# File 'utils/beaker_bootstrap.rb', line 49

def update_repo(repo_dir, repo_url, sha1_url)
  # Check out a specific repo
  sha1 = open(sha1_url).read
  unless File.directory?(repo_dir)
    FileUtils.mkdir_p(File.dirname(repo_dir))
    `git clone #{repo_url} #{repo_dir}`
  end
  `git -C #{repo_dir} checkout #{sha1}`
end

#usage(error = '') ⇒ Object

hiera_lookup: hiera lookup tool



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# File 'utils/hiera_lookup.rb', line 5

def usage(error = '')
  msg = <<-end
hiera_lookup: hiera lookup tool

Usage:
     hiera_lookup [OPTIONS] KEY

Mandatory arguments:

  --fqdn=<FQDN>  Fully Qualified Domain Name
  KEY            Key to lookup in hiera

Optional arguments:

  -v                          Verbose mode
  --site=<SITE>               Wikimedia site (ex: eqiad)
  --roles=<ROLE>[,<ROLE>]...  Puppet role(s) class(es)
  -h, --help                  Display this help and exit

If SITE is not provided, it will be derived from the FQDN whenever possible.
For Wmflabs a FQDN should be: <hostname>.<project>.<site>.wmflabs

Examples:
  hiera_lookup --fqdn=mw1020.eqiad.wmnet --roles=mediawiki::appserver admin::groups
  hiera_lookup --fqdn=host.tools.eqiad1.wikimedia.cloud classes


end
  msg << error
end

#verify_if_created?(agent, resource_type, resource_name, vg = nil, properties = nil) ⇒ Boolean

Verify if a physical volume, volume group, logical volume, or filesystem resource type is created

Attributes

  • resource_type - resorce type, i.e 'physical_volume', 'volume_group', 'logical_volume', 'filesystem',

  • 'aix_physical_volume', 'aix_volume_group', or 'aix_logical_volume'.

  • resource_name - The name of resource type, i.e '/dev/sdb' for physical volume, vg_1234 for volume group

  • vg - volume group name associated with logical volume (if any)

  • properties - a matching string or regular expression in logical volume properties

Returns

nil

Raises

assert_match failure message

Examples

verify_if_created?(agent, 'physical_volume', /dev/sdb', VolumeGroup_123, “Size 7GB”)

Returns:

  • (Boolean)


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
# File 'vendor_modules/lvm/tests/beaker/lib/lvm_helper.rb', line 19

def verify_if_created?(agent, resource_type, resource_name, vg=nil, properties=nil)
  case resource_type
    when 'physical_volume'
      on(agent, "pvdisplay") do |result|
        assert_match(/#{resource_name}/, result.stdout, 'Unexpected error was detected')
      end
    when 'volume_group'
      on(agent, "vgdisplay") do |result|
        assert_match(/#{resource_name}/, result.stdout, 'Unexpected error was detected')
      end
    when 'logical_volume'
      raise ArgumentError, 'Missing volume group that the logical volume is associated with' unless vg
      on(agent, "lvdisplay /dev/#{vg}/#{resource_name}") do |result|
        assert_match(/#{resource_name}/, result.stdout, 'Unexpected error was detected')
        if properties
          assert_match(/#{properties}/, result.stdout, 'Unexpected error was detected')
        end
      end
    when 'aix_physical_volume'
      on(agent, "lspv #{resource_name}") do |result|
        assert_match(/Physical volume #{resource_name} is not assigned to/, result.stdout, 'Unexpected error was detected')
      end
    when 'aix_volume_group'
      on(agent, "lsvg") do |result|
        assert_match(/#{resource_name}/, result.stdout, 'Unexpected error was detected')
      end
    when 'aix_logical_volume'
      raise ArgumentError, 'Missing volume group that the logical volume is associated with' unless vg
      on(agent, "lslv #{resource_name}") do |result|
        assert_match(/#{resource_name}/, result.stdout, 'Unexpected error was detected')
        if properties
          assert_match(/#{properties}/, result.stdout, 'Unexpected error was detected')
        end
      end
  end
end

#write_hieraObject



11
12
13
14
15
16
# File 'utils/beaker_bootstrap.rb', line 11

def write_hiera
  # Write out a custome hiera file with the beaker overrides taking precedence
  hiera_config = YAML.load_file(File.join(__dir__, '../modules/puppetmaster/files/production.hiera.yaml'))
  hiera_config['hierarchy'].insert(0, {'name' => 'beaker overrides', 'path' => 'beaker.yaml'})
  File.open('/etc/puppet/hiera.yaml', 'w') { |f| YAML.dump(hiera_config, f) }
end

#write_interfacesObject



18
19
20
21
22
23
24
25
26
27
28
29
30
31
# File 'utils/beaker_bootstrap.rb', line 18

def write_interfaces
  # Write out an interfaces file, this is needed so that augeas can parse and update the file
  default_route = `ip route list default`.strip.split
  default_iface = default_route[-1]
  default_route_addr = default_route[2]
  iface = Socket.getifaddrs.select { |i| i.name == default_iface && i.addr.ipv4? }[0]
  content = <<~IFACE
  iface #{default_iface} inet static
    address #{iface.addr.ip_address}
    netmask #{iface.netmask.ip_address}
    gateway #{default_route_addr}
  IFACE
  File.open('/etc/network/interfaces', 'w') { |f| f.write(content) }
end