cilium

1
# -*- mode: ruby -*-
2
# vi: set ft=ruby :
3

4
# The source of truth for vagrant box versions.
5
# Sets SERVER_BOX, SERVER_VERSION, NETNEXT_SERVER_BOX and NETNEXT_SERVER_VERSION
6
# Accepts overrides from env variables
7
require_relative 'vagrant_box_defaults.rb'
8
$SERVER_BOX = (ENV['SERVER_BOX'] || $SERVER_BOX)
9
$SERVER_VERSION= (ENV['SERVER_VERSION'] || $SERVER_VERSION)
10
$NETNEXT_SERVER_BOX = (ENV['NETNEXT_SERVER_BOX'] || $NETNEXT_SERVER_BOX)
11
$NETNEXT_SERVER_VERSION= (ENV['NETNEXT_SERVER_VERSION'] || $NETNEXT_SERVER_VERSION)
12
$NO_BUILD = (ENV['NO_BUILD'] || "0")
13

14
if ENV['NETNEXT'] == "true" || ENV['NETNEXT'] == "1" then
15
    $SERVER_BOX = $NETNEXT_SERVER_BOX
16
    $SERVER_VERSION = $NETNEXT_SERVER_VERSION
17
    $vm_kernel = '+'
18
end
19

20
Vagrant.require_version ">= 2.0.0"
21

22
if ARGV.first == "up" && ENV['CILIUM_SCRIPT'] != 'true'
23
    raise Vagrant::Errors::VagrantError.new, <<END
24
Calling 'vagrant up' directly is not supported.  Instead, please run the
25
following to deploy a single-node cluster:
26
$ ./contrib/vagrant/start.sh
27

28
See the development guide for available configuration options:
29
https://docs.cilium.io/en/latest/contributing/development/dev_setup
30
END
31
end
32

33
if ENV['IPV4'] == '0'
34
    raise Vagrant::Errors::VagrantError.new, <<END
35
Disabling IPv4 is currently not allowed until k8s 1.9 is released
36
END
37
end
38

39
# Workaround issue as described here:
40
# https://github.com/cilium/cilium/pull/12520
41
class VagrantPlugins::ProviderVirtualBox::Action::Network
42
  def dhcp_server_matches_config?(dhcp_server, config)
43
    true
44
  end
45
end
46

47
$cleanup = <<SCRIPT
48
i=1
49
while [ "$i" -le "$((num_workers+1))" ]; do
50
    VBoxManage natnetwork add --netname natnet$i --network 192.168.0.0/16 --ipv6 on --enable
51
    i=$((i+1))
52
done 2>/dev/null
53

54
res=0
55
while [ "$res" == "0" ]; do
56
    VBoxManage natnetwork remove --netname natnet$i
57
    res=$?
58
    i=$((i+1))
59
done 2>/dev/null
60
SCRIPT
61

62
$bootstrap = <<SCRIPT
63
set -o errexit
64
set -o nounset
65
set -o pipefail
66

67
if [ -x /home/vagrant/go/src/github.com/cilium/cilium/.devvmrc ] ; then
68
   echo "----------------------------------------------------------------"
69
   echo "Executing .devvmrc"
70
   /home/vagrant/go/src/github.com/cilium/cilium/.devvmrc || true
71
fi
72

73
echo "----------------------------------------------------------------"
74
export PATH=/home/vagrant/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games
75

76
echo "editing journald configuration"
77
bash -c "echo RateLimitIntervalSec=1s >> /etc/systemd/journald.conf"
78
bash -c "echo RateLimitBurst=10000 >> /etc/systemd/journald.conf"
79
echo "restarting systemd-journald"
80
systemctl restart systemd-journald
81
echo "getting status of systemd-journald"
82
service systemd-journald status
83
echo "done configuring journald"
84

85
service docker restart
86
echo 'cd ~/go/src/github.com/cilium/cilium' >> /home/vagrant/.bashrc
87
echo 'export GOPATH=$(go env GOPATH)' >> /home/vagrant/.bashrc
88
chown -R vagrant:vagrant /home/vagrant 2>/dev/null || true
89
curl -SsL https://github.com/cilium/bpf-map/releases/download/v1.0/bpf-map -o bpf-map
90
chmod +x bpf-map
91
mv bpf-map /usr/bin
92
SCRIPT
93

94
$makeclean = ENV['MAKECLEAN'] ? "export MAKECLEAN=1" : ""
95
$build = <<SCRIPT
96
set -o errexit
97
set -o nounset
98
set -o pipefail
99

100
export PATH=/home/vagrant/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games
101
#{$makeclean}
102
~/go/src/github.com/cilium/cilium/contrib/vagrant/build.sh
103
rm -fr ~/go/bin/cilium*
104
SCRIPT
105

106
$install = <<SCRIPT
107
set -o errexit
108
set -o nounset
109
set -o pipefail
110

111
# Add an exception for the cilium repo for the root user to fix the
112
# "fatal: unsafe repository ('/home/vagrant/go/src/github.com/cilium/cilium' is owned by someone else)"
113
# error condition when running `sudo make install`
114
git config --global --add safe.directory /home/vagrant/go/src/github.com/cilium/cilium
115

116
sudo -E make -C /home/vagrant/go/src/github.com/cilium/cilium/ install
117

118
sudo mkdir -p /etc/sysconfig
119
sudo cp /home/vagrant/go/src/github.com/cilium/cilium/contrib/systemd/cilium-consul.service /lib/systemd/system
120
sudo cp /home/vagrant/go/src/github.com/cilium/cilium/contrib/systemd/cilium-docker.service /lib/systemd/system
121
sudo cp /home/vagrant/go/src/github.com/cilium/cilium/contrib/systemd/cilium-etcd.service /lib/systemd/system
122
sudo cp /home/vagrant/go/src/github.com/cilium/cilium/contrib/systemd/cilium.service /lib/systemd/system
123
sudo cp /home/vagrant/go/src/github.com/cilium/cilium/contrib/systemd/cilium-operator.service /lib/systemd/system
124
sudo cp /home/vagrant/go/src/github.com/cilium/cilium/contrib/systemd/cilium /etc/sysconfig
125

126
getent group cilium >/dev/null || sudo groupadd -r cilium
127
sudo usermod -a -G cilium vagrant
128
SCRIPT
129

130
$node_ip_base = ENV['IPV4_BASE_ADDR'] || ""
131
$node_nfs_base_ip = ENV['IPV4_BASE_ADDR_NFS'] || ""
132
$num_workers = (ENV['NWORKERS'] || 0).to_i
133
$workers_ipv4_addrs = $num_workers.times.collect { |n| $node_ip_base + "#{n+(ENV['FIRST_IP_SUFFIX']).to_i+1}" }
134
$workers_ipv4_addrs_nfs = $num_workers.times.collect { |n| $node_nfs_base_ip + "#{n+(ENV['FIRST_IP_SUFFIX_NFS']).to_i+1}" }
135
$master_ip = ENV['MASTER_IPV4']
136
$master_ipv6 = ENV['MASTER_IPV6_PUBLIC']
137
$workers_ipv6_addrs_str = ENV['IPV6_PUBLIC_WORKERS_ADDRS'] || ""
138
$workers_ipv6_addrs = $workers_ipv6_addrs_str.split(' ')
139

140
# Create unique ID for use in vboxnet name so Jenkins pipeline can have concurrent builds.
141
$job_name = ENV['JOB_BASE_NAME'] || "local"
142

143
$build_number = ENV['BUILD_NUMBER'] || "0"
144
$build_id = "#{$job_name}-#{$build_number}"
145

146
# Only create the build_id_name for Jenkins environment so that
147
# we can run VMs locally without having any the `build_id` in the name.
148
if ENV['BUILD_NUMBER'] then
149
    $build_id_name = "-build-#{$build_id}"
150
end
151

152
if ENV['K8S'] then
153
    $vm_base_name = "k8s"
154
else
155
    $vm_base_name = "runtime"
156
end
157

158
# Set locate to en_US.UTF-8
159
ENV["LC_ALL"] = "en_US.UTF-8"
160
ENV["LC_CTYPE"] = "en_US.UTF-8"
161

162
if ENV['CILIUM_SCRIPT'] != 'true' then
163
    Vagrant.configure(2) do |config|
164
        config.vm.define "runtime1"
165
        config.vm.define "k8s1"
166
        config.vm.define "k8s2"
167
        config.vm.define "k8s1+"
168
        config.vm.define "k8s2+"
169
    end
170
end
171

172
Vagrant.configure(2) do |config|
173
    config.trigger.before :up, :provision, :reload do |trigger|
174
        trigger.run = {inline: "bash -c '#{$cleanup}'"}
175
    end
176

177
    config.vm.provision "bootstrap", type: "shell", inline: $bootstrap
178
    if $NO_BUILD == "0" then
179
        config.vm.provision "build", type: "shell", run: "always", privileged: false, inline: $build
180
    end
181
    config.vm.provision "install", type: "shell", run: "always", privileged: false, inline: $install
182
    config.vm.box_check_update = false
183

184
    config.vm.provider "virtualbox" do |vb|
185
        # Do not inherit DNS server from host, use proxy
186
        vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
187
        vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
188

189
        # Prevent VirtualBox from interfering with host audio stack
190
        vb.customize ["modifyvm", :id, "--audio", "none"]
191

192
        # Enable HPET, the Go scheduler heavily relies on accurate timers.
193
        vb.customize ["modifyvm", :id, "--hpet", "on"]
194

195
        config.vm.box = $SERVER_BOX
196
        config.vm.box_version = $SERVER_VERSION
197
        vb.memory = ENV['VM_MEMORY'].to_i
198
        vb.cpus = ENV['VM_CPUS'].to_i
199
    end
200

201
    master_vm_name = "#{$vm_base_name}1#{$build_id_name}#{$vm_kernel}"
202
    config.vm.define master_vm_name, primary: true do |cm|
203
        node_ip = "#{$master_ip}"
204
        node_ipv6 = "#{$master_ipv6}"
205
        cm.vm.network "forwarded_port", guest: 6443, host: 7443, auto_correct: true
206
        cm.vm.network "forwarded_port", guest: 9081, host: 9081, auto_correct: true
207
        # 2345 is the default delv server port
208
        cm.vm.network "forwarded_port", guest: 2345, host: 2345, auto_correct: true
209
        cm.vm.network "private_network", ip: "#{$master_ip}",
210
            virtualbox__intnet: "cilium-test-#{$build_id}"
211
        if ENV['FIRST_IP_SUFFIX_NFS'] then
212
            $nfs_ipv4_master_addr = $node_nfs_base_ip + "#{ENV['FIRST_IP_SUFFIX_NFS']}"
213
        end
214
        cm.vm.network "private_network", ip: "#{$nfs_ipv4_master_addr}", bridge: "enp0s9"
215
        # Add IPv6 address this way or we get hit by a virtualbox bug
216
        cm.vm.provision "ipv6-config",
217
            type: "shell",
218
            run: "always",
219
            inline: "ip -6 a a #{$master_ipv6}/16 dev enp0s9"
220
        if ENV["IPV6_EXT"] then
221
            node_ip = "#{$master_ipv6}"
222
        end
223

224
        # Interface for the IPv6 NAT Service. The IP address doesn't matter as
225
        # it won't be used. We use an IPv4 address as newer versions of VBox
226
        # reject all IPv6 addresses.
227
        cm.vm.network "private_network",
228
            ip: "192.168.59.15"
229
        cm.vm.provider "virtualbox" do |vb|
230
            vb.customize ["modifyvm", :id, "--nic4", "natnetwork"]
231
            vb.customize ["modifyvm", :id, "--nat-network4", "natnet1"]
232
        end
233
        cm.vm.provision "ipv6-nat-config",
234
            type: "shell",
235
            run: "always",
236
            inline: "ip -6 r a default via fd17:625c:f037:2::1 dev enp0s10 || true"
237

238
        cm.vm.hostname = "#{$vm_base_name}1"
239
        if ENV['CILIUM_TEMP'] then
240
           if ENV["K8S"] then
241
               k8sinstall = "#{ENV['CILIUM_TEMP']}/cilium-k8s-install-1st-part.sh"
242
               cm.vm.provision "k8s-install-master-part-1",
243
                   type: "shell",
244
                   run: "always",
245
                   env: {"node_ip" => node_ip, "node_ipv6" => node_ipv6},
246
                   privileged: true,
247
                   path: k8sinstall
248
           end
249
           script = "#{ENV['CILIUM_TEMP']}/node-1.sh"
250
           cm.vm.provision "config-install", type: "shell", privileged: true, run: "always", path: script
251
           # In k8s mode cilium needs etcd in order to run which was started in
252
           # the first part of the script. The 2nd part will install the
253
           # policies into kubernetes and cilium.
254
           if ENV["K8S"] then
255
               k8sinstall = "#{ENV['CILIUM_TEMP']}/cilium-k8s-install-2nd-part.sh"
256
               cm.vm.provision "k8s-install-master-part-2",
257
                   type: "shell",
258
                   run: "always",
259
                   env: {"node_ip" => node_ip, "node_ipv6" => node_ipv6},
260
                   privileged: true,
261
                   path: k8sinstall
262
           end
263
        end
264
    end
265

266
    $num_workers.times do |n|
267
        # n starts with 0
268
        node_vm_name = "#{$vm_base_name}#{n+2}#{$build_id_name}#{$vm_kernel}"
269
        node_hostname = "#{$vm_base_name}#{n+2}"
270
        config.vm.define node_vm_name do |node|
271
            node_ip = $workers_ipv4_addrs[n]
272
            node_ipv6 = $workers_ipv6_addrs[n]
273
            node.vm.network "private_network", ip: "#{node_ip}",
274
                virtualbox__intnet: "cilium-test-#{$build_id}"
275
            nfs_ipv4_addr = $workers_ipv4_addrs_nfs[n]
276
            node.vm.network "private_network", ip: "#{nfs_ipv4_addr}", bridge: "enp0s9"
277
            # Add IPv6 address this way or we get hit by a virtualbox bug
278
            node.vm.provision "ipv6-config",
279
                type: "shell",
280
                run: "always",
281
                inline: "ip -6 a a #{node_ipv6}/16 dev enp0s9"
282

283
            # Interface for the IPv6 NAT Service. The IP address doesn't matter
284
            # as it won't be used. We use an IPv4 address as newer versions of
285
            # VBox reject all IPv6 addresses.
286
            node.vm.network "private_network",
287
                ip: "192.168.59.15"
288
            node.vm.provider "virtualbox" do |vb|
289
                vb.customize ["modifyvm", :id, "--nic4", "natnetwork"]
290
                vb.customize ["modifyvm", :id, "--nat-network4", "natnet#{n+2}"]
291
            end
292
            node.vm.provision "ipv6-nat-config",
293
                type: "shell",
294
                run: "always",
295
                inline: "ip -6 r a default via fd17:625c:f037:2::1 dev enp0s10 || true"
296

297
            if ENV["IPV6_EXT"] then
298
                node_ip = "#{node_ipv6}"
299
            end
300
            node.vm.hostname = "#{node_hostname}"
301
            if ENV['CILIUM_TEMP'] then
302
                if ENV["K8S"] then
303
                    k8sinstall = "#{ENV['CILIUM_TEMP']}/cilium-k8s-install-1st-part.sh"
304
                    node.vm.provision "k8s-install-node-part-1",
305
                        type: "shell",
306
                        run: "always",
307
                        env: {"node_ip" => node_ip, "node_ipv6" => node_ipv6},
308
                        privileged: true,
309
                        path: k8sinstall
310
                end
311
                script = "#{ENV['CILIUM_TEMP']}/node-#{n+2}.sh"
312
                node.vm.provision "config-install", type: "shell", privileged: true, run: "always", path: script
313
                if ENV["K8S"] then
314
                    k8sinstall = "#{ENV['CILIUM_TEMP']}/cilium-k8s-install-2nd-part.sh"
315
                    node.vm.provision "k8s-install-node-part-2",
316
                        type: "shell",
317
                        run: "always",
318
                        env: {"node_ip" => node_ip, "node_ipv6" => node_ipv6},
319
                        privileged: true,
320
                        path: k8sinstall
321
                end
322
            end
323
        end
324
    end
325
    cilium_dir = '.'
326
    cilium_path = '/home/vagrant/go/src/github.com/cilium/cilium'
327
    if ENV["SHARE_PARENT"] == "2" then
328
      cilium_dir = '../..'
329
      cilium_path = '/home/vagrant/go/src/github.com'
330
    elsif ENV["SHARE_PARENT"] then
331
      cilium_dir = '..'
332
      cilium_path = '/home/vagrant/go/src/github.com/cilium'
333
    end
334
    config.vm.synced_folder cilium_dir, cilium_path, type: "nfs", nfs_udp: false
335
    # Don't forget to enable this ports on your host before starting the VM
336
    # in order to have nfs working
337
    # iptables -I INPUT -s 192.168.61.0/24 -j ACCEPT"
338
    # if using nftables, in Fedora (with firewalld), use:
339
    # nft -f ./contrib/vagrant/nftables.rules
340

341
    if ENV['USER_MOUNTS'] then
342
        # Allow multiple mounts divided by commas
343
        ENV['USER_MOUNTS'].split(",").each do |mnt|
344
            # Split "<to>=<from>"
345
            user_mount = mnt.split("=", 2)
346
            # Only one element, assume a path relative to home directories in both ends
347
            if user_mount.length == 1 then
348
                user_mount_to = "/home/vagrant/" + user_mount[0]
349
                user_mount_from = "~/" + user_mount[0]
350
            else
351
                user_mount_to = user_mount[0]
352
                # Remove "~/" prefix if any.
353
                if user_mount_to.start_with?('~/') then
354
                    user_mount_to[0..1] = ''
355
                end
356
                # Add home directory prefix for non-absolute paths
357
                if !user_mount_to.start_with?('/') then
358
                    user_mount_to = "/home/vagrant/" + user_mount_to
359
                end
360
                user_mount_from = user_mount[1]
361
                # Add home prefix for host for any path in the project directory
362
                # as it is already mounted.
363
                if !user_mount_from.start_with?('/', '.', '~') then
364
                    user_mount_from = "~/" + user_mount_from
365
                end
366
            end
367
            config.vm.synced_folder "#{user_mount_from}", "#{user_mount_to}", type: "nfs", nfs_udp: false
368
        end
369
    end
370
end
371