From 0b3e78743773b0c8cc21f392dc7b2ae6574f89af Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Mon, 23 Sep 2024 11:38:09 +0900 Subject: [PATCH 1/3] gh-87: CICD added - Client-to-LoxiLB IPsec with HA in K8s --- cicd/k8s-calico-ipsec-ha/README | 19 + cicd/k8s-calico-ipsec-ha/Vagrantfile | 95 ++++ .../k8s-calico-ipsec-ha/bird_config/bird.conf | 254 +++++++++++ cicd/k8s-calico-ipsec-ha/config.sh | 63 +++ cicd/k8s-calico-ipsec-ha/configs/config | 19 + cicd/k8s-calico-ipsec-ha/configs/join.sh | 1 + .../host_ipsec_config/charon.conf | 408 ++++++++++++++++++ .../host_ipsec_config/ipsec.conf | 129 ++++++ .../host_ipsec_config/ipsec.secrets | 4 + cicd/k8s-calico-ipsec-ha/host_validation.sh | 28 ++ cicd/k8s-calico-ipsec-ha/host_validation2.sh | 49 +++ .../host_validation2_with_sctp.sh | 81 ++++ .../host_validation_with_sctp.sh | 37 ++ .../llb1_ipsec_config/charon.conf | 376 ++++++++++++++++ .../llb1_ipsec_config/ipsec.conf | 77 ++++ .../llb1_ipsec_config/ipsec.secrets | 3 + .../llb2_ipsec_config/charon.conf | 376 ++++++++++++++++ .../llb2_ipsec_config/ipsec.conf | 82 ++++ .../llb2_ipsec_config/ipsec.secrets | 3 + .../node_scripts/common.sh | 93 ++++ cicd/k8s-calico-ipsec-ha/node_scripts/host.sh | 29 ++ .../node_scripts/loxilb1.sh | 23 + .../node_scripts/loxilb2.sh | 26 ++ .../node_scripts/master.sh | 60 +++ .../node_scripts/worker.sh | 18 + cicd/k8s-calico-ipsec-ha/rmconfig.sh | 7 + cicd/k8s-calico-ipsec-ha/validation.sh | 121 ++++++ .../validation_with_sctp.sh | 121 ++++++ cicd/k8s-calico-ipsec-ha/yaml/kube-loxilb.yml | 135 ++++++ .../yaml/kubeadm-config.yaml | 69 +++ cicd/k8s-calico-ipsec-ha/yaml/loxilb-peer.yml | 75 ++++ .../k8s-calico-ipsec-ha/yaml/sctp_default.yml | 35 ++ .../k8s-calico-ipsec-ha/yaml/sctp_fullnat.yml | 35 ++ cicd/k8s-calico-ipsec-ha/yaml/settings.yaml | 45 ++ cicd/k8s-calico-ipsec-ha/yaml/tcp_default.yml | 33 ++ cicd/k8s-calico-ipsec-ha/yaml/tcp_fullnat.yml | 33 ++ cicd/k8s-calico-ipsec-ha/yaml/udp_fullnat.yml | 30 ++ 37 files changed, 3092 insertions(+) create mode 100644 cicd/k8s-calico-ipsec-ha/README create mode 100644 cicd/k8s-calico-ipsec-ha/Vagrantfile create mode 100644 cicd/k8s-calico-ipsec-ha/bird_config/bird.conf create mode 100755 cicd/k8s-calico-ipsec-ha/config.sh create mode 100644 cicd/k8s-calico-ipsec-ha/configs/config create mode 100755 cicd/k8s-calico-ipsec-ha/configs/join.sh create mode 100644 cicd/k8s-calico-ipsec-ha/host_ipsec_config/charon.conf create mode 100644 cicd/k8s-calico-ipsec-ha/host_ipsec_config/ipsec.conf create mode 100644 cicd/k8s-calico-ipsec-ha/host_ipsec_config/ipsec.secrets create mode 100755 cicd/k8s-calico-ipsec-ha/host_validation.sh create mode 100755 cicd/k8s-calico-ipsec-ha/host_validation2.sh create mode 100755 cicd/k8s-calico-ipsec-ha/host_validation2_with_sctp.sh create mode 100755 cicd/k8s-calico-ipsec-ha/host_validation_with_sctp.sh create mode 100644 cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/charon.conf create mode 100644 cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/ipsec.conf create mode 100644 cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/ipsec.secrets create mode 100644 cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/charon.conf create mode 100644 cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/ipsec.conf create mode 100644 cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/ipsec.secrets create mode 100644 cicd/k8s-calico-ipsec-ha/node_scripts/common.sh create mode 100755 cicd/k8s-calico-ipsec-ha/node_scripts/host.sh create mode 100644 cicd/k8s-calico-ipsec-ha/node_scripts/loxilb1.sh create mode 100644 cicd/k8s-calico-ipsec-ha/node_scripts/loxilb2.sh create mode 100644 cicd/k8s-calico-ipsec-ha/node_scripts/master.sh create mode 100644 cicd/k8s-calico-ipsec-ha/node_scripts/worker.sh create mode 100755 cicd/k8s-calico-ipsec-ha/rmconfig.sh create mode 100755 cicd/k8s-calico-ipsec-ha/validation.sh create mode 100755 cicd/k8s-calico-ipsec-ha/validation_with_sctp.sh create mode 100644 cicd/k8s-calico-ipsec-ha/yaml/kube-loxilb.yml create mode 100644 cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml create mode 100644 cicd/k8s-calico-ipsec-ha/yaml/loxilb-peer.yml create mode 100644 cicd/k8s-calico-ipsec-ha/yaml/sctp_default.yml create mode 100644 cicd/k8s-calico-ipsec-ha/yaml/sctp_fullnat.yml create mode 100644 cicd/k8s-calico-ipsec-ha/yaml/settings.yaml create mode 100644 cicd/k8s-calico-ipsec-ha/yaml/tcp_default.yml create mode 100644 cicd/k8s-calico-ipsec-ha/yaml/tcp_fullnat.yml create mode 100644 cicd/k8s-calico-ipsec-ha/yaml/udp_fullnat.yml diff --git a/cicd/k8s-calico-ipsec-ha/README b/cicd/k8s-calico-ipsec-ha/README new file mode 100644 index 000000000..2edcbf3e1 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/README @@ -0,0 +1,19 @@ +## Test Case Description + +This scenario will demonstrate LoxiLB with ipsec in HA mode(clustering). The setup will have 2 LoxiLB nodes, K8s(1 Master Nodes & 2 Worker Nodes) cluster with Calico CNI in ipvs mode. LoxiLB will be running as external Service LB. Workloads will be spawned in all the cluster nodes. + +Client will be connected to the LoxiLB with L3 network over IPSec tunnels. Client and LoxiLB will do eBGP peering over IPSec tunnels where Cluster nodes and LoxiLB will do iBGP. LoxiLB will advertise the Service CIDR or VirtualIP to the client and cluster nodes. + +Service CIDR will also be a Virtual IP, different from the K8s cluster network. + +In scenarios where LoxiLB runs outside of the cluster in HA mode, it is advised to create LB services in fullnat mode for ease of connectivity. + +Please follow the link for detailed explanation about similar scenario(except ipsec): https://www.loxilb.io/post/k8s-deploying-hitless-and-ha-load-balancing + +If you wish to create this scenario in your lab then install Vagrant and follow the steps below: + +1. Run ./config.sh to setup the K8s cluster, client and LoxiLB nodes + +2. Run ./validation.sh to run the TCP HA test or ./validation_with_sctp.sh to run TCP & SCTP HA Test. Test Results will be displayed at the end. + +3. Run ./rmconfig.sh to cleanup the setup. diff --git a/cicd/k8s-calico-ipsec-ha/Vagrantfile b/cicd/k8s-calico-ipsec-ha/Vagrantfile new file mode 100644 index 000000000..f2a5f620a --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/Vagrantfile @@ -0,0 +1,95 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +require "yaml" +settings = YAML.load_file "yaml/settings.yaml" + +workers = settings["nodes"]["workers"]["count"] +loxilbs = (ENV['LOXILBS'] || "2").to_i + +Vagrant.configure("2") do |config| + + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + config.vm.define "host" do |host| + host.vm.hostname = 'host1' + host.vm.box = settings["software"]["cluster"]["box"] + host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0" + host.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0" + host.vm.provision :shell, :path => "node_scripts/host.sh" + host.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 2048] + vbox.customize ["modifyvm", :id, "--cpus", 1] + end + end + + (1..loxilbs).each do |node_number| + config.vm.define "llb#{node_number}" do |loxilb| + loxilb.vm.box = settings["software"]["loxilb"]["box"]["name"] + loxilb.vm.box_version = settings["software"]["loxilb"]["box"]["version"] + loxilb.vm.hostname = "llb#{node_number}" + ip = node_number + 251 + loxilb.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0" + loxilb.vm.network :private_network, ip: "192.168.90.#{ip}", :netmask => "255.255.255.0" + loxilb.vm.provision :shell, :path => "node_scripts/loxilb#{node_number}.sh" + loxilb.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 4096] + vbox.customize ["modifyvm", :id, "--cpus", 2] + vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + end + end + end + + config.vm.define "master" do |master| + master.vm.box = settings["software"]["cluster"]["box"] + master.vm.hostname = 'master' + master.vm.network :private_network, ip: settings["network"]["control_ip"], :netmask => "255.255.255.0" + master.vm.provision "shell", + env: { + "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "), + "ENVIRONMENT" => settings["environment"], + "KUBERNETES_VERSION" => settings["software"]["kubernetes"], + "OS" => settings["software"]["os"] + }, + path: "node_scripts/common.sh" + master.vm.provision "shell", + env: { + "CALICO_VERSION" => settings["software"]["calico"], + "CONTROL_IP" => settings["network"]["control_ip"], + "POD_CIDR" => settings["network"]["pod_cidr"], + "SERVICE_CIDR" => settings["network"]["service_cidr"] + }, + path: "node_scripts/master.sh" + + master.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 4096] + vbox.customize ["modifyvm", :id, "--cpus", 2] + vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + end + end + + (1..workers).each do |node_number| + config.vm.define "worker#{node_number}" do |worker| + worker.vm.box = settings["software"]["cluster"]["box"] + worker.vm.hostname = "worker#{node_number}" + ip = node_number + 200 + worker.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0" + worker.vm.provision "shell", + env: { + "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "), + "ENVIRONMENT" => settings["environment"], + "KUBERNETES_VERSION" => settings["software"]["kubernetes"], + "OS" => settings["software"]["os"] + }, + path: "node_scripts/common.sh" + worker.vm.provision "shell", path: "node_scripts/worker.sh" + + worker.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 4096] + vbox.customize ["modifyvm", :id, "--cpus", 2] + vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + end + end + end +end diff --git a/cicd/k8s-calico-ipsec-ha/bird_config/bird.conf b/cicd/k8s-calico-ipsec-ha/bird_config/bird.conf new file mode 100644 index 000000000..c4e7ef827 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/bird_config/bird.conf @@ -0,0 +1,254 @@ +# This is a basic configuration file, which contains boilerplate options and +# some basic examples. It allows the BIRD daemon to start but will not cause +# anything else to happen. +# +# Please refer to the BIRD User's Guide documentation, which is also available +# online at http://bird.network.cz/ in HTML format, for more information on +# configuring BIRD and adding routing protocols. + +# Configure logging +#log syslog all; +log "/var/log/bird.log" { debug, trace, info, remote, warning, error, auth, fatal, bug }; + +# Set router ID. It is a unique identification of your router, usually one of +# IPv4 addresses of the router. It is recommended to configure it explicitly. +router id 192.168.90.9; + +# Turn on global debugging of all protocols (all messages or just selected classes) +# debug protocols all; +# debug protocols { events, states }; + +# Turn on internal watchdog +# watchdog warning 5 s; +# watchdog timeout 30 s; + +# You can define your own constants +# define my_asn = 65000; +# define my_addr = 198.51.100.1; + +# Tables master4 and master6 are defined by default +# ipv4 table master4; +# ipv6 table master6; + +# Define more tables, e.g. for policy routing or as MRIB +# ipv4 table mrib4; +# ipv6 table mrib6; + +# The Device protocol is not a real routing protocol. It does not generate any +# routes and it only serves as a module for getting information about network +# interfaces from the kernel. It is necessary in almost any configuration. +protocol device { +} + +# The direct protocol is not a real routing protocol. It automatically generates +# direct routes to all network interfaces. Can exist in as many instances as you +# wish if you want to populate multiple routing tables with direct routes. +protocol direct { + #disabled; # Disable by default + ipv4; # Connect to default IPv4 table + #ipv6; # ... and to default IPv6 table +} + +# The Kernel protocol is not a real routing protocol. Instead of communicating +# with other routers in the network, it performs synchronization of BIRD +# routing tables with the OS kernel. One instance per table. +protocol kernel { + ipv4 { # Connect protocol to IPv4 table by channel +# table master4; # Default IPv4 table is master4 +# import all; # Import to table, default is import all + export all; # Export to protocol. default is export none + }; +# learn; # Learn alien routes from the kernel +# kernel table 10; # Kernel table to synchronize with (default: main) + merge paths on; +} + +# Another instance for IPv6, skipping default options +protocol kernel { + ipv6 { export all; }; +} + +# Static routes (Again, there can be multiple instances, for different address +# families and to disable/enable various groups of static routes on the fly). +protocol static { + ipv4; # Again, IPv4 channel with default options + +# route 0.0.0.0/0 via 198.51.100.10; +# route 192.0.2.0/24 blackhole; +# route 10.0.0.0/8 unreachable; +# route 10.2.0.0/24 via "eth0"; +# # Static routes can be defined with optional attributes +# route 10.1.1.0/24 via 198.51.100.3 { rip_metric = 3; }; +# route 10.1.2.0/24 via 198.51.100.3 { ospf_metric1 = 100; }; +# route 10.1.3.0/24 via 198.51.100.4 { ospf_metric2 = 100; }; +} + +# Pipe protocol connects two routing tables. Beware of loops. +# protocol pipe { +# table master4; # No ipv4/ipv6 channel definition like in other protocols +# peer table mrib4; +# import all; # Direction peer table -> table +# export all; # Direction table -> peer table +# } + +# RIP example, both RIP and RIPng are supported +# protocol rip { +# ipv4 { +# # Export direct, static routes and ones from RIP itself +# import all; +# export where source ~ [ RTS_DEVICE, RTS_STATIC, RTS_RIP ]; +# }; +# interface "eth*" { +# update time 10; # Default period is 30 +# timeout time 60; # Default timeout is 180 +# authentication cryptographic; # No authentication by default +# password "hello" { algorithm hmac sha256; }; # Default is MD5 +# }; +# } + +# OSPF example, both OSPFv2 and OSPFv3 are supported +# protocol ospf v3 { +# ipv6 { +# import all; +# export where source = RTS_STATIC; +# }; +# area 0 { +# interface "eth*" { +# type broadcast; # Detected by default +# cost 10; # Interface metric +# hello 5; # Default hello perid 10 is too long +# }; +# interface "tun*" { +# type ptp; # PtP mode, avoids DR selection +# cost 100; # Interface metric +# hello 5; # Default hello perid 10 is too long +# }; +# interface "dummy0" { +# stub; # Stub interface, just propagate it +# }; +# }; +#} + +# Define simple filter as an example for BGP import filter +# See https://gitlab.labs.nic.cz/labs/bird/wikis/BGP_filtering for more examples +# filter rt_import +# { +# if bgp_path.first != 64496 then accept; +# if bgp_path.len > 64 then accept; +# if bgp_next_hop != from then accept; +# reject; +# } + +# BGP example, explicit name 'uplink1' is used instead of default 'bgp1' +# protocol bgp uplink1 { +# description "My BGP uplink"; +# local 198.51.100.1 as 65000; +# neighbor 198.51.100.10 as 64496; +# hold time 90; # Default is 240 +# password "secret"; # Password used for MD5 authentication +# +# ipv4 { # regular IPv4 unicast (1/1) +# import filter rt_import; +# export where source ~ [ RTS_STATIC, RTS_BGP ]; +# }; +# +# ipv6 { # regular IPv6 unicast (2/1) +# import filter rt_import; +# export filter { # The same as 'where' expression above +# if source ~ [ RTS_STATIC, RTS_BGP ] +# then accept; +# else reject; +# }; +# }; +# +# ipv4 multicast { # IPv4 multicast topology (1/2) +# table mrib4; # explicit IPv4 table +# import filter rt_import; +# export all; +# }; +# +# ipv6 multicast { # IPv6 multicast topology (2/2) +# table mrib6; # explicit IPv6 table +# import filter rt_import; +# export all; +# }; +#} + +# Template example. Using templates to define IBGP route reflector clients. +# template bgp rr_clients { +# local 10.0.0.1 as 65000; +# neighbor as 65000; +# rr client; +# rr cluster id 1.0.0.1; +# +# ipv4 { +# import all; +# export where source = RTS_BGP; +# }; +# +# ipv6 { +# import all; +# export where source = RTS_BGP; +# }; +# } +# +# protocol bgp client1 from rr_clients { +# neighbor 10.0.1.1; +# } +# +# protocol bgp client2 from rr_clients { +# neighbor 10.0.2.1; +# } +# +# protocol bgp client3 from rr_clients { +# neighbor 10.0.3.1; +# } +# + +protocol static my_routes1 { + ipv4; + route 30.30.30.0/24 via 77.77.100.1; +} + +protocol static my_routes2 { + ipv4; + route 30.30.30.0/24 via 77.77.101.1; +} + +filter export_my_routes1 { + if proto = "my_routes1" then { + accept; + } + reject; +} + +filter export_my_routes2 { + if proto = "my_routes2" then { + accept; + } + reject; +} + + +protocol bgp llb1 { + local as 64512; + #neighbor 192.168.90.252 as 64511; + neighbor 77.77.100.254 as 64511; + + ipv4 { + import all; + export filter export_my_routes1; + }; +} + +protocol bgp llb2 { + local as 64512; + #neighbor 192.168.90.253 as 64511; + neighbor 77.77.101.254 as 64511; + + ipv4 { + import all; + export filter export_my_routes1; + }; +} + diff --git a/cicd/k8s-calico-ipsec-ha/config.sh b/cicd/k8s-calico-ipsec-ha/config.sh new file mode 100755 index 000000000..be48bd3bc --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/config.sh @@ -0,0 +1,63 @@ +#!/bin/bash +#VMs=$(vagrant global-status | grep -i virtualbox) +#while IFS= read -a VMs; do +# read -a vm <<< "$VMs" +# cd ${vm[4]} 2>&1>/dev/null +# echo "Destroying ${vm[1]}" +# vagrant destroy -f ${vm[1]} +# cd - 2>&1>/dev/null +#done <<< "$VMs" + +vagrant up + +for((i=1; i<=60; i++)) +do + fin=1 + pods=$(vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null | grep -v "NAMESPACE") + + while IFS= read -a pods; do + read -a pod <<< "$pods" + if [[ ${pod[3]} != *"Running"* ]]; then + echo "${pod[1]} is not UP yet" + fin=0 + fi + done <<< "$pods" + if [ $fin == 1 ]; + then + break; + fi + echo "Will try after 10s" + sleep 10 +done + +sudo sysctl net.ipv4.conf.vboxnet1.arp_accept=1 + +#Create fullnat Service +vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_fullnat.yml' 2> /dev/null +vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_default.yml' 2> /dev/null +vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_fullnat.yml' 2> /dev/null +vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_default.yml' 2> /dev/null +for((i=1; i<=60; i++)) +do + fin=1 + pods=$(vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null | grep -v "NAMESPACE") + + while IFS= read -a pods; do + read -a pod <<< "$pods" + if [[ ${pod[3]} != *"Running"* ]]; then + echo "${pod[1]} is not UP yet" + fin=0 + fi + done <<< "$pods" + if [ $fin == 1 ]; + then + break; + fi + echo "Will try after 10s" + sleep 10 +done +if [[ $fin == 0 ]]; then + echo "Cluster is not ready" + exit 1 +fi +echo "Cluster is ready" diff --git a/cicd/k8s-calico-ipsec-ha/configs/config b/cicd/k8s-calico-ipsec-ha/configs/config new file mode 100644 index 000000000..c92d9659b --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/configs/config @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJU3hkSVAzQkFxYjh3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRBNU1qRXdPREE1TXpGYUZ3MHpOREE1TVRrd09ERTBNekZhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUURGVkRob2k4ZmNHcHVXQ0ZDK1QveDVhT2pHZkZmZ2h5TWt1WTM3QWY1dDBWMitxYklPY09OWnZLeWYKTm8wNUJWZUU3UjNHVEs5TkVMRFI3Y05WclNIdldXQ3k0Qnhmb3hyNjNIRS9HQzMzTU1OczFaTmlYZk41SmRTNwp2QXdCbkJ1SFhqeHpUVkd1VnJjMWpmZWoyM1dvYkh1UHVYbUtsaG4wYm93Q3oyR3JjSURwTGNlUFljTXh0ZXdkCm5ITXJ5SDkyQWVMNVNzVFdlWDYwazlDV1VUREJoM3hKSTB0NVFCRWx6elNQb0FJcUlWWkhrVXZkUm9QWjQ2Q3EKTllCZEpoYU1MdlJVRDVhU1pRMjRPSjhnblp3WW1LK29oejV2VlRvamwvUGFzMlcybStQOXRIM216dXhjK0xEaApEUFFIVXJaQlZTMCtyUzFneDNDcHVLdWFmREtmQWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJSc0NRTlJPR2xabkdxeXB0OU94Q3haQm5KRkRUQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQVJUYjllWlJmWgpsMWFmRExtTEZzbTdnM1lQU3R6bnIyVi9RS3VXNTVUa2pvaWlOUC9kMzY5Y2U4QlJWTHBXV1VOdURRRUhWS21aCjhLSnJxbXZOMzBFaDVqM05OV1MxR1liamlTTmdnNXFiTHFUL3FWb3Z1SzAvL0NSRVEyekNtT0JIaGM5VkhJdFgKS1lmZk1TWnl3czI0WGNKYno0aEdEazFpbk9tRS9xWnFQeEJFTXNpMmZmSFBXTnRoTGh5RitvTkhucU5hN3F6dwpkRmE1RlgyTVpjN25xVUQ4TW56VnJuV2ZRRTBmT3VDVEVDZUJnSjFHYU1pK1p5SmVGU2lpc1NDbjg2enAwR0lICnlrZjhJckU1UStQV1Q0dTNxbndobjN3QzNBQ3ZkRFF0cWpIYzF1SVpIZXlXdE9TQWtQM1UyZWZPSDM1c2tKalYKcWo4MFUzTjBlcEVkCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + server: https://192.168.80.250:6443 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: kubernetes-admin + name: kubernetes-admin@kubernetes +current-context: kubernetes-admin@kubernetes +kind: Config +preferences: {} +users: +- name: kubernetes-admin + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURLRENDQWhDZ0F3SUJBZ0lIRUF3aCtWNUZ6ekFOQmdrcWhraUc5dzBCQVFzRkFEQVZNUk13RVFZRFZRUUQKRXdwcmRXSmxjbTVsZEdWek1CNFhEVEkwTURreU1UQTRNRGt6TVZvWERUSTFNRGt5TVRBNE1UUXpORm93UERFZgpNQjBHQTFVRUNoTVdhM1ZpWldGa2JUcGpiSFZ6ZEdWeUxXRmtiV2x1Y3pFWk1CY0dBMVVFQXhNUWEzVmlaWEp1ClpYUmxjeTFoWkcxcGJqQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUxhbFhzMTAKbzc4WGQyT3dWTGdITHBVSTlMR2piNnhvN3ByVDRoV25kc01BRmgvUVNYTXRaYnZJRmZPeU9ERGtWTWFhYTBXagpVU3BXalo0b2luZHVGTE5CdElRaEdZUXQzTGRYbG1jeGJpZjFQKzkwRFkxOWtZbXllU05BL0lEVHRyNHV6U0FLCmt4VjVZcFM4eFk2U1pFWExkYzlEUDNHU2tBZGhaWlRFekpOZUIyckVpMzlITzJ1R2NZVDYzaktTM09TNE4vTksKSHh2aVRmRVRPWlNKbmEvZkgvWThpWUkwVlN0blN1bTVtY284ZnA5bFhaN0lRMW1NWGtoQ1NjWFNhcldpdFM2VQpRNFI2LzlUUldBd1h5cHRuOWVYWHNZS3pkb1VrMVlhdEJWMTE4MDdIeVpqU0diOFVSOWxWUjlURER1Q3k0WXZDCjg2YmIwbFEzb2xXZEJmY0NBd0VBQWFOV01GUXdEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CTUdBMVVkSlFRTU1Bb0cKQ0NzR0FRVUZCd01DTUF3R0ExVWRFd0VCL3dRQ01BQXdId1lEVlIwakJCZ3dGb0FVYkFrRFVUaHBXWnhxc3FiZgpUc1FzV1FaeVJRMHdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBQWpjcFllUnRQU01lVk9YL3BRT2RweEY3NXVICktZbnpqU1l5VzgrSE4zdG93SGszbE1MVFJZYURVdzdLU2N1OTlSVHJxTTdaYmdLcDFMRjhWNUtkZmIzeklMaDkKTytWdEdlRGZZbWxIc0FaMS9XNWpUdkJHY3dGYTR1U1l4SW9BRzYrMkQxQ2tUM3hVc2h4QXowaXA4M2xLUWpCWQpDRTF6aVYyYUI3QXZjbXJZblRTSW9KWVFQdW5GeWFWV3Q0aEppYmVYZ0pMOFVaY2ZmeUx5RHhCOENubjdwcWdrCk1Gb004cm9mWkxxR1JMd29kVGhwQWozVGxhYW02NUo3MGh3Zys5N09JVHB2d1FkblhHRmkvWlQzbElFUTlEL0QKYmJ6N3g5Q0hKUVpBallWUURYNy8raGluTnMzMHhnMnloOVFiOGhzeWZSNGNVVXA0dFgzbFhDYkVWK1k9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdHFWZXpYU2p2eGQzWTdCVXVBY3VsUWowc2FOdnJHanVtdFBpRmFkMnd3QVdIOUJKCmN5MWx1OGdWODdJNE1PUlV4cHByUmFOUktsYU5uaWlLZDI0VXMwRzBoQ0VaaEMzY3QxZVdaekZ1Si9VLzczUU4KalgyUmliSjVJMEQ4Z05PMnZpN05JQXFURlhsaWxMekZqcEprUmN0MXowTS9jWktRQjJGbGxNVE1rMTRIYXNTTApmMGM3YTRaeGhQcmVNcExjNUxnMzgwb2ZHK0pOOFJNNWxJbWRyOThmOWp5SmdqUlZLMmRLNmJtWnlqeCtuMlZkCm5zaERXWXhlU0VKSnhkSnF0YUsxTHBSRGhIci8xTkZZREJmS20yZjE1ZGV4Z3JOMmhTVFZocTBGWFhYelRzZkoKbU5JWnZ4UkgyVlZIMU1NTzRMTGhpOEx6cHR2U1ZEZWlWWjBGOXdJREFRQUJBb0lCQURlNHQvV0NjanBvVkZvcgpmNDB0VTc3UUhxYmJMOFN2dHF1eG5MWHlnejcybGNsYTZDMmxXZll5RXRCb1BOQmR1S2Rta1BlTDNjVkc1cW1yCndLS3pDTW12Yi8wbmcvdFphbzdjdjl2M3JwMnVLclZJK0tHRC81Uzh5RXhERlVYUVVWNHdOUkJqYVlBUnFrdXEKVTdNdFlqNFExbjVIQytJQVRzU1JxRVg4ekZUZ3hPV2Z2eE0zeGE2MEJybFY0UlZxWDBONGtRNVE0R2pRSDg3dApUbG9YWWZIdFU4OWlyemJJMVdmQWhLQjkyakROWnFhUWNKbmJac0phd0hyc2tZU1pxT0xWWG4zK3dDdTFHVitYClFGSkVSU1FNdjZwUGtzaE4xMmp2eFdPblkxdWhWR2xTcTRQOHY4czI1cTNmSHZ2MWdDTEpOYXFTbmV2VUNnMEIKZm9wblZuRUNnWUVBMEN4SGx1cWRSRjJIalZ5WUNzT0ViT0RHZjF6RSt1cUsvemRiWndwWnFmQVhmM1owT1gxdAptNHpYSHNpdDBCQVVkb2FqcEhyQzMwMHM0TG5YUE9jWUNBaUlLSFo4MFBwTmNadzRjd0Z3ZWgyYnZRVzhTWHZsCitkcUk1NTBETVh3b0daU25EaWZ2dk11UzBnaGlwVkI0M3BkeWxndTRqL200aHRRRU9HYUxxVzBDZ1lFQTRKdTMKcWpRalFkS3hKMVc2d2VBLzAzcEhqUjdVcE5tR2hTZG9ONldLS3VRdGVObHR1d09zcmZ4cWsxQ3Jram9ZK3pkcgp0N3A5NUlaemVyZlA5dWpHM092REwxWUdHK0NqaXJySFdTQ2VEVkYvZERod0RTVi9rUU83L0duL3g1blprTU10CjI3SythYWN2bUFTbStwRlg4RkNCcW1FZzVHYURLL0JSTFAwblVuTUNnWUJNRE1FaWl4ZGxFRjRpOEg3Qy8rWnkKK0RMTCtKSWxzR1dURXlBYkpwYlhGRlVoUlo3MEdiUmZMVGF4a2xLZFBpS2JvbGhLRUdiOHVPNzZNaXBGbjQ4Ugo4RHY5dkR1aEJMQWlIeFlvUFpCZnJFMW00cCtFb1BURk1HYnZabGJ5VHc4L09TVkdjS1NPYmpoMjdvVkxLM2pjCnFLczkwWTVkV3JkazJqT29meDNxaVFLQmdRQ2k3ckxrS0NBdlhweWVFU1dDQUorc1ZEYlZXemtjUUtQQnNkMTcKQ0gzYk1Wei9IcExvSi9rNjJUR2luZEZvS1BiNFBiTEpPK0taUlNNK01Wc09ITHd0aEdVNTBHaHNEcC8wUWh2YwpQcm9JcFVjVTB1QU4yVmJacG1EVzhUblgzSFlqK2tJbzdsZ01vbmYzQ2VEclFVWU9rUWdPeFppcnR1V09tU3o1CnVzYXNtd0tCZ0RySFc4bS9RM2RDUlEvYkpFVjVSNG9XY3JZTDFWUzQ1T2pXU1FhSXZtNEV2K3IxL3EwR0FobWkKcWUvM21sazdCMURHdWhOZERDRjJ1QThYMUd3Y2hSSzd1MCtIUytsSytWNkhHZU0vbWcrLzB2Zk5YbDJBMGRaMwpXNE1iSVp1bWhmaExCckVZWFZOR0RGMThwK3RUSDJSY0NvNDRkRkVHR2hucW4wUXFLdmNVCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== diff --git a/cicd/k8s-calico-ipsec-ha/configs/join.sh b/cicd/k8s-calico-ipsec-ha/configs/join.sh new file mode 100755 index 000000000..e9b891f48 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/configs/join.sh @@ -0,0 +1 @@ +kubeadm join 192.168.80.250:6443 --token q7uek8.tofg1q4ikqrr9dex --discovery-token-ca-cert-hash sha256:637dfc7589f4acea45a58d14ce25f7d3d538873f60b6f6ba859b44b0ddb90ccf diff --git a/cicd/k8s-calico-ipsec-ha/host_ipsec_config/charon.conf b/cicd/k8s-calico-ipsec-ha/host_ipsec_config/charon.conf new file mode 100644 index 000000000..5b55fbee6 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/host_ipsec_config/charon.conf @@ -0,0 +1,408 @@ +# Options for the charon IKE daemon. +charon { + + # Deliberately violate the IKE standard's requirement and allow the use of + # private algorithm identifiers, even if the peer implementation is unknown. + # accept_private_algs = no + + # Accept unencrypted ID and HASH payloads in IKEv1 Main Mode. + # accept_unencrypted_mainmode_messages = no + + # Maximum number of half-open IKE_SAs for a single peer IP. + # block_threshold = 5 + + # Whether Certificate Revocation Lists (CRLs) fetched via HTTP or LDAP + # should be saved under a unique file name derived from the public key of + # the Certification Authority (CA) to /etc/ipsec.d/crls (stroke) or + # /etc/swanctl/x509crl (vici), respectively. + # cache_crls = no + + # Whether relations in validated certificate chains should be cached in + # memory. + # cert_cache = yes + + # Whether to use DPD to check if the current path still works after any + # changes to interfaces/addresses. + # check_current_path = no + + # Send the Cisco FlexVPN vendor ID payload (IKEv2 only). + # cisco_flexvpn = no + + # Send Cisco Unity vendor ID payload (IKEv1 only). + # cisco_unity = no + + # Close the IKE_SA if setup of the CHILD_SA along with IKE_AUTH failed. + # close_ike_on_child_failure = no + + # Number of half-open IKE_SAs that activate the cookie mechanism. + # cookie_threshold = 10 + + # Delete CHILD_SAs right after they got successfully rekeyed (IKEv1 only). + # delete_rekeyed = no + + # Delay in seconds until inbound IPsec SAs are deleted after rekeyings + # (IKEv2 only). + # delete_rekeyed_delay = 5 + + # Use ANSI X9.42 DH exponent size or optimum size matched to cryptographic + # strength. + # dh_exponent_ansi_x9_42 = yes + + # Use RTLD_NOW with dlopen when loading plugins and IMV/IMCs to reveal + # missing symbols immediately. + # dlopen_use_rtld_now = no + + # DNS server assigned to peer via configuration payload (CP). + # dns1 = + + # DNS server assigned to peer via configuration payload (CP). + # dns2 = + + # Enable Denial of Service protection using cookies and aggressiveness + # checks. + # dos_protection = yes + + # Free objects during authentication (might conflict with plugins). + # flush_auth_cfg = no + + # Whether to follow IKEv2 redirects (RFC 5685). + # follow_redirects = yes + + # Violate RFC 5998 and use EAP-only authentication even if the peer did not + # send an EAP_ONLY_AUTHENTICATION notify during IKE_AUTH. + # force_eap_only_authentication = no + + # Maximum size (complete IP datagram size in bytes) of a sent IKE fragment + # when using proprietary IKEv1 or standardized IKEv2 fragmentation, defaults + # to 1280 (use 0 for address family specific default values, which uses a + # lower value for IPv4). If specified this limit is used for both IPv4 and + # IPv6. + # fragment_size = 1280 + + # Name of the group the daemon changes to after startup. + # group = + + # Timeout in seconds for connecting IKE_SAs (also see IKE_SA_INIT DROPPING). + # half_open_timeout = 30 + + # Enable hash and URL support. + # hash_and_url = no + + # Allow IKEv1 Aggressive Mode with pre-shared keys as responder. + # i_dont_care_about_security_and_use_aggressive_mode_psk = no + + # Whether to ignore the traffic selectors from the kernel's acquire events + # for IKEv2 connections (they are not used for IKEv1). + # ignore_acquire_ts = no + + # A space-separated list of routing tables to be excluded from route + # lookups. + # ignore_routing_tables = + + # Maximum number of IKE_SAs that can be established at the same time before + # new connection attempts are blocked. + # ikesa_limit = 0 + + # Number of exclusively locked segments in the hash table. + # ikesa_table_segments = 1 + + # Size of the IKE_SA hash table. + # ikesa_table_size = 1 + + # Whether to close IKE_SA if the only CHILD_SA closed due to inactivity. + # inactivity_close_ike = no + + # Limit new connections based on the current number of half open IKE_SAs, + # see IKE_SA_INIT DROPPING in strongswan.conf(5). + # init_limit_half_open = 0 + + # Limit new connections based on the number of queued jobs. + # init_limit_job_load = 0 + + # Causes charon daemon to ignore IKE initiation requests. + # initiator_only = no + + # Install routes into a separate routing table for established IPsec + # tunnels. + install_routes = no + + # Install virtual IP addresses. + # install_virtual_ip = yes + + # The name of the interface on which virtual IP addresses should be + # installed. + # install_virtual_ip_on = + + # Check daemon, libstrongswan and plugin integrity at startup. + # integrity_test = no + + # A comma-separated list of network interfaces that should be ignored, if + # interfaces_use is specified this option has no effect. + # interfaces_ignore = + + # A comma-separated list of network interfaces that should be used by + # charon. All other interfaces are ignored. + # interfaces_use = + + # NAT keep alive interval. + # keep_alive = 20s + + # Number of seconds the keep alive interval may be exceeded before a DPD is + # sent instead of a NAT keep alive (0 to disable). This is only useful if a + # clock is used that includes time spent suspended (e.g. CLOCK_BOOTTIME). + # keep_alive_dpd_margin = 0s + + # Plugins to load in the IKE daemon charon. + # load = + + # Determine plugins to load via each plugin's load option. + # load_modular = no + + # Initiate IKEv2 reauthentication with a make-before-break scheme. + # make_before_break = no + + # Maximum number of IKEv1 phase 2 exchanges per IKE_SA to keep state about + # and track concurrently. + # max_ikev1_exchanges = 3 + + # Maximum packet size accepted by charon. + # max_packet = 10000 + + # Enable multiple authentication exchanges (RFC 4739). + # multiple_authentication = yes + + # WINS servers assigned to peer via configuration payload (CP). + # nbns1 = + + # WINS servers assigned to peer via configuration payload (CP). + # nbns2 = + + # UDP port used locally. If set to 0 a random port will be allocated. + # port = 500 + + # UDP port used locally in case of NAT-T. If set to 0 a random port will be + # allocated. Has to be different from charon.port, otherwise a random port + # will be allocated. + # port_nat_t = 4500 + + # Whether to prefer updating SAs to the path with the best route. + # prefer_best_path = no + + # Prefer locally configured proposals for IKE/IPsec over supplied ones as + # responder (disabling this can avoid keying retries due to + # INVALID_KE_PAYLOAD notifies). + # prefer_configured_proposals = yes + + # Controls whether permanent or temporary IPv6 addresses are used as source, + # or announced as additional addresses if MOBIKE is used. + # prefer_temporary_addrs = no + + # Process RTM_NEWROUTE and RTM_DELROUTE events. + # process_route = yes + + # How RDNs in subject DNs of certificates are matched against configured + # identities (strict, reordered, or relaxed). + # rdn_matching = strict + + # Delay in ms for receiving packets, to simulate larger RTT. + # receive_delay = 0 + + # Delay request messages. + # receive_delay_request = yes + + # Delay response messages. + # receive_delay_response = yes + + # Specific IKEv2 message type to delay, 0 for any. + # receive_delay_type = 0 + + # Size of the AH/ESP replay window, in packets. + # replay_window = 32 + + # Base to use for calculating exponential back off, see IKEv2 RETRANSMISSION + # in strongswan.conf(5). + # retransmit_base = 1.8 + + # Maximum jitter in percent to apply randomly to calculated retransmission + # timeout (0 to disable). + # retransmit_jitter = 0 + + # Upper limit in seconds for calculated retransmission timeout (0 to + # disable). + # retransmit_limit = 0 + + # Timeout in seconds before sending first retransmit. + # retransmit_timeout = 4.0 + + # Number of times to retransmit a packet before giving up. + # retransmit_tries = 5 + + # Interval in seconds to use when retrying to initiate an IKE_SA (e.g. if + # DNS resolution failed), 0 to disable retries. + # retry_initiate_interval = 0 + + # Initiate CHILD_SA within existing IKE_SAs (always enabled for IKEv1). + # reuse_ikesa = yes + + # Numerical routing table to install routes to. + # routing_table = + + # Priority of the routing table. + # routing_table_prio = + + # Whether to use RSA with PSS padding instead of PKCS#1 padding by default. + # rsa_pss = no + + # Delay in ms for sending packets, to simulate larger RTT. + # send_delay = 0 + + # Delay request messages. + # send_delay_request = yes + + # Delay response messages. + # send_delay_response = yes + + # Specific IKEv2 message type to delay, 0 for any. + # send_delay_type = 0 + + # Send strongSwan vendor ID payload + # send_vendor_id = no + + # Whether to enable Signature Authentication as per RFC 7427. + # signature_authentication = yes + + # Whether to enable constraints against IKEv2 signature schemes. + # signature_authentication_constraints = yes + + # Value mixed into the local IKE SPIs after applying spi_mask. + # spi_label = 0x0000000000000000 + + # Mask applied to local IKE SPIs before mixing in spi_label (bits set will + # be replaced with spi_label). + # spi_mask = 0x0000000000000000 + + # The upper limit for SPIs requested from the kernel for IPsec SAs. + # spi_max = 0xcfffffff + + # The lower limit for SPIs requested from the kernel for IPsec SAs. + # spi_min = 0xc0000000 + + # Number of worker threads in charon. + # threads = 16 + + # Name of the user the daemon changes to after startup. + # user = + + crypto_test { + + # Benchmark crypto algorithms and order them by efficiency. + # bench = no + + # Buffer size used for crypto benchmark. + # bench_size = 1024 + + # Time in ms during which crypto algorithm performance is measured. + # bench_time = 50 + + # Test crypto algorithms during registration (requires test vectors + # provided by the test-vectors plugin). + # on_add = no + + # Test crypto algorithms on each crypto primitive instantiation. + # on_create = no + + # Strictly require at least one test vector to enable an algorithm. + # required = no + + # Whether to test RNG with TRUE quality; requires a lot of entropy. + # rng_true = no + + } + + host_resolver { + + # Maximum number of concurrent resolver threads (they are terminated if + # unused). + # max_threads = 3 + + # Minimum number of resolver threads to keep around. + # min_threads = 0 + + } + + leak_detective { + + # Includes source file names and line numbers in leak detective output. + # detailed = yes + + # Threshold in bytes for leaks to be reported (0 to report all). + # usage_threshold = 10240 + + # Threshold in number of allocations for leaks to be reported (0 to + # report all). + # usage_threshold_count = 0 + + } + + processor { + + # Section to configure the number of reserved threads per priority class + # see JOB PRIORITY MANAGEMENT in strongswan.conf(5). + priority_threads { + + } + + } + + # Section containing a list of scripts (name = path) that are executed when + # the daemon is started. + start-scripts { + + } + + # Section containing a list of scripts (name = path) that are executed when + # the daemon is terminated. + stop-scripts { + + } + + tls { + + # List of TLS encryption ciphers. + # cipher = + + # List of TLS key exchange groups. + # ke_group = + + # List of TLS key exchange methods. + # key_exchange = + + # List of TLS MAC algorithms. + # mac = + + # Whether to include CAs in a server's CertificateRequest message. + # send_certreq_authorities = yes + + # List of TLS signature schemes. + # signature = + + # List of TLS cipher suites. + # suites = + + # Maximum TLS version to negotiate. + # version_max = 1.2 + + # Minimum TLS version to negotiate. + # version_min = 1.2 + + } + + x509 { + + # Discard certificates with unsupported or unknown critical extensions. + # enforce_critical = yes + + } + +} + diff --git a/cicd/k8s-calico-ipsec-ha/host_ipsec_config/ipsec.conf b/cicd/k8s-calico-ipsec-ha/host_ipsec_config/ipsec.conf new file mode 100644 index 000000000..e4bfeba63 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/host_ipsec_config/ipsec.conf @@ -0,0 +1,129 @@ +#@ /etc/strongswan/ipsec.conf (Centos) or /etc/ipsec.conf (Ubuntu) + +# ipsec.conf - strongSwan IPsec configuration file + +# basic configuration + +config setup + charondebug="cfg 2, ike 3" +# strictcrlpolicy=yes +# uniqueids = no + +# Add connections here. + +# Sample VPN connections + +#conn sample-self-signed +# leftsubnet=10.1.0.0/16 +# leftcert=selfCert.der +# leftsendcert=never +# right=192.168.0.2 +# rightsubnet=10.2.0.0/16 +# rightcert=peerCert.der +# auto=start + +#conn sample-with-ca-cert +# leftsubnet=10.1.0.0/16 +# leftcert=myCert.pem +# right=192.168.0.2 +# rightsubnet=10.2.0.0/16 +# rightid="C=CH, O=Linux strongSwan CN=peer name" +# auto=start + + +conn host-to-llb1 + leftauth=psk + rightauth=psk + # Encryption Algorithm : aes-128-cbc + # Authentication Algorithm : sha1 + # Perfect Forward Secrecy : Diffie-Hellman Group 2 + ike=aes256-sha256-modp2048s256,aes128-sha1-modp1024! + # Lifetime : 28800 seconds + ikelifetime=28800s + # Phase 1 Negotiation Mode : main + aggressive=no + # Protocol : esp + # Encryption Algorithm : aes-128-cbc + # Authentication Algorithm : hmac-sha1-96 + # Perfect Forward Secrecy : Diffie-Hellman Group 2 + esp=aes128-sha256-modp2048s256,aes128-sha1-modp1024! + # Lifetime : 3600 seconds + lifetime=3600s + # Mode : tunnel + type=tunnel + # DPD Interval : 10 + dpddelay=10s + # DPD Retries : 3 + dpdtimeout=30s + # Tuning Parameters for AWS Virtual Private Gateway: + keyexchange=ikev2 + #keyingtries=%forever + rekey=yes + reauth=no + dpdaction=restart + closeaction=restart + #left=%defaultroute + #leftsubnet=0.0.0.0/0,::/0 + #rightsubnet=0.0.0.0/0,::/0 + leftsubnet=30.30.30.1,77.77.100.1 + rightsubnet=20.20.20.1,77.77.100.254 + #leftupdown=/etc/strongswan/ipsec-vti.sh + left=192.168.90.9 + right=192.168.90.252 + installpolicy=yes + compress=no + mobike=no + #VTI Key + mark=100 + auto=start + +conn host-to-llb2 + leftauth=psk + rightauth=psk + # Encryption Algorithm : aes-128-cbc + # Authentication Algorithm : sha1 + # Perfect Forward Secrecy : Diffie-Hellman Group 2 + ike=aes256-sha256-modp2048s256,aes128-sha1-modp1024! + # Lifetime : 28800 seconds + ikelifetime=28800s + # Phase 1 Negotiation Mode : main + aggressive=no + # Protocol : esp + # Encryption Algorithm : aes-128-cbc + # Authentication Algorithm : hmac-sha1-96 + # Perfect Forward Secrecy : Diffie-Hellman Group 2 + esp=aes128-sha256-modp2048s256,aes128-sha1-modp1024! + # Lifetime : 3600 seconds + lifetime=3600s + # Mode : tunnel + type=tunnel + # DPD Interval : 10 + dpddelay=10s + # DPD Retries : 3 + dpdtimeout=30s + # Tuning Parameters for AWS Virtual Private Gateway: + keyexchange=ikev2 + #keyingtries=%forever + rekey=yes + reauth=no + dpdaction=restart + closeaction=restart + #left=%defaultroute + #leftsubnet=0.0.0.0/0,::/0 + #rightsubnet=0.0.0.0/0,::/0 + #leftsubnet=192.168.10.175 + #rightsubnet=192.168.10.200 + #leftupdown=/etc/strongswan/ipsec-vti.sh + #left=7.7.101.1 + #right=7.7.101.254 + leftsubnet=30.30.30.1,77.77.101.1 + rightsubnet=20.20.20.1,77.77.101.254 + #leftupdown=/etc/strongswan/ipsec-vti.sh + left=192.168.90.9 + right=192.168.90.253 + installpolicy=yes + compress=no + mobike=no + #VTI Key + mark=101 + auto=start diff --git a/cicd/k8s-calico-ipsec-ha/host_ipsec_config/ipsec.secrets b/cicd/k8s-calico-ipsec-ha/host_ipsec_config/ipsec.secrets new file mode 100644 index 000000000..bf7d0f83f --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/host_ipsec_config/ipsec.secrets @@ -0,0 +1,4 @@ +#@ /etc/strongswan/ipsec.secrets (Centos) or /etc/ipsec.secrets (Ubuntu) + +192.168.90.9 192.168.90.252 : PSK "loxilb@1234!" +192.168.90.9 192.168.90.253 : PSK "loxilb@1234!" diff --git a/cicd/k8s-calico-ipsec-ha/host_validation.sh b/cicd/k8s-calico-ipsec-ha/host_validation.sh new file mode 100755 index 000000000..efd5aa638 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/host_validation.sh @@ -0,0 +1,28 @@ +#!/bin/bash +extIP=$(cat /vagrant/extIP) + +code=0 + +echo Service IP: $extIP + +numECMP=$(birdc show route | grep $extIP -A 3 | grep via | wc -l) + +birdc show route | grep $extIP -A 3 + +if [ $numECMP == "2" ]; then + echo "Host route [OK]" +else + echo "Host route [NOK]" +fi +echo -e "\n*********************************************" +echo "Testing Service" +echo "*********************************************" + +# iperf client accessing fullnat service +stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperff.out & + +# iperf client accessing default service +stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out & + +echo "iperf client started" +echo "phase-1 done" diff --git a/cicd/k8s-calico-ipsec-ha/host_validation2.sh b/cicd/k8s-calico-ipsec-ha/host_validation2.sh new file mode 100755 index 000000000..3ee6116cd --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/host_validation2.sh @@ -0,0 +1,49 @@ +#!/bin/bash +fin=0 +for((i=0;i<50;i++)) +do + echo -e "\n --- Host Client status after HA ---\n" + echo -e "\niperf fullnat" + tail -n 1 iperff.out + + echo -e "\niperf default" + tail -n 1 iperfd.out + + ifin1=$(tail -n 5 iperff.out | grep "0.0000-100" | xargs | cut -d ' ' -f 7) + ifin2=$(tail -n 5 iperff.out | grep "0.0000-100" | xargs | cut -d ' ' -f 7) + + if [[ ! -z $ifin1 ]]; then + iperfd_res=1 + echo "iperfdefault) done." + fi + if [[ ! -z $ifin2 ]]; then + iperff_res=1 + echo "iperf(fullnat) done." + fi + + if [[ $iperfd_res == 1 && $iperfd_res == 1 ]]; then + echo "iperf done." + break + fi + sleep 5 +done + + +pkill iperf +echo -e "\n\n**********************************************************\n\n" +if [[ $iperff_res == 1 ]]; then + echo -e "K8s-calico-ipsec-ha TCP\t\t(fullnat)\t[OK]" +else + echo -e "K8s-calico-ipsec-ha TCP\t\t(fullnat)\t[FAILED]" + code=1 +fi + +if [[ $iperfd_res == 1 ]]; then + echo -e "K8s-calico-ipsec-ha TCP\t\t(default\t[OK]" +else + echo -e "K8s-calico-ipsec-ha TCP\t\t(default)\t[FAILED]" + code=1 +fi +echo -e "\n\n**********************************************************" +echo $code > /vagrant/status.txt +exit $code diff --git a/cicd/k8s-calico-ipsec-ha/host_validation2_with_sctp.sh b/cicd/k8s-calico-ipsec-ha/host_validation2_with_sctp.sh new file mode 100755 index 000000000..e1ed2501d --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/host_validation2_with_sctp.sh @@ -0,0 +1,81 @@ +#!/bin/bash +fin=0 +for((i=0;i<50;i++)) +do + echo -e "\n --- Host Client status after HA ---\n" + echo -e "\niperf fullnat" + tail -n 1 iperff.out + + echo -e "\niperf default" + tail -n 1 iperfd.out + + ifin1=$(tail -n 5 iperff.out | grep "0.0000-100" | xargs | cut -d ' ' -f 7) + ifin2=$(tail -n 5 iperff.out | grep "0.0000-100" | xargs | cut -d ' ' -f 7) + + if [[ ! -z $ifin1 ]]; then + iperfd_res=1 + echo "iperfdefault) done." + fi + if [[ ! -z $ifin2 ]]; then + iperff_res=1 + echo "iperf(fullnat) done." + fi + + echo -e "\nsctp_test fullnat" + tail -n 30 sdf.out | grep "Client: Sending packets" + + echo -e "\nsctp_test default" + tail -n 30 sdd.out | grep "Client: Sending packets" + + sfin1=`tail -n 100 sdd.out | grep "Client: Sending packets.(70000/70000)"` + sfin2=`tail -n 100 sdf.out | grep "Client: Sending packets.(70000/70000)"` + if [[ ! -z $sfin1 ]]; then + sdd_res=1 + echo "sctp_test(default) done." + fi + if [[ ! -z $sfin2 ]]; then + sdf_res=1 + echo "sctp_test(fullnat) done." + fi + + if [[ $sdd_res == 1 && $sdf_res == 1 && $iperfd_res == 1 && $iperfd_res == 1 ]]; then + echo "iperf and sctp_test done." + break + fi + sleep 5 +done + + +pkill iperf +pkill sctp_test +echo -e "\n\n**********************************************************\n\n" +if [[ $iperff_res == 1 ]]; then + echo -e "K8s-calico-ipsec-ha TCP\t\t(fullnat)\t[OK]" +else + echo -e "K8s-calico-ipsec-ha TCP\t\t(fullnat)\t[FAILED]" + code=1 +fi + +if [[ $iperfd_res == 1 ]]; then + echo -e "K8s-calico-ipsec-ha TCP\t\t(default\t[OK]" +else + echo -e "K8s-calico-ipsec-ha TCP\t\t(default)\t[FAILED]" + code=1 +fi + +if [[ $sdf_res == 1 ]]; then + echo -e "K8s-calico-ipsec-ha SCTP\t(fullnat)\t[OK]" +else + echo -e "K8s-calico-ipsec-ha SCTP\t(fullnat)\t[FAILED]" + code=1 +fi + +if [[ $sdd_res == 1 ]]; then + echo -e "K8s-calico-ipsec-ha SCTP\t(default)\t[OK]" +else + echo -e "K8s-calico-ipsec-ha SCTP\t(default)\t[FAILED]" + code=1 +fi +echo -e "\n\n**********************************************************" +echo $code > /vagrant/status.txt +exit $code diff --git a/cicd/k8s-calico-ipsec-ha/host_validation_with_sctp.sh b/cicd/k8s-calico-ipsec-ha/host_validation_with_sctp.sh new file mode 100755 index 000000000..9994a5b3f --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/host_validation_with_sctp.sh @@ -0,0 +1,37 @@ +#!/bin/bash +extIP=$(cat /vagrant/extIP) + +code=0 + +echo Service IP: $extIP + +numECMP=$(birdc show route | grep $extIP -A 3 | grep via | wc -l) + +birdc show route | grep $extIP -A 3 + +if [ $numECMP == "2" ]; then + echo "Host route [OK]" +else + echo "Host route [NOK]" +fi +echo -e "\n*********************************************" +echo "Testing Service" +echo "*********************************************" + +# iperf client accessing fullnat service +stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperff.out & + +# iperf client accessing default service +stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out & + +echo "iperf client started" + +sleep 1 + +stdbuf -oL nohup sctp_test -H 30.30.30.1 -h 20.20.20.1 -p 56004 -s -m 1400 -x 70000 &> sdf.out & +stdbuf -oL nohup sctp_test -H 30.30.30.1 -h 20.20.20.1 -p 56005 -s -m 1400 -x 70000 &> sdd.out & + +echo "sctp_test client started" + +echo "phase-1 done" +exit 0 diff --git a/cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/charon.conf b/cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/charon.conf new file mode 100644 index 000000000..926ae24aa --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/charon.conf @@ -0,0 +1,376 @@ +# Options for the charon IKE daemon. +charon { + + # Accept unencrypted ID and HASH payloads in IKEv1 Main Mode. + # accept_unencrypted_mainmode_messages = no + + # Maximum number of half-open IKE_SAs for a single peer IP. + # block_threshold = 5 + + # Whether Certificate Revocation Lists (CRLs) fetched via HTTP or LDAP + # should be saved under a unique file name derived from the public key of + # the Certification Authority (CA) to /etc/ipsec.d/crls (stroke) or + # /etc/swanctl/x509crl (vici), respectively. + # cache_crls = no + + # Whether relations in validated certificate chains should be cached in + # memory. + # cert_cache = yes + + # Send Cisco Unity vendor ID payload (IKEv1 only). + # cisco_unity = no + + # Close the IKE_SA if setup of the CHILD_SA along with IKE_AUTH failed. + # close_ike_on_child_failure = no + + # Number of half-open IKE_SAs that activate the cookie mechanism. + # cookie_threshold = 10 + + # Delete CHILD_SAs right after they got successfully rekeyed (IKEv1 only). + # delete_rekeyed = no + + # Delay in seconds until inbound IPsec SAs are deleted after rekeyings + # (IKEv2 only). + # delete_rekeyed_delay = 5 + + # Use ANSI X9.42 DH exponent size or optimum size matched to cryptographic + # strength. + # dh_exponent_ansi_x9_42 = yes + + # Use RTLD_NOW with dlopen when loading plugins and IMV/IMCs to reveal + # missing symbols immediately. + # dlopen_use_rtld_now = no + + # DNS server assigned to peer via configuration payload (CP). + # dns1 = + + # DNS server assigned to peer via configuration payload (CP). + # dns2 = + + # Enable Denial of Service protection using cookies and aggressiveness + # checks. + # dos_protection = yes + + # Compliance with the errata for RFC 4753. + # ecp_x_coordinate_only = yes + + # Free objects during authentication (might conflict with plugins). + # flush_auth_cfg = no + + # Whether to follow IKEv2 redirects (RFC 5685). + # follow_redirects = yes + + # Maximum size (complete IP datagram size in bytes) of a sent IKE fragment + # when using proprietary IKEv1 or standardized IKEv2 fragmentation, defaults + # to 1280 (use 0 for address family specific default values, which uses a + # lower value for IPv4). If specified this limit is used for both IPv4 and + # IPv6. + # fragment_size = 1280 + + # Name of the group the daemon changes to after startup. + # group = + + # Timeout in seconds for connecting IKE_SAs (also see IKE_SA_INIT DROPPING). + # half_open_timeout = 30 + + # Enable hash and URL support. + # hash_and_url = no + + # Allow IKEv1 Aggressive Mode with pre-shared keys as responder. + # i_dont_care_about_security_and_use_aggressive_mode_psk = no + + # Whether to ignore the traffic selectors from the kernel's acquire events + # for IKEv2 connections (they are not used for IKEv1). + # ignore_acquire_ts = no + + # A space-separated list of routing tables to be excluded from route + # lookups. + # ignore_routing_tables = + + # Maximum number of IKE_SAs that can be established at the same time before + # new connection attempts are blocked. + # ikesa_limit = 0 + + # Number of exclusively locked segments in the hash table. + # ikesa_table_segments = 1 + + # Size of the IKE_SA hash table. + # ikesa_table_size = 1 + + # Whether to close IKE_SA if the only CHILD_SA closed due to inactivity. + # inactivity_close_ike = no + + # Limit new connections based on the current number of half open IKE_SAs, + # see IKE_SA_INIT DROPPING in strongswan.conf(5). + # init_limit_half_open = 0 + + # Limit new connections based on the number of queued jobs. + # init_limit_job_load = 0 + + # Causes charon daemon to ignore IKE initiation requests. + # initiator_only = no + + # Install routes into a separate routing table for established IPsec + # tunnels. + install_routes = no + + # Install virtual IP addresses. + install_virtual_ip = no + + # The name of the interface on which virtual IP addresses should be + # installed. + # install_virtual_ip_on = + + # Check daemon, libstrongswan and plugin integrity at startup. + # integrity_test = no + + # A comma-separated list of network interfaces that should be ignored, if + # interfaces_use is specified this option has no effect. + # interfaces_ignore = + + # A comma-separated list of network interfaces that should be used by + # charon. All other interfaces are ignored. + # interfaces_use = + + # NAT keep alive interval. + # keep_alive = 20s + + # Plugins to load in the IKE daemon charon. + # load = + + # Determine plugins to load via each plugin's load option. + # load_modular = no + + # Initiate IKEv2 reauthentication with a make-before-break scheme. + # make_before_break = no + + # Maximum number of IKEv1 phase 2 exchanges per IKE_SA to keep state about + # and track concurrently. + # max_ikev1_exchanges = 3 + + # Maximum packet size accepted by charon. + # max_packet = 10000 + + # Enable multiple authentication exchanges (RFC 4739). + # multiple_authentication = yes + + # WINS servers assigned to peer via configuration payload (CP). + # nbns1 = + + # WINS servers assigned to peer via configuration payload (CP). + # nbns2 = + + # UDP port used locally. If set to 0 a random port will be allocated. + # port = 500 + + # UDP port used locally in case of NAT-T. If set to 0 a random port will be + # allocated. Has to be different from charon.port, otherwise a random port + # will be allocated. + # port_nat_t = 4500 + + # Whether to prefer updating SAs to the path with the best route. + # prefer_best_path = no + + # Prefer locally configured proposals for IKE/IPsec over supplied ones as + # responder (disabling this can avoid keying retries due to + # INVALID_KE_PAYLOAD notifies). + # prefer_configured_proposals = yes + + # Controls whether permanent or temporary IPv6 addresses are used as source, + # or announced as additional addresses if MOBIKE is used. + # prefer_temporary_addrs = no + + # Process RTM_NEWROUTE and RTM_DELROUTE events. + # process_route = yes + + # How RDNs in subject DNs of certificates are matched against configured + # identities (strict, reordered, or relaxed). + # rdn_matching = strict + + # Delay in ms for receiving packets, to simulate larger RTT. + # receive_delay = 0 + + # Delay request messages. + # receive_delay_request = yes + + # Delay response messages. + # receive_delay_response = yes + + # Specific IKEv2 message type to delay, 0 for any. + # receive_delay_type = 0 + + # Size of the AH/ESP replay window, in packets. + # replay_window = 32 + + # Base to use for calculating exponential back off, see IKEv2 RETRANSMISSION + # in strongswan.conf(5). + # retransmit_base = 1.8 + + # Maximum jitter in percent to apply randomly to calculated retransmission + # timeout (0 to disable). + # retransmit_jitter = 0 + + # Upper limit in seconds for calculated retransmission timeout (0 to + # disable). + # retransmit_limit = 0 + + # Timeout in seconds before sending first retransmit. + # retransmit_timeout = 4.0 + + # Number of times to retransmit a packet before giving up. + # retransmit_tries = 5 + + # Interval in seconds to use when retrying to initiate an IKE_SA (e.g. if + # DNS resolution failed), 0 to disable retries. + # retry_initiate_interval = 0 + + # Initiate CHILD_SA within existing IKE_SAs (always enabled for IKEv1). + # reuse_ikesa = yes + + # Numerical routing table to install routes to. + # routing_table = + + # Priority of the routing table. + # routing_table_prio = + + # Whether to use RSA with PSS padding instead of PKCS#1 padding by default. + # rsa_pss = no + + # Delay in ms for sending packets, to simulate larger RTT. + # send_delay = 0 + + # Delay request messages. + # send_delay_request = yes + + # Delay response messages. + # send_delay_response = yes + + # Specific IKEv2 message type to delay, 0 for any. + # send_delay_type = 0 + + # Send strongSwan vendor ID payload + # send_vendor_id = no + + # Whether to enable Signature Authentication as per RFC 7427. + # signature_authentication = yes + + # Whether to enable constraints against IKEv2 signature schemes. + # signature_authentication_constraints = yes + + # Value mixed into the local IKE SPIs after applying spi_mask. + # spi_label = 0x0000000000000000 + + # Mask applied to local IKE SPIs before mixing in spi_label (bits set will + # be replaced with spi_label). + # spi_mask = 0x0000000000000000 + + # The upper limit for SPIs requested from the kernel for IPsec SAs. + # spi_max = 0xcfffffff + + # The lower limit for SPIs requested from the kernel for IPsec SAs. + # spi_min = 0xc0000000 + + # Number of worker threads in charon. + # threads = 16 + + # Name of the user the daemon changes to after startup. + # user = + + crypto_test { + + # Benchmark crypto algorithms and order them by efficiency. + # bench = no + + # Buffer size used for crypto benchmark. + # bench_size = 1024 + + # Time in ms during which crypto algorithm performance is measured. + # bench_time = 50 + + # Test crypto algorithms during registration (requires test vectors + # provided by the test-vectors plugin). + # on_add = no + + # Test crypto algorithms on each crypto primitive instantiation. + # on_create = no + + # Strictly require at least one test vector to enable an algorithm. + # required = no + + # Whether to test RNG with TRUE quality; requires a lot of entropy. + # rng_true = no + + } + + host_resolver { + + # Maximum number of concurrent resolver threads (they are terminated if + # unused). + # max_threads = 3 + + # Minimum number of resolver threads to keep around. + # min_threads = 0 + + } + + leak_detective { + + # Includes source file names and line numbers in leak detective output. + # detailed = yes + + # Threshold in bytes for leaks to be reported (0 to report all). + # usage_threshold = 10240 + + # Threshold in number of allocations for leaks to be reported (0 to + # report all). + # usage_threshold_count = 0 + + } + + processor { + + # Section to configure the number of reserved threads per priority class + # see JOB PRIORITY MANAGEMENT in strongswan.conf(5). + priority_threads { + + } + + } + + # Section containing a list of scripts (name = path) that are executed when + # the daemon is started. + start-scripts { + + } + + # Section containing a list of scripts (name = path) that are executed when + # the daemon is terminated. + stop-scripts { + + } + + tls { + + # List of TLS encryption ciphers. + # cipher = + + # List of TLS key exchange methods. + # key_exchange = + + # List of TLS MAC algorithms. + # mac = + + # List of TLS cipher suites. + # suites = + + } + + x509 { + + # Discard certificates with unsupported or unknown critical extensions. + # enforce_critical = yes + + } + +} + diff --git a/cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/ipsec.conf b/cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/ipsec.conf new file mode 100644 index 000000000..f9973bb19 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/ipsec.conf @@ -0,0 +1,77 @@ +#@ /etc/strongswan/ipsec.conf (Centos) or /etc/ipsec.conf (Ubuntu) + +# ipsec.conf - strongSwan IPsec configuration file + +# basic configuration + +config setup + charondebug="cfg 2, ike 3" +# strictcrlpolicy=yes +# uniqueids = no + +# Add connections here. + +# Sample VPN connections + +#conn sample-self-signed +# leftsubnet=10.1.0.0/16 +# leftcert=selfCert.der +# leftsendcert=never +# right=192.168.0.2 +# rightsubnet=10.2.0.0/16 +# rightcert=peerCert.der +# auto=start + +#conn sample-with-ca-cert +# leftsubnet=10.1.0.0/16 +# leftcert=myCert.pem +# right=192.168.0.2 +# rightsubnet=10.2.0.0/16 +# rightid="C=CH, O=Linux strongSwan CN=peer name" +# auto=start + +conn llb1-to-host + leftauth=psk + rightauth=psk + # Encryption Algorithm : aes-128-cbc + # Authentication Algorithm : sha1 + # Perfect Forward Secrecy : Diffie-Hellman Group 2 + ike=aes256-sha256-modp2048s256,aes128-sha1-modp1024! + # Lifetime : 28800 seconds + ikelifetime=28800s + # Phase 1 Negotiation Mode : main + aggressive=no + # Protocol : esp + # Encryption Algorithm : aes-128-cbc + # Authentication Algorithm : hmac-sha1-96 + # Perfect Forward Secrecy : Diffie-Hellman Group 2 + esp=aes128-sha256-modp2048s256,aes128-sha1-modp1024! + # Lifetime : 3600 seconds + lifetime=3600s + # Mode : tunnel + type=tunnel + # DPD Interval : 10 + dpddelay=10s + # DPD Retries : 3 + dpdtimeout=30s + # Tuning Parameters for AWS Virtual Private Gateway: + keyexchange=ikev2 + #keyingtries=%forever + rekey=yes + reauth=no + dpdaction=restart + closeaction=restart + #left=%defaultroute + #leftsubnet=0.0.0.0/0,::/0 + #rightsubnet=0.0.0.0/0,::/0 + leftsubnet=20.20.20.1,77.77.100.254 + rightsubnet=30.30.30.1,77.77.100.1 + #leftupdown=/etc/strongswan/ipsec-vti.sh + left=192.168.90.252 + right=192.168.90.9 + installpolicy=yes + compress=no + mobike=no + #VTI Key + mark=100 + auto=start diff --git a/cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/ipsec.secrets b/cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/ipsec.secrets new file mode 100644 index 000000000..5cec0326f --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/llb1_ipsec_config/ipsec.secrets @@ -0,0 +1,3 @@ +#@ /etc/strongswan/ipsec.secrets (Centos) or /etc/ipsec.secrets (Ubuntu) + +192.168.90.252 192.168.90.9 : PSK "loxilb@1234!" diff --git a/cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/charon.conf b/cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/charon.conf new file mode 100644 index 000000000..926ae24aa --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/charon.conf @@ -0,0 +1,376 @@ +# Options for the charon IKE daemon. +charon { + + # Accept unencrypted ID and HASH payloads in IKEv1 Main Mode. + # accept_unencrypted_mainmode_messages = no + + # Maximum number of half-open IKE_SAs for a single peer IP. + # block_threshold = 5 + + # Whether Certificate Revocation Lists (CRLs) fetched via HTTP or LDAP + # should be saved under a unique file name derived from the public key of + # the Certification Authority (CA) to /etc/ipsec.d/crls (stroke) or + # /etc/swanctl/x509crl (vici), respectively. + # cache_crls = no + + # Whether relations in validated certificate chains should be cached in + # memory. + # cert_cache = yes + + # Send Cisco Unity vendor ID payload (IKEv1 only). + # cisco_unity = no + + # Close the IKE_SA if setup of the CHILD_SA along with IKE_AUTH failed. + # close_ike_on_child_failure = no + + # Number of half-open IKE_SAs that activate the cookie mechanism. + # cookie_threshold = 10 + + # Delete CHILD_SAs right after they got successfully rekeyed (IKEv1 only). + # delete_rekeyed = no + + # Delay in seconds until inbound IPsec SAs are deleted after rekeyings + # (IKEv2 only). + # delete_rekeyed_delay = 5 + + # Use ANSI X9.42 DH exponent size or optimum size matched to cryptographic + # strength. + # dh_exponent_ansi_x9_42 = yes + + # Use RTLD_NOW with dlopen when loading plugins and IMV/IMCs to reveal + # missing symbols immediately. + # dlopen_use_rtld_now = no + + # DNS server assigned to peer via configuration payload (CP). + # dns1 = + + # DNS server assigned to peer via configuration payload (CP). + # dns2 = + + # Enable Denial of Service protection using cookies and aggressiveness + # checks. + # dos_protection = yes + + # Compliance with the errata for RFC 4753. + # ecp_x_coordinate_only = yes + + # Free objects during authentication (might conflict with plugins). + # flush_auth_cfg = no + + # Whether to follow IKEv2 redirects (RFC 5685). + # follow_redirects = yes + + # Maximum size (complete IP datagram size in bytes) of a sent IKE fragment + # when using proprietary IKEv1 or standardized IKEv2 fragmentation, defaults + # to 1280 (use 0 for address family specific default values, which uses a + # lower value for IPv4). If specified this limit is used for both IPv4 and + # IPv6. + # fragment_size = 1280 + + # Name of the group the daemon changes to after startup. + # group = + + # Timeout in seconds for connecting IKE_SAs (also see IKE_SA_INIT DROPPING). + # half_open_timeout = 30 + + # Enable hash and URL support. + # hash_and_url = no + + # Allow IKEv1 Aggressive Mode with pre-shared keys as responder. + # i_dont_care_about_security_and_use_aggressive_mode_psk = no + + # Whether to ignore the traffic selectors from the kernel's acquire events + # for IKEv2 connections (they are not used for IKEv1). + # ignore_acquire_ts = no + + # A space-separated list of routing tables to be excluded from route + # lookups. + # ignore_routing_tables = + + # Maximum number of IKE_SAs that can be established at the same time before + # new connection attempts are blocked. + # ikesa_limit = 0 + + # Number of exclusively locked segments in the hash table. + # ikesa_table_segments = 1 + + # Size of the IKE_SA hash table. + # ikesa_table_size = 1 + + # Whether to close IKE_SA if the only CHILD_SA closed due to inactivity. + # inactivity_close_ike = no + + # Limit new connections based on the current number of half open IKE_SAs, + # see IKE_SA_INIT DROPPING in strongswan.conf(5). + # init_limit_half_open = 0 + + # Limit new connections based on the number of queued jobs. + # init_limit_job_load = 0 + + # Causes charon daemon to ignore IKE initiation requests. + # initiator_only = no + + # Install routes into a separate routing table for established IPsec + # tunnels. + install_routes = no + + # Install virtual IP addresses. + install_virtual_ip = no + + # The name of the interface on which virtual IP addresses should be + # installed. + # install_virtual_ip_on = + + # Check daemon, libstrongswan and plugin integrity at startup. + # integrity_test = no + + # A comma-separated list of network interfaces that should be ignored, if + # interfaces_use is specified this option has no effect. + # interfaces_ignore = + + # A comma-separated list of network interfaces that should be used by + # charon. All other interfaces are ignored. + # interfaces_use = + + # NAT keep alive interval. + # keep_alive = 20s + + # Plugins to load in the IKE daemon charon. + # load = + + # Determine plugins to load via each plugin's load option. + # load_modular = no + + # Initiate IKEv2 reauthentication with a make-before-break scheme. + # make_before_break = no + + # Maximum number of IKEv1 phase 2 exchanges per IKE_SA to keep state about + # and track concurrently. + # max_ikev1_exchanges = 3 + + # Maximum packet size accepted by charon. + # max_packet = 10000 + + # Enable multiple authentication exchanges (RFC 4739). + # multiple_authentication = yes + + # WINS servers assigned to peer via configuration payload (CP). + # nbns1 = + + # WINS servers assigned to peer via configuration payload (CP). + # nbns2 = + + # UDP port used locally. If set to 0 a random port will be allocated. + # port = 500 + + # UDP port used locally in case of NAT-T. If set to 0 a random port will be + # allocated. Has to be different from charon.port, otherwise a random port + # will be allocated. + # port_nat_t = 4500 + + # Whether to prefer updating SAs to the path with the best route. + # prefer_best_path = no + + # Prefer locally configured proposals for IKE/IPsec over supplied ones as + # responder (disabling this can avoid keying retries due to + # INVALID_KE_PAYLOAD notifies). + # prefer_configured_proposals = yes + + # Controls whether permanent or temporary IPv6 addresses are used as source, + # or announced as additional addresses if MOBIKE is used. + # prefer_temporary_addrs = no + + # Process RTM_NEWROUTE and RTM_DELROUTE events. + # process_route = yes + + # How RDNs in subject DNs of certificates are matched against configured + # identities (strict, reordered, or relaxed). + # rdn_matching = strict + + # Delay in ms for receiving packets, to simulate larger RTT. + # receive_delay = 0 + + # Delay request messages. + # receive_delay_request = yes + + # Delay response messages. + # receive_delay_response = yes + + # Specific IKEv2 message type to delay, 0 for any. + # receive_delay_type = 0 + + # Size of the AH/ESP replay window, in packets. + # replay_window = 32 + + # Base to use for calculating exponential back off, see IKEv2 RETRANSMISSION + # in strongswan.conf(5). + # retransmit_base = 1.8 + + # Maximum jitter in percent to apply randomly to calculated retransmission + # timeout (0 to disable). + # retransmit_jitter = 0 + + # Upper limit in seconds for calculated retransmission timeout (0 to + # disable). + # retransmit_limit = 0 + + # Timeout in seconds before sending first retransmit. + # retransmit_timeout = 4.0 + + # Number of times to retransmit a packet before giving up. + # retransmit_tries = 5 + + # Interval in seconds to use when retrying to initiate an IKE_SA (e.g. if + # DNS resolution failed), 0 to disable retries. + # retry_initiate_interval = 0 + + # Initiate CHILD_SA within existing IKE_SAs (always enabled for IKEv1). + # reuse_ikesa = yes + + # Numerical routing table to install routes to. + # routing_table = + + # Priority of the routing table. + # routing_table_prio = + + # Whether to use RSA with PSS padding instead of PKCS#1 padding by default. + # rsa_pss = no + + # Delay in ms for sending packets, to simulate larger RTT. + # send_delay = 0 + + # Delay request messages. + # send_delay_request = yes + + # Delay response messages. + # send_delay_response = yes + + # Specific IKEv2 message type to delay, 0 for any. + # send_delay_type = 0 + + # Send strongSwan vendor ID payload + # send_vendor_id = no + + # Whether to enable Signature Authentication as per RFC 7427. + # signature_authentication = yes + + # Whether to enable constraints against IKEv2 signature schemes. + # signature_authentication_constraints = yes + + # Value mixed into the local IKE SPIs after applying spi_mask. + # spi_label = 0x0000000000000000 + + # Mask applied to local IKE SPIs before mixing in spi_label (bits set will + # be replaced with spi_label). + # spi_mask = 0x0000000000000000 + + # The upper limit for SPIs requested from the kernel for IPsec SAs. + # spi_max = 0xcfffffff + + # The lower limit for SPIs requested from the kernel for IPsec SAs. + # spi_min = 0xc0000000 + + # Number of worker threads in charon. + # threads = 16 + + # Name of the user the daemon changes to after startup. + # user = + + crypto_test { + + # Benchmark crypto algorithms and order them by efficiency. + # bench = no + + # Buffer size used for crypto benchmark. + # bench_size = 1024 + + # Time in ms during which crypto algorithm performance is measured. + # bench_time = 50 + + # Test crypto algorithms during registration (requires test vectors + # provided by the test-vectors plugin). + # on_add = no + + # Test crypto algorithms on each crypto primitive instantiation. + # on_create = no + + # Strictly require at least one test vector to enable an algorithm. + # required = no + + # Whether to test RNG with TRUE quality; requires a lot of entropy. + # rng_true = no + + } + + host_resolver { + + # Maximum number of concurrent resolver threads (they are terminated if + # unused). + # max_threads = 3 + + # Minimum number of resolver threads to keep around. + # min_threads = 0 + + } + + leak_detective { + + # Includes source file names and line numbers in leak detective output. + # detailed = yes + + # Threshold in bytes for leaks to be reported (0 to report all). + # usage_threshold = 10240 + + # Threshold in number of allocations for leaks to be reported (0 to + # report all). + # usage_threshold_count = 0 + + } + + processor { + + # Section to configure the number of reserved threads per priority class + # see JOB PRIORITY MANAGEMENT in strongswan.conf(5). + priority_threads { + + } + + } + + # Section containing a list of scripts (name = path) that are executed when + # the daemon is started. + start-scripts { + + } + + # Section containing a list of scripts (name = path) that are executed when + # the daemon is terminated. + stop-scripts { + + } + + tls { + + # List of TLS encryption ciphers. + # cipher = + + # List of TLS key exchange methods. + # key_exchange = + + # List of TLS MAC algorithms. + # mac = + + # List of TLS cipher suites. + # suites = + + } + + x509 { + + # Discard certificates with unsupported or unknown critical extensions. + # enforce_critical = yes + + } + +} + diff --git a/cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/ipsec.conf b/cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/ipsec.conf new file mode 100644 index 000000000..98b7eb4c6 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/ipsec.conf @@ -0,0 +1,82 @@ +#@ /etc/strongswan/ipsec.conf (Centos) or /etc/ipsec.conf (Ubuntu) + +# ipsec.conf - strongSwan IPsec configuration file + +# basic configuration + +config setup + charondebug="cfg 2, ike 3" +# strictcrlpolicy=yes +# uniqueids = no + +# Add connections here. + +# Sample VPN connections + +#conn sample-self-signed +# leftsubnet=10.1.0.0/16 +# leftcert=selfCert.der +# leftsendcert=never +# right=192.168.0.2 +# rightsubnet=10.2.0.0/16 +# rightcert=peerCert.der +# auto=start + +#conn sample-with-ca-cert +# leftsubnet=10.1.0.0/16 +# leftcert=myCert.pem +# right=192.168.0.2 +# rightsubnet=10.2.0.0/16 +# rightid="C=CH, O=Linux strongSwan CN=peer name" +# auto=start + +conn llb2-to-host` + leftauth=psk + rightauth=psk + # Encryption Algorithm : aes-128-cbc + # Authentication Algorithm : sha1 + # Perfect Forward Secrecy : Diffie-Hellman Group 2 + ike=aes256-sha256-modp2048s256,aes128-sha1-modp1024! + # Lifetime : 28800 seconds + ikelifetime=28800s + # Phase 1 Negotiation Mode : main + aggressive=no + # Protocol : esp + # Encryption Algorithm : aes-128-cbc + # Authentication Algorithm : hmac-sha1-96 + # Perfect Forward Secrecy : Diffie-Hellman Group 2 + esp=aes128-sha256-modp2048s256,aes128-sha1-modp1024! + # Lifetime : 3600 seconds + lifetime=3600s + # Mode : tunnel + type=tunnel + # DPD Interval : 10 + dpddelay=10s + # DPD Retries : 3 + dpdtimeout=30s + # Tuning Parameters for AWS Virtual Private Gateway: + keyexchange=ikev2 + #keyingtries=%forever + rekey=yes + reauth=no + dpdaction=restart + closeaction=restart + #left=%defaultroute + #leftsubnet=0.0.0.0/0,::/0 + #rightsubnet=0.0.0.0/0,::/0 + #leftsubnet=192.168.10.200 + #rightsubnet=192.168.10.175 + #leftupdown=/etc/strongswan/ipsec-vti.sh + #left=7.7.101.254 + #right=7.7.101.1 + leftsubnet=20.20.20.1,77.77.101.254 + rightsubnet=30.30.30.1,77.77.101.1 + #leftupdown=/etc/strongswan/ipsec-vti.sh + left=192.168.90.253 + right=192.168.90.9 + installpolicy=yes + compress=no + mobike=no + #VTI Key + mark=101 + auto=start diff --git a/cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/ipsec.secrets b/cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/ipsec.secrets new file mode 100644 index 000000000..a13a6efd2 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/llb2_ipsec_config/ipsec.secrets @@ -0,0 +1,3 @@ +#@ /etc/strongswan/ipsec.secrets (Centos) or /etc/ipsec.secrets (Ubuntu) + +192.168.90.253 192.168.90.9 : PSK "loxilb@1234!" diff --git a/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh b/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh new file mode 100644 index 000000000..c01ad688f --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# +# Common setup for all servers (Control Plane and Nodes) + +set -euxo pipefail + +# Variable Declaration + +# DNS Setting +if [ ! -d /etc/systemd/resolved.conf.d ]; then + sudo mkdir /etc/systemd/resolved.conf.d/ +fi +cat </dev/null; echo "@reboot /sbin/swapoff -a") | crontab - || true +sudo apt-get update -y +# Install CRI-O Runtime + +VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')" +CRIO_VERSION=1.27 +# Create the .conf file to load the modules at bootup +cat <> /etc/default/crio << EOF +${ENVIRONMENT} +EOF +sudo systemctl daemon-reload +sudo systemctl enable crio --now + +echo "CRI runtime installed successfully" + +sudo apt-get update +sudo apt-get install -y apt-transport-https ca-certificates curl gpg +curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg +echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list + +sudo apt-get update -y +sudo apt-get install -y kubelet kubectl kubeadm +sudo apt-get update -y +sudo apt-get install -y jq +sudo apt-get install -y ipvsadm + +local_ip="$(ip --json a s | jq -r '.[] | if .ifname == "eth1" then .addr_info[] | if .family == "inet" then .local else empty end else empty end')" +cat > /etc/default/kubelet << EOF +KUBELET_EXTRA_ARGS=--node-ip=$local_ip +${ENVIRONMENT} +EOF diff --git a/cicd/k8s-calico-ipsec-ha/node_scripts/host.sh b/cicd/k8s-calico-ipsec-ha/node_scripts/host.sh new file mode 100755 index 000000000..deaaf1438 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/node_scripts/host.sh @@ -0,0 +1,29 @@ +# Install Bird to work with k3s +sudo apt-get update +sudo apt-get -y install bird2 lksctp-tools iperf +sudo apt-get install -y iputils-ping curl vim iptables strongswan strongswan-swanctl + +sudo ip addr add 30.30.30.1/32 dev lo + +sudo ip link add vti100 type vti key 100 remote 192.168.90.252 local 192.168.90.9 +sudo ip link set vti100 up +sudo ip addr add 77.77.100.1/24 remote 77.77.100.254/24 dev vti100 +sudo sysctl -w "net.ipv4.conf.vti100.disable_policy=1" + +sudo ip link add vti101 type vti key 101 remote 192.168.90.253 local 192.168.90.9 +sudo ip link set vti101 up +sudo ip addr add 77.77.101.1/24 remote 77.77.101.254/24 dev vti101 +sudo sysctl -w "net.ipv4.conf.vti101.disable_policy=1" + +sudo cp /vagrant/host_ipsec_config/ipsec.conf /etc/ +sudo cp /vagrant/host_ipsec_config/ipsec.secrets /etc/ +sudo cp /vagrant/host_ipsec_config/charon.conf /etc/strongswan.d/ +sudo systemctl restart strongswan-starter + +sudo cp -f /vagrant/bird_config/bird.conf /etc/bird/bird.conf +if [ ! -f /var/log/bird.log ]; then + sudo touch /var/log/bird.log +fi +sudo chown bird:bird /var/log/bird.log +sudo service bird restart +echo "Host is up" diff --git a/cicd/k8s-calico-ipsec-ha/node_scripts/loxilb1.sh b/cicd/k8s-calico-ipsec-ha/node_scripts/loxilb1.sh new file mode 100644 index 000000000..d76762c31 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/node_scripts/loxilb1.sh @@ -0,0 +1,23 @@ +export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/') + +apt-get update +apt-get install -y software-properties-common +apt-get install -y iputils-ping curl vim iptables strongswan strongswan-swanctl +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce +docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest -b --cluster=192.168.80.253 --self=0 +docker cp loxilb:/usr/local/sbin/loxicmd ./ +#docker exec -dt loxilb /root/loxilb-io/loxilb/loxilb -b --cluster=192.168.80.253 --self=0 +dexec="docker exec -dt" +$dexec loxilb ip link add vti100 type vti key 100 remote 192.168.90.9 local 192.168.90.252 +$dexec loxilb ip link set vti100 up +$dexec loxilb ip addr add 77.77.100.254/24 remote 77.77.100.1/24 dev vti100 +$dexec loxilb sysctl -w "net.ipv4.conf.vti100.disable_policy=1" + +sudo cp /vagrant/llb1_ipsec_config/ipsec.conf /etc/ +sudo cp /vagrant/llb1_ipsec_config/ipsec.secrets /etc/ +sudo cp /vagrant/llb1_ipsec_config/charon.conf /etc/strongswan.d/ +sudo systemctl restart strongswan-starter + diff --git a/cicd/k8s-calico-ipsec-ha/node_scripts/loxilb2.sh b/cicd/k8s-calico-ipsec-ha/node_scripts/loxilb2.sh new file mode 100644 index 000000000..d14946669 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/node_scripts/loxilb2.sh @@ -0,0 +1,26 @@ +export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/') + +apt-get update +apt-get install -y software-properties-common +apt-get install -y iputils-ping curl vim iptables strongswan strongswan-swanctl +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce +docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest -b --cluster=192.168.80.252 --self=1 +docker cp loxilb:/usr/local/sbin/loxicmd ./ +#docker exec -dt loxilb /root/loxilb-io/loxilb/loxilb -b --cluster=192.168.80.252 --self=1 + +dexec="docker exec -dt" +$dexec loxilb ip link add vti101 type vti key 101 remote 192.168.90.9 local 192.168.90.253 +$dexec loxilb ip link set vti101 up +$dexec loxilb ip addr add 77.77.101.254/24 remote 77.77.101.1/24 dev vti101 +$dexec loxilb sysctl -w "net.ipv4.conf.vti101.disable_policy=1" + + +sudo cp /vagrant/llb2_ipsec_config/ipsec.conf /etc/ +sudo cp /vagrant/llb2_ipsec_config/ipsec.secrets /etc/ +sudo cp /vagrant/llb2_ipsec_config/charon.conf /etc/strongswan.d/ +sudo systemctl restart strongswan-starter + + diff --git a/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh b/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh new file mode 100644 index 000000000..a5cdecaac --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Setup for Control Plane (Master) servers + +set -euxo pipefail + +NODENAME=$(hostname -s) + +sudo kubeadm config images pull + +echo "Preflight Check Passed: Downloaded All Required Images" + +#sudo kubeadm init --apiserver-advertise-address=$CONTROL_IP --apiserver-cert-extra-sans=$CONTROL_IP --pod-network-cidr=$POD_CIDR --service-cidr=$SERVICE_CIDR --node-name "$NODENAME" --ignore-preflight-errors Swap +sudo kubeadm init --ignore-preflight-errors Swap --config /vagrant/yaml/kubeadm-config.yaml + +mkdir -p "$HOME"/.kube +sudo cp -i /etc/kubernetes/admin.conf "$HOME"/.kube/config +sudo chown "$(id -u)":"$(id -g)" "$HOME"/.kube/config + +# Save Configs to shared /Vagrant location + +# For Vagrant re-runs, check if there is existing configs in the location and delete it for saving new configuration. + +config_path="/vagrant/configs" + +if [ -d $config_path ]; then + rm -f $config_path/* +else + mkdir -p $config_path +fi + +cp -i /etc/kubernetes/admin.conf $config_path/config +touch $config_path/join.sh +chmod +x $config_path/join.sh + +kubeadm token create --print-join-command > $config_path/join.sh + +# Install Calico Network Plugin + +curl https://raw.githubusercontent.com/projectcalico/calico/v${CALICO_VERSION}/manifests/calico.yaml -O + +kubectl apply -f calico.yaml +kubectl patch configmap/calico-config -n kube-system --type merge \ + -p '{"data":{"veth_mtu": "8900"}}' +kubectl rollout restart daemonset calico-node -n kube-system + +sudo -i -u vagrant bash << EOF +whoami +mkdir -p /home/vagrant/.kube +sudo cp -i $config_path/config /home/vagrant/.kube/ +sudo chown 1000:1000 /home/vagrant/.kube/config +EOF + +# Install Metrics Server + +kubectl apply -f https://raw.githubusercontent.com/techiescamp/kubeadm-scripts/main/manifests/metrics-server.yaml + +# Install loxilb +kubectl apply -f /vagrant/yaml/kube-loxilb.yml +kubectl apply -f /vagrant/yaml/loxilb-peer.yml diff --git a/cicd/k8s-calico-ipsec-ha/node_scripts/worker.sh b/cicd/k8s-calico-ipsec-ha/node_scripts/worker.sh new file mode 100644 index 000000000..a5754170b --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/node_scripts/worker.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Setup for Node servers + +set -euxo pipefail + +config_path="/vagrant/configs" + +/bin/bash $config_path/join.sh -v + +sudo -i -u vagrant bash << EOF +whoami +mkdir -p /home/vagrant/.kube +sudo cp -i $config_path/config /home/vagrant/.kube/ +sudo chown 1000:1000 /home/vagrant/.kube/config +NODENAME=$(hostname -s) +kubectl label node $(hostname -s) node-role.kubernetes.io/worker=worker +EOF diff --git a/cicd/k8s-calico-ipsec-ha/rmconfig.sh b/cicd/k8s-calico-ipsec-ha/rmconfig.sh new file mode 100755 index 000000000..4c990e5e0 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/rmconfig.sh @@ -0,0 +1,7 @@ +#!/bin/bash +vagrant destroy -f worker2 +vagrant destroy -f worker1 +vagrant destroy -f master +vagrant destroy -f llb1 +vagrant destroy -f llb2 +vagrant destroy -f host diff --git a/cicd/k8s-calico-ipsec-ha/validation.sh b/cicd/k8s-calico-ipsec-ha/validation.sh new file mode 100755 index 000000000..29d6c289a --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/validation.sh @@ -0,0 +1,121 @@ +#!/bin/bash +source ../common.sh +echo k8s-calico-ipsec-ha + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +for((i=0; i<120; i++)) +do + extLB=$(vagrant ssh master -c 'kubectl get svc' 2> /dev/null | grep "tcp-lb-default") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find tcp-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + break + fi + echo "No external LB allocated" + sleep 1 +done + +## Any routing updates ?? +#sleep 30 + +echo Service IP : $extIP +echo $extIP > extIP +echo -e "\nEnd Points List" +echo "******************************************************************************" +vagrant ssh master -c 'kubectl get endpoints -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nSVC List" +echo "******************************************************************************" +vagrant ssh master -c 'kubectl get svc' 2> /dev/null +echo "******************************************************************************" +echo -e "\nPod List" +echo "******************************************************************************" +vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nLB List" +echo -e "\n---- LLB1 ----" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\n---- LLB2 ----" +vagrant ssh llb2 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\nEP List" +echo -e "\n---- LLB1 ----" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\n---- LLB2 ----" +vagrant ssh llb2 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\nTEST RESULTS" +echo "******************************************************************************" + +master="llb1" +backup="llb2" + +state=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/cistate/all' -H 'accept: application/json') + +if [[ $state == *"BACKUP"* ]]; then + master="llb2" + backup="llb1" +fi + +echo -e "\n MASTER\t: $master" +echo -e " BACKUP\t: $backup\n" + +vagrant ssh host -c 'sudo /vagrant/host_validation.sh' 2> /dev/null + +sleep 15 +echo -e "phase-2 begins..\n" + +count=1 +sync=0 +while [[ $count -le 10 ]] ; do +echo -e "\nStatus at MASTER:$master\n" +vagrant ssh $master -c "sudo ./loxicmd get ct | grep est" 2> /dev/null + +echo -e "\nStatus at BACKUP:$backup\n" +vagrant ssh $backup -c "sudo ./loxicmd get ct | grep est" 2> /dev/null + +nres1=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) +nres2=$(curl -sX 'GET' 'http://192.168.80.253:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) + +if [[ ! -z $nres1 && $nres1 != 0 && $nres1 == $nres2 ]]; then + echo -e "\nConnections sync successful!!!\n" + sync=1 + break; +fi +echo -e "\nConnections sync pending.. Let's wait a little more..\n" +count=$(( $count + 1 )) +sleep 2 +done + +if [[ $sync == 0 ]]; then + echo -e "\nConnection Sync failed\n" + vagrant ssh host -c 'sudo pkill iperf; sudo pkill sctp_test; sudo rm -rf *.out' + exit 1 +fi + +echo "Restarting MASTER:$master.." +vagrant ssh $master -c 'sudo docker restart loxilb' 2> /dev/null + +sleep 30 + +sudo rm extIP +vagrant ssh host -c 'sudo /vagrant/host_validation2.sh' 2> /dev/null +code=`cat status.txt` +rm status.txt +exit $code diff --git a/cicd/k8s-calico-ipsec-ha/validation_with_sctp.sh b/cicd/k8s-calico-ipsec-ha/validation_with_sctp.sh new file mode 100755 index 000000000..1a539c856 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/validation_with_sctp.sh @@ -0,0 +1,121 @@ +#!/bin/bash +source ../common.sh +echo k8s-calico-ipsec-ha + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +for((i=0; i<120; i++)) +do + extLB=$(vagrant ssh master -c 'kubectl get svc' 2> /dev/null | grep "tcp-lb-default") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find tcp-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + break + fi + echo "No external LB allocated" + sleep 1 +done + +## Any routing updates ?? +#sleep 30 + +echo Service IP : $extIP +echo $extIP > extIP +echo -e "\nEnd Points List" +echo "******************************************************************************" +vagrant ssh master -c 'kubectl get endpoints -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nSVC List" +echo "******************************************************************************" +vagrant ssh master -c 'kubectl get svc' 2> /dev/null +echo "******************************************************************************" +echo -e "\nPod List" +echo "******************************************************************************" +vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nLB List" +echo -e "\n---- LLB1 ----" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\n---- LLB2 ----" +vagrant ssh llb2 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\nEP List" +echo -e "\n---- LLB1 ----" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\n---- LLB2 ----" +vagrant ssh llb2 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\nTEST RESULTS" +echo "******************************************************************************" + +master="llb1" +backup="llb2" + +state=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/cistate/all' -H 'accept: application/json') + +if [[ $state == *"BACKUP"* ]]; then + master="llb2" + backup="llb1" +fi + +echo -e "\n MASTER\t: $master" +echo -e " BACKUP\t: $backup\n" + +vagrant ssh host -c 'sudo /vagrant/host_validation_with_sctp.sh' 2> /dev/null + +sleep 15 +echo -e "phase-2 begins..\n" + +count=1 +sync=0 +while [[ $count -le 20 ]] ; do +echo -e "\nStatus at MASTER:$master\n" +vagrant ssh $master -c "sudo ./loxicmd get ct | grep est" 2> /dev/null + +echo -e "\nStatus at BACKUP:$backup\n" +vagrant ssh $backup -c "sudo ./loxicmd get ct | grep est" 2> /dev/null + +nres1=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) +nres2=$(curl -sX 'GET' 'http://192.168.80.253:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) + +if [[ ! -z $nres1 && $nres1 != 0 && $nres1 == $nres2 ]]; then + echo -e "\nConnections sync successful!!!\n" + sync=1 + break; +fi +echo -e "\nConnections sync pending.. Let's wait a little more..\n" +count=$(( $count + 1 )) +sleep 2 +done + +if [[ $sync == 0 ]]; then + echo -e "\nConnection Sync failed\n" + vagrant ssh host -c 'sudo pkill iperf; sudo pkill sctp_test; sudo rm -rf *.out' + exit 1 +fi + +echo "Restarting MASTER:$master.." +vagrant ssh $master -c 'sudo docker restart loxilb' 2> /dev/null + +sleep 30 + +sudo rm extIP +vagrant ssh host -c 'sudo /vagrant/host_validation2_with_sctp.sh' 2> /dev/null +code=`cat status.txt` +rm status.txt +exit $code diff --git a/cicd/k8s-calico-ipsec-ha/yaml/kube-loxilb.yml b/cicd/k8s-calico-ipsec-ha/yaml/kube-loxilb.yml new file mode 100644 index 000000000..5cfe9f0e7 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/yaml/kube-loxilb.yml @@ -0,0 +1,135 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-loxilb + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - endpoints + - services + - namespaces + - services/status + verbs: + - get + - watch + - list + - patch + - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - watch + - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-loxilb +subjects: + - kind: ServiceAccount + name: kube-loxilb + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-loxilb + namespace: kube-system + labels: + app: loxilb +spec: + replicas: 1 + selector: + matchLabels: + app: loxilb + template: + metadata: + labels: + app: loxilb + spec: + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + terminationGracePeriodSeconds: 0 + containers: + - name: kube-loxilb + image: ghcr.io/loxilb-io/kube-loxilb:latest + imagePullPolicy: Always + command: + - /bin/kube-loxilb + args: + - --loxiURL=http://192.168.80.252:11111,http://192.168.80.253:11111 + - --cidrPools=defaultPool=20.20.20.1/32 + #- --monitor + - --setBGP=64511 + - --extBGPPeers=77.77.100.1:64512,77.77.101.1:64512 + - --setRoles=0.0.0.0 + - --listenBGPPort=1791 #Mandatory to mention if running with Calico CNI + #- --monitor + #- --setBGP + - --setLBMode=2 + #- --config=/opt/loxilb/agent/kube-loxilb.conf + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + capabilities: + add: ["NET_ADMIN", "NET_RAW"] diff --git a/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml b/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml new file mode 100644 index 000000000..245a62553 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml @@ -0,0 +1,69 @@ +apiVersion: kubeadm.k8s.io/v1beta3 +bootstrapTokens: +- groups: + - system:bootstrappers:kubeadm:default-node-token + ttl: 24h0m0s + usages: + - signing + - authentication +kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: 192.168.80.250 + bindPort: 6443 +nodeRegistration: + imagePullPolicy: IfNotPresent + name: master + taints: null +--- +apiVersion: kubeadm.k8s.io/v1beta3 +certificatesDir: /etc/kubernetes/pki +kind: ClusterConfiguration +apiServer: + timeoutForControlPlane: 4m0s + certSANs: + - 192.168.80.250 +controlPlaneEndpoint: 192.168.80.250:6443 +clusterName: kubernetes +controllerManager: {} +dns: {} +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kubernetesVersion: v1.29.2 +networking: + dnsDomain: cluster.local + podSubnet: 172.16.1.0/16 + serviceSubnet: 172.17.1.0/18 +scheduler: {} +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +bindAddress: 0.0.0.0 +clientConnection: + acceptContentTypes: "" + burst: 10 + contentType: application/vnd.kubernetes.protobuf + kubeconfig: /var/lib/kube-proxy/kubeconfig.conf + qps: 5 +clusterCIDR: "" +configSyncPeriod: 15m0s +#featureGates: "SupportIPVSProxyMode=true" +mode: ipvs +enableProfiling: false +healthzBindAddress: 0.0.0.0:10256 +hostnameOverride: "" +iptables: + masqueradeAll: false + masqueradeBit: 14 + minSyncPeriod: 0s + syncPeriod: 30s +ipvs: + excludeCIDRs: null + minSyncPeriod: 0s + scheduler: "" + syncPeriod: 30s +kind: KubeProxyConfiguration +metricsBindAddress: 127.0.0.1:10249 +nodePortAddresses: null +oomScoreAdj: -999 +portRange: "" diff --git a/cicd/k8s-calico-ipsec-ha/yaml/loxilb-peer.yml b/cicd/k8s-calico-ipsec-ha/yaml/loxilb-peer.yml new file mode 100644 index 000000000..023f04f21 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/yaml/loxilb-peer.yml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: loxilb-peer + namespace: kube-system +spec: + selector: + matchLabels: + app: loxilb-peer-app + template: + metadata: + name: loxilb-peer + labels: + app: loxilb-peer-app + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Make sure loxilb gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: "node-role.kubernetes.io/master" +# operator: DoesNotExist +# - key: "node-role.kubernetes.io/control-plane" +# operator: DoesNotExist + containers: + - name: loxilb-peer-app + image: "ghcr.io/loxilb-io/loxilb:latest" + command: [ "/root/loxilb-io/loxilb/loxilb", "--peer" ] + ports: + - containerPort: 11111 + - containerPort: 1791 + - containerPort: 50051 + securityContext: + privileged: true + capabilities: + add: + - SYS_ADMIN +--- +apiVersion: v1 +kind: Service +metadata: + name: loxilb-peer-service + namespace: kube-system +spec: + clusterIP: None + selector: + app: loxilb-peer-app + ports: + - name: loxilb-peer-app + port: 11111 + targetPort: 11111 + protocol: TCP + - name: loxilb-peer-bgp + port: 1791 + targetPort: 1791 + protocol: TCP + - name: loxilb-peer-gobgp + port: 50051 + targetPort: 50051 + protocol: TCP + + diff --git a/cicd/k8s-calico-ipsec-ha/yaml/sctp_default.yml b/cicd/k8s-calico-ipsec-ha/yaml/sctp_default.yml new file mode 100644 index 000000000..324087241 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/yaml/sctp_default.yml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Service +metadata: + name: sctp-lb-default + annotations: + loxilb.io/lbmode: "default" + loxilb.io/liveness: "yes" + loxilb.io/probetype: "ping" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: sctp-default-test + ports: + - port: 56005 + protocol: SCTP + targetPort: 9999 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: sctp-default-test + labels: + what: sctp-default-test +spec: + containers: + - name: sctp-default-test + image: loxilbio/sctp-darn:latest + imagePullPolicy: Always + #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"] + #command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"] + command: ["sctp_test","-H", "0.0.0.0","-P", "9999", "-l"] + ports: + - containerPort: 9999 diff --git a/cicd/k8s-calico-ipsec-ha/yaml/sctp_fullnat.yml b/cicd/k8s-calico-ipsec-ha/yaml/sctp_fullnat.yml new file mode 100644 index 000000000..ff69e7058 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/yaml/sctp_fullnat.yml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Service +metadata: + name: sctp-lb-fullnat + annotations: + loxilb.io/lbmode: "fullnat" + loxilb.io/liveness: "yes" + loxilb.io/probetype: "ping" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: sctp-fullnat-test + ports: + - port: 56004 + protocol: SCTP + targetPort: 9999 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: sctp-fullnat-test + labels: + what: sctp-fullnat-test +spec: + containers: + - name: sctp-fullnat-test + image: loxilbio/sctp-darn:latest + imagePullPolicy: Always + #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"] + #command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"] + command: ["sctp_test","-H", "0.0.0.0","-P", "9999", "-l"] + ports: + - containerPort: 9999 diff --git a/cicd/k8s-calico-ipsec-ha/yaml/settings.yaml b/cicd/k8s-calico-ipsec-ha/yaml/settings.yaml new file mode 100644 index 000000000..492519dce --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/yaml/settings.yaml @@ -0,0 +1,45 @@ +--- +# cluster_name is used to group the nodes in a folder within VirtualBox: +cluster_name: Kubernetes Cluster +# Uncomment to set environment variables for services such as crio and kubelet. +# For example, configure the cluster to pull images via a proxy. +# environment: | +# HTTP_PROXY=http://my-proxy:8000 +# HTTPS_PROXY=http://my-proxy:8000 +# NO_PROXY=127.0.0.1,localhost,master-node,node01,node02,node03 +# All IPs/CIDRs should be private and allowed in /etc/vbox/networks.conf. +network: + iloxilb_ip: 192.168.80.253 + oloxilb_ip: 192.168.90.253 + # Worker IPs are simply incremented from the control IP. + control_ip: 192.168.80.250 + dns_servers: + - 8.8.8.8 + - 1.1.1.1 + pod_cidr: 172.16.1.0/16 + service_cidr: 172.17.1.0/18 +nodes: + control: + cpu: 2 + memory: 4096 + workers: + count: 2 + cpu: 1 + memory: 2048 +# Mount additional shared folders from the host into each virtual machine. +# Note that the project directory is automatically mounted at /vagrant. +# shared_folders: +# - host_path: ../images +# vm_path: /vagrant/images +software: + loxilb: + box: + name: sysnet4admin/Ubuntu-k8s + version: 0.7.1 + cluster: + box: bento/ubuntu-22.04 + version: 202401.31.0 + calico: 3.26.0 + # To skip the dashboard installation, set its version to an empty value or comment it out: + kubernetes: 1.29.2 + os: xUbuntu_22.04 diff --git a/cicd/k8s-calico-ipsec-ha/yaml/tcp_default.yml b/cicd/k8s-calico-ipsec-ha/yaml/tcp_default.yml new file mode 100644 index 000000000..2742671bb --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/yaml/tcp_default.yml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Service +metadata: + name: tcp-lb-default + annotations: + loxilb.io/lbmode: "default" + #loxilb.io/liveness: "yes" + #loxilb.io/probetype: "ping" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: tcp-default-test + ports: + - port: 56003 + targetPort: 5001 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: tcp-default-test + labels: + what: tcp-default-test +spec: + containers: + - name: tcp-default-test + image: eyes852/ubuntu-iperf-test:0.5 + command: + - iperf + - "-s" + ports: + - containerPort: 5001 diff --git a/cicd/k8s-calico-ipsec-ha/yaml/tcp_fullnat.yml b/cicd/k8s-calico-ipsec-ha/yaml/tcp_fullnat.yml new file mode 100644 index 000000000..0125b69b7 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/yaml/tcp_fullnat.yml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Service +metadata: + name: tcp-lb-fullnat + annotations: + loxilb.io/liveness: "yes" + loxilb.io/lbmode: "fullnat" + loxilb.io/probetype: "ping" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: tcp-fullnat-test + ports: + - port: 56002 + targetPort: 5001 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: tcp-fullnat-test + labels: + what: tcp-fullnat-test +spec: + containers: + - name: tcp-fullnat-test + image: eyes852/ubuntu-iperf-test:0.5 + command: + - iperf + - "-s" + ports: + - containerPort: 5001 diff --git a/cicd/k8s-calico-ipsec-ha/yaml/udp_fullnat.yml b/cicd/k8s-calico-ipsec-ha/yaml/udp_fullnat.yml new file mode 100644 index 000000000..833187e73 --- /dev/null +++ b/cicd/k8s-calico-ipsec-ha/yaml/udp_fullnat.yml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + name: udp-lb-onearm + annotations: + loxilb.io/liveness: "yes" + loxilb.io/lbmode: "onearm" +spec: + loadBalancerClass: loxilb.io/loxilb + externalTrafficPolicy: Local + selector: + what: udp-onearm-test + ports: + - port: 56003 + protocol: UDP + targetPort: 33333 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: udp-onearm-test + labels: + what: udp-onearm-test +spec: + containers: + - name: udp-onearm-test + image: ghcr.io/loxilb-io/udp-echo:latest + ports: + - containerPort: 33333 From 269ede3078707918d43ca20c24066f474953e6d1 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Mon, 23 Sep 2024 11:40:51 +0900 Subject: [PATCH 2/3] gh-87: CICD added - Client-to-LoxiLB IPsec with HA in K8s --- .github/workflows/k8s-calico-ipsec-ha.yml | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/k8s-calico-ipsec-ha.yml diff --git a/.github/workflows/k8s-calico-ipsec-ha.yml b/.github/workflows/k8s-calico-ipsec-ha.yml new file mode 100644 index 000000000..24d0f9201 --- /dev/null +++ b/.github/workflows/k8s-calico-ipsec-ha.yml @@ -0,0 +1,36 @@ +name: K8s-Calico-Cluster-IPSec-HA-Sanity-CI +on: + schedule: + # Runs "At 19:00 UTC every day-of-week" + - cron: '0 19 * * *' + workflow_dispatch: + inputs: + testName: + description: 'Test Run-Name' + required: true + default: 'k8s-calico-cluster-ipsec-ha' +jobs: + test-runner: + name: k8s-calico-cluster-ipsec-ha-sanity + runs-on: [self-hosted, large] + if: github.repository == 'loxilb-io/loxilb' + && github.event.inputs.tagName == '' + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Run the test + run: | + cd cicd/k8s-calico-ipsec-ha + ./config.sh + ./validation_with_sctp.sh + cd - + + - name: Clean test-bed + if: success() || failure() + run: | + cd cicd/k8s-calico-ipsec-ha || true + ./rmconfig.sh + cd - From 4f8e288d2b2c0c8537137fbd79bcf2c383e3503d Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Tue, 24 Sep 2024 17:49:39 +0900 Subject: [PATCH 3/3] gh-87 : Updating scripts inline with PR #808 --- .../node_scripts/common.sh | 81 +++++++++++-------- .../node_scripts/master.sh | 7 +- .../yaml/kubeadm-config.yaml | 5 ++ 3 files changed, 56 insertions(+), 37 deletions(-) diff --git a/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh b/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh index c01ad688f..ded6b8b5d 100644 --- a/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh +++ b/cicd/k8s-calico-ipsec-ha/node_scripts/common.sh @@ -15,6 +15,7 @@ cat </dev/null; echo "@reboot /sbin/swapoff -a") | crontab - || true sudo apt-get update -y -# Install CRI-O Runtime -VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')" -CRIO_VERSION=1.27 -# Create the .conf file to load the modules at bootup -cat <> /etc/default/crio << EOF -${ENVIRONMENT} -EOF -sudo systemctl daemon-reload -sudo systemctl enable crio --now - -echo "CRI runtime installed successfully" - -sudo apt-get update sudo apt-get install -y apt-transport-https ca-certificates curl gpg -curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg +curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --no-tty --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list sudo apt-get update -y diff --git a/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh b/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh index a5cdecaac..d4b836aef 100644 --- a/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh +++ b/cicd/k8s-calico-ipsec-ha/node_scripts/master.sh @@ -6,10 +6,6 @@ set -euxo pipefail NODENAME=$(hostname -s) -sudo kubeadm config images pull - -echo "Preflight Check Passed: Downloaded All Required Images" - #sudo kubeadm init --apiserver-advertise-address=$CONTROL_IP --apiserver-cert-extra-sans=$CONTROL_IP --pod-network-cidr=$POD_CIDR --service-cidr=$SERVICE_CIDR --node-name "$NODENAME" --ignore-preflight-errors Swap sudo kubeadm init --ignore-preflight-errors Swap --config /vagrant/yaml/kubeadm-config.yaml @@ -33,7 +29,8 @@ cp -i /etc/kubernetes/admin.conf $config_path/config touch $config_path/join.sh chmod +x $config_path/join.sh -kubeadm token create --print-join-command > $config_path/join.sh +join_cmd=`kubeadm token create --print-join-command` +echo $join_cmd "--cri-socket /var/run/cri-dockerd.sock" > $config_path/join.sh # Install Calico Network Plugin diff --git a/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml b/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml index 245a62553..20a7207ac 100644 --- a/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml +++ b/cicd/k8s-calico-ipsec-ha/yaml/kubeadm-config.yaml @@ -14,6 +14,7 @@ nodeRegistration: imagePullPolicy: IfNotPresent name: master taints: null + criSocket: unix:///var/run/cri-dockerd.sock --- apiVersion: kubeadm.k8s.io/v1beta3 certificatesDir: /etc/kubernetes/pki @@ -67,3 +68,7 @@ metricsBindAddress: 127.0.0.1:10249 nodePortAddresses: null oomScoreAdj: -999 portRange: "" +--- +kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +cgroupDriver: systemd