diff --git a/infra/ansible/k8s_lb.yaml b/infra/ansible/k8s_lb.yaml new file mode 100644 index 00000000..0bb31d0e --- /dev/null +++ b/infra/ansible/k8s_lb.yaml @@ -0,0 +1,74 @@ +- name: Setup k8s-lb + hosts: k8s-lb + vars_files: + - k8s_lb_config.yaml + tasks: + - name: Install deps + ansible.builtin.apt: + update_cache: true + pkg: + - iptables-persistent + + - name: dummy0 interface + ansible.builtin.template: + src: ./lb_config/netplan_dummy0.yaml.j2 + dest: /etc/netplan/dummy0.yaml + mode: "640" + + - name: eth0 interface + ansible.builtin.template: + src: ./lb_config/netplan_50_cloud_init.yaml.j2 + dest: /etc/netplan/50-cloud-init.yaml + mode: "640" + + - name: Install frr + ansible.builtin.apt: + update_cache: true + pkg: + - frr + + - name: Enable ospfd + ansible.builtin.lineinfile: + path: /etc/frr/daemons + search_string: ospfd=no + line: "ospfd=yes" + + - name: Config template frr + ansible.builtin.template: + src: ./lb_config/frr.conf.j2 + dest: /etc/frr/frr.conf + + - name: Iptables rules + ansible.builtin.template: + src: ./lb_config/iptables.j2 + dest: /etc/iptables/rules.v4 + + - name: Restore iptables rules + ansible.builtin.command: + cmd: "bash -c '/sbin/iptables-restore < /etc/iptables/rules.v4 && touch /tmp/firewall_set'" + creates: /tmp/firewall_set + + - name: Restore iptables rules + ansible.builtin.command: + cmd: "bash -c 'netplan apply && touch /tmp/netplan_applied'" + creates: /tmp/netplan_applied + + - name: Restart and enable iptables service + ansible.builtin.service: + name: netfilter-persistent + state: restarted + enabled: true + + - name: Restart and enable frr service + ansible.builtin.service: + name: frr + state: restarted + enabled: true + + - name: net.ipv4.ip_forward + ansible.posix.sysctl: + name: net.ipv4.ip_forward + value: '1' + sysctl_set: true + state: present + reload: true \ No newline at end of file diff --git a/infra/ansible/k8s_lb_config.yaml b/infra/ansible/k8s_lb_config.yaml new file mode 100644 index 00000000..58d6266c --- /dev/null +++ b/infra/ansible/k8s_lb_config.yaml @@ -0,0 +1,4 @@ +LB_HOSTNAME: k8s-lb +EXTERNAL_LISTEN_IP: "something" +INTERNAL_NETWORK_RANGE: 24 +INTERNAL_NETWORK_MAC_ADDRESS: "something" \ No newline at end of file diff --git a/infra/ansible/lb_config/frr.conf.j2 b/infra/ansible/lb_config/frr.conf.j2 new file mode 100644 index 00000000..3a097d8c --- /dev/null +++ b/infra/ansible/lb_config/frr.conf.j2 @@ -0,0 +1,17 @@ +frr version 8.4.4 +frr defaults traditional +hostname {{ LB_HOSTNAME }} +log syslog informational +no ip forwarding +no ipv6 forwarding +service integrated-vtysh-config +! +interface eth0 + ip ospf cost 10 +exit +! +router ospf + network {{ INTERNAL_NETWORK_RANGE }} area 0 + network {{ EXTERNAL_LISTEN_IP }}/32 area 0 +exit +! \ No newline at end of file diff --git a/infra/ansible/lb_config/iptables.j2 b/infra/ansible/lb_config/iptables.j2 new file mode 100644 index 00000000..a3d5cac3 --- /dev/null +++ b/infra/ansible/lb_config/iptables.j2 @@ -0,0 +1,11 @@ +*filter +:INPUT ACCEPT [0:0] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [0:0] +-A INPUT -d {{ EXTERNAL_LISTEN_IP }}/32 -p tcp -m tcp --dport 80 -j ACCEPT +-A INPUT -d {{ EXTERNAL_LISTEN_IP }}/32 -p tcp -m tcp --dport 443 -j ACCEPT +-A INPUT -d {{ EXTERNAL_LISTEN_IP }}/32 -j DROP +-A FORWARD -d {{ EXTERNAL_LISTEN_IP }}/32 -p tcp -m tcp --dport 80 -j ACCEPT +-A FORWARD -d {{ EXTERNAL_LISTEN_IP }}/32 -p tcp -m tcp --dport 443 -j ACCEPT +-A FORWARD -d {{ EXTERNAL_LISTEN_IP }}/32 -j DROP +COMMIT \ No newline at end of file diff --git a/infra/ansible/lb_config/netplan_50_cloud_init.yaml.j2 b/infra/ansible/lb_config/netplan_50_cloud_init.yaml.j2 new file mode 100644 index 00000000..204eedb8 --- /dev/null +++ b/infra/ansible/lb_config/netplan_50_cloud_init.yaml.j2 @@ -0,0 +1,17 @@ +# This file is generated from information provided by the datasource. Changes +# to it will not persist across an instance reboot. To disable cloud-init's +# network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +network: + version: 2 + ethernets: + eth0: + dhcp4: false + dhcp6: false + addresses: [ "{{ inventory_hostname }}/{{ INTERNAL_NETWORK_RANGE }}" ] + nameservers: + addresses: [ "10.10.10.10", "10.10.10.11" ] + match: + macaddress: {{ INTERNAL_NETWORK_MAC_ADDRESS }} + set-name: eth0 \ No newline at end of file diff --git a/infra/ansible/lb_config/netplan_dummy0.yaml.j2 b/infra/ansible/lb_config/netplan_dummy0.yaml.j2 new file mode 100644 index 00000000..3a366539 --- /dev/null +++ b/infra/ansible/lb_config/netplan_dummy0.yaml.j2 @@ -0,0 +1,9 @@ +network: + version: 2 + renderer: networkd + ethernets: + lo: + dhcp4: no + dhcp6: no + addresses: + - {{ EXTERNAL_LISTEN_IP }}/32 \ No newline at end of file diff --git a/infra/tf/example.tfvars b/infra/tf/example.tfvars index 0c33c5a1..0b4f5038 100644 --- a/infra/tf/example.tfvars +++ b/infra/tf/example.tfvars @@ -15,4 +15,5 @@ meshdb_ips = [ "10.70.90.Z", "10.70.90.A", ] +meshdb_lb_ip = "10.70.90.B" meshdb_metallb_range = "10.70.90.80/29" diff --git a/infra/tf/lb.tf b/infra/tf/lb.tf new file mode 100644 index 00000000..298fb72d --- /dev/null +++ b/infra/tf/lb.tf @@ -0,0 +1,51 @@ +resource "proxmox_vm_qemu" "k8s-lb" { + name = "k8s-lb" + desc = "router and lb for k8s" + target_node = var.meshdb_proxmox_node + + clone = var.meshdb_proxmox_template_image + + cores = 2 + sockets = 1 + memory = 2560 + os_type = "cloud-init" + agent = 1 + cloudinit_cdrom_storage = var.meshdb_proxmox_storage_location + ciuser = "${var.meshdb_local_user}" + cipassword = "${var.meshdb_local_password}" + + scsihw = "virtio-scsi-pci" + + disks { + scsi { + scsi0 { + disk { + backup = false + size = 10 + storage = var.meshdb_proxmox_storage_location + + } + } + } + } + + network { + bridge = "vmbr0" + model = "virtio" + } + + ipconfig0 = "ip=${var.meshdb_lb_ip}/${var.meshdb_networkrange},gw=${var.meshdb_gateway}" + + ssh_user = "root" + ssh_private_key = file("${path.module}/meshdb${var.meshdb_env_name}") + + sshkeys = file("${path.module}/meshdb${var.meshdb_env_name}.pub") + + serial { + id = 0 + type = "socket" + } + + tags = "meshdb${var.meshdb_env_name}" +} + diff --git a/infra/tf/output.tf b/infra/tf/output.tf index fa983545..a66dd456 100644 --- a/infra/tf/output.tf +++ b/infra/tf/output.tf @@ -12,6 +12,13 @@ output "worker_ips" { } } +output "lb_ip" { + description = "IP address of the lb" + value = { + "0" = var.meshdb_lb_ip + } +} + #locals { # concatenated_ips = concat( # [for mgr in proxmox_vm_qemu.meshdbmgr : mgr.default_ipv4_address],